Training in progress, step 500, checkpoint

checkpoint-500/model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:95441933ea86b91163c07c3fb1a6a720f6a695ac8df596b542ad1f220d91ba5f
+oid sha256:e9c28f0c4f6247a3c58ec730f099730e09c924a51cf707246da5877add436502
 size 577789320

checkpoint-500/optimizer.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:6fc8527ef8b7c491e74272e19413e7fd3fa23f5723bc4a1986494a568c0c6a41
+oid sha256:c04f7c51bd736d71b71ad0f06c9aac8ef451e000446e2ca47b49ecd46b72ec7d
 size 1155772634

checkpoint-500/rng_state.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:89f5da95b0303a02fa2a705db6e7bc7e9855f2a5c6b1511bc20c1a5758bfb737
+oid sha256:5a01255d31a9d554eaeb669901ca26ca8d30658b5b62d4d443762559ae14946d
 size 14645

checkpoint-500/scheduler.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:758b6085dd91f6211910e2d03c608ba018e8d7921e654b306e8288b2312fb4f9
+oid sha256:a93aeea283375c9143ff39cf70921ccb23681d856312c34d5f256cee429ea4a7
 size 1465

checkpoint-500/trainer_state.json

@@ -1,6 +1,6 @@
 {
   "best_global_step": 500,
-  "best_metric": 1.2534884214401245,
+  "best_metric": 0.9119711518287659,
   "best_model_checkpoint": "/kaggle/working/tts_processed/checkpoint-500",
   "epoch": 2.4752475247524752,
   "eval_steps": 500,
@@ -11,150 +11,150 @@
   "log_history": [
     {
       "epoch": 0.12376237623762376,
-      "grad_norm": 363.43377685546875,
-      "learning_rate": 4.8e-08,
-      "loss": 10.89256103515625,
+      "grad_norm": 52.214942932128906,
+      "learning_rate": 1.2000000000000002e-06,
+      "loss": 7.82237548828125,
       "step": 25
     },
     {
       "epoch": 0.24752475247524752,
-      "grad_norm": 336.6938171386719,
-      "learning_rate": 9.8e-08,
-      "loss": 10.47541259765625,
+      "grad_norm": 26.48613739013672,
+      "learning_rate": 2.4500000000000003e-06,
+      "loss": 6.6130438232421875,
       "step": 50
     },
     {
       "epoch": 0.3712871287128713,
-      "grad_norm": 255.0070343017578,
-      "learning_rate": 1.4799999999999998e-07,
-      "loss": 10.08507568359375,
+      "grad_norm": 26.04130744934082,
+      "learning_rate": 3.7e-06,
+      "loss": 5.921455688476563,
       "step": 75
     },
     {
       "epoch": 0.49504950495049505,
-      "grad_norm": 247.54541015625,
-      "learning_rate": 1.98e-07,
-      "loss": 9.282888793945313,
+      "grad_norm": 20.060731887817383,
+      "learning_rate": 4.95e-06,
+      "loss": 5.629042358398437,
       "step": 100
     },
     {
       "epoch": 0.6188118811881188,
-      "grad_norm": 124.208740234375,
-      "learning_rate": 2.48e-07,
-      "loss": 8.717689208984375,
+      "grad_norm": 26.09154510498047,
+      "learning_rate": 6.200000000000001e-06,
+      "loss": 5.37800537109375,
       "step": 125
     },
     {
       "epoch": 0.7425742574257426,
-      "grad_norm": 194.5498809814453,
-      "learning_rate": 2.98e-07,
-      "loss": 8.212156372070313,
+      "grad_norm": 15.835660934448242,
+      "learning_rate": 7.450000000000001e-06,
+      "loss": 5.193469848632812,
       "step": 150
     },
     {
       "epoch": 0.8663366336633663,
-      "grad_norm": 143.2777557373047,
-      "learning_rate": 3.4799999999999994e-07,
-      "loss": 7.8293896484375,
+      "grad_norm": 14.546422958374023,
+      "learning_rate": 8.700000000000001e-06,
+      "loss": 4.758580017089844,
       "step": 175
     },
     {
       "epoch": 0.9900990099009901,
-      "grad_norm": 55.50076675415039,
-      "learning_rate": 3.98e-07,
-      "loss": 7.3809716796875,
+      "grad_norm": 13.828607559204102,
+      "learning_rate": 9.950000000000001e-06,
+      "loss": 4.533587646484375,
       "step": 200
     },
     {
       "epoch": 1.113861386138614,
-      "grad_norm": 69.43047332763672,
-      "learning_rate": 4.48e-07,
-      "loss": 6.937587890625,
+      "grad_norm": 20.71561622619629,
+      "learning_rate": 9.999955143892542e-06,
+      "loss": 4.4758062744140625,
       "step": 225
     },
     {
       "epoch": 1.2376237623762376,
-      "grad_norm": 49.69321060180664,
-      "learning_rate": 4.979999999999999e-07,
-      "loss": 6.8110888671875,
+      "grad_norm": 13.708377838134766,
+      "learning_rate": 9.99981302256286e-06,
+      "loss": 4.330422058105468,
       "step": 250
     },
     {
       "epoch": 1.3613861386138613,
-      "grad_norm": 26.72427749633789,
-      "learning_rate": 5.48e-07,
-      "loss": 6.522671508789062,
+      "grad_norm": 15.750677108764648,
+      "learning_rate": 9.999573560902585e-06,
+      "loss": 4.3221432495117185,
       "step": 275
     },
     {
       "epoch": 1.4851485148514851,
-      "grad_norm": 51.36345672607422,
-      "learning_rate": 5.979999999999999e-07,
-      "loss": 6.398966064453125,
+      "grad_norm": 22.981563568115234,
+      "learning_rate": 9.999236763573743e-06,
+      "loss": 4.196664428710937,
       "step": 300
     },
     {
       "epoch": 1.608910891089109,
-      "grad_norm": 38.85439682006836,
-      "learning_rate": 6.48e-07,
-      "loss": 6.198662109375,
+      "grad_norm": 12.653732299804688,
+      "learning_rate": 9.998802637133376e-06,
+      "loss": 4.263923950195313,
       "step": 325
     },
     {
       "epoch": 1.7326732673267327,
-      "grad_norm": 27.508665084838867,
-      "learning_rate": 6.979999999999999e-07,
-      "loss": 6.075740356445312,
+      "grad_norm": 16.097843170166016,
+      "learning_rate": 9.998271190033401e-06,
+      "loss": 4.140713500976562,
       "step": 350
     },
     {
       "epoch": 1.8564356435643563,
-      "grad_norm": 36.8642463684082,
-      "learning_rate": 7.48e-07,
-      "loss": 6.0834722900390625,
+      "grad_norm": 15.19772720336914,
+      "learning_rate": 9.997642432620452e-06,
+      "loss": 4.2155322265625,
       "step": 375
     },
     {
       "epoch": 1.9801980198019802,
-      "grad_norm": 27.07192039489746,
-      "learning_rate": 7.98e-07,
-      "loss": 5.9166015625,
+      "grad_norm": 13.829080581665039,
+      "learning_rate": 9.996916377135675e-06,
+      "loss": 4.071525268554687,
       "step": 400
     },
     {
       "epoch": 2.103960396039604,
-      "grad_norm": 22.465463638305664,
-      "learning_rate": 8.48e-07,
-      "loss": 5.795056762695313,
+      "grad_norm": 12.264175415039062,
+      "learning_rate": 9.996093037714498e-06,
+      "loss": 4.0506112670898435,
       "step": 425
     },
     {
       "epoch": 2.227722772277228,
-      "grad_norm": 34.86723709106445,
-      "learning_rate": 8.98e-07,
-      "loss": 5.741965942382812,
+      "grad_norm": 12.264559745788574,
+      "learning_rate": 9.995172430386348e-06,
+      "loss": 4.0823095703125,
       "step": 450
     },
     {
       "epoch": 2.3514851485148514,
-      "grad_norm": 22.37932777404785,
-      "learning_rate": 9.479999999999999e-07,
-      "loss": 5.716455688476563,
+      "grad_norm": 13.590176582336426,
+      "learning_rate": 9.994154573074336e-06,
+      "loss": 4.028086853027344,
       "step": 475
     },
     {
       "epoch": 2.4752475247524752,
-      "grad_norm": 29.54213523864746,
-      "learning_rate": 9.98e-07,
-      "loss": 5.599214477539062,
+      "grad_norm": 17.2532901763916,
+      "learning_rate": 9.99303948559492e-06,
+      "loss": 4.071697387695313,
       "step": 500
     },
     {
       "epoch": 2.4752475247524752,
-      "eval_loss": 1.2534884214401245,
-      "eval_runtime": 82.4141,
-      "eval_samples_per_second": 19.608,
-      "eval_steps_per_second": 4.902,
+      "eval_loss": 0.9119711518287659,
+      "eval_runtime": 78.5598,
+      "eval_samples_per_second": 20.57,
+      "eval_steps_per_second": 5.143,
       "step": 500
     }
   ],
@@ -184,7 +184,7 @@
       "attributes": {}
     }
   },
-  "total_flos": 2146045761867744.0,
+  "total_flos": 2157149086640064.0,
   "train_batch_size": 8,
   "trial_name": null,
   "trial_params": null

checkpoint-500/training_args.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:5d34645042a62af83117180e7d8c3a51dc32127e71ed6c053e7766610460e0b1
+oid sha256:58f25dd2c08c3dff6bb26ea632a1c1ce37ce33af2257addb4718aa5dab19b66e
 size 5393