Training in progress, step 200, checkpoint

last-checkpoint/model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:95c73d234eccc64a09618013c53a92bc4a7c429ebfb1e2678f1bc84f054f04ca
+oid sha256:561331e81c6fcc3ea4bd80e90aeeb451e6195fe512d776202f16932e0c17ea6e
 size 234681136

last-checkpoint/optimizer.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:dde59450052d104d8238c08fc21017ca7929ea8c3847f36800fa1a9fca90e493
+oid sha256:7e830fa46b8c9ec45ddf2b32b75587588f6e3d46a65f46e08aad400084a9a6d5
 size 469516363

last-checkpoint/rng_state.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:1f19cd3c4a3ac433efb0482e01f74c9bb46c2bbfbf5afed45038fc450ceaa5f6
+oid sha256:9842a5fdf27ba02b590d63e6f1d65edf706e29451025042d306b57ac70d681d7
 size 14645

last-checkpoint/scheduler.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:0b3af4aaa142e4a55ebe50a8d2db120dc2b7b23b5e6bb84de2f69f7d1627803e
+oid sha256:62e2260209115d04c8ea31c913d8d38246600d352056e2ec48e197265ec64063
 size 1465

last-checkpoint/trainer_state.json

@@ -2,331 +2,175 @@
   "best_global_step": null,
   "best_metric": null,
   "best_model_checkpoint": null,
-  "epoch": 0.03636363636363636,
-  "eval_steps": 1000,
-  "global_step": 4000,
+  "epoch": 0.004,
+  "eval_steps": 100,
+  "global_step": 200,
   "is_hyper_param_search": false,
   "is_local_process_zero": true,
   "is_world_process_zero": true,
   "log_history": [
     {
-      "epoch": 0.0009090909090909091,
-      "grad_norm": 0.25,
-      "learning_rate": 0.00066,
-      "loss": 2.65932861328125,
-      "step": 100
-    },
-    {
-      "epoch": 0.0018181818181818182,
-      "grad_norm": 0.193359375,
-      "learning_rate": 0.000999999509056267,
-      "loss": 2.663287353515625,
-      "step": 200
-    },
-    {
-      "epoch": 0.0027272727272727275,
-      "grad_norm": 0.19140625,
-      "learning_rate": 0.0009999954604635122,
-      "loss": 2.6766876220703124,
-      "step": 300
-    },
-    {
-      "epoch": 0.0036363636363636364,
-      "grad_norm": 0.1630859375,
-      "learning_rate": 0.0009999873224161853,
-      "loss": 2.6875625610351563,
-      "step": 400
-    },
-    {
-      "epoch": 0.004545454545454545,
-      "grad_norm": 0.1591796875,
-      "learning_rate": 0.000999975094980847,
-      "loss": 2.6895318603515626,
-      "step": 500
-    },
-    {
-      "epoch": 0.005454545454545455,
-      "grad_norm": 0.16796875,
-      "learning_rate": 0.0009999587782575051,
-      "loss": 2.7013665771484376,
-      "step": 600
-    },
-    {
-      "epoch": 0.006363636363636364,
-      "grad_norm": 0.162109375,
-      "learning_rate": 0.0009999383723796143,
-      "loss": 2.6975772094726564,
-      "step": 700
-    },
-    {
-      "epoch": 0.007272727272727273,
-      "grad_norm": 0.17578125,
-      "learning_rate": 0.0009999138775140734,
-      "loss": 2.694542236328125,
-      "step": 800
-    },
-    {
-      "epoch": 0.008181818181818182,
-      "grad_norm": 0.181640625,
-      "learning_rate": 0.000999885293861226,
-      "loss": 2.6991891479492187,
-      "step": 900
-    },
-    {
-      "epoch": 0.00909090909090909,
-      "grad_norm": 0.1787109375,
-      "learning_rate": 0.0009998526216548572,
-      "loss": 2.6621365356445312,
-      "step": 1000
-    },
-    {
-      "epoch": 0.00909090909090909,
-      "eval_loss": 3.092919111251831,
-      "eval_runtime": 5.3241,
-      "eval_samples_per_second": 112.132,
-      "eval_steps_per_second": 28.174,
-      "step": 1000
-    },
-    {
-      "epoch": 0.01,
-      "grad_norm": 0.1591796875,
-      "learning_rate": 0.0009998158611621922,
-      "loss": 2.6619110107421875,
-      "step": 1100
-    },
-    {
-      "epoch": 0.01090909090909091,
-      "grad_norm": 0.1669921875,
-      "learning_rate": 0.0009997750126838946,
-      "loss": 2.700511779785156,
-      "step": 1200
-    },
-    {
-      "epoch": 0.011818181818181818,
-      "grad_norm": 0.1591796875,
-      "learning_rate": 0.0009997300765540635,
-      "loss": 2.689774475097656,
-      "step": 1300
-    },
-    {
-      "epoch": 0.012727272727272728,
-      "grad_norm": 0.171875,
-      "learning_rate": 0.0009996810531402306,
-      "loss": 2.69824462890625,
-      "step": 1400
-    },
-    {
-      "epoch": 0.013636363636363636,
-      "grad_norm": 0.166015625,
-      "learning_rate": 0.0009996279428433578,
-      "loss": 2.7036367797851564,
-      "step": 1500
-    },
-    {
-      "epoch": 0.014545454545454545,
-      "grad_norm": 0.1591796875,
-      "learning_rate": 0.000999570746097833,
-      "loss": 2.6719509887695314,
-      "step": 1600
-    },
-    {
-      "epoch": 0.015454545454545455,
-      "grad_norm": 0.16796875,
-      "learning_rate": 0.0009995094633714677,
-      "loss": 2.6892950439453127,
-      "step": 1700
-    },
-    {
-      "epoch": 0.016363636363636365,
-      "grad_norm": 0.23828125,
-      "learning_rate": 0.0009994440951654924,
-      "loss": 2.7064022827148437,
-      "step": 1800
+      "epoch": 0.0002,
+      "grad_norm": 2672.0,
+      "learning_rate": 6e-05,
+      "loss": 26.674310302734376,
+      "step": 10
     },
     {
-      "epoch": 0.017272727272727273,
-      "grad_norm": 0.169921875,
-      "learning_rate": 0.0009993746420145523,
-      "loss": 2.6744000244140627,
-      "step": 1900
+      "epoch": 0.0004,
+      "grad_norm": 652.0,
+      "learning_rate": 0.0001266666666666667,
+      "loss": 26.47657165527344,
+      "step": 20
     },
     {
-      "epoch": 0.01818181818181818,
-      "grad_norm": 0.1630859375,
-      "learning_rate": 0.0009993011044867038,
-      "loss": 2.69939453125,
-      "step": 2000
+      "epoch": 0.0006,
+      "grad_norm": 8256.0,
+      "learning_rate": 0.00019333333333333333,
+      "loss": 26.448770141601564,
+      "step": 30
     },
     {
-      "epoch": 0.01818181818181818,
-      "eval_loss": 3.0854060649871826,
-      "eval_runtime": 5.237,
-      "eval_samples_per_second": 113.996,
-      "eval_steps_per_second": 28.642,
-      "step": 2000
+      "epoch": 0.0008,
+      "grad_norm": 13632.0,
+      "learning_rate": 0.00026000000000000003,
+      "loss": 25.869580078125,
+      "step": 40
     },
     {
-      "epoch": 0.019090909090909092,
-      "grad_norm": 0.150390625,
-      "learning_rate": 0.000999223483183409,
-      "loss": 2.6586599731445313,
-      "step": 2100
+      "epoch": 0.001,
+      "grad_norm": 576.0,
+      "learning_rate": 0.0003266666666666667,
+      "loss": 25.283935546875,
+      "step": 50
     },
     {
-      "epoch": 0.02,
-      "grad_norm": 0.1572265625,
-      "learning_rate": 0.000999141778739531,
-      "loss": 2.689534912109375,
-      "step": 2200
+      "epoch": 0.0012,
+      "grad_norm": 212.0,
+      "learning_rate": 0.0003933333333333333,
+      "loss": 25.18727264404297,
+      "step": 60
     },
     {
-      "epoch": 0.02090909090909091,
-      "grad_norm": 0.177734375,
-      "learning_rate": 0.0009990559918233294,
-      "loss": 2.6719708251953125,
-      "step": 2300
+      "epoch": 0.0014,
+      "grad_norm": 109.5,
+      "learning_rate": 0.00046,
+      "loss": 24.912762451171876,
+      "step": 70
     },
     {
-      "epoch": 0.02181818181818182,
-      "grad_norm": 0.20703125,
-      "learning_rate": 0.0009989661231364542,
-      "loss": 2.6797967529296876,
-      "step": 2400
+      "epoch": 0.0016,
+      "grad_norm": 195.0,
+      "learning_rate": 0.0005266666666666666,
+      "loss": 24.746240234375,
+      "step": 80
     },
     {
-      "epoch": 0.022727272727272728,
-      "grad_norm": 0.173828125,
-      "learning_rate": 0.0009988721734139395,
-      "loss": 2.68374267578125,
-      "step": 2500
+      "epoch": 0.0018,
+      "grad_norm": 1032.0,
+      "learning_rate": 0.0005933333333333334,
+      "loss": 24.107341003417968,
+      "step": 90
     },
     {
-      "epoch": 0.023636363636363636,
-      "grad_norm": 0.1748046875,
-      "learning_rate": 0.0009987741434241983,
-      "loss": 2.677032775878906,
-      "step": 2600
-    },
-    {
-      "epoch": 0.024545454545454544,
-      "grad_norm": 0.2119140625,
-      "learning_rate": 0.000998672033969017,
-      "loss": 2.7009921264648438,
-      "step": 2700
-    },
-    {
-      "epoch": 0.025454545454545455,
-      "grad_norm": 0.189453125,
-      "learning_rate": 0.0009985658458835465,
-      "loss": 2.66544677734375,
-      "step": 2800
-    },
-    {
-      "epoch": 0.026363636363636363,
-      "grad_norm": 0.16796875,
-      "learning_rate": 0.0009984555800362977,
-      "loss": 2.6626568603515626,
-      "step": 2900
-    },
-    {
-      "epoch": 0.02727272727272727,
-      "grad_norm": 0.1689453125,
-      "learning_rate": 0.0009983412373291332,
-      "loss": 2.708771057128906,
-      "step": 3000
+      "epoch": 0.002,
+      "grad_norm": 135.0,
+      "learning_rate": 0.00066,
+      "loss": 23.6096923828125,
+      "step": 100
     },
     {
-      "epoch": 0.02727272727272727,
-      "eval_loss": 3.0887579917907715,
-      "eval_runtime": 5.2307,
-      "eval_samples_per_second": 114.134,
-      "eval_steps_per_second": 28.677,
-      "step": 3000
+      "epoch": 0.002,
+      "eval_loss": 3.206618070602417,
+      "eval_runtime": 4.5304,
+      "eval_samples_per_second": 65.777,
+      "eval_steps_per_second": 3.752,
+      "step": 100
     },
     {
-      "epoch": 0.028181818181818183,
-      "grad_norm": 0.169921875,
-      "learning_rate": 0.0009982228186972597,
-      "loss": 2.6861444091796876,
-      "step": 3100
+      "epoch": 0.0022,
+      "grad_norm": 55.75,
+      "learning_rate": 0.0007266666666666667,
+      "loss": 22.759126281738283,
+      "step": 110
     },
     {
-      "epoch": 0.02909090909090909,
-      "grad_norm": 0.19921875,
-      "learning_rate": 0.0009981003251092215,
-      "loss": 2.706370544433594,
-      "step": 3200
+      "epoch": 0.0024,
+      "grad_norm": 8.3125,
+      "learning_rate": 0.0007933333333333334,
+      "loss": 22.43407287597656,
+      "step": 120
     },
     {
-      "epoch": 0.03,
-      "grad_norm": 0.1708984375,
-      "learning_rate": 0.0009979737575668917,
-      "loss": 2.6617535400390624,
-      "step": 3300
+      "epoch": 0.0026,
+      "grad_norm": 2.15625,
+      "learning_rate": 0.00086,
+      "loss": 21.722286987304688,
+      "step": 130
     },
     {
-      "epoch": 0.03090909090909091,
-      "grad_norm": 0.162109375,
-      "learning_rate": 0.000997843117105464,
-      "loss": 2.673968505859375,
-      "step": 3400
+      "epoch": 0.0028,
+      "grad_norm": 1.046875,
+      "learning_rate": 0.0009266666666666667,
+      "loss": 21.480882263183595,
+      "step": 140
     },
     {
-      "epoch": 0.031818181818181815,
-      "grad_norm": 0.1533203125,
-      "learning_rate": 0.0009977084047934448,
-      "loss": 2.6613540649414062,
-      "step": 3500
+      "epoch": 0.003,
+      "grad_norm": 1.1875,
+      "learning_rate": 0.0009933333333333333,
+      "loss": 21.57379913330078,
+      "step": 150
     },
     {
-      "epoch": 0.03272727272727273,
-      "grad_norm": 0.1640625,
-      "learning_rate": 0.0009975696217326435,
-      "loss": 2.6789846801757813,
-      "step": 3600
+      "epoch": 0.0032,
+      "grad_norm": 0.86328125,
+      "learning_rate": 0.001,
+      "loss": 21.262675476074218,
+      "step": 160
     },
     {
-      "epoch": 0.03363636363636364,
-      "grad_norm": 0.1787109375,
-      "learning_rate": 0.0009974267690581644,
-      "loss": 2.664853210449219,
-      "step": 3700
+      "epoch": 0.0034,
+      "grad_norm": 0.703125,
+      "learning_rate": 0.001,
+      "loss": 21.385879516601562,
+      "step": 170
     },
     {
-      "epoch": 0.034545454545454546,
-      "grad_norm": 0.1787109375,
-      "learning_rate": 0.0009972798479383977,
-      "loss": 2.685954284667969,
-      "step": 3800
+      "epoch": 0.0036,
+      "grad_norm": 0.84765625,
+      "learning_rate": 0.001,
+      "loss": 21.12591552734375,
+      "step": 180
     },
     {
-      "epoch": 0.035454545454545454,
-      "grad_norm": 0.205078125,
-      "learning_rate": 0.0009971288595750083,
-      "loss": 2.677633056640625,
-      "step": 3900
+      "epoch": 0.0038,
+      "grad_norm": 0.78125,
+      "learning_rate": 0.001,
+      "loss": 21.397132873535156,
+      "step": 190
     },
     {
-      "epoch": 0.03636363636363636,
-      "grad_norm": 0.1630859375,
-      "learning_rate": 0.0009969738052029277,
-      "loss": 2.6985595703125,
-      "step": 4000
+      "epoch": 0.004,
+      "grad_norm": 0.67578125,
+      "learning_rate": 0.001,
+      "loss": 21.077099609375,
+      "step": 200
     },
     {
-      "epoch": 0.03636363636363636,
-      "eval_loss": 3.0944459438323975,
-      "eval_runtime": 5.3684,
-      "eval_samples_per_second": 111.206,
-      "eval_steps_per_second": 27.941,
-      "step": 4000
+      "epoch": 0.004,
+      "eval_loss": 3.0551397800445557,
+      "eval_runtime": 4.4525,
+      "eval_samples_per_second": 66.929,
+      "eval_steps_per_second": 3.818,
+      "step": 200
     }
   ],
-  "logging_steps": 100,
-  "max_steps": 110000,
+  "logging_steps": 10,
+  "max_steps": 50000,
   "num_input_tokens_seen": 0,
   "num_train_epochs": 9223372036854775807,
-  "save_steps": 4000,
+  "save_steps": 200,
   "stateful_callbacks": {
     "TrainerControl": {
       "args": {
@@ -339,8 +183,8 @@
       "attributes": {}
     }
   },
-  "total_flos": 9.9658323984384e+16,
-  "train_batch_size": 22,
+  "total_flos": 6.52309029715968e+16,
+  "train_batch_size": 18,
   "trial_name": null,
   "trial_params": null
 }

last-checkpoint/training_args.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:0e603921b5fe0b842c30ab2079e7b2171b7aa484c50ff53b17a53468fcd71ca0
-size 5201
+oid sha256:668a0e377ebad57c5d1965a2d2cc09bbdbb8e4a54e766f286eae108d1e754ff9
+size 5265