Training in progress, step 400, checkpoint

last-checkpoint/model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:561331e81c6fcc3ea4bd80e90aeeb451e6195fe512d776202f16932e0c17ea6e
+oid sha256:f421426d14674c7d4ad546e19a4e57083d7bca462ab2e13d58cc41736dd8c7fe
 size 234681136

last-checkpoint/optimizer.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:7e830fa46b8c9ec45ddf2b32b75587588f6e3d46a65f46e08aad400084a9a6d5
+oid sha256:6ccea549a034e18c4f5ecfe1285a4528b857a0949f057eddfb3c088844cb9c36
 size 469516363

last-checkpoint/rng_state.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:9842a5fdf27ba02b590d63e6f1d65edf706e29451025042d306b57ac70d681d7
+oid sha256:ca0548a1b90eaf6590b9f6afac3530aaed1157198c5a087c4229ad15001933d9
 size 14645

last-checkpoint/scheduler.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:62e2260209115d04c8ea31c913d8d38246600d352056e2ec48e197265ec64063
+oid sha256:2d251671b590d29a4724daa893ce2020a6083e1de3b017ea6154c342d8e856b9
 size 1465

last-checkpoint/trainer_state.json

@@ -2,175 +2,331 @@
   "best_global_step": null,
   "best_metric": null,
   "best_model_checkpoint": null,
-  "epoch": 0.004,
+  "epoch": 0.008,
   "eval_steps": 100,
-  "global_step": 200,
+  "global_step": 400,
   "is_hyper_param_search": false,
   "is_local_process_zero": true,
   "is_world_process_zero": true,
   "log_history": [
     {
       "epoch": 0.0002,
-      "grad_norm": 2672.0,
+      "grad_norm": 23680.0,
       "learning_rate": 6e-05,
-      "loss": 26.674310302734376,
+      "loss": 13.378594970703125,
       "step": 10
     },
     {
       "epoch": 0.0004,
-      "grad_norm": 652.0,
+      "grad_norm": 308.0,
       "learning_rate": 0.0001266666666666667,
-      "loss": 26.47657165527344,
+      "loss": 13.249179077148437,
       "step": 20
     },
     {
       "epoch": 0.0006,
-      "grad_norm": 8256.0,
+      "grad_norm": 1240.0,
       "learning_rate": 0.00019333333333333333,
-      "loss": 26.448770141601564,
+      "loss": 13.185873413085938,
       "step": 30
     },
     {
       "epoch": 0.0008,
-      "grad_norm": 13632.0,
+      "grad_norm": 54528.0,
       "learning_rate": 0.00026000000000000003,
-      "loss": 25.869580078125,
+      "loss": 12.9256103515625,
       "step": 40
     },
     {
       "epoch": 0.001,
-      "grad_norm": 576.0,
+      "grad_norm": 1120.0,
       "learning_rate": 0.0003266666666666667,
-      "loss": 25.283935546875,
+      "loss": 12.976683044433594,
       "step": 50
     },
     {
       "epoch": 0.0012,
-      "grad_norm": 212.0,
+      "grad_norm": 1012.0,
       "learning_rate": 0.0003933333333333333,
-      "loss": 25.18727264404297,
+      "loss": 13.311146545410157,
       "step": 60
     },
     {
       "epoch": 0.0014,
-      "grad_norm": 109.5,
+      "grad_norm": 1192.0,
       "learning_rate": 0.00046,
-      "loss": 24.912762451171876,
+      "loss": 13.249429321289062,
       "step": 70
     },
     {
       "epoch": 0.0016,
-      "grad_norm": 195.0,
+      "grad_norm": 560.0,
       "learning_rate": 0.0005266666666666666,
-      "loss": 24.746240234375,
+      "loss": 13.768452453613282,
       "step": 80
     },
     {
       "epoch": 0.0018,
-      "grad_norm": 1032.0,
+      "grad_norm": 764.0,
       "learning_rate": 0.0005933333333333334,
-      "loss": 24.107341003417968,
+      "loss": 14.640971374511718,
       "step": 90
     },
     {
       "epoch": 0.002,
-      "grad_norm": 135.0,
+      "grad_norm": 2736.0,
       "learning_rate": 0.00066,
-      "loss": 23.6096923828125,
+      "loss": 14.4900634765625,
       "step": 100
     },
     {
       "epoch": 0.002,
-      "eval_loss": 3.206618070602417,
-      "eval_runtime": 4.5304,
-      "eval_samples_per_second": 65.777,
-      "eval_steps_per_second": 3.752,
+      "eval_loss": 3.883816957473755,
+      "eval_runtime": 4.54,
+      "eval_samples_per_second": 65.638,
+      "eval_steps_per_second": 3.744,
       "step": 100
     },
     {
       "epoch": 0.0022,
-      "grad_norm": 55.75,
+      "grad_norm": 5280.0,
       "learning_rate": 0.0007266666666666667,
-      "loss": 22.759126281738283,
+      "loss": 14.381755065917968,
       "step": 110
     },
     {
       "epoch": 0.0024,
-      "grad_norm": 8.3125,
+      "grad_norm": 298.0,
       "learning_rate": 0.0007933333333333334,
-      "loss": 22.43407287597656,
+      "loss": 14.265592956542969,
       "step": 120
     },
     {
       "epoch": 0.0026,
-      "grad_norm": 2.15625,
+      "grad_norm": 45.0,
       "learning_rate": 0.00086,
-      "loss": 21.722286987304688,
+      "loss": 13.5162353515625,
       "step": 130
     },
     {
       "epoch": 0.0028,
-      "grad_norm": 1.046875,
+      "grad_norm": 74.5,
       "learning_rate": 0.0009266666666666667,
-      "loss": 21.480882263183595,
+      "loss": 13.169541931152343,
       "step": 140
     },
     {
       "epoch": 0.003,
-      "grad_norm": 1.1875,
+      "grad_norm": 8.125,
       "learning_rate": 0.0009933333333333333,
-      "loss": 21.57379913330078,
+      "loss": 12.071795654296874,
       "step": 150
     },
     {
       "epoch": 0.0032,
-      "grad_norm": 0.86328125,
+      "grad_norm": 2.703125,
       "learning_rate": 0.001,
-      "loss": 21.262675476074218,
+      "loss": 11.409983825683593,
       "step": 160
     },
     {
       "epoch": 0.0034,
-      "grad_norm": 0.703125,
+      "grad_norm": 1.03125,
       "learning_rate": 0.001,
-      "loss": 21.385879516601562,
+      "loss": 10.876200103759766,
       "step": 170
     },
     {
       "epoch": 0.0036,
-      "grad_norm": 0.84765625,
+      "grad_norm": 0.91796875,
       "learning_rate": 0.001,
-      "loss": 21.12591552734375,
+      "loss": 10.752876281738281,
       "step": 180
     },
     {
       "epoch": 0.0038,
-      "grad_norm": 0.78125,
+      "grad_norm": 0.71484375,
       "learning_rate": 0.001,
-      "loss": 21.397132873535156,
+      "loss": 10.67746810913086,
       "step": 190
     },
     {
       "epoch": 0.004,
-      "grad_norm": 0.67578125,
+      "grad_norm": 5.0,
       "learning_rate": 0.001,
-      "loss": 21.077099609375,
+      "loss": 10.72914810180664,
       "step": 200
     },
     {
       "epoch": 0.004,
-      "eval_loss": 3.0551397800445557,
-      "eval_runtime": 4.4525,
-      "eval_samples_per_second": 66.929,
-      "eval_steps_per_second": 3.818,
+      "eval_loss": 3.0593068599700928,
+      "eval_runtime": 4.4647,
+      "eval_samples_per_second": 66.746,
+      "eval_steps_per_second": 3.808,
       "step": 200
+    },
+    {
+      "epoch": 0.0042,
+      "grad_norm": 12.25,
+      "learning_rate": 0.001,
+      "loss": 10.68040771484375,
+      "step": 210
+    },
+    {
+      "epoch": 0.0044,
+      "grad_norm": 0.6640625,
+      "learning_rate": 0.001,
+      "loss": 10.56131820678711,
+      "step": 220
+    },
+    {
+      "epoch": 0.0046,
+      "grad_norm": 1.0703125,
+      "learning_rate": 0.001,
+      "loss": 10.673944854736328,
+      "step": 230
+    },
+    {
+      "epoch": 0.0048,
+      "grad_norm": 0.59375,
+      "learning_rate": 0.001,
+      "loss": 10.737371826171875,
+      "step": 240
+    },
+    {
+      "epoch": 0.005,
+      "grad_norm": 0.546875,
+      "learning_rate": 0.001,
+      "loss": 10.560675048828125,
+      "step": 250
+    },
+    {
+      "epoch": 0.0052,
+      "grad_norm": 0.470703125,
+      "learning_rate": 0.001,
+      "loss": 10.704891204833984,
+      "step": 260
+    },
+    {
+      "epoch": 0.0054,
+      "grad_norm": 0.494140625,
+      "learning_rate": 0.001,
+      "loss": 10.544506072998047,
+      "step": 270
+    },
+    {
+      "epoch": 0.0056,
+      "grad_norm": 0.421875,
+      "learning_rate": 0.001,
+      "loss": 10.619854736328126,
+      "step": 280
+    },
+    {
+      "epoch": 0.0058,
+      "grad_norm": 44.5,
+      "learning_rate": 0.001,
+      "loss": 10.606621551513673,
+      "step": 290
+    },
+    {
+      "epoch": 0.006,
+      "grad_norm": 0.400390625,
+      "learning_rate": 0.001,
+      "loss": 10.764106750488281,
+      "step": 300
+    },
+    {
+      "epoch": 0.006,
+      "eval_loss": 3.049805164337158,
+      "eval_runtime": 4.4123,
+      "eval_samples_per_second": 67.538,
+      "eval_steps_per_second": 3.853,
+      "step": 300
+    },
+    {
+      "epoch": 0.0062,
+      "grad_norm": 0.45703125,
+      "learning_rate": 0.001,
+      "loss": 10.587713623046875,
+      "step": 310
+    },
+    {
+      "epoch": 0.0064,
+      "grad_norm": 0.50390625,
+      "learning_rate": 0.001,
+      "loss": 10.587606811523438,
+      "step": 320
+    },
+    {
+      "epoch": 0.0066,
+      "grad_norm": 0.373046875,
+      "learning_rate": 0.001,
+      "loss": 10.690779876708984,
+      "step": 330
+    },
+    {
+      "epoch": 0.0068,
+      "grad_norm": 0.33984375,
+      "learning_rate": 0.001,
+      "loss": 10.637954711914062,
+      "step": 340
+    },
+    {
+      "epoch": 0.007,
+      "grad_norm": 0.443359375,
+      "learning_rate": 0.001,
+      "loss": 10.537116241455077,
+      "step": 350
+    },
+    {
+      "epoch": 0.0072,
+      "grad_norm": 0.5,
+      "learning_rate": 0.001,
+      "loss": 10.539633178710938,
+      "step": 360
+    },
+    {
+      "epoch": 0.0074,
+      "grad_norm": 0.72265625,
+      "learning_rate": 0.001,
+      "loss": 10.701078033447265,
+      "step": 370
+    },
+    {
+      "epoch": 0.0076,
+      "grad_norm": 0.36328125,
+      "learning_rate": 0.001,
+      "loss": 10.648773193359375,
+      "step": 380
+    },
+    {
+      "epoch": 0.0078,
+      "grad_norm": 0.380859375,
+      "learning_rate": 0.001,
+      "loss": 10.477677154541016,
+      "step": 390
+    },
+    {
+      "epoch": 0.008,
+      "grad_norm": 7.6875,
+      "learning_rate": 0.001,
+      "loss": 10.57955093383789,
+      "step": 400
+    },
+    {
+      "epoch": 0.008,
+      "eval_loss": 3.0504510402679443,
+      "eval_runtime": 4.4431,
+      "eval_samples_per_second": 67.07,
+      "eval_steps_per_second": 3.826,
+      "step": 400
     }
   ],
   "logging_steps": 10,
   "max_steps": 50000,
   "num_input_tokens_seen": 0,
   "num_train_epochs": 9223372036854775807,
-  "save_steps": 200,
+  "save_steps": 400,
   "stateful_callbacks": {
     "TrainerControl": {
       "args": {

last-checkpoint/training_args.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:668a0e377ebad57c5d1965a2d2cc09bbdbb8e4a54e766f286eae108d1e754ff9
+oid sha256:666419280b80b3d3942f48be91e30de30a6906e1b7b476ee09f4d3f438f4356a
 size 5265