Training in progress, step 1000, checkpoint

last-checkpoint/model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:f421426d14674c7d4ad546e19a4e57083d7bca462ab2e13d58cc41736dd8c7fe
+oid sha256:9d91f361723cdb3da7122704a5a753827705ee51e8d9db8d95d17fa16e1214a1
 size 234681136

last-checkpoint/optimizer.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:6ccea549a034e18c4f5ecfe1285a4528b857a0949f057eddfb3c088844cb9c36
+oid sha256:be60ae141d864e2c961c2d98241d3c95b42ab05195017b53e1dd1fb3455bc2c7
 size 469516363

last-checkpoint/rng_state.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:ca0548a1b90eaf6590b9f6afac3530aaed1157198c5a087c4229ad15001933d9
+oid sha256:76cd281a3ce995fc434ac757f1094066d955dd6b7188cf148f0dd1523cb9cd80
 size 14645

last-checkpoint/scheduler.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:2d251671b590d29a4724daa893ce2020a6083e1de3b017ea6154c342d8e856b9
+oid sha256:dfc5c6949cf8a9eca4d88710f0c9c06f410fb63454fb60c644727889ad457648
 size 1465

last-checkpoint/trainer_state.json

@@ -2,331 +2,409 @@
   "best_global_step": null,
   "best_metric": null,
   "best_model_checkpoint": null,
-  "epoch": 0.008,
-  "eval_steps": 100,
-  "global_step": 400,
+  "epoch": 0.05555555555555555,
+  "eval_steps": 200,
+  "global_step": 1000,
   "is_hyper_param_search": false,
   "is_local_process_zero": true,
   "is_world_process_zero": true,
   "log_history": [
     {
-      "epoch": 0.0002,
-      "grad_norm": 23680.0,
-      "learning_rate": 6e-05,
-      "loss": 13.378594970703125,
-      "step": 10
+      "epoch": 0.0011111111111111111,
+      "grad_norm": 95.5,
+      "learning_rate": 0.00011399999999999999,
+      "loss": 3.3263259887695313,
+      "step": 20
     },
     {
-      "epoch": 0.0004,
-      "grad_norm": 308.0,
-      "learning_rate": 0.0001266666666666667,
-      "loss": 13.249179077148437,
-      "step": 20
+      "epoch": 0.0022222222222222222,
+      "grad_norm": 37.75,
+      "learning_rate": 0.000234,
+      "loss": 3.246612548828125,
+      "step": 40
     },
     {
-      "epoch": 0.0006,
-      "grad_norm": 1240.0,
-      "learning_rate": 0.00019333333333333333,
-      "loss": 13.185873413085938,
-      "step": 30
+      "epoch": 0.0033333333333333335,
+      "grad_norm": 72.0,
+      "learning_rate": 0.0003,
+      "loss": 3.197848892211914,
+      "step": 60
     },
     {
-      "epoch": 0.0008,
-      "grad_norm": 54528.0,
-      "learning_rate": 0.00026000000000000003,
-      "loss": 12.9256103515625,
-      "step": 40
+      "epoch": 0.0044444444444444444,
+      "grad_norm": 137.0,
+      "learning_rate": 0.0003,
+      "loss": 3.0936569213867187,
+      "step": 80
     },
     {
-      "epoch": 0.001,
-      "grad_norm": 1120.0,
-      "learning_rate": 0.0003266666666666667,
-      "loss": 12.976683044433594,
-      "step": 50
+      "epoch": 0.005555555555555556,
+      "grad_norm": 13.8125,
+      "learning_rate": 0.0003,
+      "loss": 3.0532867431640627,
+      "step": 100
     },
     {
-      "epoch": 0.0012,
-      "grad_norm": 1012.0,
-      "learning_rate": 0.0003933333333333333,
-      "loss": 13.311146545410157,
-      "step": 60
+      "epoch": 0.006666666666666667,
+      "grad_norm": 11.875,
+      "learning_rate": 0.0003,
+      "loss": 3.0499711990356446,
+      "step": 120
     },
     {
-      "epoch": 0.0014,
-      "grad_norm": 1192.0,
-      "learning_rate": 0.00046,
-      "loss": 13.249429321289062,
-      "step": 70
+      "epoch": 0.0077777777777777776,
+      "grad_norm": 34.0,
+      "learning_rate": 0.0003,
+      "loss": 2.9780717849731446,
+      "step": 140
     },
     {
-      "epoch": 0.0016,
-      "grad_norm": 560.0,
-      "learning_rate": 0.0005266666666666666,
-      "loss": 13.768452453613282,
-      "step": 80
+      "epoch": 0.008888888888888889,
+      "grad_norm": 16.375,
+      "learning_rate": 0.0003,
+      "loss": 2.9729740142822267,
+      "step": 160
     },
     {
-      "epoch": 0.0018,
-      "grad_norm": 764.0,
-      "learning_rate": 0.0005933333333333334,
-      "loss": 14.640971374511718,
-      "step": 90
+      "epoch": 0.01,
+      "grad_norm": 1.6640625,
+      "learning_rate": 0.0003,
+      "loss": 2.905041313171387,
+      "step": 180
     },
     {
-      "epoch": 0.002,
-      "grad_norm": 2736.0,
-      "learning_rate": 0.00066,
-      "loss": 14.4900634765625,
-      "step": 100
+      "epoch": 0.011111111111111112,
+      "grad_norm": 2.5625,
+      "learning_rate": 0.0003,
+      "loss": 2.8482807159423826,
+      "step": 200
     },
     {
-      "epoch": 0.002,
-      "eval_loss": 3.883816957473755,
-      "eval_runtime": 4.54,
-      "eval_samples_per_second": 65.638,
-      "eval_steps_per_second": 3.744,
-      "step": 100
+      "epoch": 0.011111111111111112,
+      "eval_loss": 3.1418681144714355,
+      "eval_runtime": 4.509,
+      "eval_samples_per_second": 66.091,
+      "eval_steps_per_second": 3.77,
+      "step": 200
     },
     {
-      "epoch": 0.0022,
-      "grad_norm": 5280.0,
-      "learning_rate": 0.0007266666666666667,
-      "loss": 14.381755065917968,
-      "step": 110
+      "epoch": 0.012222222222222223,
+      "grad_norm": 1.6875,
+      "learning_rate": 0.0003,
+      "loss": 2.763028144836426,
+      "step": 220
     },
     {
-      "epoch": 0.0024,
-      "grad_norm": 298.0,
-      "learning_rate": 0.0007933333333333334,
-      "loss": 14.265592956542969,
-      "step": 120
+      "epoch": 0.013333333333333334,
+      "grad_norm": 0.5078125,
+      "learning_rate": 0.0003,
+      "loss": 2.7428956985473634,
+      "step": 240
     },
     {
-      "epoch": 0.0026,
-      "grad_norm": 45.0,
-      "learning_rate": 0.00086,
-      "loss": 13.5162353515625,
-      "step": 130
+      "epoch": 0.014444444444444444,
+      "grad_norm": 0.2578125,
+      "learning_rate": 0.0003,
+      "loss": 2.6967418670654295,
+      "step": 260
     },
     {
-      "epoch": 0.0028,
-      "grad_norm": 74.5,
-      "learning_rate": 0.0009266666666666667,
-      "loss": 13.169541931152343,
-      "step": 140
+      "epoch": 0.015555555555555555,
+      "grad_norm": 0.357421875,
+      "learning_rate": 0.0003,
+      "loss": 2.666821098327637,
+      "step": 280
     },
     {
-      "epoch": 0.003,
-      "grad_norm": 8.125,
-      "learning_rate": 0.0009933333333333333,
-      "loss": 12.071795654296874,
-      "step": 150
+      "epoch": 0.016666666666666666,
+      "grad_norm": 0.404296875,
+      "learning_rate": 0.0003,
+      "loss": 2.6833669662475588,
+      "step": 300
     },
     {
-      "epoch": 0.0032,
-      "grad_norm": 2.703125,
-      "learning_rate": 0.001,
-      "loss": 11.409983825683593,
-      "step": 160
+      "epoch": 0.017777777777777778,
+      "grad_norm": 0.142578125,
+      "learning_rate": 0.0003,
+      "loss": 2.6547067642211912,
+      "step": 320
     },
     {
-      "epoch": 0.0034,
-      "grad_norm": 1.03125,
-      "learning_rate": 0.001,
-      "loss": 10.876200103759766,
-      "step": 170
+      "epoch": 0.01888888888888889,
+      "grad_norm": 0.10986328125,
+      "learning_rate": 0.0003,
+      "loss": 2.6726154327392577,
+      "step": 340
     },
     {
-      "epoch": 0.0036,
-      "grad_norm": 0.91796875,
-      "learning_rate": 0.001,
-      "loss": 10.752876281738281,
-      "step": 180
+      "epoch": 0.02,
+      "grad_norm": 0.310546875,
+      "learning_rate": 0.0003,
+      "loss": 2.639314079284668,
+      "step": 360
     },
     {
-      "epoch": 0.0038,
-      "grad_norm": 0.71484375,
-      "learning_rate": 0.001,
-      "loss": 10.67746810913086,
-      "step": 190
+      "epoch": 0.021111111111111112,
+      "grad_norm": 0.265625,
+      "learning_rate": 0.0003,
+      "loss": 2.6715606689453124,
+      "step": 380
     },
     {
-      "epoch": 0.004,
-      "grad_norm": 5.0,
-      "learning_rate": 0.001,
-      "loss": 10.72914810180664,
-      "step": 200
+      "epoch": 0.022222222222222223,
+      "grad_norm": 0.15234375,
+      "learning_rate": 0.0003,
+      "loss": 2.633011817932129,
+      "step": 400
     },
     {
-      "epoch": 0.004,
-      "eval_loss": 3.0593068599700928,
-      "eval_runtime": 4.4647,
-      "eval_samples_per_second": 66.746,
-      "eval_steps_per_second": 3.808,
-      "step": 200
+      "epoch": 0.022222222222222223,
+      "eval_loss": 3.0355894565582275,
+      "eval_runtime": 4.4466,
+      "eval_samples_per_second": 67.017,
+      "eval_steps_per_second": 3.823,
+      "step": 400
     },
     {
-      "epoch": 0.0042,
-      "grad_norm": 12.25,
-      "learning_rate": 0.001,
-      "loss": 10.68040771484375,
-      "step": 210
+      "epoch": 0.023333333333333334,
+      "grad_norm": 0.9296875,
+      "learning_rate": 0.0003,
+      "loss": 2.6349510192871093,
+      "step": 420
     },
     {
-      "epoch": 0.0044,
-      "grad_norm": 0.6640625,
-      "learning_rate": 0.001,
-      "loss": 10.56131820678711,
-      "step": 220
+      "epoch": 0.024444444444444446,
+      "grad_norm": 0.25,
+      "learning_rate": 0.0003,
+      "loss": 2.6378190994262694,
+      "step": 440
     },
     {
-      "epoch": 0.0046,
-      "grad_norm": 1.0703125,
-      "learning_rate": 0.001,
-      "loss": 10.673944854736328,
-      "step": 230
+      "epoch": 0.025555555555555557,
+      "grad_norm": 0.091796875,
+      "learning_rate": 0.0003,
+      "loss": 2.645473670959473,
+      "step": 460
     },
     {
-      "epoch": 0.0048,
-      "grad_norm": 0.59375,
-      "learning_rate": 0.001,
-      "loss": 10.737371826171875,
-      "step": 240
+      "epoch": 0.02666666666666667,
+      "grad_norm": 0.7109375,
+      "learning_rate": 0.0003,
+      "loss": 2.6475948333740233,
+      "step": 480
     },
     {
-      "epoch": 0.005,
-      "grad_norm": 0.546875,
-      "learning_rate": 0.001,
-      "loss": 10.560675048828125,
-      "step": 250
+      "epoch": 0.027777777777777776,
+      "grad_norm": 0.0927734375,
+      "learning_rate": 0.0003,
+      "loss": 2.652556228637695,
+      "step": 500
     },
     {
-      "epoch": 0.0052,
-      "grad_norm": 0.470703125,
-      "learning_rate": 0.001,
-      "loss": 10.704891204833984,
-      "step": 260
+      "epoch": 0.028888888888888888,
+      "grad_norm": 0.10400390625,
+      "learning_rate": 0.0003,
+      "loss": 2.6353578567504883,
+      "step": 520
     },
     {
-      "epoch": 0.0054,
-      "grad_norm": 0.494140625,
-      "learning_rate": 0.001,
-      "loss": 10.544506072998047,
-      "step": 270
+      "epoch": 0.03,
+      "grad_norm": 0.11767578125,
+      "learning_rate": 0.0003,
+      "loss": 2.617463493347168,
+      "step": 540
     },
     {
-      "epoch": 0.0056,
-      "grad_norm": 0.421875,
-      "learning_rate": 0.001,
-      "loss": 10.619854736328126,
-      "step": 280
+      "epoch": 0.03111111111111111,
+      "grad_norm": 0.09619140625,
+      "learning_rate": 0.0003,
+      "loss": 2.6145917892456056,
+      "step": 560
     },
     {
-      "epoch": 0.0058,
-      "grad_norm": 44.5,
-      "learning_rate": 0.001,
-      "loss": 10.606621551513673,
-      "step": 290
+      "epoch": 0.03222222222222222,
+      "grad_norm": 0.291015625,
+      "learning_rate": 0.0003,
+      "loss": 2.6116378784179686,
+      "step": 580
     },
     {
-      "epoch": 0.006,
-      "grad_norm": 0.400390625,
-      "learning_rate": 0.001,
-      "loss": 10.764106750488281,
-      "step": 300
+      "epoch": 0.03333333333333333,
+      "grad_norm": 0.09423828125,
+      "learning_rate": 0.0003,
+      "loss": 2.6324670791625975,
+      "step": 600
     },
     {
-      "epoch": 0.006,
-      "eval_loss": 3.049805164337158,
-      "eval_runtime": 4.4123,
-      "eval_samples_per_second": 67.538,
-      "eval_steps_per_second": 3.853,
-      "step": 300
+      "epoch": 0.03333333333333333,
+      "eval_loss": 3.0325393676757812,
+      "eval_runtime": 4.4279,
+      "eval_samples_per_second": 67.301,
+      "eval_steps_per_second": 3.839,
+      "step": 600
     },
     {
-      "epoch": 0.0062,
-      "grad_norm": 0.45703125,
-      "learning_rate": 0.001,
-      "loss": 10.587713623046875,
-      "step": 310
+      "epoch": 0.034444444444444444,
+      "grad_norm": 0.09912109375,
+      "learning_rate": 0.0003,
+      "loss": 2.627296257019043,
+      "step": 620
     },
     {
-      "epoch": 0.0064,
-      "grad_norm": 0.50390625,
-      "learning_rate": 0.001,
-      "loss": 10.587606811523438,
-      "step": 320
+      "epoch": 0.035555555555555556,
+      "grad_norm": 0.1064453125,
+      "learning_rate": 0.0003,
+      "loss": 2.615768623352051,
+      "step": 640
     },
     {
-      "epoch": 0.0066,
-      "grad_norm": 0.373046875,
-      "learning_rate": 0.001,
-      "loss": 10.690779876708984,
-      "step": 330
+      "epoch": 0.03666666666666667,
+      "grad_norm": 0.0830078125,
+      "learning_rate": 0.0003,
+      "loss": 2.630397415161133,
+      "step": 660
     },
     {
-      "epoch": 0.0068,
-      "grad_norm": 0.33984375,
-      "learning_rate": 0.001,
-      "loss": 10.637954711914062,
-      "step": 340
+      "epoch": 0.03777777777777778,
+      "grad_norm": 2.34375,
+      "learning_rate": 0.0003,
+      "loss": 2.6295297622680662,
+      "step": 680
     },
     {
-      "epoch": 0.007,
-      "grad_norm": 0.443359375,
-      "learning_rate": 0.001,
-      "loss": 10.537116241455077,
-      "step": 350
+      "epoch": 0.03888888888888889,
+      "grad_norm": 0.091796875,
+      "learning_rate": 0.0003,
+      "loss": 2.6139799118041993,
+      "step": 700
     },
     {
-      "epoch": 0.0072,
-      "grad_norm": 0.5,
-      "learning_rate": 0.001,
-      "loss": 10.539633178710938,
-      "step": 360
+      "epoch": 0.04,
+      "grad_norm": 0.177734375,
+      "learning_rate": 0.0003,
+      "loss": 2.612909507751465,
+      "step": 720
     },
     {
-      "epoch": 0.0074,
-      "grad_norm": 0.72265625,
-      "learning_rate": 0.001,
-      "loss": 10.701078033447265,
-      "step": 370
+      "epoch": 0.04111111111111111,
+      "grad_norm": 0.08984375,
+      "learning_rate": 0.0003,
+      "loss": 2.637497901916504,
+      "step": 740
     },
     {
-      "epoch": 0.0076,
-      "grad_norm": 0.36328125,
-      "learning_rate": 0.001,
-      "loss": 10.648773193359375,
-      "step": 380
+      "epoch": 0.042222222222222223,
+      "grad_norm": 0.091796875,
+      "learning_rate": 0.0003,
+      "loss": 2.6311773300170898,
+      "step": 760
     },
     {
-      "epoch": 0.0078,
-      "grad_norm": 0.380859375,
-      "learning_rate": 0.001,
-      "loss": 10.477677154541016,
-      "step": 390
+      "epoch": 0.043333333333333335,
+      "grad_norm": 0.1748046875,
+      "learning_rate": 0.0003,
+      "loss": 2.631211280822754,
+      "step": 780
     },
     {
-      "epoch": 0.008,
-      "grad_norm": 7.6875,
-      "learning_rate": 0.001,
-      "loss": 10.57955093383789,
-      "step": 400
+      "epoch": 0.044444444444444446,
+      "grad_norm": 0.08154296875,
+      "learning_rate": 0.0003,
+      "loss": 2.64044075012207,
+      "step": 800
     },
     {
-      "epoch": 0.008,
-      "eval_loss": 3.0504510402679443,
-      "eval_runtime": 4.4431,
-      "eval_samples_per_second": 67.07,
-      "eval_steps_per_second": 3.826,
-      "step": 400
+      "epoch": 0.044444444444444446,
+      "eval_loss": 3.029780149459839,
+      "eval_runtime": 4.4461,
+      "eval_samples_per_second": 67.025,
+      "eval_steps_per_second": 3.824,
+      "step": 800
+    },
+    {
+      "epoch": 0.04555555555555556,
+      "grad_norm": 0.0810546875,
+      "learning_rate": 0.0003,
+      "loss": 2.6481130599975584,
+      "step": 820
+    },
+    {
+      "epoch": 0.04666666666666667,
+      "grad_norm": 0.06884765625,
+      "learning_rate": 0.0003,
+      "loss": 2.6261564254760743,
+      "step": 840
+    },
+    {
+      "epoch": 0.04777777777777778,
+      "grad_norm": 0.07373046875,
+      "learning_rate": 0.0003,
+      "loss": 2.611224365234375,
+      "step": 860
+    },
+    {
+      "epoch": 0.04888888888888889,
+      "grad_norm": 0.119140625,
+      "learning_rate": 0.0003,
+      "loss": 2.636740875244141,
+      "step": 880
+    },
+    {
+      "epoch": 0.05,
+      "grad_norm": 0.06884765625,
+      "learning_rate": 0.0003,
+      "loss": 2.614007568359375,
+      "step": 900
+    },
+    {
+      "epoch": 0.051111111111111114,
+      "grad_norm": 0.068359375,
+      "learning_rate": 0.0003,
+      "loss": 2.616529846191406,
+      "step": 920
+    },
+    {
+      "epoch": 0.052222222222222225,
+      "grad_norm": 0.07470703125,
+      "learning_rate": 0.0003,
+      "loss": 2.6233530044555664,
+      "step": 940
+    },
+    {
+      "epoch": 0.05333333333333334,
+      "grad_norm": 0.06689453125,
+      "learning_rate": 0.0003,
+      "loss": 2.600985527038574,
+      "step": 960
+    },
+    {
+      "epoch": 0.05444444444444444,
+      "grad_norm": 0.0791015625,
+      "learning_rate": 0.0003,
+      "loss": 2.6462072372436523,
+      "step": 980
+    },
+    {
+      "epoch": 0.05555555555555555,
+      "grad_norm": 0.07470703125,
+      "learning_rate": 0.0003,
+      "loss": 2.6183977127075195,
+      "step": 1000
+    },
+    {
+      "epoch": 0.05555555555555555,
+      "eval_loss": 3.0301759243011475,
+      "eval_runtime": 4.4446,
+      "eval_samples_per_second": 67.048,
+      "eval_steps_per_second": 3.825,
+      "step": 1000
     }
   ],
-  "logging_steps": 10,
-  "max_steps": 50000,
+  "logging_steps": 20,
+  "max_steps": 18000,
   "num_input_tokens_seen": 0,
   "num_train_epochs": 9223372036854775807,
-  "save_steps": 400,
+  "save_steps": 1000,
   "stateful_callbacks": {
     "TrainerControl": {
       "args": {
@@ -339,7 +417,7 @@
       "attributes": {}
     }
   },
-  "total_flos": 6.52309029715968e+16,
+  "total_flos": 1.63077257428992e+17,
   "train_batch_size": 18,
   "trial_name": null,
   "trial_params": null

last-checkpoint/training_args.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:666419280b80b3d3942f48be91e30de30a6906e1b7b476ee09f4d3f438f4356a
+oid sha256:4d0fa3459db214f81ac56235c0274e72db303ff14a77456f1fa013b5e4128f2e
 size 5265