Training in progress, step 500, checkpoint

last-checkpoint/model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:9d91f361723cdb3da7122704a5a753827705ee51e8d9db8d95d17fa16e1214a1
+oid sha256:5dd7f230851339b861737ac7f67ae0ed9375ce1b110bdab60cbcb0423e7fa234
 size 234681136

last-checkpoint/optimizer.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:be60ae141d864e2c961c2d98241d3c95b42ab05195017b53e1dd1fb3455bc2c7
+oid sha256:9b6baba95860d4d92494572f788ea8c87710f9e3ab9a13bcb44e1ff7ec7c834a
 size 469516363

last-checkpoint/scheduler.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:dfc5c6949cf8a9eca4d88710f0c9c06f410fb63454fb60c644727889ad457648
+oid sha256:1ba487b3335620f3a6e123aeda5007c1c83d8788fc08709b71729143cfcd3d74
 size 1465

last-checkpoint/trainer_state.json

@@ -2,409 +2,409 @@
   "best_global_step": null,
   "best_metric": null,
   "best_model_checkpoint": null,
-  "epoch": 0.05555555555555555,
-  "eval_steps": 200,
-  "global_step": 1000,
+  "epoch": 0.07142857142857142,
+  "eval_steps": 100,
+  "global_step": 500,
   "is_hyper_param_search": false,
   "is_local_process_zero": true,
   "is_world_process_zero": true,
   "log_history": [
     {
-      "epoch": 0.0011111111111111111,
-      "grad_norm": 95.5,
+      "epoch": 0.0014285714285714286,
+      "grad_norm": 160.0,
+      "learning_rate": 5.399999999999999e-05,
+      "loss": 3.334370803833008,
+      "step": 10
+    },
+    {
+      "epoch": 0.002857142857142857,
+      "grad_norm": 143.0,
       "learning_rate": 0.00011399999999999999,
-      "loss": 3.3263259887695313,
+      "loss": 3.312285232543945,
       "step": 20
     },
     {
-      "epoch": 0.0022222222222222222,
-      "grad_norm": 37.75,
-      "learning_rate": 0.000234,
-      "loss": 3.246612548828125,
-      "step": 40
+      "epoch": 0.004285714285714286,
+      "grad_norm": 91.0,
+      "learning_rate": 0.00017399999999999997,
+      "loss": 3.3183692932128905,
+      "step": 30
     },
     {
-      "epoch": 0.0033333333333333335,
-      "grad_norm": 72.0,
-      "learning_rate": 0.0003,
-      "loss": 3.197848892211914,
-      "step": 60
+      "epoch": 0.005714285714285714,
+      "grad_norm": 306.0,
+      "learning_rate": 0.000234,
+      "loss": 3.232763671875,
+      "step": 40
     },
     {
-      "epoch": 0.0044444444444444444,
-      "grad_norm": 137.0,
-      "learning_rate": 0.0003,
-      "loss": 3.0936569213867187,
-      "step": 80
+      "epoch": 0.007142857142857143,
+      "grad_norm": 114.5,
+      "learning_rate": 0.000294,
+      "loss": 3.1770435333251954,
+      "step": 50
     },
     {
-      "epoch": 0.005555555555555556,
-      "grad_norm": 13.8125,
-      "learning_rate": 0.0003,
-      "loss": 3.0532867431640627,
-      "step": 100
-    },
-    {
-      "epoch": 0.006666666666666667,
-      "grad_norm": 11.875,
-      "learning_rate": 0.0003,
-      "loss": 3.0499711990356446,
-      "step": 120
+      "epoch": 0.008571428571428572,
+      "grad_norm": 148.0,
+      "learning_rate": 0.00029999875870267496,
+      "loss": 3.1800256729125977,
+      "step": 60
     },
     {
-      "epoch": 0.0077777777777777776,
-      "grad_norm": 34.0,
-      "learning_rate": 0.0003,
-      "loss": 2.9780717849731446,
-      "step": 140
+      "epoch": 0.01,
+      "grad_norm": 25.0,
+      "learning_rate": 0.0002999944678247173,
+      "loss": 3.166558837890625,
+      "step": 70
     },
     {
-      "epoch": 0.008888888888888889,
-      "grad_norm": 16.375,
-      "learning_rate": 0.0003,
-      "loss": 2.9729740142822267,
-      "step": 160
+      "epoch": 0.011428571428571429,
+      "grad_norm": 22.75,
+      "learning_rate": 0.0002999871121291227,
+      "loss": 3.203672409057617,
+      "step": 80
     },
     {
-      "epoch": 0.01,
-      "grad_norm": 1.6640625,
-      "learning_rate": 0.0003,
-      "loss": 2.905041313171387,
-      "step": 180
+      "epoch": 0.012857142857142857,
+      "grad_norm": 6.4375,
+      "learning_rate": 0.00029997669176618923,
+      "loss": 3.155362129211426,
+      "step": 90
     },
     {
-      "epoch": 0.011111111111111112,
-      "grad_norm": 2.5625,
-      "learning_rate": 0.0003,
-      "loss": 2.8482807159423826,
-      "step": 200
+      "epoch": 0.014285714285714285,
+      "grad_norm": 142.0,
+      "learning_rate": 0.0002999632069488347,
+      "loss": 3.1418533325195312,
+      "step": 100
     },
     {
-      "epoch": 0.011111111111111112,
-      "eval_loss": 3.1418681144714355,
-      "eval_runtime": 4.509,
-      "eval_samples_per_second": 66.091,
-      "eval_steps_per_second": 3.77,
-      "step": 200
+      "epoch": 0.014285714285714285,
+      "eval_loss": 3.406630277633667,
+      "eval_runtime": 4.5564,
+      "eval_samples_per_second": 65.402,
+      "eval_steps_per_second": 3.731,
+      "step": 100
     },
     {
-      "epoch": 0.012222222222222223,
-      "grad_norm": 1.6875,
-      "learning_rate": 0.0003,
-      "loss": 2.763028144836426,
-      "step": 220
+      "epoch": 0.015714285714285715,
+      "grad_norm": 200.0,
+      "learning_rate": 0.00029994665795259274,
+      "loss": 3.106761932373047,
+      "step": 110
     },
     {
-      "epoch": 0.013333333333333334,
-      "grad_norm": 0.5078125,
-      "learning_rate": 0.0003,
-      "loss": 2.7428956985473634,
-      "step": 240
+      "epoch": 0.017142857142857144,
+      "grad_norm": 193.0,
+      "learning_rate": 0.0002999270451156068,
+      "loss": 3.1113397598266603,
+      "step": 120
     },
     {
-      "epoch": 0.014444444444444444,
-      "grad_norm": 0.2578125,
-      "learning_rate": 0.0003,
-      "loss": 2.6967418670654295,
-      "step": 260
+      "epoch": 0.018571428571428572,
+      "grad_norm": 39.5,
+      "learning_rate": 0.0002999043688386235,
+      "loss": 3.0628725051879884,
+      "step": 130
     },
     {
-      "epoch": 0.015555555555555555,
-      "grad_norm": 0.357421875,
-      "learning_rate": 0.0003,
-      "loss": 2.666821098327637,
-      "step": 280
+      "epoch": 0.02,
+      "grad_norm": 3.125,
+      "learning_rate": 0.0002998786295849843,
+      "loss": 3.030979347229004,
+      "step": 140
     },
     {
-      "epoch": 0.016666666666666666,
-      "grad_norm": 0.404296875,
-      "learning_rate": 0.0003,
-      "loss": 2.6833669662475588,
-      "step": 300
+      "epoch": 0.02142857142857143,
+      "grad_norm": 10.0625,
+      "learning_rate": 0.000299849827880616,
+      "loss": 3.0148197174072267,
+      "step": 150
     },
     {
-      "epoch": 0.017777777777777778,
-      "grad_norm": 0.142578125,
-      "learning_rate": 0.0003,
-      "loss": 2.6547067642211912,
-      "step": 320
+      "epoch": 0.022857142857142857,
+      "grad_norm": 4.03125,
+      "learning_rate": 0.00029981796431402015,
+      "loss": 2.963352394104004,
+      "step": 160
     },
     {
-      "epoch": 0.01888888888888889,
-      "grad_norm": 0.10986328125,
-      "learning_rate": 0.0003,
-      "loss": 2.6726154327392577,
-      "step": 340
+      "epoch": 0.024285714285714285,
+      "grad_norm": 28.5,
+      "learning_rate": 0.0002997830395362608,
+      "loss": 2.951630401611328,
+      "step": 170
     },
     {
-      "epoch": 0.02,
-      "grad_norm": 0.310546875,
-      "learning_rate": 0.0003,
-      "loss": 2.639314079284668,
-      "step": 360
+      "epoch": 0.025714285714285714,
+      "grad_norm": 5.15625,
+      "learning_rate": 0.00029974505426095166,
+      "loss": 2.903683662414551,
+      "step": 180
     },
     {
-      "epoch": 0.021111111111111112,
-      "grad_norm": 0.265625,
-      "learning_rate": 0.0003,
-      "loss": 2.6715606689453124,
-      "step": 380
+      "epoch": 0.027142857142857142,
+      "grad_norm": 2.359375,
+      "learning_rate": 0.0002997040092642407,
+      "loss": 2.911179542541504,
+      "step": 190
     },
     {
-      "epoch": 0.022222222222222223,
-      "grad_norm": 0.15234375,
-      "learning_rate": 0.0003,
-      "loss": 2.633011817932129,
-      "step": 400
+      "epoch": 0.02857142857142857,
+      "grad_norm": 1.484375,
+      "learning_rate": 0.00029965990538479526,
+      "loss": 2.8417181015014648,
+      "step": 200
     },
     {
-      "epoch": 0.022222222222222223,
-      "eval_loss": 3.0355894565582275,
-      "eval_runtime": 4.4466,
-      "eval_samples_per_second": 67.017,
-      "eval_steps_per_second": 3.823,
-      "step": 400
+      "epoch": 0.02857142857142857,
+      "eval_loss": 3.159050703048706,
+      "eval_runtime": 4.4564,
+      "eval_samples_per_second": 66.87,
+      "eval_steps_per_second": 3.815,
+      "step": 200
     },
     {
-      "epoch": 0.023333333333333334,
-      "grad_norm": 0.9296875,
-      "learning_rate": 0.0003,
-      "loss": 2.6349510192871093,
-      "step": 420
+      "epoch": 0.03,
+      "grad_norm": 1.515625,
+      "learning_rate": 0.0002996127435237841,
+      "loss": 2.815742301940918,
+      "step": 210
     },
     {
-      "epoch": 0.024444444444444446,
-      "grad_norm": 0.25,
-      "learning_rate": 0.0003,
-      "loss": 2.6378190994262694,
-      "step": 440
+      "epoch": 0.03142857142857143,
+      "grad_norm": 1.71875,
+      "learning_rate": 0.0002995625246448595,
+      "loss": 2.7929365158081056,
+      "step": 220
     },
     {
-      "epoch": 0.025555555555555557,
-      "grad_norm": 0.091796875,
-      "learning_rate": 0.0003,
-      "loss": 2.645473670959473,
-      "step": 460
+      "epoch": 0.032857142857142856,
+      "grad_norm": 0.765625,
+      "learning_rate": 0.00029950924977413735,
+      "loss": 2.7637331008911135,
+      "step": 230
     },
     {
-      "epoch": 0.02666666666666667,
-      "grad_norm": 0.7109375,
-      "learning_rate": 0.0003,
-      "loss": 2.6475948333740233,
-      "step": 480
+      "epoch": 0.03428571428571429,
+      "grad_norm": 0.609375,
+      "learning_rate": 0.0002994529200001762,
+      "loss": 2.73776912689209,
+      "step": 240
     },
     {
-      "epoch": 0.027777777777777776,
-      "grad_norm": 0.0927734375,
-      "learning_rate": 0.0003,
-      "loss": 2.652556228637695,
-      "step": 500
+      "epoch": 0.03571428571428571,
+      "grad_norm": 1.5703125,
+      "learning_rate": 0.00029939353647395506,
+      "loss": 2.723506736755371,
+      "step": 250
     },
     {
-      "epoch": 0.028888888888888888,
-      "grad_norm": 0.10400390625,
-      "learning_rate": 0.0003,
-      "loss": 2.6353578567504883,
-      "step": 520
+      "epoch": 0.037142857142857144,
+      "grad_norm": 0.66015625,
+      "learning_rate": 0.00029933110040884987,
+      "loss": 2.6915863037109373,
+      "step": 260
     },
     {
-      "epoch": 0.03,
-      "grad_norm": 0.11767578125,
-      "learning_rate": 0.0003,
-      "loss": 2.617463493347168,
-      "step": 540
+      "epoch": 0.03857142857142857,
+      "grad_norm": 0.9609375,
+      "learning_rate": 0.00029926561308060874,
+      "loss": 2.663774871826172,
+      "step": 270
     },
     {
-      "epoch": 0.03111111111111111,
-      "grad_norm": 0.09619140625,
-      "learning_rate": 0.0003,
-      "loss": 2.6145917892456056,
-      "step": 560
+      "epoch": 0.04,
+      "grad_norm": 0.65234375,
+      "learning_rate": 0.00029919707582732577,
+      "loss": 2.6552204132080077,
+      "step": 280
     },
     {
-      "epoch": 0.03222222222222222,
-      "grad_norm": 0.291015625,
-      "learning_rate": 0.0003,
-      "loss": 2.6116378784179686,
-      "step": 580
+      "epoch": 0.041428571428571426,
+      "grad_norm": 0.1708984375,
+      "learning_rate": 0.0002991254900494139,
+      "loss": 2.6489795684814452,
+      "step": 290
     },
     {
-      "epoch": 0.03333333333333333,
-      "grad_norm": 0.09423828125,
-      "learning_rate": 0.0003,
-      "loss": 2.6324670791625975,
-      "step": 600
+      "epoch": 0.04285714285714286,
+      "grad_norm": 0.984375,
+      "learning_rate": 0.000299050857209576,
+      "loss": 2.6665163040161133,
+      "step": 300
     },
     {
-      "epoch": 0.03333333333333333,
-      "eval_loss": 3.0325393676757812,
-      "eval_runtime": 4.4279,
-      "eval_samples_per_second": 67.301,
-      "eval_steps_per_second": 3.839,
-      "step": 600
+      "epoch": 0.04285714285714286,
+      "eval_loss": 3.0404820442199707,
+      "eval_runtime": 4.4652,
+      "eval_samples_per_second": 66.739,
+      "eval_steps_per_second": 3.807,
+      "step": 300
     },
     {
-      "epoch": 0.034444444444444444,
-      "grad_norm": 0.09912109375,
-      "learning_rate": 0.0003,
-      "loss": 2.627296257019043,
-      "step": 620
+      "epoch": 0.04428571428571428,
+      "grad_norm": 1.2890625,
+      "learning_rate": 0.00029897317883277537,
+      "loss": 2.6572853088378907,
+      "step": 310
     },
     {
-      "epoch": 0.035555555555555556,
-      "grad_norm": 0.1064453125,
-      "learning_rate": 0.0003,
-      "loss": 2.615768623352051,
-      "step": 640
+      "epoch": 0.045714285714285714,
+      "grad_norm": 0.2265625,
+      "learning_rate": 0.00029889245650620413,
+      "loss": 2.642633056640625,
+      "step": 320
     },
     {
-      "epoch": 0.03666666666666667,
-      "grad_norm": 0.0830078125,
-      "learning_rate": 0.0003,
-      "loss": 2.630397415161133,
-      "step": 660
+      "epoch": 0.047142857142857146,
+      "grad_norm": 0.6484375,
+      "learning_rate": 0.0002988086918792514,
+      "loss": 2.6547801971435545,
+      "step": 330
     },
     {
-      "epoch": 0.03777777777777778,
-      "grad_norm": 2.34375,
-      "learning_rate": 0.0003,
-      "loss": 2.6295297622680662,
-      "step": 680
+      "epoch": 0.04857142857142857,
+      "grad_norm": 4.625,
+      "learning_rate": 0.0002987218866634688,
+      "loss": 2.651763153076172,
+      "step": 340
     },
     {
-      "epoch": 0.03888888888888889,
-      "grad_norm": 0.091796875,
-      "learning_rate": 0.0003,
-      "loss": 2.6139799118041993,
-      "step": 700
+      "epoch": 0.05,
+      "grad_norm": 0.2275390625,
+      "learning_rate": 0.00029863204263253624,
+      "loss": 2.635426902770996,
+      "step": 350
     },
     {
-      "epoch": 0.04,
-      "grad_norm": 0.177734375,
-      "learning_rate": 0.0003,
-      "loss": 2.612909507751465,
-      "step": 720
+      "epoch": 0.05142857142857143,
+      "grad_norm": 0.11328125,
+      "learning_rate": 0.0002985391616222252,
+      "loss": 2.6323591232299806,
+      "step": 360
     },
     {
-      "epoch": 0.04111111111111111,
-      "grad_norm": 0.08984375,
-      "learning_rate": 0.0003,
-      "loss": 2.637497901916504,
-      "step": 740
+      "epoch": 0.05285714285714286,
+      "grad_norm": 3.125,
+      "learning_rate": 0.0002984432455303614,
+      "loss": 2.655726432800293,
+      "step": 370
     },
     {
-      "epoch": 0.042222222222222223,
-      "grad_norm": 0.091796875,
-      "learning_rate": 0.0003,
-      "loss": 2.6311773300170898,
-      "step": 760
+      "epoch": 0.054285714285714284,
+      "grad_norm": 0.3359375,
+      "learning_rate": 0.00029834429631678597,
+      "loss": 2.6482254028320313,
+      "step": 380
     },
     {
-      "epoch": 0.043333333333333335,
-      "grad_norm": 0.1748046875,
-      "learning_rate": 0.0003,
-      "loss": 2.631211280822754,
-      "step": 780
+      "epoch": 0.055714285714285716,
+      "grad_norm": 0.54296875,
+      "learning_rate": 0.00029824231600331547,
+      "loss": 2.6480077743530273,
+      "step": 390
     },
     {
-      "epoch": 0.044444444444444446,
-      "grad_norm": 0.08154296875,
-      "learning_rate": 0.0003,
-      "loss": 2.64044075012207,
-      "step": 800
+      "epoch": 0.05714285714285714,
+      "grad_norm": 0.166015625,
+      "learning_rate": 0.0002981373066737005,
+      "loss": 2.6565391540527346,
+      "step": 400
     },
     {
-      "epoch": 0.044444444444444446,
-      "eval_loss": 3.029780149459839,
-      "eval_runtime": 4.4461,
-      "eval_samples_per_second": 67.025,
-      "eval_steps_per_second": 3.824,
-      "step": 800
+      "epoch": 0.05714285714285714,
+      "eval_loss": 3.0312750339508057,
+      "eval_runtime": 4.4433,
+      "eval_samples_per_second": 67.068,
+      "eval_steps_per_second": 3.826,
+      "step": 400
     },
     {
-      "epoch": 0.04555555555555556,
-      "grad_norm": 0.0810546875,
-      "learning_rate": 0.0003,
-      "loss": 2.6481130599975584,
-      "step": 820
+      "epoch": 0.05857142857142857,
+      "grad_norm": 0.255859375,
+      "learning_rate": 0.0002980292704735831,
+      "loss": 2.663307762145996,
+      "step": 410
     },
     {
-      "epoch": 0.04666666666666667,
-      "grad_norm": 0.06884765625,
-      "learning_rate": 0.0003,
-      "loss": 2.6261564254760743,
-      "step": 840
+      "epoch": 0.06,
+      "grad_norm": 0.0966796875,
+      "learning_rate": 0.00029791820961045317,
+      "loss": 2.6410280227661134,
+      "step": 420
     },
     {
-      "epoch": 0.04777777777777778,
-      "grad_norm": 0.07373046875,
-      "learning_rate": 0.0003,
-      "loss": 2.611224365234375,
-      "step": 860
+      "epoch": 0.06142857142857143,
+      "grad_norm": 0.3125,
+      "learning_rate": 0.0002978041263536029,
+      "loss": 2.6257579803466795,
+      "step": 430
     },
     {
-      "epoch": 0.04888888888888889,
-      "grad_norm": 0.119140625,
-      "learning_rate": 0.0003,
-      "loss": 2.636740875244141,
-      "step": 880
+      "epoch": 0.06285714285714286,
+      "grad_norm": 0.310546875,
+      "learning_rate": 0.0002976870230340808,
+      "loss": 2.650602912902832,
+      "step": 440
     },
     {
-      "epoch": 0.05,
-      "grad_norm": 0.06884765625,
-      "learning_rate": 0.0003,
-      "loss": 2.614007568359375,
-      "step": 900
+      "epoch": 0.06428571428571428,
+      "grad_norm": 0.115234375,
+      "learning_rate": 0.0002975669020446439,
+      "loss": 2.6274070739746094,
+      "step": 450
     },
     {
-      "epoch": 0.051111111111111114,
-      "grad_norm": 0.068359375,
-      "learning_rate": 0.0003,
-      "loss": 2.616529846191406,
-      "step": 920
+      "epoch": 0.06571428571428571,
+      "grad_norm": 0.10693359375,
+      "learning_rate": 0.00029744376583970897,
+      "loss": 2.629538154602051,
+      "step": 460
     },
     {
-      "epoch": 0.052222222222222225,
-      "grad_norm": 0.07470703125,
-      "learning_rate": 0.0003,
-      "loss": 2.6233530044555664,
-      "step": 940
+      "epoch": 0.06714285714285714,
+      "grad_norm": 0.111328125,
+      "learning_rate": 0.0002973176169353022,
+      "loss": 2.6362348556518556,
+      "step": 470
     },
     {
-      "epoch": 0.05333333333333334,
-      "grad_norm": 0.06689453125,
-      "learning_rate": 0.0003,
-      "loss": 2.600985527038574,
-      "step": 960
+      "epoch": 0.06857142857142857,
+      "grad_norm": 1.1015625,
+      "learning_rate": 0.00029718845790900785,
+      "loss": 2.613158416748047,
+      "step": 480
     },
     {
-      "epoch": 0.05444444444444444,
-      "grad_norm": 0.0791015625,
-      "learning_rate": 0.0003,
-      "loss": 2.6462072372436523,
-      "step": 980
+      "epoch": 0.07,
+      "grad_norm": 0.412109375,
+      "learning_rate": 0.00029705629139991567,
+      "loss": 2.658156967163086,
+      "step": 490
     },
     {
-      "epoch": 0.05555555555555555,
-      "grad_norm": 0.07470703125,
-      "learning_rate": 0.0003,
-      "loss": 2.6183977127075195,
-      "step": 1000
+      "epoch": 0.07142857142857142,
+      "grad_norm": 0.42578125,
+      "learning_rate": 0.000296921120108567,
+      "loss": 2.6301057815551756,
+      "step": 500
     },
     {
-      "epoch": 0.05555555555555555,
-      "eval_loss": 3.0301759243011475,
-      "eval_runtime": 4.4446,
-      "eval_samples_per_second": 67.048,
-      "eval_steps_per_second": 3.825,
-      "step": 1000
+      "epoch": 0.07142857142857142,
+      "eval_loss": 3.0305709838867188,
+      "eval_runtime": 4.4742,
+      "eval_samples_per_second": 66.604,
+      "eval_steps_per_second": 3.8,
+      "step": 500
     }
   ],
-  "logging_steps": 20,
-  "max_steps": 18000,
+  "logging_steps": 10,
+  "max_steps": 7000,
   "num_input_tokens_seen": 0,
   "num_train_epochs": 9223372036854775807,
-  "save_steps": 1000,
+  "save_steps": 500,
   "stateful_callbacks": {
     "TrainerControl": {
       "args": {

last-checkpoint/training_args.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:4d0fa3459db214f81ac56235c0274e72db303ff14a77456f1fa013b5e4128f2e
-size 5265
+oid sha256:6f167c9a1f8ccc5220a546dd5b0acec4ebf0eadee91fee1ea2fa171733ead18e
+size 5201