Training in progress, step 56000, checkpoint

last-checkpoint/model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:e2c14fd1bee72f4ae520b48b248b2ea9e307a5e6c6de8beaabe8a16fe62005b7
+oid sha256:37d8cb0a48975371483c8d0f0ea6cbaf3751d2262e07e187669c5fa919e08bdd
 size 373077376

last-checkpoint/optimizer.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:eefb6b7646dd239517efc08c52c05e50bd2e554a46e9a05729931b19fbb006cd
+oid sha256:0698df592b4199333ec83aae72823b0afe639bf7b1724f696793eea67074662d
 size 422377931

last-checkpoint/rng_state_0.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:5f5bdfa2877904e7b9a7b53aade30f914468c44c9bd87ddaf3d868c947122bdc
+oid sha256:1b1f38a3490635b3698eb8dc53557f282297c0ad5c2a0688b8ef197ae933742a
 size 15365

last-checkpoint/rng_state_1.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:afd1a1a2628858c949d2caf414f3cde378351d0ea08a296d07ed8c1eca505698
+oid sha256:79ac1919bb29ab4ae2b1b0fd8994bb6245a7e65efdd03caed0c32a8061880453
 size 15365

last-checkpoint/rng_state_2.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:cb73362b1face5d1a476a56ab0566aef0055e01bb21637b1e4295a0b2bea00c6
+oid sha256:8d1a2675a27b09581871a511c3a1b6270fe19135aa45d2b488c95610aa4071bf
 size 15365

last-checkpoint/rng_state_3.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:3af950afbfdd102f1586489bf8551b3cbee62c9fe5ac201ea0bda312d056d639
+oid sha256:093642c6af6e0eff31531a5cc5f33bd12f66b88e390ca8a0b353843eca5d3d88
 size 15365

last-checkpoint/scheduler.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:ec23b225f5d4e10d609ff2c03cba3f552baeaa7a8e06ceae9d6f9bd12845a59c
+oid sha256:5a43a1adab58b12e1b9f398e0b2131e69a1f26da62e19f0d8154a7da3af2a7c5
 size 1401

last-checkpoint/trainer_state.json

@@ -2,9 +2,9 @@
   "best_global_step": 55000,
   "best_metric": 2.553321599960327,
   "best_model_checkpoint": "./artifacts/models/pretrain-4gpu-8k-ckpt/checkpoint-55000",
-  "epoch": 0.9633236417136651,
+  "epoch": 0.98083861701755,
   "eval_steps": 1000,
-  "global_step": 55000,
+  "global_step": 56000,
   "is_hyper_param_search": false,
   "is_local_process_zero": true,
   "is_world_process_zero": true,
@@ -15848,6 +15848,294 @@
       "eval_samples_per_second": 89.359,
       "eval_steps_per_second": 2.812,
       "step": 55000
+    },
+    {
+      "epoch": 0.9637615160962623,
+      "grad_norm": 0.2177734375,
+      "learning_rate": 3.589523287436003e-06,
+      "loss": 2.5365,
+      "step": 55025
+    },
+    {
+      "epoch": 0.9641993904788594,
+      "grad_norm": 0.212890625,
+      "learning_rate": 3.5034441832159647e-06,
+      "loss": 2.5433,
+      "step": 55050
+    },
+    {
+      "epoch": 0.9646372648614565,
+      "grad_norm": 0.2177734375,
+      "learning_rate": 3.418406131370844e-06,
+      "loss": 2.5401,
+      "step": 55075
+    },
+    {
+      "epoch": 0.9650751392440536,
+      "grad_norm": 0.21484375,
+      "learning_rate": 3.334409310208175e-06,
+      "loss": 2.5335,
+      "step": 55100
+    },
+    {
+      "epoch": 0.9655130136266508,
+      "grad_norm": 0.2158203125,
+      "learning_rate": 3.2514538958521857e-06,
+      "loss": 2.534,
+      "step": 55125
+    },
+    {
+      "epoch": 0.9659508880092479,
+      "grad_norm": 0.2177734375,
+      "learning_rate": 3.1695400622435145e-06,
+      "loss": 2.5357,
+      "step": 55150
+    },
+    {
+      "epoch": 0.966388762391845,
+      "grad_norm": 0.208984375,
+      "learning_rate": 3.088667981138826e-06,
+      "loss": 2.5522,
+      "step": 55175
+    },
+    {
+      "epoch": 0.9668266367744421,
+      "grad_norm": 0.21484375,
+      "learning_rate": 3.008837822110533e-06,
+      "loss": 2.5374,
+      "step": 55200
+    },
+    {
+      "epoch": 0.9672645111570393,
+      "grad_norm": 0.2119140625,
+      "learning_rate": 2.9300497525461266e-06,
+      "loss": 2.5592,
+      "step": 55225
+    },
+    {
+      "epoch": 0.9677023855396364,
+      "grad_norm": 0.234375,
+      "learning_rate": 2.8523039376482373e-06,
+      "loss": 2.534,
+      "step": 55250
+    },
+    {
+      "epoch": 0.9681402599222335,
+      "grad_norm": 0.2099609375,
+      "learning_rate": 2.77560054043402e-06,
+      "loss": 2.5422,
+      "step": 55275
+    },
+    {
+      "epoch": 0.9685781343048306,
+      "grad_norm": 0.2099609375,
+      "learning_rate": 2.6999397217348788e-06,
+      "loss": 2.55,
+      "step": 55300
+    },
+    {
+      "epoch": 0.9690160086874278,
+      "grad_norm": 0.2158203125,
+      "learning_rate": 2.625321640196299e-06,
+      "loss": 2.5261,
+      "step": 55325
+    },
+    {
+      "epoch": 0.9694538830700249,
+      "grad_norm": 0.2265625,
+      "learning_rate": 2.551746452277126e-06,
+      "loss": 2.5446,
+      "step": 55350
+    },
+    {
+      "epoch": 0.969891757452622,
+      "grad_norm": 0.2177734375,
+      "learning_rate": 2.479214312249567e-06,
+      "loss": 2.5474,
+      "step": 55375
+    },
+    {
+      "epoch": 0.9703296318352191,
+      "grad_norm": 0.208984375,
+      "learning_rate": 2.4077253721988545e-06,
+      "loss": 2.5297,
+      "step": 55400
+    },
+    {
+      "epoch": 0.9707675062178163,
+      "grad_norm": 0.216796875,
+      "learning_rate": 2.3372797820226943e-06,
+      "loss": 2.5263,
+      "step": 55425
+    },
+    {
+      "epoch": 0.9712053806004134,
+      "grad_norm": 0.2060546875,
+      "learning_rate": 2.267877689431208e-06,
+      "loss": 2.5389,
+      "step": 55450
+    },
+    {
+      "epoch": 0.9716432549830105,
+      "grad_norm": 0.216796875,
+      "learning_rate": 2.19951923994649e-06,
+      "loss": 2.5364,
+      "step": 55475
+    },
+    {
+      "epoch": 0.9720811293656076,
+      "grad_norm": 0.2109375,
+      "learning_rate": 2.1322045769022745e-06,
+      "loss": 2.5467,
+      "step": 55500
+    },
+    {
+      "epoch": 0.9725190037482048,
+      "grad_norm": 0.2119140625,
+      "learning_rate": 2.065933841443768e-06,
+      "loss": 2.5398,
+      "step": 55525
+    },
+    {
+      "epoch": 0.9729568781308018,
+      "grad_norm": 0.2197265625,
+      "learning_rate": 2.000707172527205e-06,
+      "loss": 2.5369,
+      "step": 55550
+    },
+    {
+      "epoch": 0.9733947525133989,
+      "grad_norm": 0.2158203125,
+      "learning_rate": 1.9365247069196847e-06,
+      "loss": 2.5471,
+      "step": 55575
+    },
+    {
+      "epoch": 0.973832626895996,
+      "grad_norm": 0.2080078125,
+      "learning_rate": 1.8733865791987769e-06,
+      "loss": 2.5409,
+      "step": 55600
+    },
+    {
+      "epoch": 0.9742705012785932,
+      "grad_norm": 0.2099609375,
+      "learning_rate": 1.811292921752361e-06,
+      "loss": 2.5313,
+      "step": 55625
+    },
+    {
+      "epoch": 0.9747083756611903,
+      "grad_norm": 0.212890625,
+      "learning_rate": 1.7502438647781227e-06,
+      "loss": 2.5568,
+      "step": 55650
+    },
+    {
+      "epoch": 0.9751462500437874,
+      "grad_norm": 0.2099609375,
+      "learning_rate": 1.6902395362835555e-06,
+      "loss": 2.5366,
+      "step": 55675
+    },
+    {
+      "epoch": 0.9755841244263845,
+      "grad_norm": 0.2138671875,
+      "learning_rate": 1.631280062085516e-06,
+      "loss": 2.5387,
+      "step": 55700
+    },
+    {
+      "epoch": 0.9760219988089817,
+      "grad_norm": 0.21484375,
+      "learning_rate": 1.5733655658101698e-06,
+      "loss": 2.5226,
+      "step": 55725
+    },
+    {
+      "epoch": 0.9764598731915788,
+      "grad_norm": 0.21484375,
+      "learning_rate": 1.5164961688922673e-06,
+      "loss": 2.5414,
+      "step": 55750
+    },
+    {
+      "epoch": 0.9768977475741759,
+      "grad_norm": 0.2158203125,
+      "learning_rate": 1.460671990575535e-06,
+      "loss": 2.5334,
+      "step": 55775
+    },
+    {
+      "epoch": 0.977335621956773,
+      "grad_norm": 0.23046875,
+      "learning_rate": 1.405893147911841e-06,
+      "loss": 2.5389,
+      "step": 55800
+    },
+    {
+      "epoch": 0.9777734963393702,
+      "grad_norm": 0.21484375,
+      "learning_rate": 1.352159755761251e-06,
+      "loss": 2.539,
+      "step": 55825
+    },
+    {
+      "epoch": 0.9782113707219673,
+      "grad_norm": 0.2080078125,
+      "learning_rate": 1.299471926791862e-06,
+      "loss": 2.5344,
+      "step": 55850
+    },
+    {
+      "epoch": 0.9786492451045644,
+      "grad_norm": 0.208984375,
+      "learning_rate": 1.247829771479303e-06,
+      "loss": 2.5352,
+      "step": 55875
+    },
+    {
+      "epoch": 0.9790871194871615,
+      "grad_norm": 0.212890625,
+      "learning_rate": 1.1972333981066785e-06,
+      "loss": 2.5426,
+      "step": 55900
+    },
+    {
+      "epoch": 0.9795249938697587,
+      "grad_norm": 0.23046875,
+      "learning_rate": 1.147682912764292e-06,
+      "loss": 2.5364,
+      "step": 55925
+    },
+    {
+      "epoch": 0.9799628682523558,
+      "grad_norm": 0.216796875,
+      "learning_rate": 1.0991784193494224e-06,
+      "loss": 2.5522,
+      "step": 55950
+    },
+    {
+      "epoch": 0.9804007426349529,
+      "grad_norm": 0.21484375,
+      "learning_rate": 1.0517200195661603e-06,
+      "loss": 2.5417,
+      "step": 55975
+    },
+    {
+      "epoch": 0.98083861701755,
+      "grad_norm": 0.2158203125,
+      "learning_rate": 1.0053078129251824e-06,
+      "loss": 2.5455,
+      "step": 56000
+    },
+    {
+      "epoch": 0.98083861701755,
+      "eval_loss": 2.5533249378204346,
+      "eval_runtime": 36.4148,
+      "eval_samples_per_second": 100.371,
+      "eval_steps_per_second": 3.158,
+      "step": 56000
     }
   ],
   "logging_steps": 25,
@@ -15867,7 +16155,7 @@
       "attributes": {}
     }
   },
-  "total_flos": 2.789592043844377e+19,
+  "total_flos": 2.8403068460916015e+19,
   "train_batch_size": 16,
   "trial_name": null,
   "trial_params": null