Training in progress, step 700, checkpoint

last-checkpoint/config.json

@@ -2,9 +2,13 @@
   "architectures": [
     "ModernLLMForCausalLM"
   ],
-  "bos_token_id": 1,
+  "auto_map": {
+    "AutoConfig": "configuration_modern_llm.ModernLLMConfig",
+    "AutoModelForCausalLM": "modeling_modern_llm.ModernLLMForCausalLM"
+  },
+  "bos_token_id": 151643,
   "dtype": "float32",
-  "eos_token_id": 2,
+  "eos_token_id": 151643,
   "hidden_size": 768,
   "intermediate_size": 2048,
   "max_position_embeddings": 2048,
@@ -12,7 +16,7 @@
   "num_attention_heads": 12,
   "num_hidden_layers": 12,
   "num_key_value_heads": 4,
-  "pad_token_id": 0,
+  "pad_token_id": 151643,
   "rms_norm_eps": 1e-06,
   "rope_theta": 1000000.0,
   "transformers_version": "5.15.1",

last-checkpoint/model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:224bbc547320cbc7a57a274bef9f525f2e4be737031df14b601959f9380cb929
+oid sha256:58886de0a0264c68415f0973d23f1c25b70abbb55d1ed2f9efc5c3bf293fcdc2
 size 1235573136

last-checkpoint/optimizer.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:f014f1f8cf2eb9a4ff44ad059ad7d7ab8356457d91805882d791c32ab849f992
+oid sha256:1900f7ef4a97b52f3f05b63a249031ce17f304a92cddfd0a50690e346510fc3e
 size 2471218763

last-checkpoint/scaler.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:68b03f74e10591ff22477365642275703166a34b306471da2a9df3e87497356f
+oid sha256:e153e7ac99a45d390bef333d0c839de8b829e6cf174508b6f317b3fb7550ce99
 size 1383

last-checkpoint/scheduler.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:0471ce3745a36e49a3e0d19eac020ddc42dff8b92447d06fa38e6a682e228b19
+oid sha256:26190536bb49f49f7e44c120697c0e6b6c7e45ebca58eedf944e71fd9beb322d
 size 1465

last-checkpoint/trainer_state.json

@@ -2,9 +2,9 @@
   "best_global_step": null,
   "best_metric": null,
   "best_model_checkpoint": null,
-  "epoch": 0.01,
+  "epoch": 0.014,
   "eval_steps": 500,
-  "global_step": 500,
+  "global_step": 700,
   "is_hyper_param_search": false,
   "is_local_process_zero": true,
   "is_world_process_zero": true,
@@ -358,6 +358,146 @@
       "learning_rate": 0.0002976012024048096,
       "loss": 78.9918212890625,
       "step": 500
+    },
+    {
+      "epoch": 0.0102,
+      "grad_norm": 15.467362403869629,
+      "learning_rate": 0.00029754108216432863,
+      "loss": 124.401513671875,
+      "step": 510
+    },
+    {
+      "epoch": 0.0104,
+      "grad_norm": 13.991318702697754,
+      "learning_rate": 0.0002974809619238477,
+      "loss": 110.567431640625,
+      "step": 520
+    },
+    {
+      "epoch": 0.0106,
+      "grad_norm": 17.036069869995117,
+      "learning_rate": 0.0002974208416833667,
+      "loss": 104.7319580078125,
+      "step": 530
+    },
+    {
+      "epoch": 0.0108,
+      "grad_norm": 13.922304153442383,
+      "learning_rate": 0.0002973607214428857,
+      "loss": 100.63988647460937,
+      "step": 540
+    },
+    {
+      "epoch": 0.011,
+      "grad_norm": 13.223531723022461,
+      "learning_rate": 0.0002973006012024048,
+      "loss": 99.96443481445313,
+      "step": 550
+    },
+    {
+      "epoch": 0.0112,
+      "grad_norm": 14.53707504272461,
+      "learning_rate": 0.0002972404809619238,
+      "loss": 95.74251098632813,
+      "step": 560
+    },
+    {
+      "epoch": 0.0114,
+      "grad_norm": 14.91770076751709,
+      "learning_rate": 0.00029718036072144287,
+      "loss": 94.661767578125,
+      "step": 570
+    },
+    {
+      "epoch": 0.0116,
+      "grad_norm": 13.45614242553711,
+      "learning_rate": 0.0002971202404809619,
+      "loss": 90.941796875,
+      "step": 580
+    },
+    {
+      "epoch": 0.0118,
+      "grad_norm": 13.695131301879883,
+      "learning_rate": 0.00029706012024048096,
+      "loss": 92.9096923828125,
+      "step": 590
+    },
+    {
+      "epoch": 0.012,
+      "grad_norm": 16.406902313232422,
+      "learning_rate": 0.00029699999999999996,
+      "loss": 90.14566650390626,
+      "step": 600
+    },
+    {
+      "epoch": 0.0122,
+      "grad_norm": 11.779080390930176,
+      "learning_rate": 0.000296939879759519,
+      "loss": 86.8746826171875,
+      "step": 610
+    },
+    {
+      "epoch": 0.0124,
+      "grad_norm": 13.758212089538574,
+      "learning_rate": 0.00029687975951903805,
+      "loss": 87.3355224609375,
+      "step": 620
+    },
+    {
+      "epoch": 0.0126,
+      "grad_norm": 12.833357810974121,
+      "learning_rate": 0.0002968196392785571,
+      "loss": 89.81148071289063,
+      "step": 630
+    },
+    {
+      "epoch": 0.0128,
+      "grad_norm": 14.684094429016113,
+      "learning_rate": 0.0002967595190380761,
+      "loss": 89.44540405273438,
+      "step": 640
+    },
+    {
+      "epoch": 0.013,
+      "grad_norm": 10.780083656311035,
+      "learning_rate": 0.0002966993987975952,
+      "loss": 88.07953491210938,
+      "step": 650
+    },
+    {
+      "epoch": 0.0132,
+      "grad_norm": 11.908385276794434,
+      "learning_rate": 0.0002966392785571142,
+      "loss": 87.04054565429688,
+      "step": 660
+    },
+    {
+      "epoch": 0.0134,
+      "grad_norm": 15.518954277038574,
+      "learning_rate": 0.00029657915831663323,
+      "loss": 85.9423583984375,
+      "step": 670
+    },
+    {
+      "epoch": 0.0136,
+      "grad_norm": 10.570508003234863,
+      "learning_rate": 0.0002965190380761523,
+      "loss": 86.79979858398437,
+      "step": 680
+    },
+    {
+      "epoch": 0.0138,
+      "grad_norm": 14.696393013000488,
+      "learning_rate": 0.00029645891783567133,
+      "loss": 79.76475219726562,
+      "step": 690
+    },
+    {
+      "epoch": 0.014,
+      "grad_norm": 12.92392349243164,
+      "learning_rate": 0.0002963987975951904,
+      "loss": 87.0448974609375,
+      "step": 700
     }
   ],
   "logging_steps": 10,
@@ -377,7 +517,7 @@
       "attributes": {}
     }
   },
-  "total_flos": 8161860202859520.0,
+  "total_flos": 1.154174487899904e+16,
   "train_batch_size": 2,
   "trial_name": null,
   "trial_params": null

last-checkpoint/training_args.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:3ead25e90961f39a7cf235a53c52302322d446327bb423dad890f04c47276211
+oid sha256:cba5aee8ada95f513ddc2ee5204de606b5a10fa33758d2e9c8bcbc512ffe2d6b
 size 5201