Training in progress, step 1100

last-checkpoint/config.json

@@ -2,13 +2,9 @@
   "architectures": [
     "ModernLLMForCausalLM"
   ],
-  "auto_map": {
-    "AutoConfig": "configuration_modern_llm.ModernLLMConfig",
-    "AutoModelForCausalLM": "modeling_modern_llm.ModernLLMForCausalLM"
-  },
-  "bos_token_id": 151643,
+  "bos_token_id": 1,
   "dtype": "float32",
-  "eos_token_id": 151643,
+  "eos_token_id": 2,
   "hidden_size": 768,
   "intermediate_size": 2048,
   "max_position_embeddings": 2048,
@@ -16,7 +12,7 @@
   "num_attention_heads": 12,
   "num_hidden_layers": 12,
   "num_key_value_heads": 4,
-  "pad_token_id": 151643,
+  "pad_token_id": 0,
   "rms_norm_eps": 1e-06,
   "rope_theta": 1000000.0,
   "transformers_version": "5.15.1",

last-checkpoint/model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:5a3ad66771ef5b8246c59acc3ea2a9104f762ec13d612e6e588d54d7c66836e3
+oid sha256:224bbc547320cbc7a57a274bef9f525f2e4be737031df14b601959f9380cb929
 size 1235573136

last-checkpoint/optimizer.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:87af1234b6b81d7ca503ab8b876edeee1375b5e739dae1d471d4d0c94c2cdc13
+oid sha256:f014f1f8cf2eb9a4ff44ad059ad7d7ab8356457d91805882d791c32ab849f992
 size 2471218763

last-checkpoint/scaler.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:9aa7e5f7f3366b7db6f25a9e3fa739116674bd641ce589a5940ff73f382fec0c
+oid sha256:68b03f74e10591ff22477365642275703166a34b306471da2a9df3e87497356f
 size 1383

last-checkpoint/scheduler.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:4835dfebca207db20e97bc75a0cdfb3c9040e987a06d53d84bc51055c8c81f56
+oid sha256:0471ce3745a36e49a3e0d19eac020ddc42dff8b92447d06fa38e6a682e228b19
 size 1465

last-checkpoint/trainer_state.json

@@ -2,9 +2,9 @@
   "best_global_step": null,
   "best_metric": null,
   "best_model_checkpoint": null,
-  "epoch": 0.02,
+  "epoch": 0.01,
   "eval_steps": 500,
-  "global_step": 1000,
+  "global_step": 500,
   "is_hyper_param_search": false,
   "is_local_process_zero": true,
   "is_world_process_zero": true,
@@ -358,356 +358,6 @@
       "learning_rate": 0.0002976012024048096,
       "loss": 78.9918212890625,
       "step": 500
-    },
-    {
-      "epoch": 0.0102,
-      "grad_norm": 15.467362403869629,
-      "learning_rate": 0.00029754108216432863,
-      "loss": 124.401513671875,
-      "step": 510
-    },
-    {
-      "epoch": 0.0104,
-      "grad_norm": 13.991318702697754,
-      "learning_rate": 0.0002974809619238477,
-      "loss": 110.567431640625,
-      "step": 520
-    },
-    {
-      "epoch": 0.0106,
-      "grad_norm": 17.036069869995117,
-      "learning_rate": 0.0002974208416833667,
-      "loss": 104.7319580078125,
-      "step": 530
-    },
-    {
-      "epoch": 0.0108,
-      "grad_norm": 13.922304153442383,
-      "learning_rate": 0.0002973607214428857,
-      "loss": 100.63988647460937,
-      "step": 540
-    },
-    {
-      "epoch": 0.011,
-      "grad_norm": 13.223531723022461,
-      "learning_rate": 0.0002973006012024048,
-      "loss": 99.96443481445313,
-      "step": 550
-    },
-    {
-      "epoch": 0.0112,
-      "grad_norm": 14.53707504272461,
-      "learning_rate": 0.0002972404809619238,
-      "loss": 95.74251098632813,
-      "step": 560
-    },
-    {
-      "epoch": 0.0114,
-      "grad_norm": 14.91770076751709,
-      "learning_rate": 0.00029718036072144287,
-      "loss": 94.661767578125,
-      "step": 570
-    },
-    {
-      "epoch": 0.0116,
-      "grad_norm": 13.45614242553711,
-      "learning_rate": 0.0002971202404809619,
-      "loss": 90.941796875,
-      "step": 580
-    },
-    {
-      "epoch": 0.0118,
-      "grad_norm": 13.695131301879883,
-      "learning_rate": 0.00029706012024048096,
-      "loss": 92.9096923828125,
-      "step": 590
-    },
-    {
-      "epoch": 0.012,
-      "grad_norm": 16.406902313232422,
-      "learning_rate": 0.00029699999999999996,
-      "loss": 90.14566650390626,
-      "step": 600
-    },
-    {
-      "epoch": 0.0122,
-      "grad_norm": 11.779080390930176,
-      "learning_rate": 0.000296939879759519,
-      "loss": 86.8746826171875,
-      "step": 610
-    },
-    {
-      "epoch": 0.0124,
-      "grad_norm": 13.758212089538574,
-      "learning_rate": 0.00029687975951903805,
-      "loss": 87.3355224609375,
-      "step": 620
-    },
-    {
-      "epoch": 0.0126,
-      "grad_norm": 12.833357810974121,
-      "learning_rate": 0.0002968196392785571,
-      "loss": 89.81148071289063,
-      "step": 630
-    },
-    {
-      "epoch": 0.0128,
-      "grad_norm": 14.684094429016113,
-      "learning_rate": 0.0002967595190380761,
-      "loss": 89.44540405273438,
-      "step": 640
-    },
-    {
-      "epoch": 0.013,
-      "grad_norm": 10.780083656311035,
-      "learning_rate": 0.0002966993987975952,
-      "loss": 88.07953491210938,
-      "step": 650
-    },
-    {
-      "epoch": 0.0132,
-      "grad_norm": 11.908385276794434,
-      "learning_rate": 0.0002966392785571142,
-      "loss": 87.04054565429688,
-      "step": 660
-    },
-    {
-      "epoch": 0.0134,
-      "grad_norm": 15.518954277038574,
-      "learning_rate": 0.00029657915831663323,
-      "loss": 85.9423583984375,
-      "step": 670
-    },
-    {
-      "epoch": 0.0136,
-      "grad_norm": 10.570508003234863,
-      "learning_rate": 0.0002965190380761523,
-      "loss": 86.79979858398437,
-      "step": 680
-    },
-    {
-      "epoch": 0.0138,
-      "grad_norm": 14.696393013000488,
-      "learning_rate": 0.00029645891783567133,
-      "loss": 79.76475219726562,
-      "step": 690
-    },
-    {
-      "epoch": 0.014,
-      "grad_norm": 12.92392349243164,
-      "learning_rate": 0.0002963987975951904,
-      "loss": 87.0448974609375,
-      "step": 700
-    },
-    {
-      "epoch": 0.0142,
-      "grad_norm": 13.465226173400879,
-      "learning_rate": 0.00029633867735470937,
-      "loss": 90.22592163085938,
-      "step": 710
-    },
-    {
-      "epoch": 0.0144,
-      "grad_norm": 12.435464859008789,
-      "learning_rate": 0.0002962785571142284,
-      "loss": 86.32198486328124,
-      "step": 720
-    },
-    {
-      "epoch": 0.0146,
-      "grad_norm": 12.340784072875977,
-      "learning_rate": 0.00029621843687374747,
-      "loss": 89.89630737304688,
-      "step": 730
-    },
-    {
-      "epoch": 0.0148,
-      "grad_norm": 14.63569164276123,
-      "learning_rate": 0.0002961583166332665,
-      "loss": 86.31669921875,
-      "step": 740
-    },
-    {
-      "epoch": 0.015,
-      "grad_norm": 10.646244049072266,
-      "learning_rate": 0.00029609819639278556,
-      "loss": 85.20569458007813,
-      "step": 750
-    },
-    {
-      "epoch": 0.0152,
-      "grad_norm": 8.930062294006348,
-      "learning_rate": 0.0002960380761523046,
-      "loss": 86.44678955078125,
-      "step": 760
-    },
-    {
-      "epoch": 0.0154,
-      "grad_norm": 12.228261947631836,
-      "learning_rate": 0.0002959779559118236,
-      "loss": 84.96986083984375,
-      "step": 770
-    },
-    {
-      "epoch": 0.0156,
-      "grad_norm": 9.189741134643555,
-      "learning_rate": 0.00029591783567134265,
-      "loss": 86.03868408203125,
-      "step": 780
-    },
-    {
-      "epoch": 0.0158,
-      "grad_norm": 12.331954956054688,
-      "learning_rate": 0.0002958577154308617,
-      "loss": 85.13253784179688,
-      "step": 790
-    },
-    {
-      "epoch": 0.016,
-      "grad_norm": 14.345333099365234,
-      "learning_rate": 0.00029579759519038075,
-      "loss": 86.58322143554688,
-      "step": 800
-    },
-    {
-      "epoch": 0.0162,
-      "grad_norm": 10.961579322814941,
-      "learning_rate": 0.00029573747494989974,
-      "loss": 84.83054809570312,
-      "step": 810
-    },
-    {
-      "epoch": 0.0164,
-      "grad_norm": 11.578486442565918,
-      "learning_rate": 0.0002956773547094188,
-      "loss": 87.94888305664062,
-      "step": 820
-    },
-    {
-      "epoch": 0.0166,
-      "grad_norm": 13.537806510925293,
-      "learning_rate": 0.00029561723446893784,
-      "loss": 87.522900390625,
-      "step": 830
-    },
-    {
-      "epoch": 0.0168,
-      "grad_norm": 10.907570838928223,
-      "learning_rate": 0.0002955571142284569,
-      "loss": 84.80013427734374,
-      "step": 840
-    },
-    {
-      "epoch": 0.017,
-      "grad_norm": 13.713836669921875,
-      "learning_rate": 0.00029549699398797593,
-      "loss": 86.79594116210937,
-      "step": 850
-    },
-    {
-      "epoch": 0.0172,
-      "grad_norm": 12.571099281311035,
-      "learning_rate": 0.000295436873747495,
-      "loss": 84.85569458007812,
-      "step": 860
-    },
-    {
-      "epoch": 0.0174,
-      "grad_norm": 9.433479309082031,
-      "learning_rate": 0.000295376753507014,
-      "loss": 85.20677490234375,
-      "step": 870
-    },
-    {
-      "epoch": 0.0176,
-      "grad_norm": 12.85809326171875,
-      "learning_rate": 0.000295316633266533,
-      "loss": 79.00835571289062,
-      "step": 880
-    },
-    {
-      "epoch": 0.0178,
-      "grad_norm": 11.075084686279297,
-      "learning_rate": 0.00029525651302605207,
-      "loss": 80.06212158203125,
-      "step": 890
-    },
-    {
-      "epoch": 0.018,
-      "grad_norm": 14.64054012298584,
-      "learning_rate": 0.0002951963927855711,
-      "loss": 83.31229248046876,
-      "step": 900
-    },
-    {
-      "epoch": 0.0182,
-      "grad_norm": 11.558423042297363,
-      "learning_rate": 0.00029513627254509016,
-      "loss": 78.82657470703126,
-      "step": 910
-    },
-    {
-      "epoch": 0.0184,
-      "grad_norm": 22.44339942932129,
-      "learning_rate": 0.00029507615230460916,
-      "loss": 81.31973266601562,
-      "step": 920
-    },
-    {
-      "epoch": 0.0186,
-      "grad_norm": 10.331223487854004,
-      "learning_rate": 0.0002950160320641282,
-      "loss": 83.343115234375,
-      "step": 930
-    },
-    {
-      "epoch": 0.0188,
-      "grad_norm": 9.869694709777832,
-      "learning_rate": 0.0002949559118236473,
-      "loss": 79.85993041992188,
-      "step": 940
-    },
-    {
-      "epoch": 0.019,
-      "grad_norm": 10.701475143432617,
-      "learning_rate": 0.0002948957915831663,
-      "loss": 84.02029418945312,
-      "step": 950
-    },
-    {
-      "epoch": 0.0192,
-      "grad_norm": 12.234188079833984,
-      "learning_rate": 0.00029483567134268535,
-      "loss": 83.42827758789062,
-      "step": 960
-    },
-    {
-      "epoch": 0.0194,
-      "grad_norm": 11.610219955444336,
-      "learning_rate": 0.0002947755511022044,
-      "loss": 80.161669921875,
-      "step": 970
-    },
-    {
-      "epoch": 0.0196,
-      "grad_norm": 8.672530174255371,
-      "learning_rate": 0.00029471543086172344,
-      "loss": 81.43499145507812,
-      "step": 980
-    },
-    {
-      "epoch": 0.0198,
-      "grad_norm": 11.546252250671387,
-      "learning_rate": 0.00029465531062124244,
-      "loss": 79.0343017578125,
-      "step": 990
-    },
-    {
-      "epoch": 0.02,
-      "grad_norm": 11.632270812988281,
-      "learning_rate": 0.0002945951903807615,
-      "loss": 80.40287475585937,
-      "step": 1000
     }
   ],
   "logging_steps": 10,
@@ -727,7 +377,7 @@
       "attributes": {}
     }
   },
-  "total_flos": 1.65498647609088e+16,
+  "total_flos": 8161860202859520.0,
   "train_batch_size": 2,
   "trial_name": null,
   "trial_params": null

last-checkpoint/training_args.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:cba5aee8ada95f513ddc2ee5204de606b5a10fa33758d2e9c8bcbc512ffe2d6b
+oid sha256:3ead25e90961f39a7cf235a53c52302322d446327bb423dad890f04c47276211
 size 5201

model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:5a3ad66771ef5b8246c59acc3ea2a9104f762ec13d612e6e588d54d7c66836e3
+oid sha256:5115379d755d2923b9e3aee82fa06ce8b6ba2890c3351d2cad9f1e33c87b9712
 size 1235573136