Training in progress, step 500, checkpoint

last-checkpoint/config.json

@@ -0,0 +1,29 @@
+{
+  "architectures": [
+    "LlamaForCausalLM"
+  ],
+  "attention_bias": false,
+  "attention_dropout": 0.0,
+  "bos_token_id": 1,
+  "dtype": "float32",
+  "eos_token_id": 2,
+  "head_dim": 64,
+  "hidden_act": "silu",
+  "hidden_size": 384,
+  "initializer_range": 0.02,
+  "intermediate_size": 1024,
+  "max_position_embeddings": 1024,
+  "mlp_bias": false,
+  "model_type": "llama",
+  "num_attention_heads": 6,
+  "num_hidden_layers": 15,
+  "num_key_value_heads": 1,
+  "pretraining_tp": 1,
+  "rms_norm_eps": 1e-05,
+  "rope_scaling": null,
+  "rope_theta": 10000.0,
+  "tie_word_embeddings": true,
+  "transformers_version": "4.57.6",
+  "use_cache": true,
+  "vocab_size": 32000
+}

last-checkpoint/generation_config.json

@@ -0,0 +1,8 @@
+{
+  "_from_model_config": true,
+  "bos_token_id": 1,
+  "eos_token_id": [
+    2
+  ],
+  "transformers_version": "4.57.6"
+}

last-checkpoint/model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:74f03a7cae6f59d5162106ba624865f0712b3ce00ebffa4abcaf9dc5a605315d
+size 140637408

last-checkpoint/optimizer.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c6c79565252c3803d96ba398472eccf87cf739191272381560ad07d68c20c9ca
+size 281362507

last-checkpoint/rng_state.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:098b29492211804ab324a36f37466821d948280bb74fce4ba895c03f13ecd878
+size 14645

last-checkpoint/scheduler.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:350811cf0b8d7ebb0375f99d7c5be37c2b3fc48fc087f648711ef9871266ac16
+size 1465

last-checkpoint/special_tokens_map.json

@@ -0,0 +1,23 @@
+{
+  "bos_token": {
+    "content": "<s>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "eos_token": {
+    "content": "</s>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "unk_token": {
+    "content": "<unk>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  }
+}

last-checkpoint/tokenizer.model

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9e556afd44213b6bd1be2b850ebbbd98f5481437a8021afaf58ee7fb1818d347
+size 499723

last-checkpoint/tokenizer_config.json

@@ -0,0 +1,43 @@
+{
+  "add_bos_token": true,
+  "add_eos_token": false,
+  "add_prefix_space": null,
+  "added_tokens_decoder": {
+    "0": {
+      "content": "<unk>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "1": {
+      "content": "<s>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "2": {
+      "content": "</s>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    }
+  },
+  "bos_token": "<s>",
+  "clean_up_tokenization_spaces": false,
+  "eos_token": "</s>",
+  "extra_special_tokens": {},
+  "legacy": false,
+  "model_max_length": 1000000000000000019884624838656,
+  "pad_token": null,
+  "padding_side": "right",
+  "sp_model_kwargs": {},
+  "tokenizer_class": "LlamaTokenizer",
+  "unk_token": "<unk>",
+  "use_default_system_prompt": false
+}

last-checkpoint/trainer_state.json

@@ -0,0 +1,393 @@
+{
+  "best_global_step": null,
+  "best_metric": null,
+  "best_model_checkpoint": null,
+  "epoch": 0.0065535953023828876,
+  "eval_steps": 500,
+  "global_step": 500,
+  "is_hyper_param_search": false,
+  "is_local_process_zero": true,
+  "is_world_process_zero": true,
+  "log_history": [
+    {
+      "epoch": 0.00013107190604765774,
+      "grad_norm": 1.6560418605804443,
+      "learning_rate": 5.399999999999999e-06,
+      "loss": 10.4207,
+      "step": 10
+    },
+    {
+      "epoch": 0.0002621438120953155,
+      "grad_norm": 2.0435309410095215,
+      "learning_rate": 1.14e-05,
+      "loss": 10.3202,
+      "step": 20
+    },
+    {
+      "epoch": 0.00039321571814297324,
+      "grad_norm": 1.6376142501831055,
+      "learning_rate": 1.74e-05,
+      "loss": 10.104,
+      "step": 30
+    },
+    {
+      "epoch": 0.000524287624190631,
+      "grad_norm": 1.467596411705017,
+      "learning_rate": 2.34e-05,
+      "loss": 9.894,
+      "step": 40
+    },
+    {
+      "epoch": 0.0006553595302382887,
+      "grad_norm": 1.453574538230896,
+      "learning_rate": 2.94e-05,
+      "loss": 9.7455,
+      "step": 50
+    },
+    {
+      "epoch": 0.0007864314362859465,
+      "grad_norm": 1.4320564270019531,
+      "learning_rate": 3.539999999999999e-05,
+      "loss": 9.6131,
+      "step": 60
+    },
+    {
+      "epoch": 0.0009175033423336042,
+      "grad_norm": 1.4820325374603271,
+      "learning_rate": 4.14e-05,
+      "loss": 9.4517,
+      "step": 70
+    },
+    {
+      "epoch": 0.001048575248381262,
+      "grad_norm": 1.4840964078903198,
+      "learning_rate": 4.7399999999999993e-05,
+      "loss": 9.2702,
+      "step": 80
+    },
+    {
+      "epoch": 0.0011796471544289198,
+      "grad_norm": 1.404575228691101,
+      "learning_rate": 5.339999999999999e-05,
+      "loss": 9.0683,
+      "step": 90
+    },
+    {
+      "epoch": 0.0013107190604765774,
+      "grad_norm": 1.39352285861969,
+      "learning_rate": 5.94e-05,
+      "loss": 8.8432,
+      "step": 100
+    },
+    {
+      "epoch": 0.0014417909665242353,
+      "grad_norm": 1.5529464483261108,
+      "learning_rate": 6.539999999999999e-05,
+      "loss": 8.5904,
+      "step": 110
+    },
+    {
+      "epoch": 0.001572862872571893,
+      "grad_norm": 1.4607466459274292,
+      "learning_rate": 7.139999999999999e-05,
+      "loss": 8.3305,
+      "step": 120
+    },
+    {
+      "epoch": 0.0017039347786195506,
+      "grad_norm": 1.279509425163269,
+      "learning_rate": 7.74e-05,
+      "loss": 8.1079,
+      "step": 130
+    },
+    {
+      "epoch": 0.0018350066846672085,
+      "grad_norm": 1.2591545581817627,
+      "learning_rate": 8.34e-05,
+      "loss": 7.8698,
+      "step": 140
+    },
+    {
+      "epoch": 0.0019660785907148664,
+      "grad_norm": 1.019737958908081,
+      "learning_rate": 8.939999999999999e-05,
+      "loss": 7.6731,
+      "step": 150
+    },
+    {
+      "epoch": 0.002097150496762524,
+      "grad_norm": 0.8820710778236389,
+      "learning_rate": 9.539999999999999e-05,
+      "loss": 7.4882,
+      "step": 160
+    },
+    {
+      "epoch": 0.0022282224028101817,
+      "grad_norm": 0.7616180181503296,
+      "learning_rate": 0.0001014,
+      "loss": 7.3478,
+      "step": 170
+    },
+    {
+      "epoch": 0.0023592943088578395,
+      "grad_norm": 0.8487536907196045,
+      "learning_rate": 0.00010739999999999998,
+      "loss": 7.228,
+      "step": 180
+    },
+    {
+      "epoch": 0.002490366214905497,
+      "grad_norm": 0.7819269299507141,
+      "learning_rate": 0.00011339999999999999,
+      "loss": 7.0934,
+      "step": 190
+    },
+    {
+      "epoch": 0.002621438120953155,
+      "grad_norm": 0.8089203238487244,
+      "learning_rate": 0.0001194,
+      "loss": 7.0225,
+      "step": 200
+    },
+    {
+      "epoch": 0.0027525100270008127,
+      "grad_norm": 0.6102705597877502,
+      "learning_rate": 0.00012539999999999999,
+      "loss": 6.9299,
+      "step": 210
+    },
+    {
+      "epoch": 0.0028835819330484706,
+      "grad_norm": 0.8053275346755981,
+      "learning_rate": 0.0001314,
+      "loss": 6.8061,
+      "step": 220
+    },
+    {
+      "epoch": 0.003014653839096128,
+      "grad_norm": 0.853755533695221,
+      "learning_rate": 0.0001374,
+      "loss": 6.7241,
+      "step": 230
+    },
+    {
+      "epoch": 0.003145725745143786,
+      "grad_norm": 0.845130443572998,
+      "learning_rate": 0.0001434,
+      "loss": 6.6139,
+      "step": 240
+    },
+    {
+      "epoch": 0.0032767976511914438,
+      "grad_norm": 0.9666972160339355,
+      "learning_rate": 0.0001494,
+      "loss": 6.5051,
+      "step": 250
+    },
+    {
+      "epoch": 0.0034078695572391012,
+      "grad_norm": 0.8451218008995056,
+      "learning_rate": 0.00015539999999999998,
+      "loss": 6.429,
+      "step": 260
+    },
+    {
+      "epoch": 0.003538941463286759,
+      "grad_norm": 1.2488940954208374,
+      "learning_rate": 0.0001614,
+      "loss": 6.3691,
+      "step": 270
+    },
+    {
+      "epoch": 0.003670013369334417,
+      "grad_norm": 1.445700764656067,
+      "learning_rate": 0.0001674,
+      "loss": 6.3267,
+      "step": 280
+    },
+    {
+      "epoch": 0.0038010852753820744,
+      "grad_norm": 0.8115171194076538,
+      "learning_rate": 0.00017339999999999996,
+      "loss": 6.2537,
+      "step": 290
+    },
+    {
+      "epoch": 0.003932157181429733,
+      "grad_norm": 1.0047732591629028,
+      "learning_rate": 0.00017939999999999997,
+      "loss": 6.2275,
+      "step": 300
+    },
+    {
+      "epoch": 0.00406322908747739,
+      "grad_norm": 0.8865781426429749,
+      "learning_rate": 0.00018539999999999998,
+      "loss": 6.1809,
+      "step": 310
+    },
+    {
+      "epoch": 0.004194300993525048,
+      "grad_norm": 0.8600600361824036,
+      "learning_rate": 0.0001914,
+      "loss": 6.0634,
+      "step": 320
+    },
+    {
+      "epoch": 0.0043253728995727055,
+      "grad_norm": 0.9079091548919678,
+      "learning_rate": 0.0001974,
+      "loss": 6.0005,
+      "step": 330
+    },
+    {
+      "epoch": 0.004456444805620363,
+      "grad_norm": 0.9107650518417358,
+      "learning_rate": 0.00020339999999999998,
+      "loss": 5.9971,
+      "step": 340
+    },
+    {
+      "epoch": 0.004587516711668021,
+      "grad_norm": 0.9232202768325806,
+      "learning_rate": 0.00020939999999999997,
+      "loss": 5.9575,
+      "step": 350
+    },
+    {
+      "epoch": 0.004718588617715679,
+      "grad_norm": 1.1863384246826172,
+      "learning_rate": 0.00021539999999999998,
+      "loss": 5.905,
+      "step": 360
+    },
+    {
+      "epoch": 0.004849660523763337,
+      "grad_norm": 1.2496181726455688,
+      "learning_rate": 0.0002214,
+      "loss": 5.8543,
+      "step": 370
+    },
+    {
+      "epoch": 0.004980732429810994,
+      "grad_norm": 1.1276811361312866,
+      "learning_rate": 0.00022739999999999997,
+      "loss": 5.8504,
+      "step": 380
+    },
+    {
+      "epoch": 0.005111804335858652,
+      "grad_norm": 0.952456533908844,
+      "learning_rate": 0.00023339999999999998,
+      "loss": 5.7879,
+      "step": 390
+    },
+    {
+      "epoch": 0.00524287624190631,
+      "grad_norm": 1.126177430152893,
+      "learning_rate": 0.0002394,
+      "loss": 5.7604,
+      "step": 400
+    },
+    {
+      "epoch": 0.005373948147953968,
+      "grad_norm": 0.8310980200767517,
+      "learning_rate": 0.00024539999999999995,
+      "loss": 5.7287,
+      "step": 410
+    },
+    {
+      "epoch": 0.0055050200540016254,
+      "grad_norm": 0.9914957880973816,
+      "learning_rate": 0.0002514,
+      "loss": 5.6483,
+      "step": 420
+    },
+    {
+      "epoch": 0.005636091960049283,
+      "grad_norm": 1.1346724033355713,
+      "learning_rate": 0.00025739999999999997,
+      "loss": 5.612,
+      "step": 430
+    },
+    {
+      "epoch": 0.005767163866096941,
+      "grad_norm": 0.9475547671318054,
+      "learning_rate": 0.00026339999999999995,
+      "loss": 5.6072,
+      "step": 440
+    },
+    {
+      "epoch": 0.005898235772144598,
+      "grad_norm": 1.1116561889648438,
+      "learning_rate": 0.0002694,
+      "loss": 5.5409,
+      "step": 450
+    },
+    {
+      "epoch": 0.006029307678192256,
+      "grad_norm": 0.9825686812400818,
+      "learning_rate": 0.00027539999999999997,
+      "loss": 5.5056,
+      "step": 460
+    },
+    {
+      "epoch": 0.006160379584239914,
+      "grad_norm": 1.2326000928878784,
+      "learning_rate": 0.00028139999999999996,
+      "loss": 5.4939,
+      "step": 470
+    },
+    {
+      "epoch": 0.006291451490287572,
+      "grad_norm": 1.417091965675354,
+      "learning_rate": 0.00028739999999999994,
+      "loss": 5.4776,
+      "step": 480
+    },
+    {
+      "epoch": 0.00642252339633523,
+      "grad_norm": 1.6037111282348633,
+      "learning_rate": 0.0002934,
+      "loss": 5.4743,
+      "step": 490
+    },
+    {
+      "epoch": 0.0065535953023828876,
+      "grad_norm": 0.9632703065872192,
+      "learning_rate": 0.00029939999999999996,
+      "loss": 5.4157,
+      "step": 500
+    },
+    {
+      "epoch": 0.0065535953023828876,
+      "eval_accuracy": 0.19562903225806452,
+      "eval_loss": 5.403072357177734,
+      "eval_runtime": 57.8092,
+      "eval_samples_per_second": 34.597,
+      "eval_steps_per_second": 0.554,
+      "step": 500
+    }
+  ],
+  "logging_steps": 10,
+  "max_steps": 76294,
+  "num_input_tokens_seen": 0,
+  "num_train_epochs": 9223372036854775807,
+  "save_steps": 500,
+  "stateful_callbacks": {
+    "TrainerControl": {
+      "args": {
+        "should_epoch_stop": false,
+        "should_evaluate": false,
+        "should_log": false,
+        "should_save": true,
+        "should_training_stop": false
+      },
+      "attributes": {}
+    }
+  },
+  "total_flos": 8991899910144000.0,
+  "train_batch_size": 128,
+  "trial_name": null,
+  "trial_params": null
+}

last-checkpoint/training_args.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:3d543c88f7fef36ef68b6d45e6baf53067caa045005adb7fff867ae9be65dfe2
+size 5905