diff --git a/.gitattributes b/.gitattributes
index a6344aac8c09253b3b630fb776ae94478aa0275b..ecdcc5b06fb9a6799e1087cebda215fb47f1a118 100644
--- a/.gitattributes
+++ b/.gitattributes
@@ -33,3 +33,8 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *.zip filter=lfs diff=lfs merge=lfs -text
 *.zst filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
+checkpoint-1500/tokenizer.json filter=lfs diff=lfs merge=lfs -text
+checkpoint-2000/tokenizer.json filter=lfs diff=lfs merge=lfs -text
+checkpoint-2500/tokenizer.json filter=lfs diff=lfs merge=lfs -text
+checkpoint-3000/tokenizer.json filter=lfs diff=lfs merge=lfs -text
+tokenizer.json filter=lfs diff=lfs merge=lfs -text
diff --git a/checkpoint-1500/config.json b/checkpoint-1500/config.json
new file mode 100644
index 0000000000000000000000000000000000000000..703de243df321b85c26832f48c923eb928e5ea16
--- /dev/null
+++ b/checkpoint-1500/config.json
@@ -0,0 +1,36 @@
+{
+  "architectures": [
+    "LlamaForCausalLM"
+  ],
+  "attention_bias": false,
+  "attention_dropout": 0.0,
+  "bos_token_id": 128000,
+  "dtype": "bfloat16",
+  "eos_token_id": 128001,
+  "head_dim": 128,
+  "hidden_act": "silu",
+  "hidden_size": 3072,
+  "initializer_range": 0.02,
+  "intermediate_size": 8192,
+  "max_position_embeddings": 131072,
+  "mlp_bias": false,
+  "model_type": "llama",
+  "num_attention_heads": 24,
+  "num_hidden_layers": 28,
+  "num_key_value_heads": 8,
+  "pad_token_id": null,
+  "pretraining_tp": 1,
+  "rms_norm_eps": 1e-05,
+  "rope_parameters": {
+    "factor": 32.0,
+    "high_freq_factor": 4.0,
+    "low_freq_factor": 1.0,
+    "original_max_position_embeddings": 8192,
+    "rope_theta": 500000.0,
+    "rope_type": "llama3"
+  },
+  "tie_word_embeddings": true,
+  "transformers_version": "5.2.0",
+  "use_cache": false,
+  "vocab_size": 128256
+}
diff --git a/checkpoint-1500/generation_config.json b/checkpoint-1500/generation_config.json
new file mode 100644
index 0000000000000000000000000000000000000000..1dd5c214ba6644c149e92991d8f72378202332d1
--- /dev/null
+++ b/checkpoint-1500/generation_config.json
@@ -0,0 +1,9 @@
+{
+  "_from_model_config": true,
+  "bos_token_id": 128000,
+  "do_sample": true,
+  "eos_token_id": 128001,
+  "temperature": 0.6,
+  "top_p": 0.9,
+  "transformers_version": "5.2.0"
+}
diff --git a/checkpoint-1500/model.safetensors b/checkpoint-1500/model.safetensors
new file mode 100644
index 0000000000000000000000000000000000000000..f5f085ecccbb212ff213ff2f50d6ea196da585df
--- /dev/null
+++ b/checkpoint-1500/model.safetensors
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:325d5be9c15855ea5e4c42eb51b617547901ad05aac3da9c7359f5398c149f8c
+size 6425529112
diff --git a/checkpoint-1500/optimizer.pt b/checkpoint-1500/optimizer.pt
new file mode 100644
index 0000000000000000000000000000000000000000..7bd53ec678220a73cb3b76455d1b72599a98fc49
--- /dev/null
+++ b/checkpoint-1500/optimizer.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e398f350c15a2f4c18086da618a0d9a8e282aa9883ac519e9d03680a5befca89
+size 12851155703
diff --git a/checkpoint-1500/rng_state_0.pth b/checkpoint-1500/rng_state_0.pth
new file mode 100644
index 0000000000000000000000000000000000000000..3cd72c974e82825e0615a259e4bcca95a3a3273c
--- /dev/null
+++ b/checkpoint-1500/rng_state_0.pth
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b80829a78a755ba0cf5e985f9212f164229ce25f7c02fa877b4cdebf230922b5
+size 15429
diff --git a/checkpoint-1500/rng_state_1.pth b/checkpoint-1500/rng_state_1.pth
new file mode 100644
index 0000000000000000000000000000000000000000..2749e3e0a290757087922def08bd7cfa7e218b5e
--- /dev/null
+++ b/checkpoint-1500/rng_state_1.pth
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:0c6f4074cb5241eff8959c1f45b18c23c893119f6c03fa2da27359dbe7dba66c
+size 15429
diff --git a/checkpoint-1500/rng_state_2.pth b/checkpoint-1500/rng_state_2.pth
new file mode 100644
index 0000000000000000000000000000000000000000..c4dcd559098a83c8196eebb4aa54e5ee6c3f2a8f
--- /dev/null
+++ b/checkpoint-1500/rng_state_2.pth
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f1e84026d823a1c04897f710cd119a65b654c08ad443e97c4486fc7a26f6e64b
+size 15429
diff --git a/checkpoint-1500/rng_state_3.pth b/checkpoint-1500/rng_state_3.pth
new file mode 100644
index 0000000000000000000000000000000000000000..57161b2dc15e410494ead6a7f9ba4524f334abae
--- /dev/null
+++ b/checkpoint-1500/rng_state_3.pth
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d9b60a41c989678f542cc75416e61e7a43f81d6221d259490134253d1f38a920
+size 15429
diff --git a/checkpoint-1500/scheduler.pt b/checkpoint-1500/scheduler.pt
new file mode 100644
index 0000000000000000000000000000000000000000..dec628d8a36a55cc56cf47eb85f701a81e6b94ee
--- /dev/null
+++ b/checkpoint-1500/scheduler.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:50b73634fbfc9590d5b52ce9c3911c77c2f560280e7738d0cfe33d18ede80d89
+size 1593
diff --git a/checkpoint-1500/tokenizer.json b/checkpoint-1500/tokenizer.json
new file mode 100644
index 0000000000000000000000000000000000000000..1c1d8d5c9024994f1d3b00f9662b8dd89ca13cf2
--- /dev/null
+++ b/checkpoint-1500/tokenizer.json
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6b9e4e7fb171f92fd137b777cc2714bf87d11576700a1dcd7a399e7bbe39537b
+size 17209920
diff --git a/checkpoint-1500/tokenizer_config.json b/checkpoint-1500/tokenizer_config.json
new file mode 100644
index 0000000000000000000000000000000000000000..35f0a8972c032adea64d32508952119451312298
--- /dev/null
+++ b/checkpoint-1500/tokenizer_config.json
@@ -0,0 +1,14 @@
+{
+  "backend": "tokenizers",
+  "bos_token": "<|begin_of_text|>",
+  "clean_up_tokenization_spaces": true,
+  "eos_token": "<|end_of_text|>",
+  "is_local": true,
+  "model_input_names": [
+    "input_ids",
+    "attention_mask"
+  ],
+  "model_max_length": 131072,
+  "pad_token": "<|end_of_text|>",
+  "tokenizer_class": "TokenizersBackend"
+}
diff --git a/checkpoint-1500/trainer_state.json b/checkpoint-1500/trainer_state.json
new file mode 100644
index 0000000000000000000000000000000000000000..3db077fa9dd6caf0c12c7da6bc8cc50c66fef6f6
--- /dev/null
+++ b/checkpoint-1500/trainer_state.json
@@ -0,0 +1,244 @@
+{
+  "best_global_step": null,
+  "best_metric": null,
+  "best_model_checkpoint": null,
+  "epoch": 4.0,
+  "eval_steps": 500,
+  "global_step": 1500,
+  "is_hyper_param_search": false,
+  "is_local_process_zero": true,
+  "is_world_process_zero": true,
+  "log_history": [
+    {
+      "epoch": 0.13333333333333333,
+      "grad_norm": 1.4375,
+      "learning_rate": 9.8e-06,
+      "loss": 1.436378936767578,
+      "step": 50
+    },
+    {
+      "epoch": 0.26666666666666666,
+      "grad_norm": 1.1484375,
+      "learning_rate": 1.98e-05,
+      "loss": 1.2349661254882813,
+      "step": 100
+    },
+    {
+      "epoch": 0.4,
+      "grad_norm": 1.171875,
+      "learning_rate": 2.98e-05,
+      "loss": 1.1963430786132812,
+      "step": 150
+    },
+    {
+      "epoch": 0.5333333333333333,
+      "grad_norm": 1.0,
+      "learning_rate": 2.997812450643705e-05,
+      "loss": 1.1292117309570313,
+      "step": 200
+    },
+    {
+      "epoch": 0.6666666666666666,
+      "grad_norm": 1.1484375,
+      "learning_rate": 2.991077005423577e-05,
+      "loss": 1.1315299987792968,
+      "step": 250
+    },
+    {
+      "epoch": 0.8,
+      "grad_norm": 1.1484375,
+      "learning_rate": 2.979813208344657e-05,
+      "loss": 1.1493177795410157,
+      "step": 300
+    },
+    {
+      "epoch": 0.9333333333333333,
+      "grad_norm": 1.078125,
+      "learning_rate": 2.964055267188472e-05,
+      "loss": 1.1162089538574218,
+      "step": 350
+    },
+    {
+      "epoch": 1.0666666666666667,
+      "grad_norm": 1.0546875,
+      "learning_rate": 2.943851038304401e-05,
+      "loss": 1.0408141326904297,
+      "step": 400
+    },
+    {
+      "epoch": 1.2,
+      "grad_norm": 1.125,
+      "learning_rate": 2.9192618812715216e-05,
+      "loss": 0.9248477172851562,
+      "step": 450
+    },
+    {
+      "epoch": 1.3333333333333333,
+      "grad_norm": 1.1796875,
+      "learning_rate": 2.8903624725515786e-05,
+      "loss": 0.8885752868652343,
+      "step": 500
+    },
+    {
+      "epoch": 1.4666666666666668,
+      "grad_norm": 1.1484375,
+      "learning_rate": 2.8572405786990293e-05,
+      "loss": 0.8980158996582032,
+      "step": 550
+    },
+    {
+      "epoch": 1.6,
+      "grad_norm": 1.125,
+      "learning_rate": 2.819996789816898e-05,
+      "loss": 0.9338175964355468,
+      "step": 600
+    },
+    {
+      "epoch": 1.7333333333333334,
+      "grad_norm": 1.1953125,
+      "learning_rate": 2.7787442140679422e-05,
+      "loss": 0.8838777160644531,
+      "step": 650
+    },
+    {
+      "epoch": 1.8666666666666667,
+      "grad_norm": 1.2265625,
+      "learning_rate": 2.7336081341688787e-05,
+      "loss": 0.9308477783203125,
+      "step": 700
+    },
+    {
+      "epoch": 2.0,
+      "grad_norm": 1.1484375,
+      "learning_rate": 2.684725626910889e-05,
+      "loss": 0.9126625061035156,
+      "step": 750
+    },
+    {
+      "epoch": 2.1333333333333333,
+      "grad_norm": 1.3203125,
+      "learning_rate": 2.632245146861912e-05,
+      "loss": 0.6762647247314453,
+      "step": 800
+    },
+    {
+      "epoch": 2.2666666666666666,
+      "grad_norm": 1.4140625,
+      "learning_rate": 2.5763260755150013e-05,
+      "loss": 0.6678054809570313,
+      "step": 850
+    },
+    {
+      "epoch": 2.4,
+      "grad_norm": 1.3203125,
+      "learning_rate": 2.5171382372519647e-05,
+      "loss": 0.6882208251953125,
+      "step": 900
+    },
+    {
+      "epoch": 2.533333333333333,
+      "grad_norm": 1.2109375,
+      "learning_rate": 2.4548613835923026e-05,
+      "loss": 0.6924208068847656,
+      "step": 950
+    },
+    {
+      "epoch": 2.6666666666666665,
+      "grad_norm": 1.2578125,
+      "learning_rate": 2.389684647293749e-05,
+      "loss": 0.6841564178466797,
+      "step": 1000
+    },
+    {
+      "epoch": 2.8,
+      "grad_norm": 1.578125,
+      "learning_rate": 2.3218059679623026e-05,
+      "loss": 0.6747154998779297,
+      "step": 1050
+    },
+    {
+      "epoch": 2.9333333333333336,
+      "grad_norm": 1.25,
+      "learning_rate": 2.2514314909161486e-05,
+      "loss": 0.6903182983398437,
+      "step": 1100
+    },
+    {
+      "epoch": 3.066666666666667,
+      "grad_norm": 1.46875,
+      "learning_rate": 2.1787749411291056e-05,
+      "loss": 0.6022007751464844,
+      "step": 1150
+    },
+    {
+      "epoch": 3.2,
+      "grad_norm": 1.5625,
+      "learning_rate": 2.1040569741549002e-05,
+      "loss": 0.4994393920898437,
+      "step": 1200
+    },
+    {
+      "epoch": 3.3333333333333335,
+      "grad_norm": 1.671875,
+      "learning_rate": 2.0275045060035044e-05,
+      "loss": 0.5102932357788086,
+      "step": 1250
+    },
+    {
+      "epoch": 3.466666666666667,
+      "grad_norm": 1.5625,
+      "learning_rate": 1.9493500240046727e-05,
+      "loss": 0.5024614334106445,
+      "step": 1300
+    },
+    {
+      "epoch": 3.6,
+      "grad_norm": 1.34375,
+      "learning_rate": 1.8698308807515613e-05,
+      "loss": 0.5217875671386719,
+      "step": 1350
+    },
+    {
+      "epoch": 3.7333333333333334,
+      "grad_norm": 1.453125,
+      "learning_rate": 1.7891885732687026e-05,
+      "loss": 0.5089986419677734,
+      "step": 1400
+    },
+    {
+      "epoch": 3.8666666666666667,
+      "grad_norm": 1.328125,
+      "learning_rate": 1.7076680095934813e-05,
+      "loss": 0.5084488677978516,
+      "step": 1450
+    },
+    {
+      "epoch": 4.0,
+      "grad_norm": 1.4140625,
+      "learning_rate": 1.6255167649984738e-05,
+      "loss": 0.5237356567382813,
+      "step": 1500
+    }
+  ],
+  "logging_steps": 50,
+  "max_steps": 3000,
+  "num_input_tokens_seen": 0,
+  "num_train_epochs": 8,
+  "save_steps": 500,
+  "stateful_callbacks": {
+    "TrainerControl": {
+      "args": {
+        "should_epoch_stop": false,
+        "should_evaluate": false,
+        "should_log": false,
+        "should_save": true,
+        "should_training_stop": false
+      },
+      "attributes": {}
+    }
+  },
+  "total_flos": 1.6625649181104538e+18,
+  "train_batch_size": 4,
+  "trial_name": null,
+  "trial_params": null
+}
diff --git a/checkpoint-1500/training_args.bin b/checkpoint-1500/training_args.bin
new file mode 100644
index 0000000000000000000000000000000000000000..37b73168988e1fd9436d6ccab0dabbe39f411e05
--- /dev/null
+++ b/checkpoint-1500/training_args.bin
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d087fa6937562656f59245cc94d04789d9768311661d6a35d5adfe9f5ea1cb39
+size 5265
diff --git a/checkpoint-2000/config.json b/checkpoint-2000/config.json
new file mode 100644
index 0000000000000000000000000000000000000000..703de243df321b85c26832f48c923eb928e5ea16
--- /dev/null
+++ b/checkpoint-2000/config.json
@@ -0,0 +1,36 @@
+{
+  "architectures": [
+    "LlamaForCausalLM"
+  ],
+  "attention_bias": false,
+  "attention_dropout": 0.0,
+  "bos_token_id": 128000,
+  "dtype": "bfloat16",
+  "eos_token_id": 128001,
+  "head_dim": 128,
+  "hidden_act": "silu",
+  "hidden_size": 3072,
+  "initializer_range": 0.02,
+  "intermediate_size": 8192,
+  "max_position_embeddings": 131072,
+  "mlp_bias": false,
+  "model_type": "llama",
+  "num_attention_heads": 24,
+  "num_hidden_layers": 28,
+  "num_key_value_heads": 8,
+  "pad_token_id": null,
+  "pretraining_tp": 1,
+  "rms_norm_eps": 1e-05,
+  "rope_parameters": {
+    "factor": 32.0,
+    "high_freq_factor": 4.0,
+    "low_freq_factor": 1.0,
+    "original_max_position_embeddings": 8192,
+    "rope_theta": 500000.0,
+    "rope_type": "llama3"
+  },
+  "tie_word_embeddings": true,
+  "transformers_version": "5.2.0",
+  "use_cache": false,
+  "vocab_size": 128256
+}
diff --git a/checkpoint-2000/generation_config.json b/checkpoint-2000/generation_config.json
new file mode 100644
index 0000000000000000000000000000000000000000..1dd5c214ba6644c149e92991d8f72378202332d1
--- /dev/null
+++ b/checkpoint-2000/generation_config.json
@@ -0,0 +1,9 @@
+{
+  "_from_model_config": true,
+  "bos_token_id": 128000,
+  "do_sample": true,
+  "eos_token_id": 128001,
+  "temperature": 0.6,
+  "top_p": 0.9,
+  "transformers_version": "5.2.0"
+}
diff --git a/checkpoint-2000/model.safetensors b/checkpoint-2000/model.safetensors
new file mode 100644
index 0000000000000000000000000000000000000000..b824f4e48afcfabbe71950107ea3bf3042638010
--- /dev/null
+++ b/checkpoint-2000/model.safetensors
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9a0b6a01a55023051df9271482a986b9c1fccca7074a091e1d22c9ade8c8a764
+size 6425529112
diff --git a/checkpoint-2000/optimizer.pt b/checkpoint-2000/optimizer.pt
new file mode 100644
index 0000000000000000000000000000000000000000..b378d527d0d85150f018b628d3d2da874d75e170
--- /dev/null
+++ b/checkpoint-2000/optimizer.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:40afb3608230d525b2423be2e8adb4a3500bff83da64e18b51c9c04a34d75706
+size 12851155703
diff --git a/checkpoint-2000/rng_state_0.pth b/checkpoint-2000/rng_state_0.pth
new file mode 100644
index 0000000000000000000000000000000000000000..afcc75952a72312775719a5b7436d424ac06a7be
--- /dev/null
+++ b/checkpoint-2000/rng_state_0.pth
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:485c3eb7b8ebe39984117ce19c256db4c11dba6dcd8d32908be6d94ca7a4d529
+size 15429
diff --git a/checkpoint-2000/rng_state_1.pth b/checkpoint-2000/rng_state_1.pth
new file mode 100644
index 0000000000000000000000000000000000000000..187216abeb18820101c39ef4d7489769e9c8b018
--- /dev/null
+++ b/checkpoint-2000/rng_state_1.pth
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f7cd0007903b1a0344b4ba578f518e1366c3607febc6b70080704119da49e8fc
+size 15429
diff --git a/checkpoint-2000/rng_state_2.pth b/checkpoint-2000/rng_state_2.pth
new file mode 100644
index 0000000000000000000000000000000000000000..cfc3b0628b54f6466425b592bb5d9cd311937f5a
--- /dev/null
+++ b/checkpoint-2000/rng_state_2.pth
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a453120913ecd5657ebaf5fb69504cf1a158e4ff4015f66acf4be6e06bc270e0
+size 15429
diff --git a/checkpoint-2000/rng_state_3.pth b/checkpoint-2000/rng_state_3.pth
new file mode 100644
index 0000000000000000000000000000000000000000..b116710f2349218d380226e98e466d0cb6e39731
--- /dev/null
+++ b/checkpoint-2000/rng_state_3.pth
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:8c18edf48cc3adf4eb70b31f8e96972c6e25718fd8fcd0866b9ace124a9269d2
+size 15429
diff --git a/checkpoint-2000/scheduler.pt b/checkpoint-2000/scheduler.pt
new file mode 100644
index 0000000000000000000000000000000000000000..d78ec6723ef65941576c66e19aaa8d8e19566a7e
--- /dev/null
+++ b/checkpoint-2000/scheduler.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:3a060b9c14b1aaf1552c000b7f613919d318da04ea38667603c1ee8aa367f966
+size 1593
diff --git a/checkpoint-2000/tokenizer.json b/checkpoint-2000/tokenizer.json
new file mode 100644
index 0000000000000000000000000000000000000000..1c1d8d5c9024994f1d3b00f9662b8dd89ca13cf2
--- /dev/null
+++ b/checkpoint-2000/tokenizer.json
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6b9e4e7fb171f92fd137b777cc2714bf87d11576700a1dcd7a399e7bbe39537b
+size 17209920
diff --git a/checkpoint-2000/tokenizer_config.json b/checkpoint-2000/tokenizer_config.json
new file mode 100644
index 0000000000000000000000000000000000000000..35f0a8972c032adea64d32508952119451312298
--- /dev/null
+++ b/checkpoint-2000/tokenizer_config.json
@@ -0,0 +1,14 @@
+{
+  "backend": "tokenizers",
+  "bos_token": "<|begin_of_text|>",
+  "clean_up_tokenization_spaces": true,
+  "eos_token": "<|end_of_text|>",
+  "is_local": true,
+  "model_input_names": [
+    "input_ids",
+    "attention_mask"
+  ],
+  "model_max_length": 131072,
+  "pad_token": "<|end_of_text|>",
+  "tokenizer_class": "TokenizersBackend"
+}
diff --git a/checkpoint-2000/trainer_state.json b/checkpoint-2000/trainer_state.json
new file mode 100644
index 0000000000000000000000000000000000000000..1e5d996dbe2493fc89637210b7397b3f3c3fba7e
--- /dev/null
+++ b/checkpoint-2000/trainer_state.json
@@ -0,0 +1,314 @@
+{
+  "best_global_step": null,
+  "best_metric": null,
+  "best_model_checkpoint": null,
+  "epoch": 5.333333333333333,
+  "eval_steps": 500,
+  "global_step": 2000,
+  "is_hyper_param_search": false,
+  "is_local_process_zero": true,
+  "is_world_process_zero": true,
+  "log_history": [
+    {
+      "epoch": 0.13333333333333333,
+      "grad_norm": 1.4375,
+      "learning_rate": 9.8e-06,
+      "loss": 1.436378936767578,
+      "step": 50
+    },
+    {
+      "epoch": 0.26666666666666666,
+      "grad_norm": 1.1484375,
+      "learning_rate": 1.98e-05,
+      "loss": 1.2349661254882813,
+      "step": 100
+    },
+    {
+      "epoch": 0.4,
+      "grad_norm": 1.171875,
+      "learning_rate": 2.98e-05,
+      "loss": 1.1963430786132812,
+      "step": 150
+    },
+    {
+      "epoch": 0.5333333333333333,
+      "grad_norm": 1.0,
+      "learning_rate": 2.997812450643705e-05,
+      "loss": 1.1292117309570313,
+      "step": 200
+    },
+    {
+      "epoch": 0.6666666666666666,
+      "grad_norm": 1.1484375,
+      "learning_rate": 2.991077005423577e-05,
+      "loss": 1.1315299987792968,
+      "step": 250
+    },
+    {
+      "epoch": 0.8,
+      "grad_norm": 1.1484375,
+      "learning_rate": 2.979813208344657e-05,
+      "loss": 1.1493177795410157,
+      "step": 300
+    },
+    {
+      "epoch": 0.9333333333333333,
+      "grad_norm": 1.078125,
+      "learning_rate": 2.964055267188472e-05,
+      "loss": 1.1162089538574218,
+      "step": 350
+    },
+    {
+      "epoch": 1.0666666666666667,
+      "grad_norm": 1.0546875,
+      "learning_rate": 2.943851038304401e-05,
+      "loss": 1.0408141326904297,
+      "step": 400
+    },
+    {
+      "epoch": 1.2,
+      "grad_norm": 1.125,
+      "learning_rate": 2.9192618812715216e-05,
+      "loss": 0.9248477172851562,
+      "step": 450
+    },
+    {
+      "epoch": 1.3333333333333333,
+      "grad_norm": 1.1796875,
+      "learning_rate": 2.8903624725515786e-05,
+      "loss": 0.8885752868652343,
+      "step": 500
+    },
+    {
+      "epoch": 1.4666666666666668,
+      "grad_norm": 1.1484375,
+      "learning_rate": 2.8572405786990293e-05,
+      "loss": 0.8980158996582032,
+      "step": 550
+    },
+    {
+      "epoch": 1.6,
+      "grad_norm": 1.125,
+      "learning_rate": 2.819996789816898e-05,
+      "loss": 0.9338175964355468,
+      "step": 600
+    },
+    {
+      "epoch": 1.7333333333333334,
+      "grad_norm": 1.1953125,
+      "learning_rate": 2.7787442140679422e-05,
+      "loss": 0.8838777160644531,
+      "step": 650
+    },
+    {
+      "epoch": 1.8666666666666667,
+      "grad_norm": 1.2265625,
+      "learning_rate": 2.7336081341688787e-05,
+      "loss": 0.9308477783203125,
+      "step": 700
+    },
+    {
+      "epoch": 2.0,
+      "grad_norm": 1.1484375,
+      "learning_rate": 2.684725626910889e-05,
+      "loss": 0.9126625061035156,
+      "step": 750
+    },
+    {
+      "epoch": 2.1333333333333333,
+      "grad_norm": 1.3203125,
+      "learning_rate": 2.632245146861912e-05,
+      "loss": 0.6762647247314453,
+      "step": 800
+    },
+    {
+      "epoch": 2.2666666666666666,
+      "grad_norm": 1.4140625,
+      "learning_rate": 2.5763260755150013e-05,
+      "loss": 0.6678054809570313,
+      "step": 850
+    },
+    {
+      "epoch": 2.4,
+      "grad_norm": 1.3203125,
+      "learning_rate": 2.5171382372519647e-05,
+      "loss": 0.6882208251953125,
+      "step": 900
+    },
+    {
+      "epoch": 2.533333333333333,
+      "grad_norm": 1.2109375,
+      "learning_rate": 2.4548613835923026e-05,
+      "loss": 0.6924208068847656,
+      "step": 950
+    },
+    {
+      "epoch": 2.6666666666666665,
+      "grad_norm": 1.2578125,
+      "learning_rate": 2.389684647293749e-05,
+      "loss": 0.6841564178466797,
+      "step": 1000
+    },
+    {
+      "epoch": 2.8,
+      "grad_norm": 1.578125,
+      "learning_rate": 2.3218059679623026e-05,
+      "loss": 0.6747154998779297,
+      "step": 1050
+    },
+    {
+      "epoch": 2.9333333333333336,
+      "grad_norm": 1.25,
+      "learning_rate": 2.2514314909161486e-05,
+      "loss": 0.6903182983398437,
+      "step": 1100
+    },
+    {
+      "epoch": 3.066666666666667,
+      "grad_norm": 1.46875,
+      "learning_rate": 2.1787749411291056e-05,
+      "loss": 0.6022007751464844,
+      "step": 1150
+    },
+    {
+      "epoch": 3.2,
+      "grad_norm": 1.5625,
+      "learning_rate": 2.1040569741549002e-05,
+      "loss": 0.4994393920898437,
+      "step": 1200
+    },
+    {
+      "epoch": 3.3333333333333335,
+      "grad_norm": 1.671875,
+      "learning_rate": 2.0275045060035044e-05,
+      "loss": 0.5102932357788086,
+      "step": 1250
+    },
+    {
+      "epoch": 3.466666666666667,
+      "grad_norm": 1.5625,
+      "learning_rate": 1.9493500240046727e-05,
+      "loss": 0.5024614334106445,
+      "step": 1300
+    },
+    {
+      "epoch": 3.6,
+      "grad_norm": 1.34375,
+      "learning_rate": 1.8698308807515613e-05,
+      "loss": 0.5217875671386719,
+      "step": 1350
+    },
+    {
+      "epoch": 3.7333333333333334,
+      "grad_norm": 1.453125,
+      "learning_rate": 1.7891885732687026e-05,
+      "loss": 0.5089986419677734,
+      "step": 1400
+    },
+    {
+      "epoch": 3.8666666666666667,
+      "grad_norm": 1.328125,
+      "learning_rate": 1.7076680095934813e-05,
+      "loss": 0.5084488677978516,
+      "step": 1450
+    },
+    {
+      "epoch": 4.0,
+      "grad_norm": 1.4140625,
+      "learning_rate": 1.6255167649984738e-05,
+      "loss": 0.5237356567382813,
+      "step": 1500
+    },
+    {
+      "epoch": 4.133333333333334,
+      "grad_norm": 1.4453125,
+      "learning_rate": 1.5429843301134782e-05,
+      "loss": 0.4232036590576172,
+      "step": 1550
+    },
+    {
+      "epoch": 4.266666666666667,
+      "grad_norm": 1.421875,
+      "learning_rate": 1.4603213532306782e-05,
+      "loss": 0.4021181106567383,
+      "step": 1600
+    },
+    {
+      "epoch": 4.4,
+      "grad_norm": 1.3359375,
+      "learning_rate": 1.3777788790940136e-05,
+      "loss": 0.3950656890869141,
+      "step": 1650
+    },
+    {
+      "epoch": 4.533333333333333,
+      "grad_norm": 1.2890625,
+      "learning_rate": 1.2956075864845435e-05,
+      "loss": 0.37584991455078126,
+      "step": 1700
+    },
+    {
+      "epoch": 4.666666666666667,
+      "grad_norm": 1.671875,
+      "learning_rate": 1.21405702691723e-05,
+      "loss": 0.4060086441040039,
+      "step": 1750
+    },
+    {
+      "epoch": 4.8,
+      "grad_norm": 1.59375,
+      "learning_rate": 1.1333748667611854e-05,
+      "loss": 0.3911821746826172,
+      "step": 1800
+    },
+    {
+      "epoch": 4.933333333333334,
+      "grad_norm": 1.484375,
+      "learning_rate": 1.0538061350850545e-05,
+      "loss": 0.4123883438110352,
+      "step": 1850
+    },
+    {
+      "epoch": 5.066666666666666,
+      "grad_norm": 1.3671875,
+      "learning_rate": 9.755924795117798e-06,
+      "loss": 0.3774250030517578,
+      "step": 1900
+    },
+    {
+      "epoch": 5.2,
+      "grad_norm": 1.5859375,
+      "learning_rate": 8.989714323427191e-06,
+      "loss": 0.34358409881591795,
+      "step": 1950
+    },
+    {
+      "epoch": 5.333333333333333,
+      "grad_norm": 1.3515625,
+      "learning_rate": 8.241756891798485e-06,
+      "loss": 0.3409086227416992,
+      "step": 2000
+    }
+  ],
+  "logging_steps": 50,
+  "max_steps": 3000,
+  "num_input_tokens_seen": 0,
+  "num_train_epochs": 8,
+  "save_steps": 500,
+  "stateful_callbacks": {
+    "TrainerControl": {
+      "args": {
+        "should_epoch_stop": false,
+        "should_evaluate": false,
+        "should_log": false,
+        "should_save": true,
+        "should_training_stop": false
+      },
+      "attributes": {}
+    }
+  },
+  "total_flos": 2.2167532241472717e+18,
+  "train_batch_size": 4,
+  "trial_name": null,
+  "trial_params": null
+}
diff --git a/checkpoint-2000/training_args.bin b/checkpoint-2000/training_args.bin
new file mode 100644
index 0000000000000000000000000000000000000000..37b73168988e1fd9436d6ccab0dabbe39f411e05
--- /dev/null
+++ b/checkpoint-2000/training_args.bin
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d087fa6937562656f59245cc94d04789d9768311661d6a35d5adfe9f5ea1cb39
+size 5265
diff --git a/checkpoint-2500/config.json b/checkpoint-2500/config.json
new file mode 100644
index 0000000000000000000000000000000000000000..703de243df321b85c26832f48c923eb928e5ea16
--- /dev/null
+++ b/checkpoint-2500/config.json
@@ -0,0 +1,36 @@
+{
+  "architectures": [
+    "LlamaForCausalLM"
+  ],
+  "attention_bias": false,
+  "attention_dropout": 0.0,
+  "bos_token_id": 128000,
+  "dtype": "bfloat16",
+  "eos_token_id": 128001,
+  "head_dim": 128,
+  "hidden_act": "silu",
+  "hidden_size": 3072,
+  "initializer_range": 0.02,
+  "intermediate_size": 8192,
+  "max_position_embeddings": 131072,
+  "mlp_bias": false,
+  "model_type": "llama",
+  "num_attention_heads": 24,
+  "num_hidden_layers": 28,
+  "num_key_value_heads": 8,
+  "pad_token_id": null,
+  "pretraining_tp": 1,
+  "rms_norm_eps": 1e-05,
+  "rope_parameters": {
+    "factor": 32.0,
+    "high_freq_factor": 4.0,
+    "low_freq_factor": 1.0,
+    "original_max_position_embeddings": 8192,
+    "rope_theta": 500000.0,
+    "rope_type": "llama3"
+  },
+  "tie_word_embeddings": true,
+  "transformers_version": "5.2.0",
+  "use_cache": false,
+  "vocab_size": 128256
+}
diff --git a/checkpoint-2500/generation_config.json b/checkpoint-2500/generation_config.json
new file mode 100644
index 0000000000000000000000000000000000000000..1dd5c214ba6644c149e92991d8f72378202332d1
--- /dev/null
+++ b/checkpoint-2500/generation_config.json
@@ -0,0 +1,9 @@
+{
+  "_from_model_config": true,
+  "bos_token_id": 128000,
+  "do_sample": true,
+  "eos_token_id": 128001,
+  "temperature": 0.6,
+  "top_p": 0.9,
+  "transformers_version": "5.2.0"
+}
diff --git a/checkpoint-2500/model.safetensors b/checkpoint-2500/model.safetensors
new file mode 100644
index 0000000000000000000000000000000000000000..58167762a40186aa9903ca4e6959d2d64a7f54c0
--- /dev/null
+++ b/checkpoint-2500/model.safetensors
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:3b001821269dd0aa11c97e859ec00db509784a0c4280403c3aea0bbafbc7f3e3
+size 6425529112
diff --git a/checkpoint-2500/optimizer.pt b/checkpoint-2500/optimizer.pt
new file mode 100644
index 0000000000000000000000000000000000000000..6d6bde4c90579eec6bd4be17099f28ce315d1a75
--- /dev/null
+++ b/checkpoint-2500/optimizer.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e0d569d0caa108588374963e56ff29f50d9e6b3bac73abcf145cdd4ee7e27a3f
+size 12851155703
diff --git a/checkpoint-2500/rng_state_0.pth b/checkpoint-2500/rng_state_0.pth
new file mode 100644
index 0000000000000000000000000000000000000000..d9c6e2fa7a82308b474a1e6c40625921e6fe2151
--- /dev/null
+++ b/checkpoint-2500/rng_state_0.pth
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:63f8c8ac7017e076bdeab5686ee808dc34f2253a11672f9168162f1c139098a8
+size 15429
diff --git a/checkpoint-2500/rng_state_1.pth b/checkpoint-2500/rng_state_1.pth
new file mode 100644
index 0000000000000000000000000000000000000000..b12387d1eece7fdf144946f0cb071ea0a6b446fe
--- /dev/null
+++ b/checkpoint-2500/rng_state_1.pth
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:da9840d31d966ef4acd9553c28e732a2f97a00ae497036cfaf9f1c9aaf7bad46
+size 15429
diff --git a/checkpoint-2500/rng_state_2.pth b/checkpoint-2500/rng_state_2.pth
new file mode 100644
index 0000000000000000000000000000000000000000..19c020cf6e41677f04cad4042ff026f710bd80b6
--- /dev/null
+++ b/checkpoint-2500/rng_state_2.pth
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:3c0d619a42d9d6b525fc902030d45e64a2643605df5c896eef51533128bd7def
+size 15429
diff --git a/checkpoint-2500/rng_state_3.pth b/checkpoint-2500/rng_state_3.pth
new file mode 100644
index 0000000000000000000000000000000000000000..7e98abe464aa16b0e983088fb02aa131798cfe57
--- /dev/null
+++ b/checkpoint-2500/rng_state_3.pth
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:5dcc41326d60bf7e5dae8feb8751b9fe5898a4cea6e4b58c37c2d6f2164d12ea
+size 15429
diff --git a/checkpoint-2500/scheduler.pt b/checkpoint-2500/scheduler.pt
new file mode 100644
index 0000000000000000000000000000000000000000..f1b5c8fc9cbde4788ddf2c84d1bcb1ff61bff515
--- /dev/null
+++ b/checkpoint-2500/scheduler.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:057893f5caed1cadb29305b8188b45a16bfd336a7ccf892232602ef08467d5ef
+size 1593
diff --git a/checkpoint-2500/tokenizer.json b/checkpoint-2500/tokenizer.json
new file mode 100644
index 0000000000000000000000000000000000000000..1c1d8d5c9024994f1d3b00f9662b8dd89ca13cf2
--- /dev/null
+++ b/checkpoint-2500/tokenizer.json
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6b9e4e7fb171f92fd137b777cc2714bf87d11576700a1dcd7a399e7bbe39537b
+size 17209920
diff --git a/checkpoint-2500/tokenizer_config.json b/checkpoint-2500/tokenizer_config.json
new file mode 100644
index 0000000000000000000000000000000000000000..35f0a8972c032adea64d32508952119451312298
--- /dev/null
+++ b/checkpoint-2500/tokenizer_config.json
@@ -0,0 +1,14 @@
+{
+  "backend": "tokenizers",
+  "bos_token": "<|begin_of_text|>",
+  "clean_up_tokenization_spaces": true,
+  "eos_token": "<|end_of_text|>",
+  "is_local": true,
+  "model_input_names": [
+    "input_ids",
+    "attention_mask"
+  ],
+  "model_max_length": 131072,
+  "pad_token": "<|end_of_text|>",
+  "tokenizer_class": "TokenizersBackend"
+}
diff --git a/checkpoint-2500/trainer_state.json b/checkpoint-2500/trainer_state.json
new file mode 100644
index 0000000000000000000000000000000000000000..04acd7bb94e652fc351e496234e107face8d56b4
--- /dev/null
+++ b/checkpoint-2500/trainer_state.json
@@ -0,0 +1,384 @@
+{
+  "best_global_step": null,
+  "best_metric": null,
+  "best_model_checkpoint": null,
+  "epoch": 6.666666666666667,
+  "eval_steps": 500,
+  "global_step": 2500,
+  "is_hyper_param_search": false,
+  "is_local_process_zero": true,
+  "is_world_process_zero": true,
+  "log_history": [
+    {
+      "epoch": 0.13333333333333333,
+      "grad_norm": 1.4375,
+      "learning_rate": 9.8e-06,
+      "loss": 1.436378936767578,
+      "step": 50
+    },
+    {
+      "epoch": 0.26666666666666666,
+      "grad_norm": 1.1484375,
+      "learning_rate": 1.98e-05,
+      "loss": 1.2349661254882813,
+      "step": 100
+    },
+    {
+      "epoch": 0.4,
+      "grad_norm": 1.171875,
+      "learning_rate": 2.98e-05,
+      "loss": 1.1963430786132812,
+      "step": 150
+    },
+    {
+      "epoch": 0.5333333333333333,
+      "grad_norm": 1.0,
+      "learning_rate": 2.997812450643705e-05,
+      "loss": 1.1292117309570313,
+      "step": 200
+    },
+    {
+      "epoch": 0.6666666666666666,
+      "grad_norm": 1.1484375,
+      "learning_rate": 2.991077005423577e-05,
+      "loss": 1.1315299987792968,
+      "step": 250
+    },
+    {
+      "epoch": 0.8,
+      "grad_norm": 1.1484375,
+      "learning_rate": 2.979813208344657e-05,
+      "loss": 1.1493177795410157,
+      "step": 300
+    },
+    {
+      "epoch": 0.9333333333333333,
+      "grad_norm": 1.078125,
+      "learning_rate": 2.964055267188472e-05,
+      "loss": 1.1162089538574218,
+      "step": 350
+    },
+    {
+      "epoch": 1.0666666666666667,
+      "grad_norm": 1.0546875,
+      "learning_rate": 2.943851038304401e-05,
+      "loss": 1.0408141326904297,
+      "step": 400
+    },
+    {
+      "epoch": 1.2,
+      "grad_norm": 1.125,
+      "learning_rate": 2.9192618812715216e-05,
+      "loss": 0.9248477172851562,
+      "step": 450
+    },
+    {
+      "epoch": 1.3333333333333333,
+      "grad_norm": 1.1796875,
+      "learning_rate": 2.8903624725515786e-05,
+      "loss": 0.8885752868652343,
+      "step": 500
+    },
+    {
+      "epoch": 1.4666666666666668,
+      "grad_norm": 1.1484375,
+      "learning_rate": 2.8572405786990293e-05,
+      "loss": 0.8980158996582032,
+      "step": 550
+    },
+    {
+      "epoch": 1.6,
+      "grad_norm": 1.125,
+      "learning_rate": 2.819996789816898e-05,
+      "loss": 0.9338175964355468,
+      "step": 600
+    },
+    {
+      "epoch": 1.7333333333333334,
+      "grad_norm": 1.1953125,
+      "learning_rate": 2.7787442140679422e-05,
+      "loss": 0.8838777160644531,
+      "step": 650
+    },
+    {
+      "epoch": 1.8666666666666667,
+      "grad_norm": 1.2265625,
+      "learning_rate": 2.7336081341688787e-05,
+      "loss": 0.9308477783203125,
+      "step": 700
+    },
+    {
+      "epoch": 2.0,
+      "grad_norm": 1.1484375,
+      "learning_rate": 2.684725626910889e-05,
+      "loss": 0.9126625061035156,
+      "step": 750
+    },
+    {
+      "epoch": 2.1333333333333333,
+      "grad_norm": 1.3203125,
+      "learning_rate": 2.632245146861912e-05,
+      "loss": 0.6762647247314453,
+      "step": 800
+    },
+    {
+      "epoch": 2.2666666666666666,
+      "grad_norm": 1.4140625,
+      "learning_rate": 2.5763260755150013e-05,
+      "loss": 0.6678054809570313,
+      "step": 850
+    },
+    {
+      "epoch": 2.4,
+      "grad_norm": 1.3203125,
+      "learning_rate": 2.5171382372519647e-05,
+      "loss": 0.6882208251953125,
+      "step": 900
+    },
+    {
+      "epoch": 2.533333333333333,
+      "grad_norm": 1.2109375,
+      "learning_rate": 2.4548613835923026e-05,
+      "loss": 0.6924208068847656,
+      "step": 950
+    },
+    {
+      "epoch": 2.6666666666666665,
+      "grad_norm": 1.2578125,
+      "learning_rate": 2.389684647293749e-05,
+      "loss": 0.6841564178466797,
+      "step": 1000
+    },
+    {
+      "epoch": 2.8,
+      "grad_norm": 1.578125,
+      "learning_rate": 2.3218059679623026e-05,
+      "loss": 0.6747154998779297,
+      "step": 1050
+    },
+    {
+      "epoch": 2.9333333333333336,
+      "grad_norm": 1.25,
+      "learning_rate": 2.2514314909161486e-05,
+      "loss": 0.6903182983398437,
+      "step": 1100
+    },
+    {
+      "epoch": 3.066666666666667,
+      "grad_norm": 1.46875,
+      "learning_rate": 2.1787749411291056e-05,
+      "loss": 0.6022007751464844,
+      "step": 1150
+    },
+    {
+      "epoch": 3.2,
+      "grad_norm": 1.5625,
+      "learning_rate": 2.1040569741549002e-05,
+      "loss": 0.4994393920898437,
+      "step": 1200
+    },
+    {
+      "epoch": 3.3333333333333335,
+      "grad_norm": 1.671875,
+      "learning_rate": 2.0275045060035044e-05,
+      "loss": 0.5102932357788086,
+      "step": 1250
+    },
+    {
+      "epoch": 3.466666666666667,
+      "grad_norm": 1.5625,
+      "learning_rate": 1.9493500240046727e-05,
+      "loss": 0.5024614334106445,
+      "step": 1300
+    },
+    {
+      "epoch": 3.6,
+      "grad_norm": 1.34375,
+      "learning_rate": 1.8698308807515613e-05,
+      "loss": 0.5217875671386719,
+      "step": 1350
+    },
+    {
+      "epoch": 3.7333333333333334,
+      "grad_norm": 1.453125,
+      "learning_rate": 1.7891885732687026e-05,
+      "loss": 0.5089986419677734,
+      "step": 1400
+    },
+    {
+      "epoch": 3.8666666666666667,
+      "grad_norm": 1.328125,
+      "learning_rate": 1.7076680095934813e-05,
+      "loss": 0.5084488677978516,
+      "step": 1450
+    },
+    {
+      "epoch": 4.0,
+      "grad_norm": 1.4140625,
+      "learning_rate": 1.6255167649984738e-05,
+      "loss": 0.5237356567382813,
+      "step": 1500
+    },
+    {
+      "epoch": 4.133333333333334,
+      "grad_norm": 1.4453125,
+      "learning_rate": 1.5429843301134782e-05,
+      "loss": 0.4232036590576172,
+      "step": 1550
+    },
+    {
+      "epoch": 4.266666666666667,
+      "grad_norm": 1.421875,
+      "learning_rate": 1.4603213532306782e-05,
+      "loss": 0.4021181106567383,
+      "step": 1600
+    },
+    {
+      "epoch": 4.4,
+      "grad_norm": 1.3359375,
+      "learning_rate": 1.3777788790940136e-05,
+      "loss": 0.3950656890869141,
+      "step": 1650
+    },
+    {
+      "epoch": 4.533333333333333,
+      "grad_norm": 1.2890625,
+      "learning_rate": 1.2956075864845435e-05,
+      "loss": 0.37584991455078126,
+      "step": 1700
+    },
+    {
+      "epoch": 4.666666666666667,
+      "grad_norm": 1.671875,
+      "learning_rate": 1.21405702691723e-05,
+      "loss": 0.4060086441040039,
+      "step": 1750
+    },
+    {
+      "epoch": 4.8,
+      "grad_norm": 1.59375,
+      "learning_rate": 1.1333748667611854e-05,
+      "loss": 0.3911821746826172,
+      "step": 1800
+    },
+    {
+      "epoch": 4.933333333333334,
+      "grad_norm": 1.484375,
+      "learning_rate": 1.0538061350850545e-05,
+      "loss": 0.4123883438110352,
+      "step": 1850
+    },
+    {
+      "epoch": 5.066666666666666,
+      "grad_norm": 1.3671875,
+      "learning_rate": 9.755924795117798e-06,
+      "loss": 0.3774250030517578,
+      "step": 1900
+    },
+    {
+      "epoch": 5.2,
+      "grad_norm": 1.5859375,
+      "learning_rate": 8.989714323427191e-06,
+      "loss": 0.34358409881591795,
+      "step": 1950
+    },
+    {
+      "epoch": 5.333333333333333,
+      "grad_norm": 1.3515625,
+      "learning_rate": 8.241756891798485e-06,
+      "loss": 0.3409086227416992,
+      "step": 2000
+    },
+    {
+      "epoch": 5.466666666666667,
+      "grad_norm": 1.6953125,
+      "learning_rate": 7.5143240223686934e-06,
+      "loss": 0.3378007888793945,
+      "step": 2050
+    },
+    {
+      "epoch": 5.6,
+      "grad_norm": 1.25,
+      "learning_rate": 6.809624904854006e-06,
+      "loss": 0.33222747802734376,
+      "step": 2100
+    },
+    {
+      "epoch": 5.733333333333333,
+      "grad_norm": 1.671875,
+      "learning_rate": 6.12979968731323e-06,
+      "loss": 0.3381657028198242,
+      "step": 2150
+    },
+    {
+      "epoch": 5.866666666666667,
+      "grad_norm": 1.4453125,
+      "learning_rate": 5.4769129765884716e-06,
+      "loss": 0.34620811462402346,
+      "step": 2200
+    },
+    {
+      "epoch": 6.0,
+      "grad_norm": 1.3046875,
+      "learning_rate": 4.852947568162011e-06,
+      "loss": 0.34920448303222656,
+      "step": 2250
+    },
+    {
+      "epoch": 6.133333333333334,
+      "grad_norm": 1.46875,
+      "learning_rate": 4.2597984244716e-06,
+      "loss": 0.3217721176147461,
+      "step": 2300
+    },
+    {
+      "epoch": 6.266666666666667,
+      "grad_norm": 1.4296875,
+      "learning_rate": 3.699266919971811e-06,
+      "loss": 0.3229765319824219,
+      "step": 2350
+    },
+    {
+      "epoch": 6.4,
+      "grad_norm": 1.3125,
+      "learning_rate": 3.1730553704190283e-06,
+      "loss": 0.31619743347167967,
+      "step": 2400
+    },
+    {
+      "epoch": 6.533333333333333,
+      "grad_norm": 1.515625,
+      "learning_rate": 2.6827618629944394e-06,
+      "loss": 0.337579345703125,
+      "step": 2450
+    },
+    {
+      "epoch": 6.666666666666667,
+      "grad_norm": 1.46875,
+      "learning_rate": 2.229875402965799e-06,
+      "loss": 0.3194581031799316,
+      "step": 2500
+    }
+  ],
+  "logging_steps": 50,
+  "max_steps": 3000,
+  "num_input_tokens_seen": 0,
+  "num_train_epochs": 8,
+  "save_steps": 500,
+  "stateful_callbacks": {
+    "TrainerControl": {
+      "args": {
+        "should_epoch_stop": false,
+        "should_evaluate": false,
+        "should_log": false,
+        "should_save": true,
+        "should_training_stop": false
+      },
+      "attributes": {}
+    }
+  },
+  "total_flos": 2.7709415301840896e+18,
+  "train_batch_size": 4,
+  "trial_name": null,
+  "trial_params": null
+}
diff --git a/checkpoint-2500/training_args.bin b/checkpoint-2500/training_args.bin
new file mode 100644
index 0000000000000000000000000000000000000000..37b73168988e1fd9436d6ccab0dabbe39f411e05
--- /dev/null
+++ b/checkpoint-2500/training_args.bin
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d087fa6937562656f59245cc94d04789d9768311661d6a35d5adfe9f5ea1cb39
+size 5265
diff --git a/checkpoint-3000/config.json b/checkpoint-3000/config.json
new file mode 100644
index 0000000000000000000000000000000000000000..703de243df321b85c26832f48c923eb928e5ea16
--- /dev/null
+++ b/checkpoint-3000/config.json
@@ -0,0 +1,36 @@
+{
+  "architectures": [
+    "LlamaForCausalLM"
+  ],
+  "attention_bias": false,
+  "attention_dropout": 0.0,
+  "bos_token_id": 128000,
+  "dtype": "bfloat16",
+  "eos_token_id": 128001,
+  "head_dim": 128,
+  "hidden_act": "silu",
+  "hidden_size": 3072,
+  "initializer_range": 0.02,
+  "intermediate_size": 8192,
+  "max_position_embeddings": 131072,
+  "mlp_bias": false,
+  "model_type": "llama",
+  "num_attention_heads": 24,
+  "num_hidden_layers": 28,
+  "num_key_value_heads": 8,
+  "pad_token_id": null,
+  "pretraining_tp": 1,
+  "rms_norm_eps": 1e-05,
+  "rope_parameters": {
+    "factor": 32.0,
+    "high_freq_factor": 4.0,
+    "low_freq_factor": 1.0,
+    "original_max_position_embeddings": 8192,
+    "rope_theta": 500000.0,
+    "rope_type": "llama3"
+  },
+  "tie_word_embeddings": true,
+  "transformers_version": "5.2.0",
+  "use_cache": false,
+  "vocab_size": 128256
+}
diff --git a/checkpoint-3000/generation_config.json b/checkpoint-3000/generation_config.json
new file mode 100644
index 0000000000000000000000000000000000000000..1dd5c214ba6644c149e92991d8f72378202332d1
--- /dev/null
+++ b/checkpoint-3000/generation_config.json
@@ -0,0 +1,9 @@
+{
+  "_from_model_config": true,
+  "bos_token_id": 128000,
+  "do_sample": true,
+  "eos_token_id": 128001,
+  "temperature": 0.6,
+  "top_p": 0.9,
+  "transformers_version": "5.2.0"
+}
diff --git a/checkpoint-3000/model.safetensors b/checkpoint-3000/model.safetensors
new file mode 100644
index 0000000000000000000000000000000000000000..b81a8bdc4b64629821ceac1b877f0efa32b354f8
--- /dev/null
+++ b/checkpoint-3000/model.safetensors
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:36c74e42fe808d3c863d3f9ec7a39f10e97c379b6a3c0cf94499d82660dbc9f5
+size 6425529112
diff --git a/checkpoint-3000/optimizer.pt b/checkpoint-3000/optimizer.pt
new file mode 100644
index 0000000000000000000000000000000000000000..9867eefbadcee0500089ef89d201703e577eb6f2
--- /dev/null
+++ b/checkpoint-3000/optimizer.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:759d5dbd50b60d068c97f5d449b08ad1c0cb0d24a16e07fa0a22dc23699bb1f5
+size 12851155703
diff --git a/checkpoint-3000/rng_state_0.pth b/checkpoint-3000/rng_state_0.pth
new file mode 100644
index 0000000000000000000000000000000000000000..c2a2270750ad34278e35aa053aac5182f2a93e99
--- /dev/null
+++ b/checkpoint-3000/rng_state_0.pth
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ccdd04b664fbfa5d05e90c688a91f18c4d07f58893b9b3a4cfad78d78efbc6ac
+size 15429
diff --git a/checkpoint-3000/rng_state_1.pth b/checkpoint-3000/rng_state_1.pth
new file mode 100644
index 0000000000000000000000000000000000000000..560af6f9618784d275690dd7c0725439475c3d26
--- /dev/null
+++ b/checkpoint-3000/rng_state_1.pth
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:5a41e144bee8116d5a47773b5f326ca7bb65df8c457f0c58a2ed8003d7ffcc73
+size 15429
diff --git a/checkpoint-3000/rng_state_2.pth b/checkpoint-3000/rng_state_2.pth
new file mode 100644
index 0000000000000000000000000000000000000000..2537b73c8fbe0077e6a758fa28986d271da796bb
--- /dev/null
+++ b/checkpoint-3000/rng_state_2.pth
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:fa38d5d663317603edb36bb18e908202a95a6e6250c06e2bea719fdba77f1277
+size 15429
diff --git a/checkpoint-3000/rng_state_3.pth b/checkpoint-3000/rng_state_3.pth
new file mode 100644
index 0000000000000000000000000000000000000000..61c811fac31215597828fee76a70aef9491d6bcf
--- /dev/null
+++ b/checkpoint-3000/rng_state_3.pth
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:dbe1f72976acfa5e0e28d92c61725f779f476ad95d06ce9b63a6d9978dd7b9ba
+size 15429
diff --git a/checkpoint-3000/scheduler.pt b/checkpoint-3000/scheduler.pt
new file mode 100644
index 0000000000000000000000000000000000000000..67346ee1bc6cee143f08f2a60dd95a91d5a3dac3
--- /dev/null
+++ b/checkpoint-3000/scheduler.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:28160ec34ab2dbc6795aa3084822335bb413441d4b6e1213d49f0c9bbb2b88e6
+size 1593
diff --git a/checkpoint-3000/tokenizer.json b/checkpoint-3000/tokenizer.json
new file mode 100644
index 0000000000000000000000000000000000000000..1c1d8d5c9024994f1d3b00f9662b8dd89ca13cf2
--- /dev/null
+++ b/checkpoint-3000/tokenizer.json
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6b9e4e7fb171f92fd137b777cc2714bf87d11576700a1dcd7a399e7bbe39537b
+size 17209920
diff --git a/checkpoint-3000/tokenizer_config.json b/checkpoint-3000/tokenizer_config.json
new file mode 100644
index 0000000000000000000000000000000000000000..35f0a8972c032adea64d32508952119451312298
--- /dev/null
+++ b/checkpoint-3000/tokenizer_config.json
@@ -0,0 +1,14 @@
+{
+  "backend": "tokenizers",
+  "bos_token": "<|begin_of_text|>",
+  "clean_up_tokenization_spaces": true,
+  "eos_token": "<|end_of_text|>",
+  "is_local": true,
+  "model_input_names": [
+    "input_ids",
+    "attention_mask"
+  ],
+  "model_max_length": 131072,
+  "pad_token": "<|end_of_text|>",
+  "tokenizer_class": "TokenizersBackend"
+}
diff --git a/checkpoint-3000/trainer_state.json b/checkpoint-3000/trainer_state.json
new file mode 100644
index 0000000000000000000000000000000000000000..c2719bccac437b2d19bc538b1cf26f922e08121b
--- /dev/null
+++ b/checkpoint-3000/trainer_state.json
@@ -0,0 +1,454 @@
+{
+  "best_global_step": null,
+  "best_metric": null,
+  "best_model_checkpoint": null,
+  "epoch": 8.0,
+  "eval_steps": 500,
+  "global_step": 3000,
+  "is_hyper_param_search": false,
+  "is_local_process_zero": true,
+  "is_world_process_zero": true,
+  "log_history": [
+    {
+      "epoch": 0.13333333333333333,
+      "grad_norm": 1.4375,
+      "learning_rate": 9.8e-06,
+      "loss": 1.436378936767578,
+      "step": 50
+    },
+    {
+      "epoch": 0.26666666666666666,
+      "grad_norm": 1.1484375,
+      "learning_rate": 1.98e-05,
+      "loss": 1.2349661254882813,
+      "step": 100
+    },
+    {
+      "epoch": 0.4,
+      "grad_norm": 1.171875,
+      "learning_rate": 2.98e-05,
+      "loss": 1.1963430786132812,
+      "step": 150
+    },
+    {
+      "epoch": 0.5333333333333333,
+      "grad_norm": 1.0,
+      "learning_rate": 2.997812450643705e-05,
+      "loss": 1.1292117309570313,
+      "step": 200
+    },
+    {
+      "epoch": 0.6666666666666666,
+      "grad_norm": 1.1484375,
+      "learning_rate": 2.991077005423577e-05,
+      "loss": 1.1315299987792968,
+      "step": 250
+    },
+    {
+      "epoch": 0.8,
+      "grad_norm": 1.1484375,
+      "learning_rate": 2.979813208344657e-05,
+      "loss": 1.1493177795410157,
+      "step": 300
+    },
+    {
+      "epoch": 0.9333333333333333,
+      "grad_norm": 1.078125,
+      "learning_rate": 2.964055267188472e-05,
+      "loss": 1.1162089538574218,
+      "step": 350
+    },
+    {
+      "epoch": 1.0666666666666667,
+      "grad_norm": 1.0546875,
+      "learning_rate": 2.943851038304401e-05,
+      "loss": 1.0408141326904297,
+      "step": 400
+    },
+    {
+      "epoch": 1.2,
+      "grad_norm": 1.125,
+      "learning_rate": 2.9192618812715216e-05,
+      "loss": 0.9248477172851562,
+      "step": 450
+    },
+    {
+      "epoch": 1.3333333333333333,
+      "grad_norm": 1.1796875,
+      "learning_rate": 2.8903624725515786e-05,
+      "loss": 0.8885752868652343,
+      "step": 500
+    },
+    {
+      "epoch": 1.4666666666666668,
+      "grad_norm": 1.1484375,
+      "learning_rate": 2.8572405786990293e-05,
+      "loss": 0.8980158996582032,
+      "step": 550
+    },
+    {
+      "epoch": 1.6,
+      "grad_norm": 1.125,
+      "learning_rate": 2.819996789816898e-05,
+      "loss": 0.9338175964355468,
+      "step": 600
+    },
+    {
+      "epoch": 1.7333333333333334,
+      "grad_norm": 1.1953125,
+      "learning_rate": 2.7787442140679422e-05,
+      "loss": 0.8838777160644531,
+      "step": 650
+    },
+    {
+      "epoch": 1.8666666666666667,
+      "grad_norm": 1.2265625,
+      "learning_rate": 2.7336081341688787e-05,
+      "loss": 0.9308477783203125,
+      "step": 700
+    },
+    {
+      "epoch": 2.0,
+      "grad_norm": 1.1484375,
+      "learning_rate": 2.684725626910889e-05,
+      "loss": 0.9126625061035156,
+      "step": 750
+    },
+    {
+      "epoch": 2.1333333333333333,
+      "grad_norm": 1.3203125,
+      "learning_rate": 2.632245146861912e-05,
+      "loss": 0.6762647247314453,
+      "step": 800
+    },
+    {
+      "epoch": 2.2666666666666666,
+      "grad_norm": 1.4140625,
+      "learning_rate": 2.5763260755150013e-05,
+      "loss": 0.6678054809570313,
+      "step": 850
+    },
+    {
+      "epoch": 2.4,
+      "grad_norm": 1.3203125,
+      "learning_rate": 2.5171382372519647e-05,
+      "loss": 0.6882208251953125,
+      "step": 900
+    },
+    {
+      "epoch": 2.533333333333333,
+      "grad_norm": 1.2109375,
+      "learning_rate": 2.4548613835923026e-05,
+      "loss": 0.6924208068847656,
+      "step": 950
+    },
+    {
+      "epoch": 2.6666666666666665,
+      "grad_norm": 1.2578125,
+      "learning_rate": 2.389684647293749e-05,
+      "loss": 0.6841564178466797,
+      "step": 1000
+    },
+    {
+      "epoch": 2.8,
+      "grad_norm": 1.578125,
+      "learning_rate": 2.3218059679623026e-05,
+      "loss": 0.6747154998779297,
+      "step": 1050
+    },
+    {
+      "epoch": 2.9333333333333336,
+      "grad_norm": 1.25,
+      "learning_rate": 2.2514314909161486e-05,
+      "loss": 0.6903182983398437,
+      "step": 1100
+    },
+    {
+      "epoch": 3.066666666666667,
+      "grad_norm": 1.46875,
+      "learning_rate": 2.1787749411291056e-05,
+      "loss": 0.6022007751464844,
+      "step": 1150
+    },
+    {
+      "epoch": 3.2,
+      "grad_norm": 1.5625,
+      "learning_rate": 2.1040569741549002e-05,
+      "loss": 0.4994393920898437,
+      "step": 1200
+    },
+    {
+      "epoch": 3.3333333333333335,
+      "grad_norm": 1.671875,
+      "learning_rate": 2.0275045060035044e-05,
+      "loss": 0.5102932357788086,
+      "step": 1250
+    },
+    {
+      "epoch": 3.466666666666667,
+      "grad_norm": 1.5625,
+      "learning_rate": 1.9493500240046727e-05,
+      "loss": 0.5024614334106445,
+      "step": 1300
+    },
+    {
+      "epoch": 3.6,
+      "grad_norm": 1.34375,
+      "learning_rate": 1.8698308807515613e-05,
+      "loss": 0.5217875671386719,
+      "step": 1350
+    },
+    {
+      "epoch": 3.7333333333333334,
+      "grad_norm": 1.453125,
+      "learning_rate": 1.7891885732687026e-05,
+      "loss": 0.5089986419677734,
+      "step": 1400
+    },
+    {
+      "epoch": 3.8666666666666667,
+      "grad_norm": 1.328125,
+      "learning_rate": 1.7076680095934813e-05,
+      "loss": 0.5084488677978516,
+      "step": 1450
+    },
+    {
+      "epoch": 4.0,
+      "grad_norm": 1.4140625,
+      "learning_rate": 1.6255167649984738e-05,
+      "loss": 0.5237356567382813,
+      "step": 1500
+    },
+    {
+      "epoch": 4.133333333333334,
+      "grad_norm": 1.4453125,
+      "learning_rate": 1.5429843301134782e-05,
+      "loss": 0.4232036590576172,
+      "step": 1550
+    },
+    {
+      "epoch": 4.266666666666667,
+      "grad_norm": 1.421875,
+      "learning_rate": 1.4603213532306782e-05,
+      "loss": 0.4021181106567383,
+      "step": 1600
+    },
+    {
+      "epoch": 4.4,
+      "grad_norm": 1.3359375,
+      "learning_rate": 1.3777788790940136e-05,
+      "loss": 0.3950656890869141,
+      "step": 1650
+    },
+    {
+      "epoch": 4.533333333333333,
+      "grad_norm": 1.2890625,
+      "learning_rate": 1.2956075864845435e-05,
+      "loss": 0.37584991455078126,
+      "step": 1700
+    },
+    {
+      "epoch": 4.666666666666667,
+      "grad_norm": 1.671875,
+      "learning_rate": 1.21405702691723e-05,
+      "loss": 0.4060086441040039,
+      "step": 1750
+    },
+    {
+      "epoch": 4.8,
+      "grad_norm": 1.59375,
+      "learning_rate": 1.1333748667611854e-05,
+      "loss": 0.3911821746826172,
+      "step": 1800
+    },
+    {
+      "epoch": 4.933333333333334,
+      "grad_norm": 1.484375,
+      "learning_rate": 1.0538061350850545e-05,
+      "loss": 0.4123883438110352,
+      "step": 1850
+    },
+    {
+      "epoch": 5.066666666666666,
+      "grad_norm": 1.3671875,
+      "learning_rate": 9.755924795117798e-06,
+      "loss": 0.3774250030517578,
+      "step": 1900
+    },
+    {
+      "epoch": 5.2,
+      "grad_norm": 1.5859375,
+      "learning_rate": 8.989714323427191e-06,
+      "loss": 0.34358409881591795,
+      "step": 1950
+    },
+    {
+      "epoch": 5.333333333333333,
+      "grad_norm": 1.3515625,
+      "learning_rate": 8.241756891798485e-06,
+      "loss": 0.3409086227416992,
+      "step": 2000
+    },
+    {
+      "epoch": 5.466666666666667,
+      "grad_norm": 1.6953125,
+      "learning_rate": 7.5143240223686934e-06,
+      "loss": 0.3378007888793945,
+      "step": 2050
+    },
+    {
+      "epoch": 5.6,
+      "grad_norm": 1.25,
+      "learning_rate": 6.809624904854006e-06,
+      "loss": 0.33222747802734376,
+      "step": 2100
+    },
+    {
+      "epoch": 5.733333333333333,
+      "grad_norm": 1.671875,
+      "learning_rate": 6.12979968731323e-06,
+      "loss": 0.3381657028198242,
+      "step": 2150
+    },
+    {
+      "epoch": 5.866666666666667,
+      "grad_norm": 1.4453125,
+      "learning_rate": 5.4769129765884716e-06,
+      "loss": 0.34620811462402346,
+      "step": 2200
+    },
+    {
+      "epoch": 6.0,
+      "grad_norm": 1.3046875,
+      "learning_rate": 4.852947568162011e-06,
+      "loss": 0.34920448303222656,
+      "step": 2250
+    },
+    {
+      "epoch": 6.133333333333334,
+      "grad_norm": 1.46875,
+      "learning_rate": 4.2597984244716e-06,
+      "loss": 0.3217721176147461,
+      "step": 2300
+    },
+    {
+      "epoch": 6.266666666666667,
+      "grad_norm": 1.4296875,
+      "learning_rate": 3.699266919971811e-06,
+      "loss": 0.3229765319824219,
+      "step": 2350
+    },
+    {
+      "epoch": 6.4,
+      "grad_norm": 1.3125,
+      "learning_rate": 3.1730553704190283e-06,
+      "loss": 0.31619743347167967,
+      "step": 2400
+    },
+    {
+      "epoch": 6.533333333333333,
+      "grad_norm": 1.515625,
+      "learning_rate": 2.6827618629944394e-06,
+      "loss": 0.337579345703125,
+      "step": 2450
+    },
+    {
+      "epoch": 6.666666666666667,
+      "grad_norm": 1.46875,
+      "learning_rate": 2.229875402965799e-06,
+      "loss": 0.3194581031799316,
+      "step": 2500
+    },
+    {
+      "epoch": 6.8,
+      "grad_norm": 1.328125,
+      "learning_rate": 1.8157713916273593e-06,
+      "loss": 0.3173810005187988,
+      "step": 2550
+    },
+    {
+      "epoch": 6.933333333333334,
+      "grad_norm": 1.5546875,
+      "learning_rate": 1.4417074492513516e-06,
+      "loss": 0.3110699462890625,
+      "step": 2600
+    },
+    {
+      "epoch": 7.066666666666666,
+      "grad_norm": 1.3125,
+      "learning_rate": 1.10881959573655e-06,
+      "loss": 0.3092642593383789,
+      "step": 2650
+    },
+    {
+      "epoch": 7.2,
+      "grad_norm": 1.5703125,
+      "learning_rate": 8.181188005531748e-07,
+      "loss": 0.32044502258300783,
+      "step": 2700
+    },
+    {
+      "epoch": 7.333333333333333,
+      "grad_norm": 1.40625,
+      "learning_rate": 5.704879124618434e-07,
+      "loss": 0.312000617980957,
+      "step": 2750
+    },
+    {
+      "epoch": 7.466666666666667,
+      "grad_norm": 1.796875,
+      "learning_rate": 3.666789783308955e-07,
+      "loss": 0.30869159698486326,
+      "step": 2800
+    },
+    {
+      "epoch": 7.6,
+      "grad_norm": 1.234375,
+      "learning_rate": 2.073109591947725e-07,
+      "loss": 0.32180892944335937,
+      "step": 2850
+    },
+    {
+      "epoch": 7.733333333333333,
+      "grad_norm": 1.5,
+      "learning_rate": 9.286785048970181e-08,
+      "loss": 0.3131877899169922,
+      "step": 2900
+    },
+    {
+      "epoch": 7.866666666666667,
+      "grad_norm": 1.5390625,
+      "learning_rate": 2.3697212175460835e-08,
+      "loss": 0.31590654373168947,
+      "step": 2950
+    },
+    {
+      "epoch": 8.0,
+      "grad_norm": 1.4140625,
+      "learning_rate": 9.113207203870211e-12,
+      "loss": 0.32530906677246096,
+      "step": 3000
+    }
+  ],
+  "logging_steps": 50,
+  "max_steps": 3000,
+  "num_input_tokens_seen": 0,
+  "num_train_epochs": 8,
+  "save_steps": 500,
+  "stateful_callbacks": {
+    "TrainerControl": {
+      "args": {
+        "should_epoch_stop": false,
+        "should_evaluate": false,
+        "should_log": false,
+        "should_save": true,
+        "should_training_stop": true
+      },
+      "attributes": {}
+    }
+  },
+  "total_flos": 3.3251298362209075e+18,
+  "train_batch_size": 4,
+  "trial_name": null,
+  "trial_params": null
+}
diff --git a/checkpoint-3000/training_args.bin b/checkpoint-3000/training_args.bin
new file mode 100644
index 0000000000000000000000000000000000000000..37b73168988e1fd9436d6ccab0dabbe39f411e05
--- /dev/null
+++ b/checkpoint-3000/training_args.bin
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d087fa6937562656f59245cc94d04789d9768311661d6a35d5adfe9f5ea1cb39
+size 5265
diff --git a/config.json b/config.json
new file mode 100644
index 0000000000000000000000000000000000000000..703de243df321b85c26832f48c923eb928e5ea16
--- /dev/null
+++ b/config.json
@@ -0,0 +1,36 @@
+{
+  "architectures": [
+    "LlamaForCausalLM"
+  ],
+  "attention_bias": false,
+  "attention_dropout": 0.0,
+  "bos_token_id": 128000,
+  "dtype": "bfloat16",
+  "eos_token_id": 128001,
+  "head_dim": 128,
+  "hidden_act": "silu",
+  "hidden_size": 3072,
+  "initializer_range": 0.02,
+  "intermediate_size": 8192,
+  "max_position_embeddings": 131072,
+  "mlp_bias": false,
+  "model_type": "llama",
+  "num_attention_heads": 24,
+  "num_hidden_layers": 28,
+  "num_key_value_heads": 8,
+  "pad_token_id": null,
+  "pretraining_tp": 1,
+  "rms_norm_eps": 1e-05,
+  "rope_parameters": {
+    "factor": 32.0,
+    "high_freq_factor": 4.0,
+    "low_freq_factor": 1.0,
+    "original_max_position_embeddings": 8192,
+    "rope_theta": 500000.0,
+    "rope_type": "llama3"
+  },
+  "tie_word_embeddings": true,
+  "transformers_version": "5.2.0",
+  "use_cache": false,
+  "vocab_size": 128256
+}
diff --git a/generation_config.json b/generation_config.json
new file mode 100644
index 0000000000000000000000000000000000000000..1dd5c214ba6644c149e92991d8f72378202332d1
--- /dev/null
+++ b/generation_config.json
@@ -0,0 +1,9 @@
+{
+  "_from_model_config": true,
+  "bos_token_id": 128000,
+  "do_sample": true,
+  "eos_token_id": 128001,
+  "temperature": 0.6,
+  "top_p": 0.9,
+  "transformers_version": "5.2.0"
+}
diff --git a/model.safetensors b/model.safetensors
new file mode 100644
index 0000000000000000000000000000000000000000..b81a8bdc4b64629821ceac1b877f0efa32b354f8
--- /dev/null
+++ b/model.safetensors
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:36c74e42fe808d3c863d3f9ec7a39f10e97c379b6a3c0cf94499d82660dbc9f5
+size 6425529112
diff --git a/tokenizer.json b/tokenizer.json
new file mode 100644
index 0000000000000000000000000000000000000000..1c1d8d5c9024994f1d3b00f9662b8dd89ca13cf2
--- /dev/null
+++ b/tokenizer.json
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6b9e4e7fb171f92fd137b777cc2714bf87d11576700a1dcd7a399e7bbe39537b
+size 17209920
diff --git a/tokenizer_config.json b/tokenizer_config.json
new file mode 100644
index 0000000000000000000000000000000000000000..35f0a8972c032adea64d32508952119451312298
--- /dev/null
+++ b/tokenizer_config.json
@@ -0,0 +1,14 @@
+{
+  "backend": "tokenizers",
+  "bos_token": "<|begin_of_text|>",
+  "clean_up_tokenization_spaces": true,
+  "eos_token": "<|end_of_text|>",
+  "is_local": true,
+  "model_input_names": [
+    "input_ids",
+    "attention_mask"
+  ],
+  "model_max_length": 131072,
+  "pad_token": "<|end_of_text|>",
+  "tokenizer_class": "TokenizersBackend"
+}
diff --git a/training_args.bin b/training_args.bin
new file mode 100644
index 0000000000000000000000000000000000000000..37b73168988e1fd9436d6ccab0dabbe39f411e05
--- /dev/null
+++ b/training_args.bin
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d087fa6937562656f59245cc94d04789d9768311661d6a35d5adfe9f5ea1cb39
+size 5265