diff --git a/.gitattributes b/.gitattributes
index a6344aac8c09253b3b630fb776ae94478aa0275b..e10e872d1ac1ff292132ebb15571344de2f2ee6d 100644
--- a/.gitattributes
+++ b/.gitattributes
@@ -33,3 +33,17 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
*.zip filter=lfs diff=lfs merge=lfs -text
*.zst filter=lfs diff=lfs merge=lfs -text
*tfevents* filter=lfs diff=lfs merge=lfs -text
+checkpoint-1000/tokenizer.json filter=lfs diff=lfs merge=lfs -text
+checkpoint-1500/tokenizer.json filter=lfs diff=lfs merge=lfs -text
+checkpoint-2000/tokenizer.json filter=lfs diff=lfs merge=lfs -text
+checkpoint-2500/tokenizer.json filter=lfs diff=lfs merge=lfs -text
+checkpoint-3000/tokenizer.json filter=lfs diff=lfs merge=lfs -text
+checkpoint-3500/tokenizer.json filter=lfs diff=lfs merge=lfs -text
+checkpoint-4000/tokenizer.json filter=lfs diff=lfs merge=lfs -text
+checkpoint-4500/tokenizer.json filter=lfs diff=lfs merge=lfs -text
+checkpoint-500/tokenizer.json filter=lfs diff=lfs merge=lfs -text
+checkpoint-5000/tokenizer.json filter=lfs diff=lfs merge=lfs -text
+checkpoint-5500/tokenizer.json filter=lfs diff=lfs merge=lfs -text
+checkpoint-6000/tokenizer.json filter=lfs diff=lfs merge=lfs -text
+checkpoint-6241/tokenizer.json filter=lfs diff=lfs merge=lfs -text
+tokenizer.json filter=lfs diff=lfs merge=lfs -text
diff --git a/README.md b/README.md
new file mode 100644
index 0000000000000000000000000000000000000000..6d664d394dfc10d0d50c820a51bb367ed02e3c33
--- /dev/null
+++ b/README.md
@@ -0,0 +1,58 @@
+---
+base_model: Qwen/Qwen3-1.7B
+library_name: transformers
+model_name: general_knowledge_model
+tags:
+- generated_from_trainer
+- sft
+- trl
+licence: license
+---
+
+# Model Card for general_knowledge_model
+
+This model is a fine-tuned version of [Qwen/Qwen3-1.7B](https://huggingface.co/Qwen/Qwen3-1.7B).
+It has been trained using [TRL](https://github.com/huggingface/trl).
+
+## Quick start
+
+```python
+from transformers import pipeline
+
+question = "If you had a time machine, but could only go to the past or the future once and never return, which would you choose and why?"
+generator = pipeline("text-generation", model="None", device="cuda")
+output = generator([{"role": "user", "content": question}], max_new_tokens=128, return_full_text=False)[0]
+print(output["generated_text"])
+```
+
+## Training procedure
+
+
+
+
+
+This model was trained with SFT.
+
+### Framework versions
+
+- TRL: 1.3.0
+- Transformers: 5.7.0
+- Pytorch: 2.10.0+cu128
+- Datasets: 4.8.5
+- Tokenizers: 0.22.2
+
+## Citations
+
+
+
+Cite TRL as:
+
+```bibtex
+@software{vonwerra2020trl,
+ title = {{TRL: Transformers Reinforcement Learning}},
+ author = {von Werra, Leandro and Belkada, Younes and Tunstall, Lewis and Beeching, Edward and Thrush, Tristan and Lambert, Nathan and Huang, Shengyi and Rasul, Kashif and Gallouédec, Quentin},
+ license = {Apache-2.0},
+ url = {https://github.com/huggingface/trl},
+ year = {2020}
+}
+```
\ No newline at end of file
diff --git a/chat_template.jinja b/chat_template.jinja
new file mode 100644
index 0000000000000000000000000000000000000000..01be9b307daa2d425f7c168c9fb145a286e0afb4
--- /dev/null
+++ b/chat_template.jinja
@@ -0,0 +1,89 @@
+{%- if tools %}
+ {{- '<|im_start|>system\n' }}
+ {%- if messages[0].role == 'system' %}
+ {{- messages[0].content + '\n\n' }}
+ {%- endif %}
+ {{- "# Tools\n\nYou may call one or more functions to assist with the user query.\n\nYou are provided with function signatures within XML tags:\n" }}
+ {%- for tool in tools %}
+ {{- "\n" }}
+ {{- tool | tojson }}
+ {%- endfor %}
+ {{- "\n\n\nFor each function call, return a json object with function name and arguments within XML tags:\n\n{\"name\": , \"arguments\": }\n<|im_end|>\n" }}
+{%- else %}
+ {%- if messages[0].role == 'system' %}
+ {{- '<|im_start|>system\n' + messages[0].content + '<|im_end|>\n' }}
+ {%- endif %}
+{%- endif %}
+{%- set ns = namespace(multi_step_tool=true, last_query_index=messages|length - 1) %}
+{%- for message in messages[::-1] %}
+ {%- set index = (messages|length - 1) - loop.index0 %}
+ {%- if ns.multi_step_tool and message.role == "user" and message.content is string and not(message.content.startswith('') and message.content.endswith('')) %}
+ {%- set ns.multi_step_tool = false %}
+ {%- set ns.last_query_index = index %}
+ {%- endif %}
+{%- endfor %}
+{%- for message in messages %}
+ {%- if message.content is string %}
+ {%- set content = message.content %}
+ {%- else %}
+ {%- set content = '' %}
+ {%- endif %}
+ {%- if (message.role == "user") or (message.role == "system" and not loop.first) %}
+ {{- '<|im_start|>' + message.role + '\n' + content + '<|im_end|>' + '\n' }}
+ {%- elif message.role == "assistant" %}
+ {%- set reasoning_content = '' %}
+ {%- if message.reasoning_content is string %}
+ {%- set reasoning_content = message.reasoning_content %}
+ {%- else %}
+ {%- if '' in content %}
+ {%- set reasoning_content = content.split('')[0].rstrip('\n').split('')[-1].lstrip('\n') %}
+ {%- set content = content.split('')[-1].lstrip('\n') %}
+ {%- endif %}
+ {%- endif %}
+ {%- if loop.index0 > ns.last_query_index %}
+ {%- if loop.last or (not loop.last and reasoning_content) %}
+ {{- '<|im_start|>' + message.role + '\n\n' + reasoning_content.strip('\n') + '\n\n\n' + content.lstrip('\n') }}
+ {%- else %}
+ {{- '<|im_start|>' + message.role + '\n' + content }}
+ {%- endif %}
+ {%- else %}
+ {{- '<|im_start|>' + message.role + '\n' + content }}
+ {%- endif %}
+ {%- if message.tool_calls %}
+ {%- for tool_call in message.tool_calls %}
+ {%- if (loop.first and content) or (not loop.first) %}
+ {{- '\n' }}
+ {%- endif %}
+ {%- if tool_call.function %}
+ {%- set tool_call = tool_call.function %}
+ {%- endif %}
+ {{- '\n{"name": "' }}
+ {{- tool_call.name }}
+ {{- '", "arguments": ' }}
+ {%- if tool_call.arguments is string %}
+ {{- tool_call.arguments }}
+ {%- else %}
+ {{- tool_call.arguments | tojson }}
+ {%- endif %}
+ {{- '}\n' }}
+ {%- endfor %}
+ {%- endif %}
+ {{- '<|im_end|>\n' }}
+ {%- elif message.role == "tool" %}
+ {%- if loop.first or (messages[loop.index0 - 1].role != "tool") %}
+ {{- '<|im_start|>user' }}
+ {%- endif %}
+ {{- '\n\n' }}
+ {{- content }}
+ {{- '\n' }}
+ {%- if loop.last or (messages[loop.index0 + 1].role != "tool") %}
+ {{- '<|im_end|>\n' }}
+ {%- endif %}
+ {%- endif %}
+{%- endfor %}
+{%- if add_generation_prompt %}
+ {{- '<|im_start|>assistant\n' }}
+ {%- if enable_thinking is defined and enable_thinking is false %}
+ {{- '\n\n\n\n' }}
+ {%- endif %}
+{%- endif %}
\ No newline at end of file
diff --git a/checkpoint-1000/chat_template.jinja b/checkpoint-1000/chat_template.jinja
new file mode 100644
index 0000000000000000000000000000000000000000..01be9b307daa2d425f7c168c9fb145a286e0afb4
--- /dev/null
+++ b/checkpoint-1000/chat_template.jinja
@@ -0,0 +1,89 @@
+{%- if tools %}
+ {{- '<|im_start|>system\n' }}
+ {%- if messages[0].role == 'system' %}
+ {{- messages[0].content + '\n\n' }}
+ {%- endif %}
+ {{- "# Tools\n\nYou may call one or more functions to assist with the user query.\n\nYou are provided with function signatures within XML tags:\n" }}
+ {%- for tool in tools %}
+ {{- "\n" }}
+ {{- tool | tojson }}
+ {%- endfor %}
+ {{- "\n\n\nFor each function call, return a json object with function name and arguments within XML tags:\n\n{\"name\": , \"arguments\": }\n<|im_end|>\n" }}
+{%- else %}
+ {%- if messages[0].role == 'system' %}
+ {{- '<|im_start|>system\n' + messages[0].content + '<|im_end|>\n' }}
+ {%- endif %}
+{%- endif %}
+{%- set ns = namespace(multi_step_tool=true, last_query_index=messages|length - 1) %}
+{%- for message in messages[::-1] %}
+ {%- set index = (messages|length - 1) - loop.index0 %}
+ {%- if ns.multi_step_tool and message.role == "user" and message.content is string and not(message.content.startswith('') and message.content.endswith('')) %}
+ {%- set ns.multi_step_tool = false %}
+ {%- set ns.last_query_index = index %}
+ {%- endif %}
+{%- endfor %}
+{%- for message in messages %}
+ {%- if message.content is string %}
+ {%- set content = message.content %}
+ {%- else %}
+ {%- set content = '' %}
+ {%- endif %}
+ {%- if (message.role == "user") or (message.role == "system" and not loop.first) %}
+ {{- '<|im_start|>' + message.role + '\n' + content + '<|im_end|>' + '\n' }}
+ {%- elif message.role == "assistant" %}
+ {%- set reasoning_content = '' %}
+ {%- if message.reasoning_content is string %}
+ {%- set reasoning_content = message.reasoning_content %}
+ {%- else %}
+ {%- if '' in content %}
+ {%- set reasoning_content = content.split('')[0].rstrip('\n').split('')[-1].lstrip('\n') %}
+ {%- set content = content.split('')[-1].lstrip('\n') %}
+ {%- endif %}
+ {%- endif %}
+ {%- if loop.index0 > ns.last_query_index %}
+ {%- if loop.last or (not loop.last and reasoning_content) %}
+ {{- '<|im_start|>' + message.role + '\n\n' + reasoning_content.strip('\n') + '\n\n\n' + content.lstrip('\n') }}
+ {%- else %}
+ {{- '<|im_start|>' + message.role + '\n' + content }}
+ {%- endif %}
+ {%- else %}
+ {{- '<|im_start|>' + message.role + '\n' + content }}
+ {%- endif %}
+ {%- if message.tool_calls %}
+ {%- for tool_call in message.tool_calls %}
+ {%- if (loop.first and content) or (not loop.first) %}
+ {{- '\n' }}
+ {%- endif %}
+ {%- if tool_call.function %}
+ {%- set tool_call = tool_call.function %}
+ {%- endif %}
+ {{- '\n{"name": "' }}
+ {{- tool_call.name }}
+ {{- '", "arguments": ' }}
+ {%- if tool_call.arguments is string %}
+ {{- tool_call.arguments }}
+ {%- else %}
+ {{- tool_call.arguments | tojson }}
+ {%- endif %}
+ {{- '}\n' }}
+ {%- endfor %}
+ {%- endif %}
+ {{- '<|im_end|>\n' }}
+ {%- elif message.role == "tool" %}
+ {%- if loop.first or (messages[loop.index0 - 1].role != "tool") %}
+ {{- '<|im_start|>user' }}
+ {%- endif %}
+ {{- '\n\n' }}
+ {{- content }}
+ {{- '\n' }}
+ {%- if loop.last or (messages[loop.index0 + 1].role != "tool") %}
+ {{- '<|im_end|>\n' }}
+ {%- endif %}
+ {%- endif %}
+{%- endfor %}
+{%- if add_generation_prompt %}
+ {{- '<|im_start|>assistant\n' }}
+ {%- if enable_thinking is defined and enable_thinking is false %}
+ {{- '\n\n\n\n' }}
+ {%- endif %}
+{%- endif %}
\ No newline at end of file
diff --git a/checkpoint-1000/config.json b/checkpoint-1000/config.json
new file mode 100644
index 0000000000000000000000000000000000000000..1d219d6a6d288625229a71646710b85cf7ae5def
--- /dev/null
+++ b/checkpoint-1000/config.json
@@ -0,0 +1,63 @@
+{
+ "architectures": [
+ "Qwen3ForCausalLM"
+ ],
+ "attention_bias": false,
+ "attention_dropout": 0.0,
+ "bos_token_id": null,
+ "dtype": "bfloat16",
+ "eos_token_id": 151645,
+ "head_dim": 128,
+ "hidden_act": "silu",
+ "hidden_size": 2048,
+ "initializer_range": 0.02,
+ "intermediate_size": 6144,
+ "layer_types": [
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention"
+ ],
+ "max_position_embeddings": 40960,
+ "max_window_layers": 28,
+ "model_type": "qwen3",
+ "num_attention_heads": 16,
+ "num_hidden_layers": 28,
+ "num_key_value_heads": 8,
+ "pad_token_id": 151643,
+ "rms_norm_eps": 1e-06,
+ "rope_parameters": {
+ "rope_theta": 1000000,
+ "rope_type": "default"
+ },
+ "sliding_window": null,
+ "tie_word_embeddings": true,
+ "transformers_version": "5.7.0",
+ "use_cache": false,
+ "use_sliding_window": false,
+ "vocab_size": 151936
+}
diff --git a/checkpoint-1000/generation_config.json b/checkpoint-1000/generation_config.json
new file mode 100644
index 0000000000000000000000000000000000000000..5ec133d867b1427d43a5ee0ea332bd088fc12013
--- /dev/null
+++ b/checkpoint-1000/generation_config.json
@@ -0,0 +1,12 @@
+{
+ "do_sample": true,
+ "eos_token_id": [
+ 151645,
+ 151643
+ ],
+ "pad_token_id": 151643,
+ "temperature": 0.6,
+ "top_k": 20,
+ "top_p": 0.95,
+ "transformers_version": "5.7.0"
+}
diff --git a/checkpoint-1000/model.safetensors b/checkpoint-1000/model.safetensors
new file mode 100644
index 0000000000000000000000000000000000000000..e8a65e51d1324957a142464f07d7b68e67f54e74
--- /dev/null
+++ b/checkpoint-1000/model.safetensors
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:fa6a5ffa325a42521faee189eed67b3922773291e206e209e90b3c770b4d4539
+size 3441185608
diff --git a/checkpoint-1000/optimizer.pt b/checkpoint-1000/optimizer.pt
new file mode 100644
index 0000000000000000000000000000000000000000..f40d49ecdbeba9232f57176dd4aecf00dd0fab66
--- /dev/null
+++ b/checkpoint-1000/optimizer.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:2c51b9e4e783c1596ac7d185b6f7589123a938675998d0d32a5f077dfa2dfefd
+size 6882572207
diff --git a/checkpoint-1000/rng_state.pth b/checkpoint-1000/rng_state.pth
new file mode 100644
index 0000000000000000000000000000000000000000..5a8f17addf8aaee350625e9847751582bc9f86dd
--- /dev/null
+++ b/checkpoint-1000/rng_state.pth
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:61c19bab1174704a4a4441475683bf1270277af15d2e2c95e964789128e482c4
+size 14645
diff --git a/checkpoint-1000/scheduler.pt b/checkpoint-1000/scheduler.pt
new file mode 100644
index 0000000000000000000000000000000000000000..97d98d53eee142ff760a681dfa6b52857b945259
--- /dev/null
+++ b/checkpoint-1000/scheduler.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:03766410d0ee20761a88c2327709b696e608f6be1dc35ba4d82dcb2eea2325f7
+size 1465
diff --git a/checkpoint-1000/tokenizer.json b/checkpoint-1000/tokenizer.json
new file mode 100644
index 0000000000000000000000000000000000000000..c7afbed2efcdf019f88ab0572ec29d3bf595dfe2
--- /dev/null
+++ b/checkpoint-1000/tokenizer.json
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:be75606093db2094d7cd20f3c2f385c212750648bd6ea4fb2bf507a6a4c55506
+size 11422650
diff --git a/checkpoint-1000/tokenizer_config.json b/checkpoint-1000/tokenizer_config.json
new file mode 100644
index 0000000000000000000000000000000000000000..770e41d6c92519d525eede4cbcf3ba27f6425311
--- /dev/null
+++ b/checkpoint-1000/tokenizer_config.json
@@ -0,0 +1,30 @@
+{
+ "add_prefix_space": false,
+ "backend": "tokenizers",
+ "bos_token": null,
+ "clean_up_tokenization_spaces": false,
+ "eos_token": "<|im_end|>",
+ "errors": "replace",
+ "extra_special_tokens": [
+ "<|im_start|>",
+ "<|im_end|>",
+ "<|object_ref_start|>",
+ "<|object_ref_end|>",
+ "<|box_start|>",
+ "<|box_end|>",
+ "<|quad_start|>",
+ "<|quad_end|>",
+ "<|vision_start|>",
+ "<|vision_end|>",
+ "<|vision_pad|>",
+ "<|image_pad|>",
+ "<|video_pad|>"
+ ],
+ "is_local": false,
+ "local_files_only": false,
+ "model_max_length": 131072,
+ "pad_token": "<|endoftext|>",
+ "split_special_tokens": false,
+ "tokenizer_class": "Qwen2Tokenizer",
+ "unk_token": null
+}
diff --git a/checkpoint-1000/trainer_state.json b/checkpoint-1000/trainer_state.json
new file mode 100644
index 0000000000000000000000000000000000000000..5654c8a22c4fb689dbf227d17dd558c8a986f937
--- /dev/null
+++ b/checkpoint-1000/trainer_state.json
@@ -0,0 +1,234 @@
+{
+ "best_global_step": null,
+ "best_metric": null,
+ "best_model_checkpoint": null,
+ "epoch": 0.16024998998437562,
+ "eval_steps": 500,
+ "global_step": 1000,
+ "is_hyper_param_search": false,
+ "is_local_process_zero": true,
+ "is_world_process_zero": true,
+ "log_history": [
+ {
+ "entropy": 2.223189059495926,
+ "epoch": 0.008012499499218782,
+ "grad_norm": 2.875,
+ "learning_rate": 1.9842973882390643e-05,
+ "loss": 2.3428109741210936,
+ "mean_token_accuracy": 0.4963963569700718,
+ "num_tokens": 305366.0,
+ "step": 50
+ },
+ {
+ "entropy": 2.188130117058754,
+ "epoch": 0.016024998998437564,
+ "grad_norm": 3.328125,
+ "learning_rate": 1.96827431501362e-05,
+ "loss": 2.225358734130859,
+ "mean_token_accuracy": 0.5110921229422093,
+ "num_tokens": 605042.0,
+ "step": 100
+ },
+ {
+ "entropy": 2.197347580194473,
+ "epoch": 0.024037498497656344,
+ "grad_norm": 3.453125,
+ "learning_rate": 1.952251241788175e-05,
+ "loss": 2.2299420166015627,
+ "mean_token_accuracy": 0.5093490639328957,
+ "num_tokens": 906888.0,
+ "step": 150
+ },
+ {
+ "entropy": 2.1913950395584108,
+ "epoch": 0.03204999799687513,
+ "grad_norm": 2.90625,
+ "learning_rate": 1.9362281685627303e-05,
+ "loss": 2.210995941162109,
+ "mean_token_accuracy": 0.5097909501194954,
+ "num_tokens": 1213125.0,
+ "step": 200
+ },
+ {
+ "entropy": 2.1611998677253723,
+ "epoch": 0.040062497496093905,
+ "grad_norm": 3.25,
+ "learning_rate": 1.920205095337286e-05,
+ "loss": 2.1867474365234374,
+ "mean_token_accuracy": 0.5154410660266876,
+ "num_tokens": 1509026.0,
+ "step": 250
+ },
+ {
+ "entropy": 2.168051454424858,
+ "epoch": 0.04807499699531269,
+ "grad_norm": 2.9375,
+ "learning_rate": 1.904182022111841e-05,
+ "loss": 2.190071716308594,
+ "mean_token_accuracy": 0.5156884534657001,
+ "num_tokens": 1806850.0,
+ "step": 300
+ },
+ {
+ "entropy": 2.1737154543399813,
+ "epoch": 0.05608749649453147,
+ "grad_norm": 3.421875,
+ "learning_rate": 1.8881589488863967e-05,
+ "loss": 2.199147186279297,
+ "mean_token_accuracy": 0.514045044630766,
+ "num_tokens": 2104505.0,
+ "step": 350
+ },
+ {
+ "entropy": 2.148039159178734,
+ "epoch": 0.06409999599375026,
+ "grad_norm": 3.109375,
+ "learning_rate": 1.872135875660952e-05,
+ "loss": 2.176946105957031,
+ "mean_token_accuracy": 0.5177112428843975,
+ "num_tokens": 2391088.0,
+ "step": 400
+ },
+ {
+ "entropy": 2.194890711903572,
+ "epoch": 0.07211249549296903,
+ "grad_norm": 3.28125,
+ "learning_rate": 1.856112802435507e-05,
+ "loss": 2.216288757324219,
+ "mean_token_accuracy": 0.5110335703194141,
+ "num_tokens": 2691272.0,
+ "step": 450
+ },
+ {
+ "entropy": 2.1557786524295808,
+ "epoch": 0.08012499499218781,
+ "grad_norm": 4.15625,
+ "learning_rate": 1.8400897292100627e-05,
+ "loss": 2.1812680053710936,
+ "mean_token_accuracy": 0.5176697050035,
+ "num_tokens": 2991855.0,
+ "step": 500
+ },
+ {
+ "entropy": 2.1708167713880537,
+ "epoch": 0.08813749449140659,
+ "grad_norm": 4.34375,
+ "learning_rate": 1.824066655984618e-05,
+ "loss": 2.1921002197265627,
+ "mean_token_accuracy": 0.5156507430970669,
+ "num_tokens": 3295049.0,
+ "step": 550
+ },
+ {
+ "entropy": 2.1686314886808393,
+ "epoch": 0.09614999399062538,
+ "grad_norm": 2.875,
+ "learning_rate": 1.8080435827591735e-05,
+ "loss": 2.1914617919921877,
+ "mean_token_accuracy": 0.5139613692462445,
+ "num_tokens": 3597206.0,
+ "step": 600
+ },
+ {
+ "entropy": 2.15742906332016,
+ "epoch": 0.10416249348984416,
+ "grad_norm": 3.109375,
+ "learning_rate": 1.7920205095337287e-05,
+ "loss": 2.174853973388672,
+ "mean_token_accuracy": 0.5159518288075924,
+ "num_tokens": 3905268.0,
+ "step": 650
+ },
+ {
+ "entropy": 2.162363375425339,
+ "epoch": 0.11217499298906294,
+ "grad_norm": 3.171875,
+ "learning_rate": 1.7759974363082843e-05,
+ "loss": 2.184231872558594,
+ "mean_token_accuracy": 0.516631413847208,
+ "num_tokens": 4207934.0,
+ "step": 700
+ },
+ {
+ "entropy": 2.1516289287805557,
+ "epoch": 0.12018749248828171,
+ "grad_norm": 3.609375,
+ "learning_rate": 1.7599743630828395e-05,
+ "loss": 2.167020263671875,
+ "mean_token_accuracy": 0.516358617246151,
+ "num_tokens": 4515774.0,
+ "step": 750
+ },
+ {
+ "entropy": 2.1549544781446457,
+ "epoch": 0.1281999919875005,
+ "grad_norm": 3.28125,
+ "learning_rate": 1.7439512898573947e-05,
+ "loss": 2.181917877197266,
+ "mean_token_accuracy": 0.5182220858335495,
+ "num_tokens": 4814709.0,
+ "step": 800
+ },
+ {
+ "entropy": 2.1532615756988527,
+ "epoch": 0.1362124914867193,
+ "grad_norm": 3.078125,
+ "learning_rate": 1.7279282166319503e-05,
+ "loss": 2.1659007263183594,
+ "mean_token_accuracy": 0.5180425970256328,
+ "num_tokens": 5112585.0,
+ "step": 850
+ },
+ {
+ "entropy": 2.1704017293453215,
+ "epoch": 0.14422499098593805,
+ "grad_norm": 2.921875,
+ "learning_rate": 1.7119051434065055e-05,
+ "loss": 2.1934996032714844,
+ "mean_token_accuracy": 0.5154964691400528,
+ "num_tokens": 5411369.0,
+ "step": 900
+ },
+ {
+ "entropy": 2.1349107217788696,
+ "epoch": 0.15223749048515683,
+ "grad_norm": 2.984375,
+ "learning_rate": 1.6958820701810608e-05,
+ "loss": 2.1598545837402345,
+ "mean_token_accuracy": 0.5208920259773732,
+ "num_tokens": 5707060.0,
+ "step": 950
+ },
+ {
+ "entropy": 2.1541705918312073,
+ "epoch": 0.16024998998437562,
+ "grad_norm": 3.203125,
+ "learning_rate": 1.6798589969556163e-05,
+ "loss": 2.170956573486328,
+ "mean_token_accuracy": 0.5204938448965549,
+ "num_tokens": 6004079.0,
+ "step": 1000
+ }
+ ],
+ "logging_steps": 50,
+ "max_steps": 6241,
+ "num_input_tokens_seen": 0,
+ "num_train_epochs": 1,
+ "save_steps": 500,
+ "stateful_callbacks": {
+ "TrainerControl": {
+ "args": {
+ "should_epoch_stop": false,
+ "should_evaluate": false,
+ "should_log": false,
+ "should_save": true,
+ "should_training_stop": false
+ },
+ "attributes": {}
+ }
+ },
+ "total_flos": 7.095551986099814e+16,
+ "train_batch_size": 4,
+ "trial_name": null,
+ "trial_params": null
+}
diff --git a/checkpoint-1000/training_args.bin b/checkpoint-1000/training_args.bin
new file mode 100644
index 0000000000000000000000000000000000000000..4b409d5a147856f0d5a3d3da0f3b4922a9def7dd
--- /dev/null
+++ b/checkpoint-1000/training_args.bin
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:78622f63304ec0913ed0b4ea7a1b26c08a5621b3661a380d798c4ea426235671
+size 5713
diff --git a/checkpoint-1500/chat_template.jinja b/checkpoint-1500/chat_template.jinja
new file mode 100644
index 0000000000000000000000000000000000000000..01be9b307daa2d425f7c168c9fb145a286e0afb4
--- /dev/null
+++ b/checkpoint-1500/chat_template.jinja
@@ -0,0 +1,89 @@
+{%- if tools %}
+ {{- '<|im_start|>system\n' }}
+ {%- if messages[0].role == 'system' %}
+ {{- messages[0].content + '\n\n' }}
+ {%- endif %}
+ {{- "# Tools\n\nYou may call one or more functions to assist with the user query.\n\nYou are provided with function signatures within XML tags:\n" }}
+ {%- for tool in tools %}
+ {{- "\n" }}
+ {{- tool | tojson }}
+ {%- endfor %}
+ {{- "\n\n\nFor each function call, return a json object with function name and arguments within XML tags:\n\n{\"name\": , \"arguments\": }\n<|im_end|>\n" }}
+{%- else %}
+ {%- if messages[0].role == 'system' %}
+ {{- '<|im_start|>system\n' + messages[0].content + '<|im_end|>\n' }}
+ {%- endif %}
+{%- endif %}
+{%- set ns = namespace(multi_step_tool=true, last_query_index=messages|length - 1) %}
+{%- for message in messages[::-1] %}
+ {%- set index = (messages|length - 1) - loop.index0 %}
+ {%- if ns.multi_step_tool and message.role == "user" and message.content is string and not(message.content.startswith('') and message.content.endswith('')) %}
+ {%- set ns.multi_step_tool = false %}
+ {%- set ns.last_query_index = index %}
+ {%- endif %}
+{%- endfor %}
+{%- for message in messages %}
+ {%- if message.content is string %}
+ {%- set content = message.content %}
+ {%- else %}
+ {%- set content = '' %}
+ {%- endif %}
+ {%- if (message.role == "user") or (message.role == "system" and not loop.first) %}
+ {{- '<|im_start|>' + message.role + '\n' + content + '<|im_end|>' + '\n' }}
+ {%- elif message.role == "assistant" %}
+ {%- set reasoning_content = '' %}
+ {%- if message.reasoning_content is string %}
+ {%- set reasoning_content = message.reasoning_content %}
+ {%- else %}
+ {%- if '' in content %}
+ {%- set reasoning_content = content.split('')[0].rstrip('\n').split('')[-1].lstrip('\n') %}
+ {%- set content = content.split('')[-1].lstrip('\n') %}
+ {%- endif %}
+ {%- endif %}
+ {%- if loop.index0 > ns.last_query_index %}
+ {%- if loop.last or (not loop.last and reasoning_content) %}
+ {{- '<|im_start|>' + message.role + '\n\n' + reasoning_content.strip('\n') + '\n\n\n' + content.lstrip('\n') }}
+ {%- else %}
+ {{- '<|im_start|>' + message.role + '\n' + content }}
+ {%- endif %}
+ {%- else %}
+ {{- '<|im_start|>' + message.role + '\n' + content }}
+ {%- endif %}
+ {%- if message.tool_calls %}
+ {%- for tool_call in message.tool_calls %}
+ {%- if (loop.first and content) or (not loop.first) %}
+ {{- '\n' }}
+ {%- endif %}
+ {%- if tool_call.function %}
+ {%- set tool_call = tool_call.function %}
+ {%- endif %}
+ {{- '\n{"name": "' }}
+ {{- tool_call.name }}
+ {{- '", "arguments": ' }}
+ {%- if tool_call.arguments is string %}
+ {{- tool_call.arguments }}
+ {%- else %}
+ {{- tool_call.arguments | tojson }}
+ {%- endif %}
+ {{- '}\n' }}
+ {%- endfor %}
+ {%- endif %}
+ {{- '<|im_end|>\n' }}
+ {%- elif message.role == "tool" %}
+ {%- if loop.first or (messages[loop.index0 - 1].role != "tool") %}
+ {{- '<|im_start|>user' }}
+ {%- endif %}
+ {{- '\n\n' }}
+ {{- content }}
+ {{- '\n' }}
+ {%- if loop.last or (messages[loop.index0 + 1].role != "tool") %}
+ {{- '<|im_end|>\n' }}
+ {%- endif %}
+ {%- endif %}
+{%- endfor %}
+{%- if add_generation_prompt %}
+ {{- '<|im_start|>assistant\n' }}
+ {%- if enable_thinking is defined and enable_thinking is false %}
+ {{- '\n\n\n\n' }}
+ {%- endif %}
+{%- endif %}
\ No newline at end of file
diff --git a/checkpoint-1500/config.json b/checkpoint-1500/config.json
new file mode 100644
index 0000000000000000000000000000000000000000..1d219d6a6d288625229a71646710b85cf7ae5def
--- /dev/null
+++ b/checkpoint-1500/config.json
@@ -0,0 +1,63 @@
+{
+ "architectures": [
+ "Qwen3ForCausalLM"
+ ],
+ "attention_bias": false,
+ "attention_dropout": 0.0,
+ "bos_token_id": null,
+ "dtype": "bfloat16",
+ "eos_token_id": 151645,
+ "head_dim": 128,
+ "hidden_act": "silu",
+ "hidden_size": 2048,
+ "initializer_range": 0.02,
+ "intermediate_size": 6144,
+ "layer_types": [
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention"
+ ],
+ "max_position_embeddings": 40960,
+ "max_window_layers": 28,
+ "model_type": "qwen3",
+ "num_attention_heads": 16,
+ "num_hidden_layers": 28,
+ "num_key_value_heads": 8,
+ "pad_token_id": 151643,
+ "rms_norm_eps": 1e-06,
+ "rope_parameters": {
+ "rope_theta": 1000000,
+ "rope_type": "default"
+ },
+ "sliding_window": null,
+ "tie_word_embeddings": true,
+ "transformers_version": "5.7.0",
+ "use_cache": false,
+ "use_sliding_window": false,
+ "vocab_size": 151936
+}
diff --git a/checkpoint-1500/generation_config.json b/checkpoint-1500/generation_config.json
new file mode 100644
index 0000000000000000000000000000000000000000..5ec133d867b1427d43a5ee0ea332bd088fc12013
--- /dev/null
+++ b/checkpoint-1500/generation_config.json
@@ -0,0 +1,12 @@
+{
+ "do_sample": true,
+ "eos_token_id": [
+ 151645,
+ 151643
+ ],
+ "pad_token_id": 151643,
+ "temperature": 0.6,
+ "top_k": 20,
+ "top_p": 0.95,
+ "transformers_version": "5.7.0"
+}
diff --git a/checkpoint-1500/model.safetensors b/checkpoint-1500/model.safetensors
new file mode 100644
index 0000000000000000000000000000000000000000..a549be5c1246b9ef87c7c3cd8f8202372803ec74
--- /dev/null
+++ b/checkpoint-1500/model.safetensors
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:438e34183f5e94f608c7669e0b2cb247069e6a47ebcded96167ec3afb7af8101
+size 3441185608
diff --git a/checkpoint-1500/optimizer.pt b/checkpoint-1500/optimizer.pt
new file mode 100644
index 0000000000000000000000000000000000000000..9f3c9c682f688f32fbd21c7c5fc82c2e93b7743b
--- /dev/null
+++ b/checkpoint-1500/optimizer.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:11e561d4c48540c45fb0712b82d5ffdd92c408f08ae4ed12ecbced4e409b1208
+size 6882572207
diff --git a/checkpoint-1500/rng_state.pth b/checkpoint-1500/rng_state.pth
new file mode 100644
index 0000000000000000000000000000000000000000..5a8f17addf8aaee350625e9847751582bc9f86dd
--- /dev/null
+++ b/checkpoint-1500/rng_state.pth
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:61c19bab1174704a4a4441475683bf1270277af15d2e2c95e964789128e482c4
+size 14645
diff --git a/checkpoint-1500/scheduler.pt b/checkpoint-1500/scheduler.pt
new file mode 100644
index 0000000000000000000000000000000000000000..d35c63e72fd325a05bb8cfe20af5e1ddc9a6c931
--- /dev/null
+++ b/checkpoint-1500/scheduler.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b666bb023db840fb022cfa0357e113ba9d208c4303ad969fafd6098567ba3038
+size 1465
diff --git a/checkpoint-1500/tokenizer.json b/checkpoint-1500/tokenizer.json
new file mode 100644
index 0000000000000000000000000000000000000000..c7afbed2efcdf019f88ab0572ec29d3bf595dfe2
--- /dev/null
+++ b/checkpoint-1500/tokenizer.json
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:be75606093db2094d7cd20f3c2f385c212750648bd6ea4fb2bf507a6a4c55506
+size 11422650
diff --git a/checkpoint-1500/tokenizer_config.json b/checkpoint-1500/tokenizer_config.json
new file mode 100644
index 0000000000000000000000000000000000000000..770e41d6c92519d525eede4cbcf3ba27f6425311
--- /dev/null
+++ b/checkpoint-1500/tokenizer_config.json
@@ -0,0 +1,30 @@
+{
+ "add_prefix_space": false,
+ "backend": "tokenizers",
+ "bos_token": null,
+ "clean_up_tokenization_spaces": false,
+ "eos_token": "<|im_end|>",
+ "errors": "replace",
+ "extra_special_tokens": [
+ "<|im_start|>",
+ "<|im_end|>",
+ "<|object_ref_start|>",
+ "<|object_ref_end|>",
+ "<|box_start|>",
+ "<|box_end|>",
+ "<|quad_start|>",
+ "<|quad_end|>",
+ "<|vision_start|>",
+ "<|vision_end|>",
+ "<|vision_pad|>",
+ "<|image_pad|>",
+ "<|video_pad|>"
+ ],
+ "is_local": false,
+ "local_files_only": false,
+ "model_max_length": 131072,
+ "pad_token": "<|endoftext|>",
+ "split_special_tokens": false,
+ "tokenizer_class": "Qwen2Tokenizer",
+ "unk_token": null
+}
diff --git a/checkpoint-1500/trainer_state.json b/checkpoint-1500/trainer_state.json
new file mode 100644
index 0000000000000000000000000000000000000000..829cf4d49c9a92cb213ee0cd4ff96ecb01dee5a0
--- /dev/null
+++ b/checkpoint-1500/trainer_state.json
@@ -0,0 +1,334 @@
+{
+ "best_global_step": null,
+ "best_metric": null,
+ "best_model_checkpoint": null,
+ "epoch": 0.24037498497656343,
+ "eval_steps": 500,
+ "global_step": 1500,
+ "is_hyper_param_search": false,
+ "is_local_process_zero": true,
+ "is_world_process_zero": true,
+ "log_history": [
+ {
+ "entropy": 2.223189059495926,
+ "epoch": 0.008012499499218782,
+ "grad_norm": 2.875,
+ "learning_rate": 1.9842973882390643e-05,
+ "loss": 2.3428109741210936,
+ "mean_token_accuracy": 0.4963963569700718,
+ "num_tokens": 305366.0,
+ "step": 50
+ },
+ {
+ "entropy": 2.188130117058754,
+ "epoch": 0.016024998998437564,
+ "grad_norm": 3.328125,
+ "learning_rate": 1.96827431501362e-05,
+ "loss": 2.225358734130859,
+ "mean_token_accuracy": 0.5110921229422093,
+ "num_tokens": 605042.0,
+ "step": 100
+ },
+ {
+ "entropy": 2.197347580194473,
+ "epoch": 0.024037498497656344,
+ "grad_norm": 3.453125,
+ "learning_rate": 1.952251241788175e-05,
+ "loss": 2.2299420166015627,
+ "mean_token_accuracy": 0.5093490639328957,
+ "num_tokens": 906888.0,
+ "step": 150
+ },
+ {
+ "entropy": 2.1913950395584108,
+ "epoch": 0.03204999799687513,
+ "grad_norm": 2.90625,
+ "learning_rate": 1.9362281685627303e-05,
+ "loss": 2.210995941162109,
+ "mean_token_accuracy": 0.5097909501194954,
+ "num_tokens": 1213125.0,
+ "step": 200
+ },
+ {
+ "entropy": 2.1611998677253723,
+ "epoch": 0.040062497496093905,
+ "grad_norm": 3.25,
+ "learning_rate": 1.920205095337286e-05,
+ "loss": 2.1867474365234374,
+ "mean_token_accuracy": 0.5154410660266876,
+ "num_tokens": 1509026.0,
+ "step": 250
+ },
+ {
+ "entropy": 2.168051454424858,
+ "epoch": 0.04807499699531269,
+ "grad_norm": 2.9375,
+ "learning_rate": 1.904182022111841e-05,
+ "loss": 2.190071716308594,
+ "mean_token_accuracy": 0.5156884534657001,
+ "num_tokens": 1806850.0,
+ "step": 300
+ },
+ {
+ "entropy": 2.1737154543399813,
+ "epoch": 0.05608749649453147,
+ "grad_norm": 3.421875,
+ "learning_rate": 1.8881589488863967e-05,
+ "loss": 2.199147186279297,
+ "mean_token_accuracy": 0.514045044630766,
+ "num_tokens": 2104505.0,
+ "step": 350
+ },
+ {
+ "entropy": 2.148039159178734,
+ "epoch": 0.06409999599375026,
+ "grad_norm": 3.109375,
+ "learning_rate": 1.872135875660952e-05,
+ "loss": 2.176946105957031,
+ "mean_token_accuracy": 0.5177112428843975,
+ "num_tokens": 2391088.0,
+ "step": 400
+ },
+ {
+ "entropy": 2.194890711903572,
+ "epoch": 0.07211249549296903,
+ "grad_norm": 3.28125,
+ "learning_rate": 1.856112802435507e-05,
+ "loss": 2.216288757324219,
+ "mean_token_accuracy": 0.5110335703194141,
+ "num_tokens": 2691272.0,
+ "step": 450
+ },
+ {
+ "entropy": 2.1557786524295808,
+ "epoch": 0.08012499499218781,
+ "grad_norm": 4.15625,
+ "learning_rate": 1.8400897292100627e-05,
+ "loss": 2.1812680053710936,
+ "mean_token_accuracy": 0.5176697050035,
+ "num_tokens": 2991855.0,
+ "step": 500
+ },
+ {
+ "entropy": 2.1708167713880537,
+ "epoch": 0.08813749449140659,
+ "grad_norm": 4.34375,
+ "learning_rate": 1.824066655984618e-05,
+ "loss": 2.1921002197265627,
+ "mean_token_accuracy": 0.5156507430970669,
+ "num_tokens": 3295049.0,
+ "step": 550
+ },
+ {
+ "entropy": 2.1686314886808393,
+ "epoch": 0.09614999399062538,
+ "grad_norm": 2.875,
+ "learning_rate": 1.8080435827591735e-05,
+ "loss": 2.1914617919921877,
+ "mean_token_accuracy": 0.5139613692462445,
+ "num_tokens": 3597206.0,
+ "step": 600
+ },
+ {
+ "entropy": 2.15742906332016,
+ "epoch": 0.10416249348984416,
+ "grad_norm": 3.109375,
+ "learning_rate": 1.7920205095337287e-05,
+ "loss": 2.174853973388672,
+ "mean_token_accuracy": 0.5159518288075924,
+ "num_tokens": 3905268.0,
+ "step": 650
+ },
+ {
+ "entropy": 2.162363375425339,
+ "epoch": 0.11217499298906294,
+ "grad_norm": 3.171875,
+ "learning_rate": 1.7759974363082843e-05,
+ "loss": 2.184231872558594,
+ "mean_token_accuracy": 0.516631413847208,
+ "num_tokens": 4207934.0,
+ "step": 700
+ },
+ {
+ "entropy": 2.1516289287805557,
+ "epoch": 0.12018749248828171,
+ "grad_norm": 3.609375,
+ "learning_rate": 1.7599743630828395e-05,
+ "loss": 2.167020263671875,
+ "mean_token_accuracy": 0.516358617246151,
+ "num_tokens": 4515774.0,
+ "step": 750
+ },
+ {
+ "entropy": 2.1549544781446457,
+ "epoch": 0.1281999919875005,
+ "grad_norm": 3.28125,
+ "learning_rate": 1.7439512898573947e-05,
+ "loss": 2.181917877197266,
+ "mean_token_accuracy": 0.5182220858335495,
+ "num_tokens": 4814709.0,
+ "step": 800
+ },
+ {
+ "entropy": 2.1532615756988527,
+ "epoch": 0.1362124914867193,
+ "grad_norm": 3.078125,
+ "learning_rate": 1.7279282166319503e-05,
+ "loss": 2.1659007263183594,
+ "mean_token_accuracy": 0.5180425970256328,
+ "num_tokens": 5112585.0,
+ "step": 850
+ },
+ {
+ "entropy": 2.1704017293453215,
+ "epoch": 0.14422499098593805,
+ "grad_norm": 2.921875,
+ "learning_rate": 1.7119051434065055e-05,
+ "loss": 2.1934996032714844,
+ "mean_token_accuracy": 0.5154964691400528,
+ "num_tokens": 5411369.0,
+ "step": 900
+ },
+ {
+ "entropy": 2.1349107217788696,
+ "epoch": 0.15223749048515683,
+ "grad_norm": 2.984375,
+ "learning_rate": 1.6958820701810608e-05,
+ "loss": 2.1598545837402345,
+ "mean_token_accuracy": 0.5208920259773732,
+ "num_tokens": 5707060.0,
+ "step": 950
+ },
+ {
+ "entropy": 2.1541705918312073,
+ "epoch": 0.16024998998437562,
+ "grad_norm": 3.203125,
+ "learning_rate": 1.6798589969556163e-05,
+ "loss": 2.170956573486328,
+ "mean_token_accuracy": 0.5204938448965549,
+ "num_tokens": 6004079.0,
+ "step": 1000
+ },
+ {
+ "entropy": 2.1523867017030716,
+ "epoch": 0.1682624894835944,
+ "grad_norm": 3.09375,
+ "learning_rate": 1.6638359237301716e-05,
+ "loss": 2.1695118713378907,
+ "mean_token_accuracy": 0.5177376234531402,
+ "num_tokens": 6315462.0,
+ "step": 1050
+ },
+ {
+ "entropy": 2.1356022965908052,
+ "epoch": 0.17627498898281319,
+ "grad_norm": 3.21875,
+ "learning_rate": 1.6478128505047268e-05,
+ "loss": 2.1513107299804686,
+ "mean_token_accuracy": 0.5217180585861206,
+ "num_tokens": 6613060.0,
+ "step": 1100
+ },
+ {
+ "entropy": 2.1483460652828215,
+ "epoch": 0.18428748848203197,
+ "grad_norm": 3.21875,
+ "learning_rate": 1.6317897772792824e-05,
+ "loss": 2.1671595764160156,
+ "mean_token_accuracy": 0.5199222292006016,
+ "num_tokens": 6909655.0,
+ "step": 1150
+ },
+ {
+ "entropy": 2.1074040442705155,
+ "epoch": 0.19229998798125075,
+ "grad_norm": 3.828125,
+ "learning_rate": 1.6157667040538376e-05,
+ "loss": 2.12770751953125,
+ "mean_token_accuracy": 0.5252153894305229,
+ "num_tokens": 7203263.0,
+ "step": 1200
+ },
+ {
+ "entropy": 2.1354027593135836,
+ "epoch": 0.20031248748046954,
+ "grad_norm": 2.921875,
+ "learning_rate": 1.5997436308283932e-05,
+ "loss": 2.156183166503906,
+ "mean_token_accuracy": 0.518907299786806,
+ "num_tokens": 7505411.0,
+ "step": 1250
+ },
+ {
+ "entropy": 2.142924894094467,
+ "epoch": 0.20832498697968832,
+ "grad_norm": 3.5625,
+ "learning_rate": 1.5837205576029484e-05,
+ "loss": 2.158486633300781,
+ "mean_token_accuracy": 0.5205909128487111,
+ "num_tokens": 7803562.0,
+ "step": 1300
+ },
+ {
+ "entropy": 2.1305291467905043,
+ "epoch": 0.2163374864789071,
+ "grad_norm": 3.34375,
+ "learning_rate": 1.5676974843775036e-05,
+ "loss": 2.151043701171875,
+ "mean_token_accuracy": 0.5211617235839366,
+ "num_tokens": 8108037.0,
+ "step": 1350
+ },
+ {
+ "entropy": 2.160947166085243,
+ "epoch": 0.2243499859781259,
+ "grad_norm": 3.234375,
+ "learning_rate": 1.5516744111520592e-05,
+ "loss": 2.177792663574219,
+ "mean_token_accuracy": 0.5167004990577698,
+ "num_tokens": 8408674.0,
+ "step": 1400
+ },
+ {
+ "entropy": 2.120403967499733,
+ "epoch": 0.23236248547734467,
+ "grad_norm": 3.375,
+ "learning_rate": 1.5356513379266144e-05,
+ "loss": 2.1462562561035154,
+ "mean_token_accuracy": 0.5225985661149025,
+ "num_tokens": 8707650.0,
+ "step": 1450
+ },
+ {
+ "entropy": 2.138882914185524,
+ "epoch": 0.24037498497656343,
+ "grad_norm": 3.171875,
+ "learning_rate": 1.5196282647011698e-05,
+ "loss": 2.1499905395507812,
+ "mean_token_accuracy": 0.5204668046534061,
+ "num_tokens": 9006296.0,
+ "step": 1500
+ }
+ ],
+ "logging_steps": 50,
+ "max_steps": 6241,
+ "num_input_tokens_seen": 0,
+ "num_train_epochs": 1,
+ "save_steps": 500,
+ "stateful_callbacks": {
+ "TrainerControl": {
+ "args": {
+ "should_epoch_stop": false,
+ "should_evaluate": false,
+ "should_log": false,
+ "should_save": true,
+ "should_training_stop": false
+ },
+ "attributes": {}
+ }
+ },
+ "total_flos": 1.064051705174999e+17,
+ "train_batch_size": 4,
+ "trial_name": null,
+ "trial_params": null
+}
diff --git a/checkpoint-1500/training_args.bin b/checkpoint-1500/training_args.bin
new file mode 100644
index 0000000000000000000000000000000000000000..4b409d5a147856f0d5a3d3da0f3b4922a9def7dd
--- /dev/null
+++ b/checkpoint-1500/training_args.bin
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:78622f63304ec0913ed0b4ea7a1b26c08a5621b3661a380d798c4ea426235671
+size 5713
diff --git a/checkpoint-2000/chat_template.jinja b/checkpoint-2000/chat_template.jinja
new file mode 100644
index 0000000000000000000000000000000000000000..01be9b307daa2d425f7c168c9fb145a286e0afb4
--- /dev/null
+++ b/checkpoint-2000/chat_template.jinja
@@ -0,0 +1,89 @@
+{%- if tools %}
+ {{- '<|im_start|>system\n' }}
+ {%- if messages[0].role == 'system' %}
+ {{- messages[0].content + '\n\n' }}
+ {%- endif %}
+ {{- "# Tools\n\nYou may call one or more functions to assist with the user query.\n\nYou are provided with function signatures within XML tags:\n" }}
+ {%- for tool in tools %}
+ {{- "\n" }}
+ {{- tool | tojson }}
+ {%- endfor %}
+ {{- "\n\n\nFor each function call, return a json object with function name and arguments within XML tags:\n\n{\"name\": , \"arguments\": }\n<|im_end|>\n" }}
+{%- else %}
+ {%- if messages[0].role == 'system' %}
+ {{- '<|im_start|>system\n' + messages[0].content + '<|im_end|>\n' }}
+ {%- endif %}
+{%- endif %}
+{%- set ns = namespace(multi_step_tool=true, last_query_index=messages|length - 1) %}
+{%- for message in messages[::-1] %}
+ {%- set index = (messages|length - 1) - loop.index0 %}
+ {%- if ns.multi_step_tool and message.role == "user" and message.content is string and not(message.content.startswith('') and message.content.endswith('')) %}
+ {%- set ns.multi_step_tool = false %}
+ {%- set ns.last_query_index = index %}
+ {%- endif %}
+{%- endfor %}
+{%- for message in messages %}
+ {%- if message.content is string %}
+ {%- set content = message.content %}
+ {%- else %}
+ {%- set content = '' %}
+ {%- endif %}
+ {%- if (message.role == "user") or (message.role == "system" and not loop.first) %}
+ {{- '<|im_start|>' + message.role + '\n' + content + '<|im_end|>' + '\n' }}
+ {%- elif message.role == "assistant" %}
+ {%- set reasoning_content = '' %}
+ {%- if message.reasoning_content is string %}
+ {%- set reasoning_content = message.reasoning_content %}
+ {%- else %}
+ {%- if '' in content %}
+ {%- set reasoning_content = content.split('')[0].rstrip('\n').split('')[-1].lstrip('\n') %}
+ {%- set content = content.split('')[-1].lstrip('\n') %}
+ {%- endif %}
+ {%- endif %}
+ {%- if loop.index0 > ns.last_query_index %}
+ {%- if loop.last or (not loop.last and reasoning_content) %}
+ {{- '<|im_start|>' + message.role + '\n\n' + reasoning_content.strip('\n') + '\n\n\n' + content.lstrip('\n') }}
+ {%- else %}
+ {{- '<|im_start|>' + message.role + '\n' + content }}
+ {%- endif %}
+ {%- else %}
+ {{- '<|im_start|>' + message.role + '\n' + content }}
+ {%- endif %}
+ {%- if message.tool_calls %}
+ {%- for tool_call in message.tool_calls %}
+ {%- if (loop.first and content) or (not loop.first) %}
+ {{- '\n' }}
+ {%- endif %}
+ {%- if tool_call.function %}
+ {%- set tool_call = tool_call.function %}
+ {%- endif %}
+ {{- '\n{"name": "' }}
+ {{- tool_call.name }}
+ {{- '", "arguments": ' }}
+ {%- if tool_call.arguments is string %}
+ {{- tool_call.arguments }}
+ {%- else %}
+ {{- tool_call.arguments | tojson }}
+ {%- endif %}
+ {{- '}\n' }}
+ {%- endfor %}
+ {%- endif %}
+ {{- '<|im_end|>\n' }}
+ {%- elif message.role == "tool" %}
+ {%- if loop.first or (messages[loop.index0 - 1].role != "tool") %}
+ {{- '<|im_start|>user' }}
+ {%- endif %}
+ {{- '\n\n' }}
+ {{- content }}
+ {{- '\n' }}
+ {%- if loop.last or (messages[loop.index0 + 1].role != "tool") %}
+ {{- '<|im_end|>\n' }}
+ {%- endif %}
+ {%- endif %}
+{%- endfor %}
+{%- if add_generation_prompt %}
+ {{- '<|im_start|>assistant\n' }}
+ {%- if enable_thinking is defined and enable_thinking is false %}
+ {{- '\n\n\n\n' }}
+ {%- endif %}
+{%- endif %}
\ No newline at end of file
diff --git a/checkpoint-2000/config.json b/checkpoint-2000/config.json
new file mode 100644
index 0000000000000000000000000000000000000000..1d219d6a6d288625229a71646710b85cf7ae5def
--- /dev/null
+++ b/checkpoint-2000/config.json
@@ -0,0 +1,63 @@
+{
+ "architectures": [
+ "Qwen3ForCausalLM"
+ ],
+ "attention_bias": false,
+ "attention_dropout": 0.0,
+ "bos_token_id": null,
+ "dtype": "bfloat16",
+ "eos_token_id": 151645,
+ "head_dim": 128,
+ "hidden_act": "silu",
+ "hidden_size": 2048,
+ "initializer_range": 0.02,
+ "intermediate_size": 6144,
+ "layer_types": [
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention"
+ ],
+ "max_position_embeddings": 40960,
+ "max_window_layers": 28,
+ "model_type": "qwen3",
+ "num_attention_heads": 16,
+ "num_hidden_layers": 28,
+ "num_key_value_heads": 8,
+ "pad_token_id": 151643,
+ "rms_norm_eps": 1e-06,
+ "rope_parameters": {
+ "rope_theta": 1000000,
+ "rope_type": "default"
+ },
+ "sliding_window": null,
+ "tie_word_embeddings": true,
+ "transformers_version": "5.7.0",
+ "use_cache": false,
+ "use_sliding_window": false,
+ "vocab_size": 151936
+}
diff --git a/checkpoint-2000/generation_config.json b/checkpoint-2000/generation_config.json
new file mode 100644
index 0000000000000000000000000000000000000000..5ec133d867b1427d43a5ee0ea332bd088fc12013
--- /dev/null
+++ b/checkpoint-2000/generation_config.json
@@ -0,0 +1,12 @@
+{
+ "do_sample": true,
+ "eos_token_id": [
+ 151645,
+ 151643
+ ],
+ "pad_token_id": 151643,
+ "temperature": 0.6,
+ "top_k": 20,
+ "top_p": 0.95,
+ "transformers_version": "5.7.0"
+}
diff --git a/checkpoint-2000/model.safetensors b/checkpoint-2000/model.safetensors
new file mode 100644
index 0000000000000000000000000000000000000000..809c23be0f3e1365389179819062be00c1b1f244
--- /dev/null
+++ b/checkpoint-2000/model.safetensors
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ec88ee383178456b8f8ab7ee942be2785b0a32656809f229cae94c76dff1b810
+size 3441185608
diff --git a/checkpoint-2000/optimizer.pt b/checkpoint-2000/optimizer.pt
new file mode 100644
index 0000000000000000000000000000000000000000..cc0c799b2477bd95663eea3091446b55f17181eb
--- /dev/null
+++ b/checkpoint-2000/optimizer.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ea2dc818dc38c52000eeebc5b101562caf8fb11d4790f976bb066ee636e4f14d
+size 6882572207
diff --git a/checkpoint-2000/rng_state.pth b/checkpoint-2000/rng_state.pth
new file mode 100644
index 0000000000000000000000000000000000000000..5a8f17addf8aaee350625e9847751582bc9f86dd
--- /dev/null
+++ b/checkpoint-2000/rng_state.pth
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:61c19bab1174704a4a4441475683bf1270277af15d2e2c95e964789128e482c4
+size 14645
diff --git a/checkpoint-2000/scheduler.pt b/checkpoint-2000/scheduler.pt
new file mode 100644
index 0000000000000000000000000000000000000000..91873181c98f7e9653734bba253f76e7b2bcef58
--- /dev/null
+++ b/checkpoint-2000/scheduler.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:11b128c7c709b8b18f78ca21ff87ee25957a3215823c4227b4ba298165c8d222
+size 1465
diff --git a/checkpoint-2000/tokenizer.json b/checkpoint-2000/tokenizer.json
new file mode 100644
index 0000000000000000000000000000000000000000..c7afbed2efcdf019f88ab0572ec29d3bf595dfe2
--- /dev/null
+++ b/checkpoint-2000/tokenizer.json
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:be75606093db2094d7cd20f3c2f385c212750648bd6ea4fb2bf507a6a4c55506
+size 11422650
diff --git a/checkpoint-2000/tokenizer_config.json b/checkpoint-2000/tokenizer_config.json
new file mode 100644
index 0000000000000000000000000000000000000000..770e41d6c92519d525eede4cbcf3ba27f6425311
--- /dev/null
+++ b/checkpoint-2000/tokenizer_config.json
@@ -0,0 +1,30 @@
+{
+ "add_prefix_space": false,
+ "backend": "tokenizers",
+ "bos_token": null,
+ "clean_up_tokenization_spaces": false,
+ "eos_token": "<|im_end|>",
+ "errors": "replace",
+ "extra_special_tokens": [
+ "<|im_start|>",
+ "<|im_end|>",
+ "<|object_ref_start|>",
+ "<|object_ref_end|>",
+ "<|box_start|>",
+ "<|box_end|>",
+ "<|quad_start|>",
+ "<|quad_end|>",
+ "<|vision_start|>",
+ "<|vision_end|>",
+ "<|vision_pad|>",
+ "<|image_pad|>",
+ "<|video_pad|>"
+ ],
+ "is_local": false,
+ "local_files_only": false,
+ "model_max_length": 131072,
+ "pad_token": "<|endoftext|>",
+ "split_special_tokens": false,
+ "tokenizer_class": "Qwen2Tokenizer",
+ "unk_token": null
+}
diff --git a/checkpoint-2000/trainer_state.json b/checkpoint-2000/trainer_state.json
new file mode 100644
index 0000000000000000000000000000000000000000..9db53384984c4a0f36ff3dde48960f6379438815
--- /dev/null
+++ b/checkpoint-2000/trainer_state.json
@@ -0,0 +1,434 @@
+{
+ "best_global_step": null,
+ "best_metric": null,
+ "best_model_checkpoint": null,
+ "epoch": 0.32049997996875124,
+ "eval_steps": 500,
+ "global_step": 2000,
+ "is_hyper_param_search": false,
+ "is_local_process_zero": true,
+ "is_world_process_zero": true,
+ "log_history": [
+ {
+ "entropy": 2.223189059495926,
+ "epoch": 0.008012499499218782,
+ "grad_norm": 2.875,
+ "learning_rate": 1.9842973882390643e-05,
+ "loss": 2.3428109741210936,
+ "mean_token_accuracy": 0.4963963569700718,
+ "num_tokens": 305366.0,
+ "step": 50
+ },
+ {
+ "entropy": 2.188130117058754,
+ "epoch": 0.016024998998437564,
+ "grad_norm": 3.328125,
+ "learning_rate": 1.96827431501362e-05,
+ "loss": 2.225358734130859,
+ "mean_token_accuracy": 0.5110921229422093,
+ "num_tokens": 605042.0,
+ "step": 100
+ },
+ {
+ "entropy": 2.197347580194473,
+ "epoch": 0.024037498497656344,
+ "grad_norm": 3.453125,
+ "learning_rate": 1.952251241788175e-05,
+ "loss": 2.2299420166015627,
+ "mean_token_accuracy": 0.5093490639328957,
+ "num_tokens": 906888.0,
+ "step": 150
+ },
+ {
+ "entropy": 2.1913950395584108,
+ "epoch": 0.03204999799687513,
+ "grad_norm": 2.90625,
+ "learning_rate": 1.9362281685627303e-05,
+ "loss": 2.210995941162109,
+ "mean_token_accuracy": 0.5097909501194954,
+ "num_tokens": 1213125.0,
+ "step": 200
+ },
+ {
+ "entropy": 2.1611998677253723,
+ "epoch": 0.040062497496093905,
+ "grad_norm": 3.25,
+ "learning_rate": 1.920205095337286e-05,
+ "loss": 2.1867474365234374,
+ "mean_token_accuracy": 0.5154410660266876,
+ "num_tokens": 1509026.0,
+ "step": 250
+ },
+ {
+ "entropy": 2.168051454424858,
+ "epoch": 0.04807499699531269,
+ "grad_norm": 2.9375,
+ "learning_rate": 1.904182022111841e-05,
+ "loss": 2.190071716308594,
+ "mean_token_accuracy": 0.5156884534657001,
+ "num_tokens": 1806850.0,
+ "step": 300
+ },
+ {
+ "entropy": 2.1737154543399813,
+ "epoch": 0.05608749649453147,
+ "grad_norm": 3.421875,
+ "learning_rate": 1.8881589488863967e-05,
+ "loss": 2.199147186279297,
+ "mean_token_accuracy": 0.514045044630766,
+ "num_tokens": 2104505.0,
+ "step": 350
+ },
+ {
+ "entropy": 2.148039159178734,
+ "epoch": 0.06409999599375026,
+ "grad_norm": 3.109375,
+ "learning_rate": 1.872135875660952e-05,
+ "loss": 2.176946105957031,
+ "mean_token_accuracy": 0.5177112428843975,
+ "num_tokens": 2391088.0,
+ "step": 400
+ },
+ {
+ "entropy": 2.194890711903572,
+ "epoch": 0.07211249549296903,
+ "grad_norm": 3.28125,
+ "learning_rate": 1.856112802435507e-05,
+ "loss": 2.216288757324219,
+ "mean_token_accuracy": 0.5110335703194141,
+ "num_tokens": 2691272.0,
+ "step": 450
+ },
+ {
+ "entropy": 2.1557786524295808,
+ "epoch": 0.08012499499218781,
+ "grad_norm": 4.15625,
+ "learning_rate": 1.8400897292100627e-05,
+ "loss": 2.1812680053710936,
+ "mean_token_accuracy": 0.5176697050035,
+ "num_tokens": 2991855.0,
+ "step": 500
+ },
+ {
+ "entropy": 2.1708167713880537,
+ "epoch": 0.08813749449140659,
+ "grad_norm": 4.34375,
+ "learning_rate": 1.824066655984618e-05,
+ "loss": 2.1921002197265627,
+ "mean_token_accuracy": 0.5156507430970669,
+ "num_tokens": 3295049.0,
+ "step": 550
+ },
+ {
+ "entropy": 2.1686314886808393,
+ "epoch": 0.09614999399062538,
+ "grad_norm": 2.875,
+ "learning_rate": 1.8080435827591735e-05,
+ "loss": 2.1914617919921877,
+ "mean_token_accuracy": 0.5139613692462445,
+ "num_tokens": 3597206.0,
+ "step": 600
+ },
+ {
+ "entropy": 2.15742906332016,
+ "epoch": 0.10416249348984416,
+ "grad_norm": 3.109375,
+ "learning_rate": 1.7920205095337287e-05,
+ "loss": 2.174853973388672,
+ "mean_token_accuracy": 0.5159518288075924,
+ "num_tokens": 3905268.0,
+ "step": 650
+ },
+ {
+ "entropy": 2.162363375425339,
+ "epoch": 0.11217499298906294,
+ "grad_norm": 3.171875,
+ "learning_rate": 1.7759974363082843e-05,
+ "loss": 2.184231872558594,
+ "mean_token_accuracy": 0.516631413847208,
+ "num_tokens": 4207934.0,
+ "step": 700
+ },
+ {
+ "entropy": 2.1516289287805557,
+ "epoch": 0.12018749248828171,
+ "grad_norm": 3.609375,
+ "learning_rate": 1.7599743630828395e-05,
+ "loss": 2.167020263671875,
+ "mean_token_accuracy": 0.516358617246151,
+ "num_tokens": 4515774.0,
+ "step": 750
+ },
+ {
+ "entropy": 2.1549544781446457,
+ "epoch": 0.1281999919875005,
+ "grad_norm": 3.28125,
+ "learning_rate": 1.7439512898573947e-05,
+ "loss": 2.181917877197266,
+ "mean_token_accuracy": 0.5182220858335495,
+ "num_tokens": 4814709.0,
+ "step": 800
+ },
+ {
+ "entropy": 2.1532615756988527,
+ "epoch": 0.1362124914867193,
+ "grad_norm": 3.078125,
+ "learning_rate": 1.7279282166319503e-05,
+ "loss": 2.1659007263183594,
+ "mean_token_accuracy": 0.5180425970256328,
+ "num_tokens": 5112585.0,
+ "step": 850
+ },
+ {
+ "entropy": 2.1704017293453215,
+ "epoch": 0.14422499098593805,
+ "grad_norm": 2.921875,
+ "learning_rate": 1.7119051434065055e-05,
+ "loss": 2.1934996032714844,
+ "mean_token_accuracy": 0.5154964691400528,
+ "num_tokens": 5411369.0,
+ "step": 900
+ },
+ {
+ "entropy": 2.1349107217788696,
+ "epoch": 0.15223749048515683,
+ "grad_norm": 2.984375,
+ "learning_rate": 1.6958820701810608e-05,
+ "loss": 2.1598545837402345,
+ "mean_token_accuracy": 0.5208920259773732,
+ "num_tokens": 5707060.0,
+ "step": 950
+ },
+ {
+ "entropy": 2.1541705918312073,
+ "epoch": 0.16024998998437562,
+ "grad_norm": 3.203125,
+ "learning_rate": 1.6798589969556163e-05,
+ "loss": 2.170956573486328,
+ "mean_token_accuracy": 0.5204938448965549,
+ "num_tokens": 6004079.0,
+ "step": 1000
+ },
+ {
+ "entropy": 2.1523867017030716,
+ "epoch": 0.1682624894835944,
+ "grad_norm": 3.09375,
+ "learning_rate": 1.6638359237301716e-05,
+ "loss": 2.1695118713378907,
+ "mean_token_accuracy": 0.5177376234531402,
+ "num_tokens": 6315462.0,
+ "step": 1050
+ },
+ {
+ "entropy": 2.1356022965908052,
+ "epoch": 0.17627498898281319,
+ "grad_norm": 3.21875,
+ "learning_rate": 1.6478128505047268e-05,
+ "loss": 2.1513107299804686,
+ "mean_token_accuracy": 0.5217180585861206,
+ "num_tokens": 6613060.0,
+ "step": 1100
+ },
+ {
+ "entropy": 2.1483460652828215,
+ "epoch": 0.18428748848203197,
+ "grad_norm": 3.21875,
+ "learning_rate": 1.6317897772792824e-05,
+ "loss": 2.1671595764160156,
+ "mean_token_accuracy": 0.5199222292006016,
+ "num_tokens": 6909655.0,
+ "step": 1150
+ },
+ {
+ "entropy": 2.1074040442705155,
+ "epoch": 0.19229998798125075,
+ "grad_norm": 3.828125,
+ "learning_rate": 1.6157667040538376e-05,
+ "loss": 2.12770751953125,
+ "mean_token_accuracy": 0.5252153894305229,
+ "num_tokens": 7203263.0,
+ "step": 1200
+ },
+ {
+ "entropy": 2.1354027593135836,
+ "epoch": 0.20031248748046954,
+ "grad_norm": 2.921875,
+ "learning_rate": 1.5997436308283932e-05,
+ "loss": 2.156183166503906,
+ "mean_token_accuracy": 0.518907299786806,
+ "num_tokens": 7505411.0,
+ "step": 1250
+ },
+ {
+ "entropy": 2.142924894094467,
+ "epoch": 0.20832498697968832,
+ "grad_norm": 3.5625,
+ "learning_rate": 1.5837205576029484e-05,
+ "loss": 2.158486633300781,
+ "mean_token_accuracy": 0.5205909128487111,
+ "num_tokens": 7803562.0,
+ "step": 1300
+ },
+ {
+ "entropy": 2.1305291467905043,
+ "epoch": 0.2163374864789071,
+ "grad_norm": 3.34375,
+ "learning_rate": 1.5676974843775036e-05,
+ "loss": 2.151043701171875,
+ "mean_token_accuracy": 0.5211617235839366,
+ "num_tokens": 8108037.0,
+ "step": 1350
+ },
+ {
+ "entropy": 2.160947166085243,
+ "epoch": 0.2243499859781259,
+ "grad_norm": 3.234375,
+ "learning_rate": 1.5516744111520592e-05,
+ "loss": 2.177792663574219,
+ "mean_token_accuracy": 0.5167004990577698,
+ "num_tokens": 8408674.0,
+ "step": 1400
+ },
+ {
+ "entropy": 2.120403967499733,
+ "epoch": 0.23236248547734467,
+ "grad_norm": 3.375,
+ "learning_rate": 1.5356513379266144e-05,
+ "loss": 2.1462562561035154,
+ "mean_token_accuracy": 0.5225985661149025,
+ "num_tokens": 8707650.0,
+ "step": 1450
+ },
+ {
+ "entropy": 2.138882914185524,
+ "epoch": 0.24037498497656343,
+ "grad_norm": 3.171875,
+ "learning_rate": 1.5196282647011698e-05,
+ "loss": 2.1499905395507812,
+ "mean_token_accuracy": 0.5204668046534061,
+ "num_tokens": 9006296.0,
+ "step": 1500
+ },
+ {
+ "entropy": 2.131927245259285,
+ "epoch": 0.2483874844757822,
+ "grad_norm": 3.546875,
+ "learning_rate": 1.5036051914757252e-05,
+ "loss": 2.157076416015625,
+ "mean_token_accuracy": 0.5220408822596073,
+ "num_tokens": 9307203.0,
+ "step": 1550
+ },
+ {
+ "entropy": 2.1308567851781843,
+ "epoch": 0.256399983975001,
+ "grad_norm": 3.546875,
+ "learning_rate": 1.4875821182502805e-05,
+ "loss": 2.146601104736328,
+ "mean_token_accuracy": 0.5219539681077003,
+ "num_tokens": 9610070.0,
+ "step": 1600
+ },
+ {
+ "entropy": 2.1097783839702604,
+ "epoch": 0.2644124834742198,
+ "grad_norm": 3.84375,
+ "learning_rate": 1.4715590450248359e-05,
+ "loss": 2.1268209838867187,
+ "mean_token_accuracy": 0.5236652019619942,
+ "num_tokens": 9909117.0,
+ "step": 1650
+ },
+ {
+ "entropy": 2.123499180674553,
+ "epoch": 0.2724249829734386,
+ "grad_norm": 3.46875,
+ "learning_rate": 1.4555359717993913e-05,
+ "loss": 2.1409934997558593,
+ "mean_token_accuracy": 0.5225465354323388,
+ "num_tokens": 10209461.0,
+ "step": 1700
+ },
+ {
+ "entropy": 2.13052960395813,
+ "epoch": 0.2804374824726573,
+ "grad_norm": 2.71875,
+ "learning_rate": 1.4395128985739467e-05,
+ "loss": 2.151373291015625,
+ "mean_token_accuracy": 0.5227928113937378,
+ "num_tokens": 10509282.0,
+ "step": 1750
+ },
+ {
+ "entropy": 2.1226114094257356,
+ "epoch": 0.2884499819718761,
+ "grad_norm": 3.65625,
+ "learning_rate": 1.4234898253485019e-05,
+ "loss": 2.144451141357422,
+ "mean_token_accuracy": 0.5233827362954616,
+ "num_tokens": 10811056.0,
+ "step": 1800
+ },
+ {
+ "entropy": 2.126143154501915,
+ "epoch": 0.2964624814710949,
+ "grad_norm": 3.359375,
+ "learning_rate": 1.4074667521230575e-05,
+ "loss": 2.1457598876953123,
+ "mean_token_accuracy": 0.5221967925131321,
+ "num_tokens": 11111153.0,
+ "step": 1850
+ },
+ {
+ "entropy": 2.1048326581716537,
+ "epoch": 0.30447498097031367,
+ "grad_norm": 3.59375,
+ "learning_rate": 1.3914436788976127e-05,
+ "loss": 2.1260116577148436,
+ "mean_token_accuracy": 0.5272719909250736,
+ "num_tokens": 11405606.0,
+ "step": 1900
+ },
+ {
+ "entropy": 2.1236617982387545,
+ "epoch": 0.31248748046953245,
+ "grad_norm": 3.0,
+ "learning_rate": 1.375420605672168e-05,
+ "loss": 2.151782684326172,
+ "mean_token_accuracy": 0.5227546173334122,
+ "num_tokens": 11703943.0,
+ "step": 1950
+ },
+ {
+ "entropy": 2.121563429236412,
+ "epoch": 0.32049997996875124,
+ "grad_norm": 3.34375,
+ "learning_rate": 1.3593975324467235e-05,
+ "loss": 2.1370144653320313,
+ "mean_token_accuracy": 0.5235123547911644,
+ "num_tokens": 12002522.0,
+ "step": 2000
+ }
+ ],
+ "logging_steps": 50,
+ "max_steps": 6241,
+ "num_input_tokens_seen": 0,
+ "num_train_epochs": 1,
+ "save_steps": 500,
+ "stateful_callbacks": {
+ "TrainerControl": {
+ "args": {
+ "should_epoch_stop": false,
+ "should_evaluate": false,
+ "should_log": false,
+ "should_save": true,
+ "should_training_stop": false
+ },
+ "attributes": {}
+ }
+ },
+ "total_flos": 1.4174735647666176e+17,
+ "train_batch_size": 4,
+ "trial_name": null,
+ "trial_params": null
+}
diff --git a/checkpoint-2000/training_args.bin b/checkpoint-2000/training_args.bin
new file mode 100644
index 0000000000000000000000000000000000000000..4b409d5a147856f0d5a3d3da0f3b4922a9def7dd
--- /dev/null
+++ b/checkpoint-2000/training_args.bin
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:78622f63304ec0913ed0b4ea7a1b26c08a5621b3661a380d798c4ea426235671
+size 5713
diff --git a/checkpoint-2500/chat_template.jinja b/checkpoint-2500/chat_template.jinja
new file mode 100644
index 0000000000000000000000000000000000000000..01be9b307daa2d425f7c168c9fb145a286e0afb4
--- /dev/null
+++ b/checkpoint-2500/chat_template.jinja
@@ -0,0 +1,89 @@
+{%- if tools %}
+ {{- '<|im_start|>system\n' }}
+ {%- if messages[0].role == 'system' %}
+ {{- messages[0].content + '\n\n' }}
+ {%- endif %}
+ {{- "# Tools\n\nYou may call one or more functions to assist with the user query.\n\nYou are provided with function signatures within XML tags:\n" }}
+ {%- for tool in tools %}
+ {{- "\n" }}
+ {{- tool | tojson }}
+ {%- endfor %}
+ {{- "\n\n\nFor each function call, return a json object with function name and arguments within XML tags:\n\n{\"name\": , \"arguments\": }\n<|im_end|>\n" }}
+{%- else %}
+ {%- if messages[0].role == 'system' %}
+ {{- '<|im_start|>system\n' + messages[0].content + '<|im_end|>\n' }}
+ {%- endif %}
+{%- endif %}
+{%- set ns = namespace(multi_step_tool=true, last_query_index=messages|length - 1) %}
+{%- for message in messages[::-1] %}
+ {%- set index = (messages|length - 1) - loop.index0 %}
+ {%- if ns.multi_step_tool and message.role == "user" and message.content is string and not(message.content.startswith('') and message.content.endswith('')) %}
+ {%- set ns.multi_step_tool = false %}
+ {%- set ns.last_query_index = index %}
+ {%- endif %}
+{%- endfor %}
+{%- for message in messages %}
+ {%- if message.content is string %}
+ {%- set content = message.content %}
+ {%- else %}
+ {%- set content = '' %}
+ {%- endif %}
+ {%- if (message.role == "user") or (message.role == "system" and not loop.first) %}
+ {{- '<|im_start|>' + message.role + '\n' + content + '<|im_end|>' + '\n' }}
+ {%- elif message.role == "assistant" %}
+ {%- set reasoning_content = '' %}
+ {%- if message.reasoning_content is string %}
+ {%- set reasoning_content = message.reasoning_content %}
+ {%- else %}
+ {%- if '' in content %}
+ {%- set reasoning_content = content.split('')[0].rstrip('\n').split('')[-1].lstrip('\n') %}
+ {%- set content = content.split('')[-1].lstrip('\n') %}
+ {%- endif %}
+ {%- endif %}
+ {%- if loop.index0 > ns.last_query_index %}
+ {%- if loop.last or (not loop.last and reasoning_content) %}
+ {{- '<|im_start|>' + message.role + '\n\n' + reasoning_content.strip('\n') + '\n\n\n' + content.lstrip('\n') }}
+ {%- else %}
+ {{- '<|im_start|>' + message.role + '\n' + content }}
+ {%- endif %}
+ {%- else %}
+ {{- '<|im_start|>' + message.role + '\n' + content }}
+ {%- endif %}
+ {%- if message.tool_calls %}
+ {%- for tool_call in message.tool_calls %}
+ {%- if (loop.first and content) or (not loop.first) %}
+ {{- '\n' }}
+ {%- endif %}
+ {%- if tool_call.function %}
+ {%- set tool_call = tool_call.function %}
+ {%- endif %}
+ {{- '\n{"name": "' }}
+ {{- tool_call.name }}
+ {{- '", "arguments": ' }}
+ {%- if tool_call.arguments is string %}
+ {{- tool_call.arguments }}
+ {%- else %}
+ {{- tool_call.arguments | tojson }}
+ {%- endif %}
+ {{- '}\n' }}
+ {%- endfor %}
+ {%- endif %}
+ {{- '<|im_end|>\n' }}
+ {%- elif message.role == "tool" %}
+ {%- if loop.first or (messages[loop.index0 - 1].role != "tool") %}
+ {{- '<|im_start|>user' }}
+ {%- endif %}
+ {{- '\n\n' }}
+ {{- content }}
+ {{- '\n' }}
+ {%- if loop.last or (messages[loop.index0 + 1].role != "tool") %}
+ {{- '<|im_end|>\n' }}
+ {%- endif %}
+ {%- endif %}
+{%- endfor %}
+{%- if add_generation_prompt %}
+ {{- '<|im_start|>assistant\n' }}
+ {%- if enable_thinking is defined and enable_thinking is false %}
+ {{- '\n\n\n\n' }}
+ {%- endif %}
+{%- endif %}
\ No newline at end of file
diff --git a/checkpoint-2500/config.json b/checkpoint-2500/config.json
new file mode 100644
index 0000000000000000000000000000000000000000..1d219d6a6d288625229a71646710b85cf7ae5def
--- /dev/null
+++ b/checkpoint-2500/config.json
@@ -0,0 +1,63 @@
+{
+ "architectures": [
+ "Qwen3ForCausalLM"
+ ],
+ "attention_bias": false,
+ "attention_dropout": 0.0,
+ "bos_token_id": null,
+ "dtype": "bfloat16",
+ "eos_token_id": 151645,
+ "head_dim": 128,
+ "hidden_act": "silu",
+ "hidden_size": 2048,
+ "initializer_range": 0.02,
+ "intermediate_size": 6144,
+ "layer_types": [
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention"
+ ],
+ "max_position_embeddings": 40960,
+ "max_window_layers": 28,
+ "model_type": "qwen3",
+ "num_attention_heads": 16,
+ "num_hidden_layers": 28,
+ "num_key_value_heads": 8,
+ "pad_token_id": 151643,
+ "rms_norm_eps": 1e-06,
+ "rope_parameters": {
+ "rope_theta": 1000000,
+ "rope_type": "default"
+ },
+ "sliding_window": null,
+ "tie_word_embeddings": true,
+ "transformers_version": "5.7.0",
+ "use_cache": false,
+ "use_sliding_window": false,
+ "vocab_size": 151936
+}
diff --git a/checkpoint-2500/generation_config.json b/checkpoint-2500/generation_config.json
new file mode 100644
index 0000000000000000000000000000000000000000..5ec133d867b1427d43a5ee0ea332bd088fc12013
--- /dev/null
+++ b/checkpoint-2500/generation_config.json
@@ -0,0 +1,12 @@
+{
+ "do_sample": true,
+ "eos_token_id": [
+ 151645,
+ 151643
+ ],
+ "pad_token_id": 151643,
+ "temperature": 0.6,
+ "top_k": 20,
+ "top_p": 0.95,
+ "transformers_version": "5.7.0"
+}
diff --git a/checkpoint-2500/model.safetensors b/checkpoint-2500/model.safetensors
new file mode 100644
index 0000000000000000000000000000000000000000..533aff51294bba434ff12bb4cf8751c5cc07bc75
--- /dev/null
+++ b/checkpoint-2500/model.safetensors
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:98a04b91a11ec1c0b526f91d12bdfb28ff4c3c69e104271b9e28a2f75f44f891
+size 3441185608
diff --git a/checkpoint-2500/optimizer.pt b/checkpoint-2500/optimizer.pt
new file mode 100644
index 0000000000000000000000000000000000000000..6526fb008af25c5106c3a66d354c2539c13acdbb
--- /dev/null
+++ b/checkpoint-2500/optimizer.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:7535b338ca59a9c376634a6fbbef04a836c5c21962e653fe7794832854643965
+size 6882572207
diff --git a/checkpoint-2500/rng_state.pth b/checkpoint-2500/rng_state.pth
new file mode 100644
index 0000000000000000000000000000000000000000..5a8f17addf8aaee350625e9847751582bc9f86dd
--- /dev/null
+++ b/checkpoint-2500/rng_state.pth
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:61c19bab1174704a4a4441475683bf1270277af15d2e2c95e964789128e482c4
+size 14645
diff --git a/checkpoint-2500/scheduler.pt b/checkpoint-2500/scheduler.pt
new file mode 100644
index 0000000000000000000000000000000000000000..659045e0095b9d0766900b4964b39c8c1964bd85
--- /dev/null
+++ b/checkpoint-2500/scheduler.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c863cc66645dc4964c4c4f2cf30406fef13cbea85c7c7bd73c333d72670c821e
+size 1465
diff --git a/checkpoint-2500/tokenizer.json b/checkpoint-2500/tokenizer.json
new file mode 100644
index 0000000000000000000000000000000000000000..c7afbed2efcdf019f88ab0572ec29d3bf595dfe2
--- /dev/null
+++ b/checkpoint-2500/tokenizer.json
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:be75606093db2094d7cd20f3c2f385c212750648bd6ea4fb2bf507a6a4c55506
+size 11422650
diff --git a/checkpoint-2500/tokenizer_config.json b/checkpoint-2500/tokenizer_config.json
new file mode 100644
index 0000000000000000000000000000000000000000..770e41d6c92519d525eede4cbcf3ba27f6425311
--- /dev/null
+++ b/checkpoint-2500/tokenizer_config.json
@@ -0,0 +1,30 @@
+{
+ "add_prefix_space": false,
+ "backend": "tokenizers",
+ "bos_token": null,
+ "clean_up_tokenization_spaces": false,
+ "eos_token": "<|im_end|>",
+ "errors": "replace",
+ "extra_special_tokens": [
+ "<|im_start|>",
+ "<|im_end|>",
+ "<|object_ref_start|>",
+ "<|object_ref_end|>",
+ "<|box_start|>",
+ "<|box_end|>",
+ "<|quad_start|>",
+ "<|quad_end|>",
+ "<|vision_start|>",
+ "<|vision_end|>",
+ "<|vision_pad|>",
+ "<|image_pad|>",
+ "<|video_pad|>"
+ ],
+ "is_local": false,
+ "local_files_only": false,
+ "model_max_length": 131072,
+ "pad_token": "<|endoftext|>",
+ "split_special_tokens": false,
+ "tokenizer_class": "Qwen2Tokenizer",
+ "unk_token": null
+}
diff --git a/checkpoint-2500/trainer_state.json b/checkpoint-2500/trainer_state.json
new file mode 100644
index 0000000000000000000000000000000000000000..38511de3127f94735edb18dbc73cf362755690bf
--- /dev/null
+++ b/checkpoint-2500/trainer_state.json
@@ -0,0 +1,534 @@
+{
+ "best_global_step": null,
+ "best_metric": null,
+ "best_model_checkpoint": null,
+ "epoch": 0.4006249749609391,
+ "eval_steps": 500,
+ "global_step": 2500,
+ "is_hyper_param_search": false,
+ "is_local_process_zero": true,
+ "is_world_process_zero": true,
+ "log_history": [
+ {
+ "entropy": 2.223189059495926,
+ "epoch": 0.008012499499218782,
+ "grad_norm": 2.875,
+ "learning_rate": 1.9842973882390643e-05,
+ "loss": 2.3428109741210936,
+ "mean_token_accuracy": 0.4963963569700718,
+ "num_tokens": 305366.0,
+ "step": 50
+ },
+ {
+ "entropy": 2.188130117058754,
+ "epoch": 0.016024998998437564,
+ "grad_norm": 3.328125,
+ "learning_rate": 1.96827431501362e-05,
+ "loss": 2.225358734130859,
+ "mean_token_accuracy": 0.5110921229422093,
+ "num_tokens": 605042.0,
+ "step": 100
+ },
+ {
+ "entropy": 2.197347580194473,
+ "epoch": 0.024037498497656344,
+ "grad_norm": 3.453125,
+ "learning_rate": 1.952251241788175e-05,
+ "loss": 2.2299420166015627,
+ "mean_token_accuracy": 0.5093490639328957,
+ "num_tokens": 906888.0,
+ "step": 150
+ },
+ {
+ "entropy": 2.1913950395584108,
+ "epoch": 0.03204999799687513,
+ "grad_norm": 2.90625,
+ "learning_rate": 1.9362281685627303e-05,
+ "loss": 2.210995941162109,
+ "mean_token_accuracy": 0.5097909501194954,
+ "num_tokens": 1213125.0,
+ "step": 200
+ },
+ {
+ "entropy": 2.1611998677253723,
+ "epoch": 0.040062497496093905,
+ "grad_norm": 3.25,
+ "learning_rate": 1.920205095337286e-05,
+ "loss": 2.1867474365234374,
+ "mean_token_accuracy": 0.5154410660266876,
+ "num_tokens": 1509026.0,
+ "step": 250
+ },
+ {
+ "entropy": 2.168051454424858,
+ "epoch": 0.04807499699531269,
+ "grad_norm": 2.9375,
+ "learning_rate": 1.904182022111841e-05,
+ "loss": 2.190071716308594,
+ "mean_token_accuracy": 0.5156884534657001,
+ "num_tokens": 1806850.0,
+ "step": 300
+ },
+ {
+ "entropy": 2.1737154543399813,
+ "epoch": 0.05608749649453147,
+ "grad_norm": 3.421875,
+ "learning_rate": 1.8881589488863967e-05,
+ "loss": 2.199147186279297,
+ "mean_token_accuracy": 0.514045044630766,
+ "num_tokens": 2104505.0,
+ "step": 350
+ },
+ {
+ "entropy": 2.148039159178734,
+ "epoch": 0.06409999599375026,
+ "grad_norm": 3.109375,
+ "learning_rate": 1.872135875660952e-05,
+ "loss": 2.176946105957031,
+ "mean_token_accuracy": 0.5177112428843975,
+ "num_tokens": 2391088.0,
+ "step": 400
+ },
+ {
+ "entropy": 2.194890711903572,
+ "epoch": 0.07211249549296903,
+ "grad_norm": 3.28125,
+ "learning_rate": 1.856112802435507e-05,
+ "loss": 2.216288757324219,
+ "mean_token_accuracy": 0.5110335703194141,
+ "num_tokens": 2691272.0,
+ "step": 450
+ },
+ {
+ "entropy": 2.1557786524295808,
+ "epoch": 0.08012499499218781,
+ "grad_norm": 4.15625,
+ "learning_rate": 1.8400897292100627e-05,
+ "loss": 2.1812680053710936,
+ "mean_token_accuracy": 0.5176697050035,
+ "num_tokens": 2991855.0,
+ "step": 500
+ },
+ {
+ "entropy": 2.1708167713880537,
+ "epoch": 0.08813749449140659,
+ "grad_norm": 4.34375,
+ "learning_rate": 1.824066655984618e-05,
+ "loss": 2.1921002197265627,
+ "mean_token_accuracy": 0.5156507430970669,
+ "num_tokens": 3295049.0,
+ "step": 550
+ },
+ {
+ "entropy": 2.1686314886808393,
+ "epoch": 0.09614999399062538,
+ "grad_norm": 2.875,
+ "learning_rate": 1.8080435827591735e-05,
+ "loss": 2.1914617919921877,
+ "mean_token_accuracy": 0.5139613692462445,
+ "num_tokens": 3597206.0,
+ "step": 600
+ },
+ {
+ "entropy": 2.15742906332016,
+ "epoch": 0.10416249348984416,
+ "grad_norm": 3.109375,
+ "learning_rate": 1.7920205095337287e-05,
+ "loss": 2.174853973388672,
+ "mean_token_accuracy": 0.5159518288075924,
+ "num_tokens": 3905268.0,
+ "step": 650
+ },
+ {
+ "entropy": 2.162363375425339,
+ "epoch": 0.11217499298906294,
+ "grad_norm": 3.171875,
+ "learning_rate": 1.7759974363082843e-05,
+ "loss": 2.184231872558594,
+ "mean_token_accuracy": 0.516631413847208,
+ "num_tokens": 4207934.0,
+ "step": 700
+ },
+ {
+ "entropy": 2.1516289287805557,
+ "epoch": 0.12018749248828171,
+ "grad_norm": 3.609375,
+ "learning_rate": 1.7599743630828395e-05,
+ "loss": 2.167020263671875,
+ "mean_token_accuracy": 0.516358617246151,
+ "num_tokens": 4515774.0,
+ "step": 750
+ },
+ {
+ "entropy": 2.1549544781446457,
+ "epoch": 0.1281999919875005,
+ "grad_norm": 3.28125,
+ "learning_rate": 1.7439512898573947e-05,
+ "loss": 2.181917877197266,
+ "mean_token_accuracy": 0.5182220858335495,
+ "num_tokens": 4814709.0,
+ "step": 800
+ },
+ {
+ "entropy": 2.1532615756988527,
+ "epoch": 0.1362124914867193,
+ "grad_norm": 3.078125,
+ "learning_rate": 1.7279282166319503e-05,
+ "loss": 2.1659007263183594,
+ "mean_token_accuracy": 0.5180425970256328,
+ "num_tokens": 5112585.0,
+ "step": 850
+ },
+ {
+ "entropy": 2.1704017293453215,
+ "epoch": 0.14422499098593805,
+ "grad_norm": 2.921875,
+ "learning_rate": 1.7119051434065055e-05,
+ "loss": 2.1934996032714844,
+ "mean_token_accuracy": 0.5154964691400528,
+ "num_tokens": 5411369.0,
+ "step": 900
+ },
+ {
+ "entropy": 2.1349107217788696,
+ "epoch": 0.15223749048515683,
+ "grad_norm": 2.984375,
+ "learning_rate": 1.6958820701810608e-05,
+ "loss": 2.1598545837402345,
+ "mean_token_accuracy": 0.5208920259773732,
+ "num_tokens": 5707060.0,
+ "step": 950
+ },
+ {
+ "entropy": 2.1541705918312073,
+ "epoch": 0.16024998998437562,
+ "grad_norm": 3.203125,
+ "learning_rate": 1.6798589969556163e-05,
+ "loss": 2.170956573486328,
+ "mean_token_accuracy": 0.5204938448965549,
+ "num_tokens": 6004079.0,
+ "step": 1000
+ },
+ {
+ "entropy": 2.1523867017030716,
+ "epoch": 0.1682624894835944,
+ "grad_norm": 3.09375,
+ "learning_rate": 1.6638359237301716e-05,
+ "loss": 2.1695118713378907,
+ "mean_token_accuracy": 0.5177376234531402,
+ "num_tokens": 6315462.0,
+ "step": 1050
+ },
+ {
+ "entropy": 2.1356022965908052,
+ "epoch": 0.17627498898281319,
+ "grad_norm": 3.21875,
+ "learning_rate": 1.6478128505047268e-05,
+ "loss": 2.1513107299804686,
+ "mean_token_accuracy": 0.5217180585861206,
+ "num_tokens": 6613060.0,
+ "step": 1100
+ },
+ {
+ "entropy": 2.1483460652828215,
+ "epoch": 0.18428748848203197,
+ "grad_norm": 3.21875,
+ "learning_rate": 1.6317897772792824e-05,
+ "loss": 2.1671595764160156,
+ "mean_token_accuracy": 0.5199222292006016,
+ "num_tokens": 6909655.0,
+ "step": 1150
+ },
+ {
+ "entropy": 2.1074040442705155,
+ "epoch": 0.19229998798125075,
+ "grad_norm": 3.828125,
+ "learning_rate": 1.6157667040538376e-05,
+ "loss": 2.12770751953125,
+ "mean_token_accuracy": 0.5252153894305229,
+ "num_tokens": 7203263.0,
+ "step": 1200
+ },
+ {
+ "entropy": 2.1354027593135836,
+ "epoch": 0.20031248748046954,
+ "grad_norm": 2.921875,
+ "learning_rate": 1.5997436308283932e-05,
+ "loss": 2.156183166503906,
+ "mean_token_accuracy": 0.518907299786806,
+ "num_tokens": 7505411.0,
+ "step": 1250
+ },
+ {
+ "entropy": 2.142924894094467,
+ "epoch": 0.20832498697968832,
+ "grad_norm": 3.5625,
+ "learning_rate": 1.5837205576029484e-05,
+ "loss": 2.158486633300781,
+ "mean_token_accuracy": 0.5205909128487111,
+ "num_tokens": 7803562.0,
+ "step": 1300
+ },
+ {
+ "entropy": 2.1305291467905043,
+ "epoch": 0.2163374864789071,
+ "grad_norm": 3.34375,
+ "learning_rate": 1.5676974843775036e-05,
+ "loss": 2.151043701171875,
+ "mean_token_accuracy": 0.5211617235839366,
+ "num_tokens": 8108037.0,
+ "step": 1350
+ },
+ {
+ "entropy": 2.160947166085243,
+ "epoch": 0.2243499859781259,
+ "grad_norm": 3.234375,
+ "learning_rate": 1.5516744111520592e-05,
+ "loss": 2.177792663574219,
+ "mean_token_accuracy": 0.5167004990577698,
+ "num_tokens": 8408674.0,
+ "step": 1400
+ },
+ {
+ "entropy": 2.120403967499733,
+ "epoch": 0.23236248547734467,
+ "grad_norm": 3.375,
+ "learning_rate": 1.5356513379266144e-05,
+ "loss": 2.1462562561035154,
+ "mean_token_accuracy": 0.5225985661149025,
+ "num_tokens": 8707650.0,
+ "step": 1450
+ },
+ {
+ "entropy": 2.138882914185524,
+ "epoch": 0.24037498497656343,
+ "grad_norm": 3.171875,
+ "learning_rate": 1.5196282647011698e-05,
+ "loss": 2.1499905395507812,
+ "mean_token_accuracy": 0.5204668046534061,
+ "num_tokens": 9006296.0,
+ "step": 1500
+ },
+ {
+ "entropy": 2.131927245259285,
+ "epoch": 0.2483874844757822,
+ "grad_norm": 3.546875,
+ "learning_rate": 1.5036051914757252e-05,
+ "loss": 2.157076416015625,
+ "mean_token_accuracy": 0.5220408822596073,
+ "num_tokens": 9307203.0,
+ "step": 1550
+ },
+ {
+ "entropy": 2.1308567851781843,
+ "epoch": 0.256399983975001,
+ "grad_norm": 3.546875,
+ "learning_rate": 1.4875821182502805e-05,
+ "loss": 2.146601104736328,
+ "mean_token_accuracy": 0.5219539681077003,
+ "num_tokens": 9610070.0,
+ "step": 1600
+ },
+ {
+ "entropy": 2.1097783839702604,
+ "epoch": 0.2644124834742198,
+ "grad_norm": 3.84375,
+ "learning_rate": 1.4715590450248359e-05,
+ "loss": 2.1268209838867187,
+ "mean_token_accuracy": 0.5236652019619942,
+ "num_tokens": 9909117.0,
+ "step": 1650
+ },
+ {
+ "entropy": 2.123499180674553,
+ "epoch": 0.2724249829734386,
+ "grad_norm": 3.46875,
+ "learning_rate": 1.4555359717993913e-05,
+ "loss": 2.1409934997558593,
+ "mean_token_accuracy": 0.5225465354323388,
+ "num_tokens": 10209461.0,
+ "step": 1700
+ },
+ {
+ "entropy": 2.13052960395813,
+ "epoch": 0.2804374824726573,
+ "grad_norm": 2.71875,
+ "learning_rate": 1.4395128985739467e-05,
+ "loss": 2.151373291015625,
+ "mean_token_accuracy": 0.5227928113937378,
+ "num_tokens": 10509282.0,
+ "step": 1750
+ },
+ {
+ "entropy": 2.1226114094257356,
+ "epoch": 0.2884499819718761,
+ "grad_norm": 3.65625,
+ "learning_rate": 1.4234898253485019e-05,
+ "loss": 2.144451141357422,
+ "mean_token_accuracy": 0.5233827362954616,
+ "num_tokens": 10811056.0,
+ "step": 1800
+ },
+ {
+ "entropy": 2.126143154501915,
+ "epoch": 0.2964624814710949,
+ "grad_norm": 3.359375,
+ "learning_rate": 1.4074667521230575e-05,
+ "loss": 2.1457598876953123,
+ "mean_token_accuracy": 0.5221967925131321,
+ "num_tokens": 11111153.0,
+ "step": 1850
+ },
+ {
+ "entropy": 2.1048326581716537,
+ "epoch": 0.30447498097031367,
+ "grad_norm": 3.59375,
+ "learning_rate": 1.3914436788976127e-05,
+ "loss": 2.1260116577148436,
+ "mean_token_accuracy": 0.5272719909250736,
+ "num_tokens": 11405606.0,
+ "step": 1900
+ },
+ {
+ "entropy": 2.1236617982387545,
+ "epoch": 0.31248748046953245,
+ "grad_norm": 3.0,
+ "learning_rate": 1.375420605672168e-05,
+ "loss": 2.151782684326172,
+ "mean_token_accuracy": 0.5227546173334122,
+ "num_tokens": 11703943.0,
+ "step": 1950
+ },
+ {
+ "entropy": 2.121563429236412,
+ "epoch": 0.32049997996875124,
+ "grad_norm": 3.34375,
+ "learning_rate": 1.3593975324467235e-05,
+ "loss": 2.1370144653320313,
+ "mean_token_accuracy": 0.5235123547911644,
+ "num_tokens": 12002522.0,
+ "step": 2000
+ },
+ {
+ "entropy": 2.1155461168289182,
+ "epoch": 0.32851247946797,
+ "grad_norm": 3.46875,
+ "learning_rate": 1.3433744592212787e-05,
+ "loss": 2.1364064025878906,
+ "mean_token_accuracy": 0.5237135806679726,
+ "num_tokens": 12303098.0,
+ "step": 2050
+ },
+ {
+ "entropy": 2.1386052322387696,
+ "epoch": 0.3365249789671888,
+ "grad_norm": 3.484375,
+ "learning_rate": 1.327351385995834e-05,
+ "loss": 2.149888458251953,
+ "mean_token_accuracy": 0.522669050693512,
+ "num_tokens": 12605534.0,
+ "step": 2100
+ },
+ {
+ "entropy": 2.102600795030594,
+ "epoch": 0.3445374784664076,
+ "grad_norm": 3.796875,
+ "learning_rate": 1.3113283127703895e-05,
+ "loss": 2.1236566162109374,
+ "mean_token_accuracy": 0.5279092155396938,
+ "num_tokens": 12901218.0,
+ "step": 2150
+ },
+ {
+ "entropy": 2.132645187973976,
+ "epoch": 0.35254997796562637,
+ "grad_norm": 3.53125,
+ "learning_rate": 1.2953052395449447e-05,
+ "loss": 2.1523503112792968,
+ "mean_token_accuracy": 0.5198129612207413,
+ "num_tokens": 13213018.0,
+ "step": 2200
+ },
+ {
+ "entropy": 2.1204321217536926,
+ "epoch": 0.36056247746484515,
+ "grad_norm": 3.25,
+ "learning_rate": 1.2792821663195001e-05,
+ "loss": 2.1325447082519533,
+ "mean_token_accuracy": 0.5243636940419674,
+ "num_tokens": 13514408.0,
+ "step": 2250
+ },
+ {
+ "entropy": 2.102410731315613,
+ "epoch": 0.36857497696406394,
+ "grad_norm": 3.1875,
+ "learning_rate": 1.2632590930940555e-05,
+ "loss": 2.123565368652344,
+ "mean_token_accuracy": 0.5257614709436893,
+ "num_tokens": 13818272.0,
+ "step": 2300
+ },
+ {
+ "entropy": 2.0876205080747603,
+ "epoch": 0.3765874764632827,
+ "grad_norm": 3.328125,
+ "learning_rate": 1.247236019868611e-05,
+ "loss": 2.116696472167969,
+ "mean_token_accuracy": 0.5275124031305313,
+ "num_tokens": 14110702.0,
+ "step": 2350
+ },
+ {
+ "entropy": 2.1078518223762512,
+ "epoch": 0.3845999759625015,
+ "grad_norm": 3.453125,
+ "learning_rate": 1.2312129466431662e-05,
+ "loss": 2.1291993713378905,
+ "mean_token_accuracy": 0.524967464953661,
+ "num_tokens": 14407015.0,
+ "step": 2400
+ },
+ {
+ "entropy": 2.0930898267030718,
+ "epoch": 0.3926124754617203,
+ "grad_norm": 3.5,
+ "learning_rate": 1.2151898734177216e-05,
+ "loss": 2.1094123840332033,
+ "mean_token_accuracy": 0.5290587748587131,
+ "num_tokens": 14696565.0,
+ "step": 2450
+ },
+ {
+ "entropy": 2.1109757220745085,
+ "epoch": 0.4006249749609391,
+ "grad_norm": 3.5,
+ "learning_rate": 1.199166800192277e-05,
+ "loss": 2.1246888732910154,
+ "mean_token_accuracy": 0.5258957357704639,
+ "num_tokens": 14994401.0,
+ "step": 2500
+ }
+ ],
+ "logging_steps": 50,
+ "max_steps": 6241,
+ "num_input_tokens_seen": 0,
+ "num_train_epochs": 1,
+ "save_steps": 500,
+ "stateful_callbacks": {
+ "TrainerControl": {
+ "args": {
+ "should_epoch_stop": false,
+ "should_evaluate": false,
+ "should_log": false,
+ "should_save": true,
+ "should_training_stop": false
+ },
+ "attributes": {}
+ }
+ },
+ "total_flos": 1.773461452668027e+17,
+ "train_batch_size": 4,
+ "trial_name": null,
+ "trial_params": null
+}
diff --git a/checkpoint-2500/training_args.bin b/checkpoint-2500/training_args.bin
new file mode 100644
index 0000000000000000000000000000000000000000..4b409d5a147856f0d5a3d3da0f3b4922a9def7dd
--- /dev/null
+++ b/checkpoint-2500/training_args.bin
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:78622f63304ec0913ed0b4ea7a1b26c08a5621b3661a380d798c4ea426235671
+size 5713
diff --git a/checkpoint-3000/chat_template.jinja b/checkpoint-3000/chat_template.jinja
new file mode 100644
index 0000000000000000000000000000000000000000..01be9b307daa2d425f7c168c9fb145a286e0afb4
--- /dev/null
+++ b/checkpoint-3000/chat_template.jinja
@@ -0,0 +1,89 @@
+{%- if tools %}
+ {{- '<|im_start|>system\n' }}
+ {%- if messages[0].role == 'system' %}
+ {{- messages[0].content + '\n\n' }}
+ {%- endif %}
+ {{- "# Tools\n\nYou may call one or more functions to assist with the user query.\n\nYou are provided with function signatures within XML tags:\n" }}
+ {%- for tool in tools %}
+ {{- "\n" }}
+ {{- tool | tojson }}
+ {%- endfor %}
+ {{- "\n\n\nFor each function call, return a json object with function name and arguments within XML tags:\n\n{\"name\": , \"arguments\": }\n<|im_end|>\n" }}
+{%- else %}
+ {%- if messages[0].role == 'system' %}
+ {{- '<|im_start|>system\n' + messages[0].content + '<|im_end|>\n' }}
+ {%- endif %}
+{%- endif %}
+{%- set ns = namespace(multi_step_tool=true, last_query_index=messages|length - 1) %}
+{%- for message in messages[::-1] %}
+ {%- set index = (messages|length - 1) - loop.index0 %}
+ {%- if ns.multi_step_tool and message.role == "user" and message.content is string and not(message.content.startswith('') and message.content.endswith('')) %}
+ {%- set ns.multi_step_tool = false %}
+ {%- set ns.last_query_index = index %}
+ {%- endif %}
+{%- endfor %}
+{%- for message in messages %}
+ {%- if message.content is string %}
+ {%- set content = message.content %}
+ {%- else %}
+ {%- set content = '' %}
+ {%- endif %}
+ {%- if (message.role == "user") or (message.role == "system" and not loop.first) %}
+ {{- '<|im_start|>' + message.role + '\n' + content + '<|im_end|>' + '\n' }}
+ {%- elif message.role == "assistant" %}
+ {%- set reasoning_content = '' %}
+ {%- if message.reasoning_content is string %}
+ {%- set reasoning_content = message.reasoning_content %}
+ {%- else %}
+ {%- if '' in content %}
+ {%- set reasoning_content = content.split('')[0].rstrip('\n').split('')[-1].lstrip('\n') %}
+ {%- set content = content.split('')[-1].lstrip('\n') %}
+ {%- endif %}
+ {%- endif %}
+ {%- if loop.index0 > ns.last_query_index %}
+ {%- if loop.last or (not loop.last and reasoning_content) %}
+ {{- '<|im_start|>' + message.role + '\n\n' + reasoning_content.strip('\n') + '\n\n\n' + content.lstrip('\n') }}
+ {%- else %}
+ {{- '<|im_start|>' + message.role + '\n' + content }}
+ {%- endif %}
+ {%- else %}
+ {{- '<|im_start|>' + message.role + '\n' + content }}
+ {%- endif %}
+ {%- if message.tool_calls %}
+ {%- for tool_call in message.tool_calls %}
+ {%- if (loop.first and content) or (not loop.first) %}
+ {{- '\n' }}
+ {%- endif %}
+ {%- if tool_call.function %}
+ {%- set tool_call = tool_call.function %}
+ {%- endif %}
+ {{- '\n{"name": "' }}
+ {{- tool_call.name }}
+ {{- '", "arguments": ' }}
+ {%- if tool_call.arguments is string %}
+ {{- tool_call.arguments }}
+ {%- else %}
+ {{- tool_call.arguments | tojson }}
+ {%- endif %}
+ {{- '}\n' }}
+ {%- endfor %}
+ {%- endif %}
+ {{- '<|im_end|>\n' }}
+ {%- elif message.role == "tool" %}
+ {%- if loop.first or (messages[loop.index0 - 1].role != "tool") %}
+ {{- '<|im_start|>user' }}
+ {%- endif %}
+ {{- '\n\n' }}
+ {{- content }}
+ {{- '\n' }}
+ {%- if loop.last or (messages[loop.index0 + 1].role != "tool") %}
+ {{- '<|im_end|>\n' }}
+ {%- endif %}
+ {%- endif %}
+{%- endfor %}
+{%- if add_generation_prompt %}
+ {{- '<|im_start|>assistant\n' }}
+ {%- if enable_thinking is defined and enable_thinking is false %}
+ {{- '\n\n\n\n' }}
+ {%- endif %}
+{%- endif %}
\ No newline at end of file
diff --git a/checkpoint-3000/config.json b/checkpoint-3000/config.json
new file mode 100644
index 0000000000000000000000000000000000000000..1d219d6a6d288625229a71646710b85cf7ae5def
--- /dev/null
+++ b/checkpoint-3000/config.json
@@ -0,0 +1,63 @@
+{
+ "architectures": [
+ "Qwen3ForCausalLM"
+ ],
+ "attention_bias": false,
+ "attention_dropout": 0.0,
+ "bos_token_id": null,
+ "dtype": "bfloat16",
+ "eos_token_id": 151645,
+ "head_dim": 128,
+ "hidden_act": "silu",
+ "hidden_size": 2048,
+ "initializer_range": 0.02,
+ "intermediate_size": 6144,
+ "layer_types": [
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention"
+ ],
+ "max_position_embeddings": 40960,
+ "max_window_layers": 28,
+ "model_type": "qwen3",
+ "num_attention_heads": 16,
+ "num_hidden_layers": 28,
+ "num_key_value_heads": 8,
+ "pad_token_id": 151643,
+ "rms_norm_eps": 1e-06,
+ "rope_parameters": {
+ "rope_theta": 1000000,
+ "rope_type": "default"
+ },
+ "sliding_window": null,
+ "tie_word_embeddings": true,
+ "transformers_version": "5.7.0",
+ "use_cache": false,
+ "use_sliding_window": false,
+ "vocab_size": 151936
+}
diff --git a/checkpoint-3000/generation_config.json b/checkpoint-3000/generation_config.json
new file mode 100644
index 0000000000000000000000000000000000000000..5ec133d867b1427d43a5ee0ea332bd088fc12013
--- /dev/null
+++ b/checkpoint-3000/generation_config.json
@@ -0,0 +1,12 @@
+{
+ "do_sample": true,
+ "eos_token_id": [
+ 151645,
+ 151643
+ ],
+ "pad_token_id": 151643,
+ "temperature": 0.6,
+ "top_k": 20,
+ "top_p": 0.95,
+ "transformers_version": "5.7.0"
+}
diff --git a/checkpoint-3000/model.safetensors b/checkpoint-3000/model.safetensors
new file mode 100644
index 0000000000000000000000000000000000000000..ff6cf747e5a537c98606423e8fff99beb1c0f9f9
--- /dev/null
+++ b/checkpoint-3000/model.safetensors
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d5b72e5752b081abdc1994e5b2076200e5122c26cf9578d62c85a820f8ff24de
+size 3441185608
diff --git a/checkpoint-3000/optimizer.pt b/checkpoint-3000/optimizer.pt
new file mode 100644
index 0000000000000000000000000000000000000000..85ae37cb3c4e8e59268a81a79dd602c9e4e444c2
--- /dev/null
+++ b/checkpoint-3000/optimizer.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:34d6d5e6b3baf98ae75f9122a60622751f2ca0f8eb5d5542a1fd00ed368520c2
+size 6882572207
diff --git a/checkpoint-3000/rng_state.pth b/checkpoint-3000/rng_state.pth
new file mode 100644
index 0000000000000000000000000000000000000000..5a8f17addf8aaee350625e9847751582bc9f86dd
--- /dev/null
+++ b/checkpoint-3000/rng_state.pth
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:61c19bab1174704a4a4441475683bf1270277af15d2e2c95e964789128e482c4
+size 14645
diff --git a/checkpoint-3000/scheduler.pt b/checkpoint-3000/scheduler.pt
new file mode 100644
index 0000000000000000000000000000000000000000..a884e1bc1fcc72978be98064f8e18ef2e0e149b4
--- /dev/null
+++ b/checkpoint-3000/scheduler.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:875525fc96d4b4bb5c12817e379ae11ed73b09ebd4f315c5475a43de7d437801
+size 1465
diff --git a/checkpoint-3000/tokenizer.json b/checkpoint-3000/tokenizer.json
new file mode 100644
index 0000000000000000000000000000000000000000..c7afbed2efcdf019f88ab0572ec29d3bf595dfe2
--- /dev/null
+++ b/checkpoint-3000/tokenizer.json
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:be75606093db2094d7cd20f3c2f385c212750648bd6ea4fb2bf507a6a4c55506
+size 11422650
diff --git a/checkpoint-3000/tokenizer_config.json b/checkpoint-3000/tokenizer_config.json
new file mode 100644
index 0000000000000000000000000000000000000000..770e41d6c92519d525eede4cbcf3ba27f6425311
--- /dev/null
+++ b/checkpoint-3000/tokenizer_config.json
@@ -0,0 +1,30 @@
+{
+ "add_prefix_space": false,
+ "backend": "tokenizers",
+ "bos_token": null,
+ "clean_up_tokenization_spaces": false,
+ "eos_token": "<|im_end|>",
+ "errors": "replace",
+ "extra_special_tokens": [
+ "<|im_start|>",
+ "<|im_end|>",
+ "<|object_ref_start|>",
+ "<|object_ref_end|>",
+ "<|box_start|>",
+ "<|box_end|>",
+ "<|quad_start|>",
+ "<|quad_end|>",
+ "<|vision_start|>",
+ "<|vision_end|>",
+ "<|vision_pad|>",
+ "<|image_pad|>",
+ "<|video_pad|>"
+ ],
+ "is_local": false,
+ "local_files_only": false,
+ "model_max_length": 131072,
+ "pad_token": "<|endoftext|>",
+ "split_special_tokens": false,
+ "tokenizer_class": "Qwen2Tokenizer",
+ "unk_token": null
+}
diff --git a/checkpoint-3000/trainer_state.json b/checkpoint-3000/trainer_state.json
new file mode 100644
index 0000000000000000000000000000000000000000..10b401cf1305d6a8d1d7830d0d558aa9f0342c95
--- /dev/null
+++ b/checkpoint-3000/trainer_state.json
@@ -0,0 +1,634 @@
+{
+ "best_global_step": null,
+ "best_metric": null,
+ "best_model_checkpoint": null,
+ "epoch": 0.48074996995312685,
+ "eval_steps": 500,
+ "global_step": 3000,
+ "is_hyper_param_search": false,
+ "is_local_process_zero": true,
+ "is_world_process_zero": true,
+ "log_history": [
+ {
+ "entropy": 2.223189059495926,
+ "epoch": 0.008012499499218782,
+ "grad_norm": 2.875,
+ "learning_rate": 1.9842973882390643e-05,
+ "loss": 2.3428109741210936,
+ "mean_token_accuracy": 0.4963963569700718,
+ "num_tokens": 305366.0,
+ "step": 50
+ },
+ {
+ "entropy": 2.188130117058754,
+ "epoch": 0.016024998998437564,
+ "grad_norm": 3.328125,
+ "learning_rate": 1.96827431501362e-05,
+ "loss": 2.225358734130859,
+ "mean_token_accuracy": 0.5110921229422093,
+ "num_tokens": 605042.0,
+ "step": 100
+ },
+ {
+ "entropy": 2.197347580194473,
+ "epoch": 0.024037498497656344,
+ "grad_norm": 3.453125,
+ "learning_rate": 1.952251241788175e-05,
+ "loss": 2.2299420166015627,
+ "mean_token_accuracy": 0.5093490639328957,
+ "num_tokens": 906888.0,
+ "step": 150
+ },
+ {
+ "entropy": 2.1913950395584108,
+ "epoch": 0.03204999799687513,
+ "grad_norm": 2.90625,
+ "learning_rate": 1.9362281685627303e-05,
+ "loss": 2.210995941162109,
+ "mean_token_accuracy": 0.5097909501194954,
+ "num_tokens": 1213125.0,
+ "step": 200
+ },
+ {
+ "entropy": 2.1611998677253723,
+ "epoch": 0.040062497496093905,
+ "grad_norm": 3.25,
+ "learning_rate": 1.920205095337286e-05,
+ "loss": 2.1867474365234374,
+ "mean_token_accuracy": 0.5154410660266876,
+ "num_tokens": 1509026.0,
+ "step": 250
+ },
+ {
+ "entropy": 2.168051454424858,
+ "epoch": 0.04807499699531269,
+ "grad_norm": 2.9375,
+ "learning_rate": 1.904182022111841e-05,
+ "loss": 2.190071716308594,
+ "mean_token_accuracy": 0.5156884534657001,
+ "num_tokens": 1806850.0,
+ "step": 300
+ },
+ {
+ "entropy": 2.1737154543399813,
+ "epoch": 0.05608749649453147,
+ "grad_norm": 3.421875,
+ "learning_rate": 1.8881589488863967e-05,
+ "loss": 2.199147186279297,
+ "mean_token_accuracy": 0.514045044630766,
+ "num_tokens": 2104505.0,
+ "step": 350
+ },
+ {
+ "entropy": 2.148039159178734,
+ "epoch": 0.06409999599375026,
+ "grad_norm": 3.109375,
+ "learning_rate": 1.872135875660952e-05,
+ "loss": 2.176946105957031,
+ "mean_token_accuracy": 0.5177112428843975,
+ "num_tokens": 2391088.0,
+ "step": 400
+ },
+ {
+ "entropy": 2.194890711903572,
+ "epoch": 0.07211249549296903,
+ "grad_norm": 3.28125,
+ "learning_rate": 1.856112802435507e-05,
+ "loss": 2.216288757324219,
+ "mean_token_accuracy": 0.5110335703194141,
+ "num_tokens": 2691272.0,
+ "step": 450
+ },
+ {
+ "entropy": 2.1557786524295808,
+ "epoch": 0.08012499499218781,
+ "grad_norm": 4.15625,
+ "learning_rate": 1.8400897292100627e-05,
+ "loss": 2.1812680053710936,
+ "mean_token_accuracy": 0.5176697050035,
+ "num_tokens": 2991855.0,
+ "step": 500
+ },
+ {
+ "entropy": 2.1708167713880537,
+ "epoch": 0.08813749449140659,
+ "grad_norm": 4.34375,
+ "learning_rate": 1.824066655984618e-05,
+ "loss": 2.1921002197265627,
+ "mean_token_accuracy": 0.5156507430970669,
+ "num_tokens": 3295049.0,
+ "step": 550
+ },
+ {
+ "entropy": 2.1686314886808393,
+ "epoch": 0.09614999399062538,
+ "grad_norm": 2.875,
+ "learning_rate": 1.8080435827591735e-05,
+ "loss": 2.1914617919921877,
+ "mean_token_accuracy": 0.5139613692462445,
+ "num_tokens": 3597206.0,
+ "step": 600
+ },
+ {
+ "entropy": 2.15742906332016,
+ "epoch": 0.10416249348984416,
+ "grad_norm": 3.109375,
+ "learning_rate": 1.7920205095337287e-05,
+ "loss": 2.174853973388672,
+ "mean_token_accuracy": 0.5159518288075924,
+ "num_tokens": 3905268.0,
+ "step": 650
+ },
+ {
+ "entropy": 2.162363375425339,
+ "epoch": 0.11217499298906294,
+ "grad_norm": 3.171875,
+ "learning_rate": 1.7759974363082843e-05,
+ "loss": 2.184231872558594,
+ "mean_token_accuracy": 0.516631413847208,
+ "num_tokens": 4207934.0,
+ "step": 700
+ },
+ {
+ "entropy": 2.1516289287805557,
+ "epoch": 0.12018749248828171,
+ "grad_norm": 3.609375,
+ "learning_rate": 1.7599743630828395e-05,
+ "loss": 2.167020263671875,
+ "mean_token_accuracy": 0.516358617246151,
+ "num_tokens": 4515774.0,
+ "step": 750
+ },
+ {
+ "entropy": 2.1549544781446457,
+ "epoch": 0.1281999919875005,
+ "grad_norm": 3.28125,
+ "learning_rate": 1.7439512898573947e-05,
+ "loss": 2.181917877197266,
+ "mean_token_accuracy": 0.5182220858335495,
+ "num_tokens": 4814709.0,
+ "step": 800
+ },
+ {
+ "entropy": 2.1532615756988527,
+ "epoch": 0.1362124914867193,
+ "grad_norm": 3.078125,
+ "learning_rate": 1.7279282166319503e-05,
+ "loss": 2.1659007263183594,
+ "mean_token_accuracy": 0.5180425970256328,
+ "num_tokens": 5112585.0,
+ "step": 850
+ },
+ {
+ "entropy": 2.1704017293453215,
+ "epoch": 0.14422499098593805,
+ "grad_norm": 2.921875,
+ "learning_rate": 1.7119051434065055e-05,
+ "loss": 2.1934996032714844,
+ "mean_token_accuracy": 0.5154964691400528,
+ "num_tokens": 5411369.0,
+ "step": 900
+ },
+ {
+ "entropy": 2.1349107217788696,
+ "epoch": 0.15223749048515683,
+ "grad_norm": 2.984375,
+ "learning_rate": 1.6958820701810608e-05,
+ "loss": 2.1598545837402345,
+ "mean_token_accuracy": 0.5208920259773732,
+ "num_tokens": 5707060.0,
+ "step": 950
+ },
+ {
+ "entropy": 2.1541705918312073,
+ "epoch": 0.16024998998437562,
+ "grad_norm": 3.203125,
+ "learning_rate": 1.6798589969556163e-05,
+ "loss": 2.170956573486328,
+ "mean_token_accuracy": 0.5204938448965549,
+ "num_tokens": 6004079.0,
+ "step": 1000
+ },
+ {
+ "entropy": 2.1523867017030716,
+ "epoch": 0.1682624894835944,
+ "grad_norm": 3.09375,
+ "learning_rate": 1.6638359237301716e-05,
+ "loss": 2.1695118713378907,
+ "mean_token_accuracy": 0.5177376234531402,
+ "num_tokens": 6315462.0,
+ "step": 1050
+ },
+ {
+ "entropy": 2.1356022965908052,
+ "epoch": 0.17627498898281319,
+ "grad_norm": 3.21875,
+ "learning_rate": 1.6478128505047268e-05,
+ "loss": 2.1513107299804686,
+ "mean_token_accuracy": 0.5217180585861206,
+ "num_tokens": 6613060.0,
+ "step": 1100
+ },
+ {
+ "entropy": 2.1483460652828215,
+ "epoch": 0.18428748848203197,
+ "grad_norm": 3.21875,
+ "learning_rate": 1.6317897772792824e-05,
+ "loss": 2.1671595764160156,
+ "mean_token_accuracy": 0.5199222292006016,
+ "num_tokens": 6909655.0,
+ "step": 1150
+ },
+ {
+ "entropy": 2.1074040442705155,
+ "epoch": 0.19229998798125075,
+ "grad_norm": 3.828125,
+ "learning_rate": 1.6157667040538376e-05,
+ "loss": 2.12770751953125,
+ "mean_token_accuracy": 0.5252153894305229,
+ "num_tokens": 7203263.0,
+ "step": 1200
+ },
+ {
+ "entropy": 2.1354027593135836,
+ "epoch": 0.20031248748046954,
+ "grad_norm": 2.921875,
+ "learning_rate": 1.5997436308283932e-05,
+ "loss": 2.156183166503906,
+ "mean_token_accuracy": 0.518907299786806,
+ "num_tokens": 7505411.0,
+ "step": 1250
+ },
+ {
+ "entropy": 2.142924894094467,
+ "epoch": 0.20832498697968832,
+ "grad_norm": 3.5625,
+ "learning_rate": 1.5837205576029484e-05,
+ "loss": 2.158486633300781,
+ "mean_token_accuracy": 0.5205909128487111,
+ "num_tokens": 7803562.0,
+ "step": 1300
+ },
+ {
+ "entropy": 2.1305291467905043,
+ "epoch": 0.2163374864789071,
+ "grad_norm": 3.34375,
+ "learning_rate": 1.5676974843775036e-05,
+ "loss": 2.151043701171875,
+ "mean_token_accuracy": 0.5211617235839366,
+ "num_tokens": 8108037.0,
+ "step": 1350
+ },
+ {
+ "entropy": 2.160947166085243,
+ "epoch": 0.2243499859781259,
+ "grad_norm": 3.234375,
+ "learning_rate": 1.5516744111520592e-05,
+ "loss": 2.177792663574219,
+ "mean_token_accuracy": 0.5167004990577698,
+ "num_tokens": 8408674.0,
+ "step": 1400
+ },
+ {
+ "entropy": 2.120403967499733,
+ "epoch": 0.23236248547734467,
+ "grad_norm": 3.375,
+ "learning_rate": 1.5356513379266144e-05,
+ "loss": 2.1462562561035154,
+ "mean_token_accuracy": 0.5225985661149025,
+ "num_tokens": 8707650.0,
+ "step": 1450
+ },
+ {
+ "entropy": 2.138882914185524,
+ "epoch": 0.24037498497656343,
+ "grad_norm": 3.171875,
+ "learning_rate": 1.5196282647011698e-05,
+ "loss": 2.1499905395507812,
+ "mean_token_accuracy": 0.5204668046534061,
+ "num_tokens": 9006296.0,
+ "step": 1500
+ },
+ {
+ "entropy": 2.131927245259285,
+ "epoch": 0.2483874844757822,
+ "grad_norm": 3.546875,
+ "learning_rate": 1.5036051914757252e-05,
+ "loss": 2.157076416015625,
+ "mean_token_accuracy": 0.5220408822596073,
+ "num_tokens": 9307203.0,
+ "step": 1550
+ },
+ {
+ "entropy": 2.1308567851781843,
+ "epoch": 0.256399983975001,
+ "grad_norm": 3.546875,
+ "learning_rate": 1.4875821182502805e-05,
+ "loss": 2.146601104736328,
+ "mean_token_accuracy": 0.5219539681077003,
+ "num_tokens": 9610070.0,
+ "step": 1600
+ },
+ {
+ "entropy": 2.1097783839702604,
+ "epoch": 0.2644124834742198,
+ "grad_norm": 3.84375,
+ "learning_rate": 1.4715590450248359e-05,
+ "loss": 2.1268209838867187,
+ "mean_token_accuracy": 0.5236652019619942,
+ "num_tokens": 9909117.0,
+ "step": 1650
+ },
+ {
+ "entropy": 2.123499180674553,
+ "epoch": 0.2724249829734386,
+ "grad_norm": 3.46875,
+ "learning_rate": 1.4555359717993913e-05,
+ "loss": 2.1409934997558593,
+ "mean_token_accuracy": 0.5225465354323388,
+ "num_tokens": 10209461.0,
+ "step": 1700
+ },
+ {
+ "entropy": 2.13052960395813,
+ "epoch": 0.2804374824726573,
+ "grad_norm": 2.71875,
+ "learning_rate": 1.4395128985739467e-05,
+ "loss": 2.151373291015625,
+ "mean_token_accuracy": 0.5227928113937378,
+ "num_tokens": 10509282.0,
+ "step": 1750
+ },
+ {
+ "entropy": 2.1226114094257356,
+ "epoch": 0.2884499819718761,
+ "grad_norm": 3.65625,
+ "learning_rate": 1.4234898253485019e-05,
+ "loss": 2.144451141357422,
+ "mean_token_accuracy": 0.5233827362954616,
+ "num_tokens": 10811056.0,
+ "step": 1800
+ },
+ {
+ "entropy": 2.126143154501915,
+ "epoch": 0.2964624814710949,
+ "grad_norm": 3.359375,
+ "learning_rate": 1.4074667521230575e-05,
+ "loss": 2.1457598876953123,
+ "mean_token_accuracy": 0.5221967925131321,
+ "num_tokens": 11111153.0,
+ "step": 1850
+ },
+ {
+ "entropy": 2.1048326581716537,
+ "epoch": 0.30447498097031367,
+ "grad_norm": 3.59375,
+ "learning_rate": 1.3914436788976127e-05,
+ "loss": 2.1260116577148436,
+ "mean_token_accuracy": 0.5272719909250736,
+ "num_tokens": 11405606.0,
+ "step": 1900
+ },
+ {
+ "entropy": 2.1236617982387545,
+ "epoch": 0.31248748046953245,
+ "grad_norm": 3.0,
+ "learning_rate": 1.375420605672168e-05,
+ "loss": 2.151782684326172,
+ "mean_token_accuracy": 0.5227546173334122,
+ "num_tokens": 11703943.0,
+ "step": 1950
+ },
+ {
+ "entropy": 2.121563429236412,
+ "epoch": 0.32049997996875124,
+ "grad_norm": 3.34375,
+ "learning_rate": 1.3593975324467235e-05,
+ "loss": 2.1370144653320313,
+ "mean_token_accuracy": 0.5235123547911644,
+ "num_tokens": 12002522.0,
+ "step": 2000
+ },
+ {
+ "entropy": 2.1155461168289182,
+ "epoch": 0.32851247946797,
+ "grad_norm": 3.46875,
+ "learning_rate": 1.3433744592212787e-05,
+ "loss": 2.1364064025878906,
+ "mean_token_accuracy": 0.5237135806679726,
+ "num_tokens": 12303098.0,
+ "step": 2050
+ },
+ {
+ "entropy": 2.1386052322387696,
+ "epoch": 0.3365249789671888,
+ "grad_norm": 3.484375,
+ "learning_rate": 1.327351385995834e-05,
+ "loss": 2.149888458251953,
+ "mean_token_accuracy": 0.522669050693512,
+ "num_tokens": 12605534.0,
+ "step": 2100
+ },
+ {
+ "entropy": 2.102600795030594,
+ "epoch": 0.3445374784664076,
+ "grad_norm": 3.796875,
+ "learning_rate": 1.3113283127703895e-05,
+ "loss": 2.1236566162109374,
+ "mean_token_accuracy": 0.5279092155396938,
+ "num_tokens": 12901218.0,
+ "step": 2150
+ },
+ {
+ "entropy": 2.132645187973976,
+ "epoch": 0.35254997796562637,
+ "grad_norm": 3.53125,
+ "learning_rate": 1.2953052395449447e-05,
+ "loss": 2.1523503112792968,
+ "mean_token_accuracy": 0.5198129612207413,
+ "num_tokens": 13213018.0,
+ "step": 2200
+ },
+ {
+ "entropy": 2.1204321217536926,
+ "epoch": 0.36056247746484515,
+ "grad_norm": 3.25,
+ "learning_rate": 1.2792821663195001e-05,
+ "loss": 2.1325447082519533,
+ "mean_token_accuracy": 0.5243636940419674,
+ "num_tokens": 13514408.0,
+ "step": 2250
+ },
+ {
+ "entropy": 2.102410731315613,
+ "epoch": 0.36857497696406394,
+ "grad_norm": 3.1875,
+ "learning_rate": 1.2632590930940555e-05,
+ "loss": 2.123565368652344,
+ "mean_token_accuracy": 0.5257614709436893,
+ "num_tokens": 13818272.0,
+ "step": 2300
+ },
+ {
+ "entropy": 2.0876205080747603,
+ "epoch": 0.3765874764632827,
+ "grad_norm": 3.328125,
+ "learning_rate": 1.247236019868611e-05,
+ "loss": 2.116696472167969,
+ "mean_token_accuracy": 0.5275124031305313,
+ "num_tokens": 14110702.0,
+ "step": 2350
+ },
+ {
+ "entropy": 2.1078518223762512,
+ "epoch": 0.3845999759625015,
+ "grad_norm": 3.453125,
+ "learning_rate": 1.2312129466431662e-05,
+ "loss": 2.1291993713378905,
+ "mean_token_accuracy": 0.524967464953661,
+ "num_tokens": 14407015.0,
+ "step": 2400
+ },
+ {
+ "entropy": 2.0930898267030718,
+ "epoch": 0.3926124754617203,
+ "grad_norm": 3.5,
+ "learning_rate": 1.2151898734177216e-05,
+ "loss": 2.1094123840332033,
+ "mean_token_accuracy": 0.5290587748587131,
+ "num_tokens": 14696565.0,
+ "step": 2450
+ },
+ {
+ "entropy": 2.1109757220745085,
+ "epoch": 0.4006249749609391,
+ "grad_norm": 3.5,
+ "learning_rate": 1.199166800192277e-05,
+ "loss": 2.1246888732910154,
+ "mean_token_accuracy": 0.5258957357704639,
+ "num_tokens": 14994401.0,
+ "step": 2500
+ },
+ {
+ "entropy": 2.1087811160087586,
+ "epoch": 0.40863747446015786,
+ "grad_norm": 3.1875,
+ "learning_rate": 1.1831437269668324e-05,
+ "loss": 2.12890625,
+ "mean_token_accuracy": 0.5269917643070221,
+ "num_tokens": 15292837.0,
+ "step": 2550
+ },
+ {
+ "entropy": 2.097252429127693,
+ "epoch": 0.41664997395937664,
+ "grad_norm": 3.0,
+ "learning_rate": 1.1671206537413878e-05,
+ "loss": 2.1145870971679686,
+ "mean_token_accuracy": 0.5274137698113919,
+ "num_tokens": 15590647.0,
+ "step": 2600
+ },
+ {
+ "entropy": 2.111199595332146,
+ "epoch": 0.4246624734585954,
+ "grad_norm": 3.25,
+ "learning_rate": 1.151097580515943e-05,
+ "loss": 2.1274327087402343,
+ "mean_token_accuracy": 0.5250249288976192,
+ "num_tokens": 15892772.0,
+ "step": 2650
+ },
+ {
+ "entropy": 2.1051502913236617,
+ "epoch": 0.4326749729578142,
+ "grad_norm": 3.203125,
+ "learning_rate": 1.1350745072904986e-05,
+ "loss": 2.127935333251953,
+ "mean_token_accuracy": 0.5263640700280666,
+ "num_tokens": 16191611.0,
+ "step": 2700
+ },
+ {
+ "entropy": 2.1127885603904724,
+ "epoch": 0.440687472457033,
+ "grad_norm": 3.515625,
+ "learning_rate": 1.1190514340650538e-05,
+ "loss": 2.129211883544922,
+ "mean_token_accuracy": 0.5250939278304577,
+ "num_tokens": 16483582.0,
+ "step": 2750
+ },
+ {
+ "entropy": 2.110930626988411,
+ "epoch": 0.4486999719562518,
+ "grad_norm": 3.1875,
+ "learning_rate": 1.103028360839609e-05,
+ "loss": 2.1220079040527344,
+ "mean_token_accuracy": 0.5250919266045093,
+ "num_tokens": 16787253.0,
+ "step": 2800
+ },
+ {
+ "entropy": 2.1182780677080153,
+ "epoch": 0.45671247145547056,
+ "grad_norm": 3.8125,
+ "learning_rate": 1.0870052876141646e-05,
+ "loss": 2.1358837890625,
+ "mean_token_accuracy": 0.5217854312062263,
+ "num_tokens": 17088741.0,
+ "step": 2850
+ },
+ {
+ "entropy": 2.1117833071947096,
+ "epoch": 0.46472497095468934,
+ "grad_norm": 3.171875,
+ "learning_rate": 1.0709822143887198e-05,
+ "loss": 2.125662841796875,
+ "mean_token_accuracy": 0.524649152457714,
+ "num_tokens": 17389236.0,
+ "step": 2900
+ },
+ {
+ "entropy": 2.1262730568647386,
+ "epoch": 0.47273747045390807,
+ "grad_norm": 3.625,
+ "learning_rate": 1.054959141163275e-05,
+ "loss": 2.142789611816406,
+ "mean_token_accuracy": 0.5236384868621826,
+ "num_tokens": 17693413.0,
+ "step": 2950
+ },
+ {
+ "entropy": 2.115410739183426,
+ "epoch": 0.48074996995312685,
+ "grad_norm": 3.140625,
+ "learning_rate": 1.0389360679378306e-05,
+ "loss": 2.1337106323242185,
+ "mean_token_accuracy": 0.5249222332239151,
+ "num_tokens": 17990326.0,
+ "step": 3000
+ }
+ ],
+ "logging_steps": 50,
+ "max_steps": 6241,
+ "num_input_tokens_seen": 0,
+ "num_train_epochs": 1,
+ "save_steps": 500,
+ "stateful_callbacks": {
+ "TrainerControl": {
+ "args": {
+ "should_epoch_stop": false,
+ "should_evaluate": false,
+ "should_log": false,
+ "should_save": true,
+ "should_training_stop": false
+ },
+ "attributes": {}
+ }
+ },
+ "total_flos": 2.126267005433856e+17,
+ "train_batch_size": 4,
+ "trial_name": null,
+ "trial_params": null
+}
diff --git a/checkpoint-3000/training_args.bin b/checkpoint-3000/training_args.bin
new file mode 100644
index 0000000000000000000000000000000000000000..4b409d5a147856f0d5a3d3da0f3b4922a9def7dd
--- /dev/null
+++ b/checkpoint-3000/training_args.bin
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:78622f63304ec0913ed0b4ea7a1b26c08a5621b3661a380d798c4ea426235671
+size 5713
diff --git a/checkpoint-3500/chat_template.jinja b/checkpoint-3500/chat_template.jinja
new file mode 100644
index 0000000000000000000000000000000000000000..01be9b307daa2d425f7c168c9fb145a286e0afb4
--- /dev/null
+++ b/checkpoint-3500/chat_template.jinja
@@ -0,0 +1,89 @@
+{%- if tools %}
+ {{- '<|im_start|>system\n' }}
+ {%- if messages[0].role == 'system' %}
+ {{- messages[0].content + '\n\n' }}
+ {%- endif %}
+ {{- "# Tools\n\nYou may call one or more functions to assist with the user query.\n\nYou are provided with function signatures within XML tags:\n" }}
+ {%- for tool in tools %}
+ {{- "\n" }}
+ {{- tool | tojson }}
+ {%- endfor %}
+ {{- "\n\n\nFor each function call, return a json object with function name and arguments within XML tags:\n\n{\"name\": , \"arguments\": }\n<|im_end|>\n" }}
+{%- else %}
+ {%- if messages[0].role == 'system' %}
+ {{- '<|im_start|>system\n' + messages[0].content + '<|im_end|>\n' }}
+ {%- endif %}
+{%- endif %}
+{%- set ns = namespace(multi_step_tool=true, last_query_index=messages|length - 1) %}
+{%- for message in messages[::-1] %}
+ {%- set index = (messages|length - 1) - loop.index0 %}
+ {%- if ns.multi_step_tool and message.role == "user" and message.content is string and not(message.content.startswith('') and message.content.endswith('')) %}
+ {%- set ns.multi_step_tool = false %}
+ {%- set ns.last_query_index = index %}
+ {%- endif %}
+{%- endfor %}
+{%- for message in messages %}
+ {%- if message.content is string %}
+ {%- set content = message.content %}
+ {%- else %}
+ {%- set content = '' %}
+ {%- endif %}
+ {%- if (message.role == "user") or (message.role == "system" and not loop.first) %}
+ {{- '<|im_start|>' + message.role + '\n' + content + '<|im_end|>' + '\n' }}
+ {%- elif message.role == "assistant" %}
+ {%- set reasoning_content = '' %}
+ {%- if message.reasoning_content is string %}
+ {%- set reasoning_content = message.reasoning_content %}
+ {%- else %}
+ {%- if '' in content %}
+ {%- set reasoning_content = content.split('')[0].rstrip('\n').split('')[-1].lstrip('\n') %}
+ {%- set content = content.split('')[-1].lstrip('\n') %}
+ {%- endif %}
+ {%- endif %}
+ {%- if loop.index0 > ns.last_query_index %}
+ {%- if loop.last or (not loop.last and reasoning_content) %}
+ {{- '<|im_start|>' + message.role + '\n\n' + reasoning_content.strip('\n') + '\n\n\n' + content.lstrip('\n') }}
+ {%- else %}
+ {{- '<|im_start|>' + message.role + '\n' + content }}
+ {%- endif %}
+ {%- else %}
+ {{- '<|im_start|>' + message.role + '\n' + content }}
+ {%- endif %}
+ {%- if message.tool_calls %}
+ {%- for tool_call in message.tool_calls %}
+ {%- if (loop.first and content) or (not loop.first) %}
+ {{- '\n' }}
+ {%- endif %}
+ {%- if tool_call.function %}
+ {%- set tool_call = tool_call.function %}
+ {%- endif %}
+ {{- '\n{"name": "' }}
+ {{- tool_call.name }}
+ {{- '", "arguments": ' }}
+ {%- if tool_call.arguments is string %}
+ {{- tool_call.arguments }}
+ {%- else %}
+ {{- tool_call.arguments | tojson }}
+ {%- endif %}
+ {{- '}\n' }}
+ {%- endfor %}
+ {%- endif %}
+ {{- '<|im_end|>\n' }}
+ {%- elif message.role == "tool" %}
+ {%- if loop.first or (messages[loop.index0 - 1].role != "tool") %}
+ {{- '<|im_start|>user' }}
+ {%- endif %}
+ {{- '\n\n' }}
+ {{- content }}
+ {{- '\n' }}
+ {%- if loop.last or (messages[loop.index0 + 1].role != "tool") %}
+ {{- '<|im_end|>\n' }}
+ {%- endif %}
+ {%- endif %}
+{%- endfor %}
+{%- if add_generation_prompt %}
+ {{- '<|im_start|>assistant\n' }}
+ {%- if enable_thinking is defined and enable_thinking is false %}
+ {{- '\n\n\n\n' }}
+ {%- endif %}
+{%- endif %}
\ No newline at end of file
diff --git a/checkpoint-3500/config.json b/checkpoint-3500/config.json
new file mode 100644
index 0000000000000000000000000000000000000000..1d219d6a6d288625229a71646710b85cf7ae5def
--- /dev/null
+++ b/checkpoint-3500/config.json
@@ -0,0 +1,63 @@
+{
+ "architectures": [
+ "Qwen3ForCausalLM"
+ ],
+ "attention_bias": false,
+ "attention_dropout": 0.0,
+ "bos_token_id": null,
+ "dtype": "bfloat16",
+ "eos_token_id": 151645,
+ "head_dim": 128,
+ "hidden_act": "silu",
+ "hidden_size": 2048,
+ "initializer_range": 0.02,
+ "intermediate_size": 6144,
+ "layer_types": [
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention"
+ ],
+ "max_position_embeddings": 40960,
+ "max_window_layers": 28,
+ "model_type": "qwen3",
+ "num_attention_heads": 16,
+ "num_hidden_layers": 28,
+ "num_key_value_heads": 8,
+ "pad_token_id": 151643,
+ "rms_norm_eps": 1e-06,
+ "rope_parameters": {
+ "rope_theta": 1000000,
+ "rope_type": "default"
+ },
+ "sliding_window": null,
+ "tie_word_embeddings": true,
+ "transformers_version": "5.7.0",
+ "use_cache": false,
+ "use_sliding_window": false,
+ "vocab_size": 151936
+}
diff --git a/checkpoint-3500/generation_config.json b/checkpoint-3500/generation_config.json
new file mode 100644
index 0000000000000000000000000000000000000000..5ec133d867b1427d43a5ee0ea332bd088fc12013
--- /dev/null
+++ b/checkpoint-3500/generation_config.json
@@ -0,0 +1,12 @@
+{
+ "do_sample": true,
+ "eos_token_id": [
+ 151645,
+ 151643
+ ],
+ "pad_token_id": 151643,
+ "temperature": 0.6,
+ "top_k": 20,
+ "top_p": 0.95,
+ "transformers_version": "5.7.0"
+}
diff --git a/checkpoint-3500/model.safetensors b/checkpoint-3500/model.safetensors
new file mode 100644
index 0000000000000000000000000000000000000000..60920a5177a15da3ef791f8b5c179cc6528c172a
--- /dev/null
+++ b/checkpoint-3500/model.safetensors
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:cd3ed08012c528778e23bfc40ce9263d1183db5fe1b344fcc375190ceff69329
+size 3441185608
diff --git a/checkpoint-3500/optimizer.pt b/checkpoint-3500/optimizer.pt
new file mode 100644
index 0000000000000000000000000000000000000000..eef7695aa07ae46f414f9e5003f9b273fa39c634
--- /dev/null
+++ b/checkpoint-3500/optimizer.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ff2d89dd80a7f07fe151122ae98c1d25ce93198c70301c9848ea4314b9f22671
+size 6882572207
diff --git a/checkpoint-3500/rng_state.pth b/checkpoint-3500/rng_state.pth
new file mode 100644
index 0000000000000000000000000000000000000000..5a8f17addf8aaee350625e9847751582bc9f86dd
--- /dev/null
+++ b/checkpoint-3500/rng_state.pth
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:61c19bab1174704a4a4441475683bf1270277af15d2e2c95e964789128e482c4
+size 14645
diff --git a/checkpoint-3500/scheduler.pt b/checkpoint-3500/scheduler.pt
new file mode 100644
index 0000000000000000000000000000000000000000..b2087fab20f55027ff5e3c879de4da3fe7d12892
--- /dev/null
+++ b/checkpoint-3500/scheduler.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:5e847bb7d94a2f7987ac5f9be2becb727ffced83ae79de46ff613b03a251110e
+size 1465
diff --git a/checkpoint-3500/tokenizer.json b/checkpoint-3500/tokenizer.json
new file mode 100644
index 0000000000000000000000000000000000000000..c7afbed2efcdf019f88ab0572ec29d3bf595dfe2
--- /dev/null
+++ b/checkpoint-3500/tokenizer.json
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:be75606093db2094d7cd20f3c2f385c212750648bd6ea4fb2bf507a6a4c55506
+size 11422650
diff --git a/checkpoint-3500/tokenizer_config.json b/checkpoint-3500/tokenizer_config.json
new file mode 100644
index 0000000000000000000000000000000000000000..770e41d6c92519d525eede4cbcf3ba27f6425311
--- /dev/null
+++ b/checkpoint-3500/tokenizer_config.json
@@ -0,0 +1,30 @@
+{
+ "add_prefix_space": false,
+ "backend": "tokenizers",
+ "bos_token": null,
+ "clean_up_tokenization_spaces": false,
+ "eos_token": "<|im_end|>",
+ "errors": "replace",
+ "extra_special_tokens": [
+ "<|im_start|>",
+ "<|im_end|>",
+ "<|object_ref_start|>",
+ "<|object_ref_end|>",
+ "<|box_start|>",
+ "<|box_end|>",
+ "<|quad_start|>",
+ "<|quad_end|>",
+ "<|vision_start|>",
+ "<|vision_end|>",
+ "<|vision_pad|>",
+ "<|image_pad|>",
+ "<|video_pad|>"
+ ],
+ "is_local": false,
+ "local_files_only": false,
+ "model_max_length": 131072,
+ "pad_token": "<|endoftext|>",
+ "split_special_tokens": false,
+ "tokenizer_class": "Qwen2Tokenizer",
+ "unk_token": null
+}
diff --git a/checkpoint-3500/trainer_state.json b/checkpoint-3500/trainer_state.json
new file mode 100644
index 0000000000000000000000000000000000000000..bac80aca6924f87c3785d49e4726e6a389fbede8
--- /dev/null
+++ b/checkpoint-3500/trainer_state.json
@@ -0,0 +1,734 @@
+{
+ "best_global_step": null,
+ "best_metric": null,
+ "best_model_checkpoint": null,
+ "epoch": 0.5608749649453146,
+ "eval_steps": 500,
+ "global_step": 3500,
+ "is_hyper_param_search": false,
+ "is_local_process_zero": true,
+ "is_world_process_zero": true,
+ "log_history": [
+ {
+ "entropy": 2.223189059495926,
+ "epoch": 0.008012499499218782,
+ "grad_norm": 2.875,
+ "learning_rate": 1.9842973882390643e-05,
+ "loss": 2.3428109741210936,
+ "mean_token_accuracy": 0.4963963569700718,
+ "num_tokens": 305366.0,
+ "step": 50
+ },
+ {
+ "entropy": 2.188130117058754,
+ "epoch": 0.016024998998437564,
+ "grad_norm": 3.328125,
+ "learning_rate": 1.96827431501362e-05,
+ "loss": 2.225358734130859,
+ "mean_token_accuracy": 0.5110921229422093,
+ "num_tokens": 605042.0,
+ "step": 100
+ },
+ {
+ "entropy": 2.197347580194473,
+ "epoch": 0.024037498497656344,
+ "grad_norm": 3.453125,
+ "learning_rate": 1.952251241788175e-05,
+ "loss": 2.2299420166015627,
+ "mean_token_accuracy": 0.5093490639328957,
+ "num_tokens": 906888.0,
+ "step": 150
+ },
+ {
+ "entropy": 2.1913950395584108,
+ "epoch": 0.03204999799687513,
+ "grad_norm": 2.90625,
+ "learning_rate": 1.9362281685627303e-05,
+ "loss": 2.210995941162109,
+ "mean_token_accuracy": 0.5097909501194954,
+ "num_tokens": 1213125.0,
+ "step": 200
+ },
+ {
+ "entropy": 2.1611998677253723,
+ "epoch": 0.040062497496093905,
+ "grad_norm": 3.25,
+ "learning_rate": 1.920205095337286e-05,
+ "loss": 2.1867474365234374,
+ "mean_token_accuracy": 0.5154410660266876,
+ "num_tokens": 1509026.0,
+ "step": 250
+ },
+ {
+ "entropy": 2.168051454424858,
+ "epoch": 0.04807499699531269,
+ "grad_norm": 2.9375,
+ "learning_rate": 1.904182022111841e-05,
+ "loss": 2.190071716308594,
+ "mean_token_accuracy": 0.5156884534657001,
+ "num_tokens": 1806850.0,
+ "step": 300
+ },
+ {
+ "entropy": 2.1737154543399813,
+ "epoch": 0.05608749649453147,
+ "grad_norm": 3.421875,
+ "learning_rate": 1.8881589488863967e-05,
+ "loss": 2.199147186279297,
+ "mean_token_accuracy": 0.514045044630766,
+ "num_tokens": 2104505.0,
+ "step": 350
+ },
+ {
+ "entropy": 2.148039159178734,
+ "epoch": 0.06409999599375026,
+ "grad_norm": 3.109375,
+ "learning_rate": 1.872135875660952e-05,
+ "loss": 2.176946105957031,
+ "mean_token_accuracy": 0.5177112428843975,
+ "num_tokens": 2391088.0,
+ "step": 400
+ },
+ {
+ "entropy": 2.194890711903572,
+ "epoch": 0.07211249549296903,
+ "grad_norm": 3.28125,
+ "learning_rate": 1.856112802435507e-05,
+ "loss": 2.216288757324219,
+ "mean_token_accuracy": 0.5110335703194141,
+ "num_tokens": 2691272.0,
+ "step": 450
+ },
+ {
+ "entropy": 2.1557786524295808,
+ "epoch": 0.08012499499218781,
+ "grad_norm": 4.15625,
+ "learning_rate": 1.8400897292100627e-05,
+ "loss": 2.1812680053710936,
+ "mean_token_accuracy": 0.5176697050035,
+ "num_tokens": 2991855.0,
+ "step": 500
+ },
+ {
+ "entropy": 2.1708167713880537,
+ "epoch": 0.08813749449140659,
+ "grad_norm": 4.34375,
+ "learning_rate": 1.824066655984618e-05,
+ "loss": 2.1921002197265627,
+ "mean_token_accuracy": 0.5156507430970669,
+ "num_tokens": 3295049.0,
+ "step": 550
+ },
+ {
+ "entropy": 2.1686314886808393,
+ "epoch": 0.09614999399062538,
+ "grad_norm": 2.875,
+ "learning_rate": 1.8080435827591735e-05,
+ "loss": 2.1914617919921877,
+ "mean_token_accuracy": 0.5139613692462445,
+ "num_tokens": 3597206.0,
+ "step": 600
+ },
+ {
+ "entropy": 2.15742906332016,
+ "epoch": 0.10416249348984416,
+ "grad_norm": 3.109375,
+ "learning_rate": 1.7920205095337287e-05,
+ "loss": 2.174853973388672,
+ "mean_token_accuracy": 0.5159518288075924,
+ "num_tokens": 3905268.0,
+ "step": 650
+ },
+ {
+ "entropy": 2.162363375425339,
+ "epoch": 0.11217499298906294,
+ "grad_norm": 3.171875,
+ "learning_rate": 1.7759974363082843e-05,
+ "loss": 2.184231872558594,
+ "mean_token_accuracy": 0.516631413847208,
+ "num_tokens": 4207934.0,
+ "step": 700
+ },
+ {
+ "entropy": 2.1516289287805557,
+ "epoch": 0.12018749248828171,
+ "grad_norm": 3.609375,
+ "learning_rate": 1.7599743630828395e-05,
+ "loss": 2.167020263671875,
+ "mean_token_accuracy": 0.516358617246151,
+ "num_tokens": 4515774.0,
+ "step": 750
+ },
+ {
+ "entropy": 2.1549544781446457,
+ "epoch": 0.1281999919875005,
+ "grad_norm": 3.28125,
+ "learning_rate": 1.7439512898573947e-05,
+ "loss": 2.181917877197266,
+ "mean_token_accuracy": 0.5182220858335495,
+ "num_tokens": 4814709.0,
+ "step": 800
+ },
+ {
+ "entropy": 2.1532615756988527,
+ "epoch": 0.1362124914867193,
+ "grad_norm": 3.078125,
+ "learning_rate": 1.7279282166319503e-05,
+ "loss": 2.1659007263183594,
+ "mean_token_accuracy": 0.5180425970256328,
+ "num_tokens": 5112585.0,
+ "step": 850
+ },
+ {
+ "entropy": 2.1704017293453215,
+ "epoch": 0.14422499098593805,
+ "grad_norm": 2.921875,
+ "learning_rate": 1.7119051434065055e-05,
+ "loss": 2.1934996032714844,
+ "mean_token_accuracy": 0.5154964691400528,
+ "num_tokens": 5411369.0,
+ "step": 900
+ },
+ {
+ "entropy": 2.1349107217788696,
+ "epoch": 0.15223749048515683,
+ "grad_norm": 2.984375,
+ "learning_rate": 1.6958820701810608e-05,
+ "loss": 2.1598545837402345,
+ "mean_token_accuracy": 0.5208920259773732,
+ "num_tokens": 5707060.0,
+ "step": 950
+ },
+ {
+ "entropy": 2.1541705918312073,
+ "epoch": 0.16024998998437562,
+ "grad_norm": 3.203125,
+ "learning_rate": 1.6798589969556163e-05,
+ "loss": 2.170956573486328,
+ "mean_token_accuracy": 0.5204938448965549,
+ "num_tokens": 6004079.0,
+ "step": 1000
+ },
+ {
+ "entropy": 2.1523867017030716,
+ "epoch": 0.1682624894835944,
+ "grad_norm": 3.09375,
+ "learning_rate": 1.6638359237301716e-05,
+ "loss": 2.1695118713378907,
+ "mean_token_accuracy": 0.5177376234531402,
+ "num_tokens": 6315462.0,
+ "step": 1050
+ },
+ {
+ "entropy": 2.1356022965908052,
+ "epoch": 0.17627498898281319,
+ "grad_norm": 3.21875,
+ "learning_rate": 1.6478128505047268e-05,
+ "loss": 2.1513107299804686,
+ "mean_token_accuracy": 0.5217180585861206,
+ "num_tokens": 6613060.0,
+ "step": 1100
+ },
+ {
+ "entropy": 2.1483460652828215,
+ "epoch": 0.18428748848203197,
+ "grad_norm": 3.21875,
+ "learning_rate": 1.6317897772792824e-05,
+ "loss": 2.1671595764160156,
+ "mean_token_accuracy": 0.5199222292006016,
+ "num_tokens": 6909655.0,
+ "step": 1150
+ },
+ {
+ "entropy": 2.1074040442705155,
+ "epoch": 0.19229998798125075,
+ "grad_norm": 3.828125,
+ "learning_rate": 1.6157667040538376e-05,
+ "loss": 2.12770751953125,
+ "mean_token_accuracy": 0.5252153894305229,
+ "num_tokens": 7203263.0,
+ "step": 1200
+ },
+ {
+ "entropy": 2.1354027593135836,
+ "epoch": 0.20031248748046954,
+ "grad_norm": 2.921875,
+ "learning_rate": 1.5997436308283932e-05,
+ "loss": 2.156183166503906,
+ "mean_token_accuracy": 0.518907299786806,
+ "num_tokens": 7505411.0,
+ "step": 1250
+ },
+ {
+ "entropy": 2.142924894094467,
+ "epoch": 0.20832498697968832,
+ "grad_norm": 3.5625,
+ "learning_rate": 1.5837205576029484e-05,
+ "loss": 2.158486633300781,
+ "mean_token_accuracy": 0.5205909128487111,
+ "num_tokens": 7803562.0,
+ "step": 1300
+ },
+ {
+ "entropy": 2.1305291467905043,
+ "epoch": 0.2163374864789071,
+ "grad_norm": 3.34375,
+ "learning_rate": 1.5676974843775036e-05,
+ "loss": 2.151043701171875,
+ "mean_token_accuracy": 0.5211617235839366,
+ "num_tokens": 8108037.0,
+ "step": 1350
+ },
+ {
+ "entropy": 2.160947166085243,
+ "epoch": 0.2243499859781259,
+ "grad_norm": 3.234375,
+ "learning_rate": 1.5516744111520592e-05,
+ "loss": 2.177792663574219,
+ "mean_token_accuracy": 0.5167004990577698,
+ "num_tokens": 8408674.0,
+ "step": 1400
+ },
+ {
+ "entropy": 2.120403967499733,
+ "epoch": 0.23236248547734467,
+ "grad_norm": 3.375,
+ "learning_rate": 1.5356513379266144e-05,
+ "loss": 2.1462562561035154,
+ "mean_token_accuracy": 0.5225985661149025,
+ "num_tokens": 8707650.0,
+ "step": 1450
+ },
+ {
+ "entropy": 2.138882914185524,
+ "epoch": 0.24037498497656343,
+ "grad_norm": 3.171875,
+ "learning_rate": 1.5196282647011698e-05,
+ "loss": 2.1499905395507812,
+ "mean_token_accuracy": 0.5204668046534061,
+ "num_tokens": 9006296.0,
+ "step": 1500
+ },
+ {
+ "entropy": 2.131927245259285,
+ "epoch": 0.2483874844757822,
+ "grad_norm": 3.546875,
+ "learning_rate": 1.5036051914757252e-05,
+ "loss": 2.157076416015625,
+ "mean_token_accuracy": 0.5220408822596073,
+ "num_tokens": 9307203.0,
+ "step": 1550
+ },
+ {
+ "entropy": 2.1308567851781843,
+ "epoch": 0.256399983975001,
+ "grad_norm": 3.546875,
+ "learning_rate": 1.4875821182502805e-05,
+ "loss": 2.146601104736328,
+ "mean_token_accuracy": 0.5219539681077003,
+ "num_tokens": 9610070.0,
+ "step": 1600
+ },
+ {
+ "entropy": 2.1097783839702604,
+ "epoch": 0.2644124834742198,
+ "grad_norm": 3.84375,
+ "learning_rate": 1.4715590450248359e-05,
+ "loss": 2.1268209838867187,
+ "mean_token_accuracy": 0.5236652019619942,
+ "num_tokens": 9909117.0,
+ "step": 1650
+ },
+ {
+ "entropy": 2.123499180674553,
+ "epoch": 0.2724249829734386,
+ "grad_norm": 3.46875,
+ "learning_rate": 1.4555359717993913e-05,
+ "loss": 2.1409934997558593,
+ "mean_token_accuracy": 0.5225465354323388,
+ "num_tokens": 10209461.0,
+ "step": 1700
+ },
+ {
+ "entropy": 2.13052960395813,
+ "epoch": 0.2804374824726573,
+ "grad_norm": 2.71875,
+ "learning_rate": 1.4395128985739467e-05,
+ "loss": 2.151373291015625,
+ "mean_token_accuracy": 0.5227928113937378,
+ "num_tokens": 10509282.0,
+ "step": 1750
+ },
+ {
+ "entropy": 2.1226114094257356,
+ "epoch": 0.2884499819718761,
+ "grad_norm": 3.65625,
+ "learning_rate": 1.4234898253485019e-05,
+ "loss": 2.144451141357422,
+ "mean_token_accuracy": 0.5233827362954616,
+ "num_tokens": 10811056.0,
+ "step": 1800
+ },
+ {
+ "entropy": 2.126143154501915,
+ "epoch": 0.2964624814710949,
+ "grad_norm": 3.359375,
+ "learning_rate": 1.4074667521230575e-05,
+ "loss": 2.1457598876953123,
+ "mean_token_accuracy": 0.5221967925131321,
+ "num_tokens": 11111153.0,
+ "step": 1850
+ },
+ {
+ "entropy": 2.1048326581716537,
+ "epoch": 0.30447498097031367,
+ "grad_norm": 3.59375,
+ "learning_rate": 1.3914436788976127e-05,
+ "loss": 2.1260116577148436,
+ "mean_token_accuracy": 0.5272719909250736,
+ "num_tokens": 11405606.0,
+ "step": 1900
+ },
+ {
+ "entropy": 2.1236617982387545,
+ "epoch": 0.31248748046953245,
+ "grad_norm": 3.0,
+ "learning_rate": 1.375420605672168e-05,
+ "loss": 2.151782684326172,
+ "mean_token_accuracy": 0.5227546173334122,
+ "num_tokens": 11703943.0,
+ "step": 1950
+ },
+ {
+ "entropy": 2.121563429236412,
+ "epoch": 0.32049997996875124,
+ "grad_norm": 3.34375,
+ "learning_rate": 1.3593975324467235e-05,
+ "loss": 2.1370144653320313,
+ "mean_token_accuracy": 0.5235123547911644,
+ "num_tokens": 12002522.0,
+ "step": 2000
+ },
+ {
+ "entropy": 2.1155461168289182,
+ "epoch": 0.32851247946797,
+ "grad_norm": 3.46875,
+ "learning_rate": 1.3433744592212787e-05,
+ "loss": 2.1364064025878906,
+ "mean_token_accuracy": 0.5237135806679726,
+ "num_tokens": 12303098.0,
+ "step": 2050
+ },
+ {
+ "entropy": 2.1386052322387696,
+ "epoch": 0.3365249789671888,
+ "grad_norm": 3.484375,
+ "learning_rate": 1.327351385995834e-05,
+ "loss": 2.149888458251953,
+ "mean_token_accuracy": 0.522669050693512,
+ "num_tokens": 12605534.0,
+ "step": 2100
+ },
+ {
+ "entropy": 2.102600795030594,
+ "epoch": 0.3445374784664076,
+ "grad_norm": 3.796875,
+ "learning_rate": 1.3113283127703895e-05,
+ "loss": 2.1236566162109374,
+ "mean_token_accuracy": 0.5279092155396938,
+ "num_tokens": 12901218.0,
+ "step": 2150
+ },
+ {
+ "entropy": 2.132645187973976,
+ "epoch": 0.35254997796562637,
+ "grad_norm": 3.53125,
+ "learning_rate": 1.2953052395449447e-05,
+ "loss": 2.1523503112792968,
+ "mean_token_accuracy": 0.5198129612207413,
+ "num_tokens": 13213018.0,
+ "step": 2200
+ },
+ {
+ "entropy": 2.1204321217536926,
+ "epoch": 0.36056247746484515,
+ "grad_norm": 3.25,
+ "learning_rate": 1.2792821663195001e-05,
+ "loss": 2.1325447082519533,
+ "mean_token_accuracy": 0.5243636940419674,
+ "num_tokens": 13514408.0,
+ "step": 2250
+ },
+ {
+ "entropy": 2.102410731315613,
+ "epoch": 0.36857497696406394,
+ "grad_norm": 3.1875,
+ "learning_rate": 1.2632590930940555e-05,
+ "loss": 2.123565368652344,
+ "mean_token_accuracy": 0.5257614709436893,
+ "num_tokens": 13818272.0,
+ "step": 2300
+ },
+ {
+ "entropy": 2.0876205080747603,
+ "epoch": 0.3765874764632827,
+ "grad_norm": 3.328125,
+ "learning_rate": 1.247236019868611e-05,
+ "loss": 2.116696472167969,
+ "mean_token_accuracy": 0.5275124031305313,
+ "num_tokens": 14110702.0,
+ "step": 2350
+ },
+ {
+ "entropy": 2.1078518223762512,
+ "epoch": 0.3845999759625015,
+ "grad_norm": 3.453125,
+ "learning_rate": 1.2312129466431662e-05,
+ "loss": 2.1291993713378905,
+ "mean_token_accuracy": 0.524967464953661,
+ "num_tokens": 14407015.0,
+ "step": 2400
+ },
+ {
+ "entropy": 2.0930898267030718,
+ "epoch": 0.3926124754617203,
+ "grad_norm": 3.5,
+ "learning_rate": 1.2151898734177216e-05,
+ "loss": 2.1094123840332033,
+ "mean_token_accuracy": 0.5290587748587131,
+ "num_tokens": 14696565.0,
+ "step": 2450
+ },
+ {
+ "entropy": 2.1109757220745085,
+ "epoch": 0.4006249749609391,
+ "grad_norm": 3.5,
+ "learning_rate": 1.199166800192277e-05,
+ "loss": 2.1246888732910154,
+ "mean_token_accuracy": 0.5258957357704639,
+ "num_tokens": 14994401.0,
+ "step": 2500
+ },
+ {
+ "entropy": 2.1087811160087586,
+ "epoch": 0.40863747446015786,
+ "grad_norm": 3.1875,
+ "learning_rate": 1.1831437269668324e-05,
+ "loss": 2.12890625,
+ "mean_token_accuracy": 0.5269917643070221,
+ "num_tokens": 15292837.0,
+ "step": 2550
+ },
+ {
+ "entropy": 2.097252429127693,
+ "epoch": 0.41664997395937664,
+ "grad_norm": 3.0,
+ "learning_rate": 1.1671206537413878e-05,
+ "loss": 2.1145870971679686,
+ "mean_token_accuracy": 0.5274137698113919,
+ "num_tokens": 15590647.0,
+ "step": 2600
+ },
+ {
+ "entropy": 2.111199595332146,
+ "epoch": 0.4246624734585954,
+ "grad_norm": 3.25,
+ "learning_rate": 1.151097580515943e-05,
+ "loss": 2.1274327087402343,
+ "mean_token_accuracy": 0.5250249288976192,
+ "num_tokens": 15892772.0,
+ "step": 2650
+ },
+ {
+ "entropy": 2.1051502913236617,
+ "epoch": 0.4326749729578142,
+ "grad_norm": 3.203125,
+ "learning_rate": 1.1350745072904986e-05,
+ "loss": 2.127935333251953,
+ "mean_token_accuracy": 0.5263640700280666,
+ "num_tokens": 16191611.0,
+ "step": 2700
+ },
+ {
+ "entropy": 2.1127885603904724,
+ "epoch": 0.440687472457033,
+ "grad_norm": 3.515625,
+ "learning_rate": 1.1190514340650538e-05,
+ "loss": 2.129211883544922,
+ "mean_token_accuracy": 0.5250939278304577,
+ "num_tokens": 16483582.0,
+ "step": 2750
+ },
+ {
+ "entropy": 2.110930626988411,
+ "epoch": 0.4486999719562518,
+ "grad_norm": 3.1875,
+ "learning_rate": 1.103028360839609e-05,
+ "loss": 2.1220079040527344,
+ "mean_token_accuracy": 0.5250919266045093,
+ "num_tokens": 16787253.0,
+ "step": 2800
+ },
+ {
+ "entropy": 2.1182780677080153,
+ "epoch": 0.45671247145547056,
+ "grad_norm": 3.8125,
+ "learning_rate": 1.0870052876141646e-05,
+ "loss": 2.1358837890625,
+ "mean_token_accuracy": 0.5217854312062263,
+ "num_tokens": 17088741.0,
+ "step": 2850
+ },
+ {
+ "entropy": 2.1117833071947096,
+ "epoch": 0.46472497095468934,
+ "grad_norm": 3.171875,
+ "learning_rate": 1.0709822143887198e-05,
+ "loss": 2.125662841796875,
+ "mean_token_accuracy": 0.524649152457714,
+ "num_tokens": 17389236.0,
+ "step": 2900
+ },
+ {
+ "entropy": 2.1262730568647386,
+ "epoch": 0.47273747045390807,
+ "grad_norm": 3.625,
+ "learning_rate": 1.054959141163275e-05,
+ "loss": 2.142789611816406,
+ "mean_token_accuracy": 0.5236384868621826,
+ "num_tokens": 17693413.0,
+ "step": 2950
+ },
+ {
+ "entropy": 2.115410739183426,
+ "epoch": 0.48074996995312685,
+ "grad_norm": 3.140625,
+ "learning_rate": 1.0389360679378306e-05,
+ "loss": 2.1337106323242185,
+ "mean_token_accuracy": 0.5249222332239151,
+ "num_tokens": 17990326.0,
+ "step": 3000
+ },
+ {
+ "entropy": 2.0951381689310073,
+ "epoch": 0.48876246945234564,
+ "grad_norm": 3.234375,
+ "learning_rate": 1.0229129947123859e-05,
+ "loss": 2.108144073486328,
+ "mean_token_accuracy": 0.5269971822202205,
+ "num_tokens": 18293136.0,
+ "step": 3050
+ },
+ {
+ "entropy": 2.116977259516716,
+ "epoch": 0.4967749689515644,
+ "grad_norm": 3.421875,
+ "learning_rate": 1.0068899214869413e-05,
+ "loss": 2.1327830505371095,
+ "mean_token_accuracy": 0.5238270902633667,
+ "num_tokens": 18591481.0,
+ "step": 3100
+ },
+ {
+ "entropy": 2.08598925113678,
+ "epoch": 0.5047874684507833,
+ "grad_norm": 2.96875,
+ "learning_rate": 9.908668482614967e-06,
+ "loss": 2.1032344055175782,
+ "mean_token_accuracy": 0.5289211909472943,
+ "num_tokens": 18885221.0,
+ "step": 3150
+ },
+ {
+ "entropy": 2.097712662220001,
+ "epoch": 0.512799967950002,
+ "grad_norm": 3.40625,
+ "learning_rate": 9.74843775036052e-06,
+ "loss": 2.1150633239746095,
+ "mean_token_accuracy": 0.5288272814452648,
+ "num_tokens": 19184784.0,
+ "step": 3200
+ },
+ {
+ "entropy": 2.1063760107755662,
+ "epoch": 0.5208124674492208,
+ "grad_norm": 3.28125,
+ "learning_rate": 9.588207018106075e-06,
+ "loss": 2.1217109680175783,
+ "mean_token_accuracy": 0.526290989369154,
+ "num_tokens": 19482538.0,
+ "step": 3250
+ },
+ {
+ "entropy": 2.090182801485062,
+ "epoch": 0.5288249669484396,
+ "grad_norm": 2.875,
+ "learning_rate": 9.427976285851627e-06,
+ "loss": 2.1130311584472654,
+ "mean_token_accuracy": 0.5285439921915531,
+ "num_tokens": 19782544.0,
+ "step": 3300
+ },
+ {
+ "entropy": 2.103260806798935,
+ "epoch": 0.5368374664476584,
+ "grad_norm": 3.59375,
+ "learning_rate": 9.267745553597181e-06,
+ "loss": 2.116189727783203,
+ "mean_token_accuracy": 0.5273195558786392,
+ "num_tokens": 20086408.0,
+ "step": 3350
+ },
+ {
+ "entropy": 2.0673747235536575,
+ "epoch": 0.5448499659468772,
+ "grad_norm": 3.125,
+ "learning_rate": 9.107514821342735e-06,
+ "loss": 2.087482147216797,
+ "mean_token_accuracy": 0.532562478929758,
+ "num_tokens": 20380429.0,
+ "step": 3400
+ },
+ {
+ "entropy": 2.1030789083242416,
+ "epoch": 0.552862465446096,
+ "grad_norm": 3.296875,
+ "learning_rate": 8.947284089088289e-06,
+ "loss": 2.122591247558594,
+ "mean_token_accuracy": 0.5256703786551953,
+ "num_tokens": 20682849.0,
+ "step": 3450
+ },
+ {
+ "entropy": 2.0988716661930082,
+ "epoch": 0.5608749649453146,
+ "grad_norm": 3.546875,
+ "learning_rate": 8.787053356833841e-06,
+ "loss": 2.115867156982422,
+ "mean_token_accuracy": 0.5266730943322182,
+ "num_tokens": 20979225.0,
+ "step": 3500
+ }
+ ],
+ "logging_steps": 50,
+ "max_steps": 6241,
+ "num_input_tokens_seen": 0,
+ "num_train_epochs": 1,
+ "save_steps": 500,
+ "stateful_callbacks": {
+ "TrainerControl": {
+ "args": {
+ "should_epoch_stop": false,
+ "should_evaluate": false,
+ "should_log": false,
+ "should_save": true,
+ "should_training_stop": false
+ },
+ "attributes": {}
+ }
+ },
+ "total_flos": 2.4793245607162675e+17,
+ "train_batch_size": 4,
+ "trial_name": null,
+ "trial_params": null
+}
diff --git a/checkpoint-3500/training_args.bin b/checkpoint-3500/training_args.bin
new file mode 100644
index 0000000000000000000000000000000000000000..4b409d5a147856f0d5a3d3da0f3b4922a9def7dd
--- /dev/null
+++ b/checkpoint-3500/training_args.bin
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:78622f63304ec0913ed0b4ea7a1b26c08a5621b3661a380d798c4ea426235671
+size 5713
diff --git a/checkpoint-4000/chat_template.jinja b/checkpoint-4000/chat_template.jinja
new file mode 100644
index 0000000000000000000000000000000000000000..01be9b307daa2d425f7c168c9fb145a286e0afb4
--- /dev/null
+++ b/checkpoint-4000/chat_template.jinja
@@ -0,0 +1,89 @@
+{%- if tools %}
+ {{- '<|im_start|>system\n' }}
+ {%- if messages[0].role == 'system' %}
+ {{- messages[0].content + '\n\n' }}
+ {%- endif %}
+ {{- "# Tools\n\nYou may call one or more functions to assist with the user query.\n\nYou are provided with function signatures within XML tags:\n" }}
+ {%- for tool in tools %}
+ {{- "\n" }}
+ {{- tool | tojson }}
+ {%- endfor %}
+ {{- "\n\n\nFor each function call, return a json object with function name and arguments within XML tags:\n\n{\"name\": , \"arguments\": }\n<|im_end|>\n" }}
+{%- else %}
+ {%- if messages[0].role == 'system' %}
+ {{- '<|im_start|>system\n' + messages[0].content + '<|im_end|>\n' }}
+ {%- endif %}
+{%- endif %}
+{%- set ns = namespace(multi_step_tool=true, last_query_index=messages|length - 1) %}
+{%- for message in messages[::-1] %}
+ {%- set index = (messages|length - 1) - loop.index0 %}
+ {%- if ns.multi_step_tool and message.role == "user" and message.content is string and not(message.content.startswith('') and message.content.endswith('')) %}
+ {%- set ns.multi_step_tool = false %}
+ {%- set ns.last_query_index = index %}
+ {%- endif %}
+{%- endfor %}
+{%- for message in messages %}
+ {%- if message.content is string %}
+ {%- set content = message.content %}
+ {%- else %}
+ {%- set content = '' %}
+ {%- endif %}
+ {%- if (message.role == "user") or (message.role == "system" and not loop.first) %}
+ {{- '<|im_start|>' + message.role + '\n' + content + '<|im_end|>' + '\n' }}
+ {%- elif message.role == "assistant" %}
+ {%- set reasoning_content = '' %}
+ {%- if message.reasoning_content is string %}
+ {%- set reasoning_content = message.reasoning_content %}
+ {%- else %}
+ {%- if '' in content %}
+ {%- set reasoning_content = content.split('')[0].rstrip('\n').split('')[-1].lstrip('\n') %}
+ {%- set content = content.split('')[-1].lstrip('\n') %}
+ {%- endif %}
+ {%- endif %}
+ {%- if loop.index0 > ns.last_query_index %}
+ {%- if loop.last or (not loop.last and reasoning_content) %}
+ {{- '<|im_start|>' + message.role + '\n\n' + reasoning_content.strip('\n') + '\n\n\n' + content.lstrip('\n') }}
+ {%- else %}
+ {{- '<|im_start|>' + message.role + '\n' + content }}
+ {%- endif %}
+ {%- else %}
+ {{- '<|im_start|>' + message.role + '\n' + content }}
+ {%- endif %}
+ {%- if message.tool_calls %}
+ {%- for tool_call in message.tool_calls %}
+ {%- if (loop.first and content) or (not loop.first) %}
+ {{- '\n' }}
+ {%- endif %}
+ {%- if tool_call.function %}
+ {%- set tool_call = tool_call.function %}
+ {%- endif %}
+ {{- '\n{"name": "' }}
+ {{- tool_call.name }}
+ {{- '", "arguments": ' }}
+ {%- if tool_call.arguments is string %}
+ {{- tool_call.arguments }}
+ {%- else %}
+ {{- tool_call.arguments | tojson }}
+ {%- endif %}
+ {{- '}\n' }}
+ {%- endfor %}
+ {%- endif %}
+ {{- '<|im_end|>\n' }}
+ {%- elif message.role == "tool" %}
+ {%- if loop.first or (messages[loop.index0 - 1].role != "tool") %}
+ {{- '<|im_start|>user' }}
+ {%- endif %}
+ {{- '\n\n' }}
+ {{- content }}
+ {{- '\n' }}
+ {%- if loop.last or (messages[loop.index0 + 1].role != "tool") %}
+ {{- '<|im_end|>\n' }}
+ {%- endif %}
+ {%- endif %}
+{%- endfor %}
+{%- if add_generation_prompt %}
+ {{- '<|im_start|>assistant\n' }}
+ {%- if enable_thinking is defined and enable_thinking is false %}
+ {{- '\n\n\n\n' }}
+ {%- endif %}
+{%- endif %}
\ No newline at end of file
diff --git a/checkpoint-4000/config.json b/checkpoint-4000/config.json
new file mode 100644
index 0000000000000000000000000000000000000000..1d219d6a6d288625229a71646710b85cf7ae5def
--- /dev/null
+++ b/checkpoint-4000/config.json
@@ -0,0 +1,63 @@
+{
+ "architectures": [
+ "Qwen3ForCausalLM"
+ ],
+ "attention_bias": false,
+ "attention_dropout": 0.0,
+ "bos_token_id": null,
+ "dtype": "bfloat16",
+ "eos_token_id": 151645,
+ "head_dim": 128,
+ "hidden_act": "silu",
+ "hidden_size": 2048,
+ "initializer_range": 0.02,
+ "intermediate_size": 6144,
+ "layer_types": [
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention"
+ ],
+ "max_position_embeddings": 40960,
+ "max_window_layers": 28,
+ "model_type": "qwen3",
+ "num_attention_heads": 16,
+ "num_hidden_layers": 28,
+ "num_key_value_heads": 8,
+ "pad_token_id": 151643,
+ "rms_norm_eps": 1e-06,
+ "rope_parameters": {
+ "rope_theta": 1000000,
+ "rope_type": "default"
+ },
+ "sliding_window": null,
+ "tie_word_embeddings": true,
+ "transformers_version": "5.7.0",
+ "use_cache": false,
+ "use_sliding_window": false,
+ "vocab_size": 151936
+}
diff --git a/checkpoint-4000/generation_config.json b/checkpoint-4000/generation_config.json
new file mode 100644
index 0000000000000000000000000000000000000000..5ec133d867b1427d43a5ee0ea332bd088fc12013
--- /dev/null
+++ b/checkpoint-4000/generation_config.json
@@ -0,0 +1,12 @@
+{
+ "do_sample": true,
+ "eos_token_id": [
+ 151645,
+ 151643
+ ],
+ "pad_token_id": 151643,
+ "temperature": 0.6,
+ "top_k": 20,
+ "top_p": 0.95,
+ "transformers_version": "5.7.0"
+}
diff --git a/checkpoint-4000/model.safetensors b/checkpoint-4000/model.safetensors
new file mode 100644
index 0000000000000000000000000000000000000000..d3a746c4a104f480524a7b3c4bcd478a7069e9f1
--- /dev/null
+++ b/checkpoint-4000/model.safetensors
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:59f52f23dfe48c7547727ea25697ac3a9d18e4d016c8fa4c5c9e1b4dff8c31cb
+size 3441185608
diff --git a/checkpoint-4000/optimizer.pt b/checkpoint-4000/optimizer.pt
new file mode 100644
index 0000000000000000000000000000000000000000..3903484a85f99809ced3ff4537d77e2dd50147fb
--- /dev/null
+++ b/checkpoint-4000/optimizer.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:5c313769ae550920040bf3021b0ea64cd98c9f52d74cedfbdb319bf5c5369138
+size 6882572207
diff --git a/checkpoint-4000/rng_state.pth b/checkpoint-4000/rng_state.pth
new file mode 100644
index 0000000000000000000000000000000000000000..5a8f17addf8aaee350625e9847751582bc9f86dd
--- /dev/null
+++ b/checkpoint-4000/rng_state.pth
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:61c19bab1174704a4a4441475683bf1270277af15d2e2c95e964789128e482c4
+size 14645
diff --git a/checkpoint-4000/scheduler.pt b/checkpoint-4000/scheduler.pt
new file mode 100644
index 0000000000000000000000000000000000000000..03f8dd3aaa6021d06404422b76c1765147f449bd
--- /dev/null
+++ b/checkpoint-4000/scheduler.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:78d0e196db341b3518b3d3e05e16be9cc71c297b4603d00fde73052aafae3e28
+size 1465
diff --git a/checkpoint-4000/tokenizer.json b/checkpoint-4000/tokenizer.json
new file mode 100644
index 0000000000000000000000000000000000000000..c7afbed2efcdf019f88ab0572ec29d3bf595dfe2
--- /dev/null
+++ b/checkpoint-4000/tokenizer.json
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:be75606093db2094d7cd20f3c2f385c212750648bd6ea4fb2bf507a6a4c55506
+size 11422650
diff --git a/checkpoint-4000/tokenizer_config.json b/checkpoint-4000/tokenizer_config.json
new file mode 100644
index 0000000000000000000000000000000000000000..770e41d6c92519d525eede4cbcf3ba27f6425311
--- /dev/null
+++ b/checkpoint-4000/tokenizer_config.json
@@ -0,0 +1,30 @@
+{
+ "add_prefix_space": false,
+ "backend": "tokenizers",
+ "bos_token": null,
+ "clean_up_tokenization_spaces": false,
+ "eos_token": "<|im_end|>",
+ "errors": "replace",
+ "extra_special_tokens": [
+ "<|im_start|>",
+ "<|im_end|>",
+ "<|object_ref_start|>",
+ "<|object_ref_end|>",
+ "<|box_start|>",
+ "<|box_end|>",
+ "<|quad_start|>",
+ "<|quad_end|>",
+ "<|vision_start|>",
+ "<|vision_end|>",
+ "<|vision_pad|>",
+ "<|image_pad|>",
+ "<|video_pad|>"
+ ],
+ "is_local": false,
+ "local_files_only": false,
+ "model_max_length": 131072,
+ "pad_token": "<|endoftext|>",
+ "split_special_tokens": false,
+ "tokenizer_class": "Qwen2Tokenizer",
+ "unk_token": null
+}
diff --git a/checkpoint-4000/trainer_state.json b/checkpoint-4000/trainer_state.json
new file mode 100644
index 0000000000000000000000000000000000000000..70695062e62eb521595a5bc20209e52e2715df06
--- /dev/null
+++ b/checkpoint-4000/trainer_state.json
@@ -0,0 +1,834 @@
+{
+ "best_global_step": null,
+ "best_metric": null,
+ "best_model_checkpoint": null,
+ "epoch": 0.6409999599375025,
+ "eval_steps": 500,
+ "global_step": 4000,
+ "is_hyper_param_search": false,
+ "is_local_process_zero": true,
+ "is_world_process_zero": true,
+ "log_history": [
+ {
+ "entropy": 2.223189059495926,
+ "epoch": 0.008012499499218782,
+ "grad_norm": 2.875,
+ "learning_rate": 1.9842973882390643e-05,
+ "loss": 2.3428109741210936,
+ "mean_token_accuracy": 0.4963963569700718,
+ "num_tokens": 305366.0,
+ "step": 50
+ },
+ {
+ "entropy": 2.188130117058754,
+ "epoch": 0.016024998998437564,
+ "grad_norm": 3.328125,
+ "learning_rate": 1.96827431501362e-05,
+ "loss": 2.225358734130859,
+ "mean_token_accuracy": 0.5110921229422093,
+ "num_tokens": 605042.0,
+ "step": 100
+ },
+ {
+ "entropy": 2.197347580194473,
+ "epoch": 0.024037498497656344,
+ "grad_norm": 3.453125,
+ "learning_rate": 1.952251241788175e-05,
+ "loss": 2.2299420166015627,
+ "mean_token_accuracy": 0.5093490639328957,
+ "num_tokens": 906888.0,
+ "step": 150
+ },
+ {
+ "entropy": 2.1913950395584108,
+ "epoch": 0.03204999799687513,
+ "grad_norm": 2.90625,
+ "learning_rate": 1.9362281685627303e-05,
+ "loss": 2.210995941162109,
+ "mean_token_accuracy": 0.5097909501194954,
+ "num_tokens": 1213125.0,
+ "step": 200
+ },
+ {
+ "entropy": 2.1611998677253723,
+ "epoch": 0.040062497496093905,
+ "grad_norm": 3.25,
+ "learning_rate": 1.920205095337286e-05,
+ "loss": 2.1867474365234374,
+ "mean_token_accuracy": 0.5154410660266876,
+ "num_tokens": 1509026.0,
+ "step": 250
+ },
+ {
+ "entropy": 2.168051454424858,
+ "epoch": 0.04807499699531269,
+ "grad_norm": 2.9375,
+ "learning_rate": 1.904182022111841e-05,
+ "loss": 2.190071716308594,
+ "mean_token_accuracy": 0.5156884534657001,
+ "num_tokens": 1806850.0,
+ "step": 300
+ },
+ {
+ "entropy": 2.1737154543399813,
+ "epoch": 0.05608749649453147,
+ "grad_norm": 3.421875,
+ "learning_rate": 1.8881589488863967e-05,
+ "loss": 2.199147186279297,
+ "mean_token_accuracy": 0.514045044630766,
+ "num_tokens": 2104505.0,
+ "step": 350
+ },
+ {
+ "entropy": 2.148039159178734,
+ "epoch": 0.06409999599375026,
+ "grad_norm": 3.109375,
+ "learning_rate": 1.872135875660952e-05,
+ "loss": 2.176946105957031,
+ "mean_token_accuracy": 0.5177112428843975,
+ "num_tokens": 2391088.0,
+ "step": 400
+ },
+ {
+ "entropy": 2.194890711903572,
+ "epoch": 0.07211249549296903,
+ "grad_norm": 3.28125,
+ "learning_rate": 1.856112802435507e-05,
+ "loss": 2.216288757324219,
+ "mean_token_accuracy": 0.5110335703194141,
+ "num_tokens": 2691272.0,
+ "step": 450
+ },
+ {
+ "entropy": 2.1557786524295808,
+ "epoch": 0.08012499499218781,
+ "grad_norm": 4.15625,
+ "learning_rate": 1.8400897292100627e-05,
+ "loss": 2.1812680053710936,
+ "mean_token_accuracy": 0.5176697050035,
+ "num_tokens": 2991855.0,
+ "step": 500
+ },
+ {
+ "entropy": 2.1708167713880537,
+ "epoch": 0.08813749449140659,
+ "grad_norm": 4.34375,
+ "learning_rate": 1.824066655984618e-05,
+ "loss": 2.1921002197265627,
+ "mean_token_accuracy": 0.5156507430970669,
+ "num_tokens": 3295049.0,
+ "step": 550
+ },
+ {
+ "entropy": 2.1686314886808393,
+ "epoch": 0.09614999399062538,
+ "grad_norm": 2.875,
+ "learning_rate": 1.8080435827591735e-05,
+ "loss": 2.1914617919921877,
+ "mean_token_accuracy": 0.5139613692462445,
+ "num_tokens": 3597206.0,
+ "step": 600
+ },
+ {
+ "entropy": 2.15742906332016,
+ "epoch": 0.10416249348984416,
+ "grad_norm": 3.109375,
+ "learning_rate": 1.7920205095337287e-05,
+ "loss": 2.174853973388672,
+ "mean_token_accuracy": 0.5159518288075924,
+ "num_tokens": 3905268.0,
+ "step": 650
+ },
+ {
+ "entropy": 2.162363375425339,
+ "epoch": 0.11217499298906294,
+ "grad_norm": 3.171875,
+ "learning_rate": 1.7759974363082843e-05,
+ "loss": 2.184231872558594,
+ "mean_token_accuracy": 0.516631413847208,
+ "num_tokens": 4207934.0,
+ "step": 700
+ },
+ {
+ "entropy": 2.1516289287805557,
+ "epoch": 0.12018749248828171,
+ "grad_norm": 3.609375,
+ "learning_rate": 1.7599743630828395e-05,
+ "loss": 2.167020263671875,
+ "mean_token_accuracy": 0.516358617246151,
+ "num_tokens": 4515774.0,
+ "step": 750
+ },
+ {
+ "entropy": 2.1549544781446457,
+ "epoch": 0.1281999919875005,
+ "grad_norm": 3.28125,
+ "learning_rate": 1.7439512898573947e-05,
+ "loss": 2.181917877197266,
+ "mean_token_accuracy": 0.5182220858335495,
+ "num_tokens": 4814709.0,
+ "step": 800
+ },
+ {
+ "entropy": 2.1532615756988527,
+ "epoch": 0.1362124914867193,
+ "grad_norm": 3.078125,
+ "learning_rate": 1.7279282166319503e-05,
+ "loss": 2.1659007263183594,
+ "mean_token_accuracy": 0.5180425970256328,
+ "num_tokens": 5112585.0,
+ "step": 850
+ },
+ {
+ "entropy": 2.1704017293453215,
+ "epoch": 0.14422499098593805,
+ "grad_norm": 2.921875,
+ "learning_rate": 1.7119051434065055e-05,
+ "loss": 2.1934996032714844,
+ "mean_token_accuracy": 0.5154964691400528,
+ "num_tokens": 5411369.0,
+ "step": 900
+ },
+ {
+ "entropy": 2.1349107217788696,
+ "epoch": 0.15223749048515683,
+ "grad_norm": 2.984375,
+ "learning_rate": 1.6958820701810608e-05,
+ "loss": 2.1598545837402345,
+ "mean_token_accuracy": 0.5208920259773732,
+ "num_tokens": 5707060.0,
+ "step": 950
+ },
+ {
+ "entropy": 2.1541705918312073,
+ "epoch": 0.16024998998437562,
+ "grad_norm": 3.203125,
+ "learning_rate": 1.6798589969556163e-05,
+ "loss": 2.170956573486328,
+ "mean_token_accuracy": 0.5204938448965549,
+ "num_tokens": 6004079.0,
+ "step": 1000
+ },
+ {
+ "entropy": 2.1523867017030716,
+ "epoch": 0.1682624894835944,
+ "grad_norm": 3.09375,
+ "learning_rate": 1.6638359237301716e-05,
+ "loss": 2.1695118713378907,
+ "mean_token_accuracy": 0.5177376234531402,
+ "num_tokens": 6315462.0,
+ "step": 1050
+ },
+ {
+ "entropy": 2.1356022965908052,
+ "epoch": 0.17627498898281319,
+ "grad_norm": 3.21875,
+ "learning_rate": 1.6478128505047268e-05,
+ "loss": 2.1513107299804686,
+ "mean_token_accuracy": 0.5217180585861206,
+ "num_tokens": 6613060.0,
+ "step": 1100
+ },
+ {
+ "entropy": 2.1483460652828215,
+ "epoch": 0.18428748848203197,
+ "grad_norm": 3.21875,
+ "learning_rate": 1.6317897772792824e-05,
+ "loss": 2.1671595764160156,
+ "mean_token_accuracy": 0.5199222292006016,
+ "num_tokens": 6909655.0,
+ "step": 1150
+ },
+ {
+ "entropy": 2.1074040442705155,
+ "epoch": 0.19229998798125075,
+ "grad_norm": 3.828125,
+ "learning_rate": 1.6157667040538376e-05,
+ "loss": 2.12770751953125,
+ "mean_token_accuracy": 0.5252153894305229,
+ "num_tokens": 7203263.0,
+ "step": 1200
+ },
+ {
+ "entropy": 2.1354027593135836,
+ "epoch": 0.20031248748046954,
+ "grad_norm": 2.921875,
+ "learning_rate": 1.5997436308283932e-05,
+ "loss": 2.156183166503906,
+ "mean_token_accuracy": 0.518907299786806,
+ "num_tokens": 7505411.0,
+ "step": 1250
+ },
+ {
+ "entropy": 2.142924894094467,
+ "epoch": 0.20832498697968832,
+ "grad_norm": 3.5625,
+ "learning_rate": 1.5837205576029484e-05,
+ "loss": 2.158486633300781,
+ "mean_token_accuracy": 0.5205909128487111,
+ "num_tokens": 7803562.0,
+ "step": 1300
+ },
+ {
+ "entropy": 2.1305291467905043,
+ "epoch": 0.2163374864789071,
+ "grad_norm": 3.34375,
+ "learning_rate": 1.5676974843775036e-05,
+ "loss": 2.151043701171875,
+ "mean_token_accuracy": 0.5211617235839366,
+ "num_tokens": 8108037.0,
+ "step": 1350
+ },
+ {
+ "entropy": 2.160947166085243,
+ "epoch": 0.2243499859781259,
+ "grad_norm": 3.234375,
+ "learning_rate": 1.5516744111520592e-05,
+ "loss": 2.177792663574219,
+ "mean_token_accuracy": 0.5167004990577698,
+ "num_tokens": 8408674.0,
+ "step": 1400
+ },
+ {
+ "entropy": 2.120403967499733,
+ "epoch": 0.23236248547734467,
+ "grad_norm": 3.375,
+ "learning_rate": 1.5356513379266144e-05,
+ "loss": 2.1462562561035154,
+ "mean_token_accuracy": 0.5225985661149025,
+ "num_tokens": 8707650.0,
+ "step": 1450
+ },
+ {
+ "entropy": 2.138882914185524,
+ "epoch": 0.24037498497656343,
+ "grad_norm": 3.171875,
+ "learning_rate": 1.5196282647011698e-05,
+ "loss": 2.1499905395507812,
+ "mean_token_accuracy": 0.5204668046534061,
+ "num_tokens": 9006296.0,
+ "step": 1500
+ },
+ {
+ "entropy": 2.131927245259285,
+ "epoch": 0.2483874844757822,
+ "grad_norm": 3.546875,
+ "learning_rate": 1.5036051914757252e-05,
+ "loss": 2.157076416015625,
+ "mean_token_accuracy": 0.5220408822596073,
+ "num_tokens": 9307203.0,
+ "step": 1550
+ },
+ {
+ "entropy": 2.1308567851781843,
+ "epoch": 0.256399983975001,
+ "grad_norm": 3.546875,
+ "learning_rate": 1.4875821182502805e-05,
+ "loss": 2.146601104736328,
+ "mean_token_accuracy": 0.5219539681077003,
+ "num_tokens": 9610070.0,
+ "step": 1600
+ },
+ {
+ "entropy": 2.1097783839702604,
+ "epoch": 0.2644124834742198,
+ "grad_norm": 3.84375,
+ "learning_rate": 1.4715590450248359e-05,
+ "loss": 2.1268209838867187,
+ "mean_token_accuracy": 0.5236652019619942,
+ "num_tokens": 9909117.0,
+ "step": 1650
+ },
+ {
+ "entropy": 2.123499180674553,
+ "epoch": 0.2724249829734386,
+ "grad_norm": 3.46875,
+ "learning_rate": 1.4555359717993913e-05,
+ "loss": 2.1409934997558593,
+ "mean_token_accuracy": 0.5225465354323388,
+ "num_tokens": 10209461.0,
+ "step": 1700
+ },
+ {
+ "entropy": 2.13052960395813,
+ "epoch": 0.2804374824726573,
+ "grad_norm": 2.71875,
+ "learning_rate": 1.4395128985739467e-05,
+ "loss": 2.151373291015625,
+ "mean_token_accuracy": 0.5227928113937378,
+ "num_tokens": 10509282.0,
+ "step": 1750
+ },
+ {
+ "entropy": 2.1226114094257356,
+ "epoch": 0.2884499819718761,
+ "grad_norm": 3.65625,
+ "learning_rate": 1.4234898253485019e-05,
+ "loss": 2.144451141357422,
+ "mean_token_accuracy": 0.5233827362954616,
+ "num_tokens": 10811056.0,
+ "step": 1800
+ },
+ {
+ "entropy": 2.126143154501915,
+ "epoch": 0.2964624814710949,
+ "grad_norm": 3.359375,
+ "learning_rate": 1.4074667521230575e-05,
+ "loss": 2.1457598876953123,
+ "mean_token_accuracy": 0.5221967925131321,
+ "num_tokens": 11111153.0,
+ "step": 1850
+ },
+ {
+ "entropy": 2.1048326581716537,
+ "epoch": 0.30447498097031367,
+ "grad_norm": 3.59375,
+ "learning_rate": 1.3914436788976127e-05,
+ "loss": 2.1260116577148436,
+ "mean_token_accuracy": 0.5272719909250736,
+ "num_tokens": 11405606.0,
+ "step": 1900
+ },
+ {
+ "entropy": 2.1236617982387545,
+ "epoch": 0.31248748046953245,
+ "grad_norm": 3.0,
+ "learning_rate": 1.375420605672168e-05,
+ "loss": 2.151782684326172,
+ "mean_token_accuracy": 0.5227546173334122,
+ "num_tokens": 11703943.0,
+ "step": 1950
+ },
+ {
+ "entropy": 2.121563429236412,
+ "epoch": 0.32049997996875124,
+ "grad_norm": 3.34375,
+ "learning_rate": 1.3593975324467235e-05,
+ "loss": 2.1370144653320313,
+ "mean_token_accuracy": 0.5235123547911644,
+ "num_tokens": 12002522.0,
+ "step": 2000
+ },
+ {
+ "entropy": 2.1155461168289182,
+ "epoch": 0.32851247946797,
+ "grad_norm": 3.46875,
+ "learning_rate": 1.3433744592212787e-05,
+ "loss": 2.1364064025878906,
+ "mean_token_accuracy": 0.5237135806679726,
+ "num_tokens": 12303098.0,
+ "step": 2050
+ },
+ {
+ "entropy": 2.1386052322387696,
+ "epoch": 0.3365249789671888,
+ "grad_norm": 3.484375,
+ "learning_rate": 1.327351385995834e-05,
+ "loss": 2.149888458251953,
+ "mean_token_accuracy": 0.522669050693512,
+ "num_tokens": 12605534.0,
+ "step": 2100
+ },
+ {
+ "entropy": 2.102600795030594,
+ "epoch": 0.3445374784664076,
+ "grad_norm": 3.796875,
+ "learning_rate": 1.3113283127703895e-05,
+ "loss": 2.1236566162109374,
+ "mean_token_accuracy": 0.5279092155396938,
+ "num_tokens": 12901218.0,
+ "step": 2150
+ },
+ {
+ "entropy": 2.132645187973976,
+ "epoch": 0.35254997796562637,
+ "grad_norm": 3.53125,
+ "learning_rate": 1.2953052395449447e-05,
+ "loss": 2.1523503112792968,
+ "mean_token_accuracy": 0.5198129612207413,
+ "num_tokens": 13213018.0,
+ "step": 2200
+ },
+ {
+ "entropy": 2.1204321217536926,
+ "epoch": 0.36056247746484515,
+ "grad_norm": 3.25,
+ "learning_rate": 1.2792821663195001e-05,
+ "loss": 2.1325447082519533,
+ "mean_token_accuracy": 0.5243636940419674,
+ "num_tokens": 13514408.0,
+ "step": 2250
+ },
+ {
+ "entropy": 2.102410731315613,
+ "epoch": 0.36857497696406394,
+ "grad_norm": 3.1875,
+ "learning_rate": 1.2632590930940555e-05,
+ "loss": 2.123565368652344,
+ "mean_token_accuracy": 0.5257614709436893,
+ "num_tokens": 13818272.0,
+ "step": 2300
+ },
+ {
+ "entropy": 2.0876205080747603,
+ "epoch": 0.3765874764632827,
+ "grad_norm": 3.328125,
+ "learning_rate": 1.247236019868611e-05,
+ "loss": 2.116696472167969,
+ "mean_token_accuracy": 0.5275124031305313,
+ "num_tokens": 14110702.0,
+ "step": 2350
+ },
+ {
+ "entropy": 2.1078518223762512,
+ "epoch": 0.3845999759625015,
+ "grad_norm": 3.453125,
+ "learning_rate": 1.2312129466431662e-05,
+ "loss": 2.1291993713378905,
+ "mean_token_accuracy": 0.524967464953661,
+ "num_tokens": 14407015.0,
+ "step": 2400
+ },
+ {
+ "entropy": 2.0930898267030718,
+ "epoch": 0.3926124754617203,
+ "grad_norm": 3.5,
+ "learning_rate": 1.2151898734177216e-05,
+ "loss": 2.1094123840332033,
+ "mean_token_accuracy": 0.5290587748587131,
+ "num_tokens": 14696565.0,
+ "step": 2450
+ },
+ {
+ "entropy": 2.1109757220745085,
+ "epoch": 0.4006249749609391,
+ "grad_norm": 3.5,
+ "learning_rate": 1.199166800192277e-05,
+ "loss": 2.1246888732910154,
+ "mean_token_accuracy": 0.5258957357704639,
+ "num_tokens": 14994401.0,
+ "step": 2500
+ },
+ {
+ "entropy": 2.1087811160087586,
+ "epoch": 0.40863747446015786,
+ "grad_norm": 3.1875,
+ "learning_rate": 1.1831437269668324e-05,
+ "loss": 2.12890625,
+ "mean_token_accuracy": 0.5269917643070221,
+ "num_tokens": 15292837.0,
+ "step": 2550
+ },
+ {
+ "entropy": 2.097252429127693,
+ "epoch": 0.41664997395937664,
+ "grad_norm": 3.0,
+ "learning_rate": 1.1671206537413878e-05,
+ "loss": 2.1145870971679686,
+ "mean_token_accuracy": 0.5274137698113919,
+ "num_tokens": 15590647.0,
+ "step": 2600
+ },
+ {
+ "entropy": 2.111199595332146,
+ "epoch": 0.4246624734585954,
+ "grad_norm": 3.25,
+ "learning_rate": 1.151097580515943e-05,
+ "loss": 2.1274327087402343,
+ "mean_token_accuracy": 0.5250249288976192,
+ "num_tokens": 15892772.0,
+ "step": 2650
+ },
+ {
+ "entropy": 2.1051502913236617,
+ "epoch": 0.4326749729578142,
+ "grad_norm": 3.203125,
+ "learning_rate": 1.1350745072904986e-05,
+ "loss": 2.127935333251953,
+ "mean_token_accuracy": 0.5263640700280666,
+ "num_tokens": 16191611.0,
+ "step": 2700
+ },
+ {
+ "entropy": 2.1127885603904724,
+ "epoch": 0.440687472457033,
+ "grad_norm": 3.515625,
+ "learning_rate": 1.1190514340650538e-05,
+ "loss": 2.129211883544922,
+ "mean_token_accuracy": 0.5250939278304577,
+ "num_tokens": 16483582.0,
+ "step": 2750
+ },
+ {
+ "entropy": 2.110930626988411,
+ "epoch": 0.4486999719562518,
+ "grad_norm": 3.1875,
+ "learning_rate": 1.103028360839609e-05,
+ "loss": 2.1220079040527344,
+ "mean_token_accuracy": 0.5250919266045093,
+ "num_tokens": 16787253.0,
+ "step": 2800
+ },
+ {
+ "entropy": 2.1182780677080153,
+ "epoch": 0.45671247145547056,
+ "grad_norm": 3.8125,
+ "learning_rate": 1.0870052876141646e-05,
+ "loss": 2.1358837890625,
+ "mean_token_accuracy": 0.5217854312062263,
+ "num_tokens": 17088741.0,
+ "step": 2850
+ },
+ {
+ "entropy": 2.1117833071947096,
+ "epoch": 0.46472497095468934,
+ "grad_norm": 3.171875,
+ "learning_rate": 1.0709822143887198e-05,
+ "loss": 2.125662841796875,
+ "mean_token_accuracy": 0.524649152457714,
+ "num_tokens": 17389236.0,
+ "step": 2900
+ },
+ {
+ "entropy": 2.1262730568647386,
+ "epoch": 0.47273747045390807,
+ "grad_norm": 3.625,
+ "learning_rate": 1.054959141163275e-05,
+ "loss": 2.142789611816406,
+ "mean_token_accuracy": 0.5236384868621826,
+ "num_tokens": 17693413.0,
+ "step": 2950
+ },
+ {
+ "entropy": 2.115410739183426,
+ "epoch": 0.48074996995312685,
+ "grad_norm": 3.140625,
+ "learning_rate": 1.0389360679378306e-05,
+ "loss": 2.1337106323242185,
+ "mean_token_accuracy": 0.5249222332239151,
+ "num_tokens": 17990326.0,
+ "step": 3000
+ },
+ {
+ "entropy": 2.0951381689310073,
+ "epoch": 0.48876246945234564,
+ "grad_norm": 3.234375,
+ "learning_rate": 1.0229129947123859e-05,
+ "loss": 2.108144073486328,
+ "mean_token_accuracy": 0.5269971822202205,
+ "num_tokens": 18293136.0,
+ "step": 3050
+ },
+ {
+ "entropy": 2.116977259516716,
+ "epoch": 0.4967749689515644,
+ "grad_norm": 3.421875,
+ "learning_rate": 1.0068899214869413e-05,
+ "loss": 2.1327830505371095,
+ "mean_token_accuracy": 0.5238270902633667,
+ "num_tokens": 18591481.0,
+ "step": 3100
+ },
+ {
+ "entropy": 2.08598925113678,
+ "epoch": 0.5047874684507833,
+ "grad_norm": 2.96875,
+ "learning_rate": 9.908668482614967e-06,
+ "loss": 2.1032344055175782,
+ "mean_token_accuracy": 0.5289211909472943,
+ "num_tokens": 18885221.0,
+ "step": 3150
+ },
+ {
+ "entropy": 2.097712662220001,
+ "epoch": 0.512799967950002,
+ "grad_norm": 3.40625,
+ "learning_rate": 9.74843775036052e-06,
+ "loss": 2.1150633239746095,
+ "mean_token_accuracy": 0.5288272814452648,
+ "num_tokens": 19184784.0,
+ "step": 3200
+ },
+ {
+ "entropy": 2.1063760107755662,
+ "epoch": 0.5208124674492208,
+ "grad_norm": 3.28125,
+ "learning_rate": 9.588207018106075e-06,
+ "loss": 2.1217109680175783,
+ "mean_token_accuracy": 0.526290989369154,
+ "num_tokens": 19482538.0,
+ "step": 3250
+ },
+ {
+ "entropy": 2.090182801485062,
+ "epoch": 0.5288249669484396,
+ "grad_norm": 2.875,
+ "learning_rate": 9.427976285851627e-06,
+ "loss": 2.1130311584472654,
+ "mean_token_accuracy": 0.5285439921915531,
+ "num_tokens": 19782544.0,
+ "step": 3300
+ },
+ {
+ "entropy": 2.103260806798935,
+ "epoch": 0.5368374664476584,
+ "grad_norm": 3.59375,
+ "learning_rate": 9.267745553597181e-06,
+ "loss": 2.116189727783203,
+ "mean_token_accuracy": 0.5273195558786392,
+ "num_tokens": 20086408.0,
+ "step": 3350
+ },
+ {
+ "entropy": 2.0673747235536575,
+ "epoch": 0.5448499659468772,
+ "grad_norm": 3.125,
+ "learning_rate": 9.107514821342735e-06,
+ "loss": 2.087482147216797,
+ "mean_token_accuracy": 0.532562478929758,
+ "num_tokens": 20380429.0,
+ "step": 3400
+ },
+ {
+ "entropy": 2.1030789083242416,
+ "epoch": 0.552862465446096,
+ "grad_norm": 3.296875,
+ "learning_rate": 8.947284089088289e-06,
+ "loss": 2.122591247558594,
+ "mean_token_accuracy": 0.5256703786551953,
+ "num_tokens": 20682849.0,
+ "step": 3450
+ },
+ {
+ "entropy": 2.0988716661930082,
+ "epoch": 0.5608749649453146,
+ "grad_norm": 3.546875,
+ "learning_rate": 8.787053356833841e-06,
+ "loss": 2.115867156982422,
+ "mean_token_accuracy": 0.5266730943322182,
+ "num_tokens": 20979225.0,
+ "step": 3500
+ },
+ {
+ "entropy": 2.0954072701931,
+ "epoch": 0.5688874644445334,
+ "grad_norm": 3.859375,
+ "learning_rate": 8.626822624579395e-06,
+ "loss": 2.1116868591308595,
+ "mean_token_accuracy": 0.5267495161294937,
+ "num_tokens": 21284166.0,
+ "step": 3550
+ },
+ {
+ "entropy": 2.11238471031189,
+ "epoch": 0.5768999639437522,
+ "grad_norm": 3.484375,
+ "learning_rate": 8.466591892324949e-06,
+ "loss": 2.1292352294921875,
+ "mean_token_accuracy": 0.5249243360757828,
+ "num_tokens": 21584612.0,
+ "step": 3600
+ },
+ {
+ "entropy": 2.091464368700981,
+ "epoch": 0.584912463442971,
+ "grad_norm": 3.359375,
+ "learning_rate": 8.306361160070501e-06,
+ "loss": 2.1099319458007812,
+ "mean_token_accuracy": 0.5292035435140133,
+ "num_tokens": 21879345.0,
+ "step": 3650
+ },
+ {
+ "entropy": 2.095204224586487,
+ "epoch": 0.5929249629421898,
+ "grad_norm": 2.96875,
+ "learning_rate": 8.146130427816055e-06,
+ "loss": 2.1198716735839844,
+ "mean_token_accuracy": 0.5255991271138192,
+ "num_tokens": 22183486.0,
+ "step": 3700
+ },
+ {
+ "entropy": 2.0666573679447175,
+ "epoch": 0.6009374624414086,
+ "grad_norm": 2.90625,
+ "learning_rate": 7.98589969556161e-06,
+ "loss": 2.08689697265625,
+ "mean_token_accuracy": 0.5318664258718491,
+ "num_tokens": 22476438.0,
+ "step": 3750
+ },
+ {
+ "entropy": 2.122153177857399,
+ "epoch": 0.6089499619406273,
+ "grad_norm": 3.40625,
+ "learning_rate": 7.825668963307162e-06,
+ "loss": 2.138288879394531,
+ "mean_token_accuracy": 0.5232584895193577,
+ "num_tokens": 22780385.0,
+ "step": 3800
+ },
+ {
+ "entropy": 2.1190748488903046,
+ "epoch": 0.6169624614398461,
+ "grad_norm": 3.46875,
+ "learning_rate": 7.665438231052716e-06,
+ "loss": 2.139790344238281,
+ "mean_token_accuracy": 0.5243287613987923,
+ "num_tokens": 23080847.0,
+ "step": 3850
+ },
+ {
+ "entropy": 2.1312570214271545,
+ "epoch": 0.6249749609390649,
+ "grad_norm": 3.765625,
+ "learning_rate": 7.5052074987982706e-06,
+ "loss": 2.1471080017089843,
+ "mean_token_accuracy": 0.5232046774029732,
+ "num_tokens": 23384738.0,
+ "step": 3900
+ },
+ {
+ "entropy": 2.0779008239507677,
+ "epoch": 0.6329874604382837,
+ "grad_norm": 3.171875,
+ "learning_rate": 7.344976766543823e-06,
+ "loss": 2.0974729919433592,
+ "mean_token_accuracy": 0.5305448272824287,
+ "num_tokens": 23682750.0,
+ "step": 3950
+ },
+ {
+ "entropy": 2.1121455317735673,
+ "epoch": 0.6409999599375025,
+ "grad_norm": 3.921875,
+ "learning_rate": 7.184746034289377e-06,
+ "loss": 2.127069549560547,
+ "mean_token_accuracy": 0.5252936625480652,
+ "num_tokens": 23989029.0,
+ "step": 4000
+ }
+ ],
+ "logging_steps": 50,
+ "max_steps": 6241,
+ "num_input_tokens_seen": 0,
+ "num_train_epochs": 1,
+ "save_steps": 500,
+ "stateful_callbacks": {
+ "TrainerControl": {
+ "args": {
+ "should_epoch_stop": false,
+ "should_evaluate": false,
+ "should_log": false,
+ "should_save": true,
+ "should_training_stop": false
+ },
+ "attributes": {}
+ }
+ },
+ "total_flos": 2.832566128574546e+17,
+ "train_batch_size": 4,
+ "trial_name": null,
+ "trial_params": null
+}
diff --git a/checkpoint-4000/training_args.bin b/checkpoint-4000/training_args.bin
new file mode 100644
index 0000000000000000000000000000000000000000..4b409d5a147856f0d5a3d3da0f3b4922a9def7dd
--- /dev/null
+++ b/checkpoint-4000/training_args.bin
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:78622f63304ec0913ed0b4ea7a1b26c08a5621b3661a380d798c4ea426235671
+size 5713
diff --git a/checkpoint-4500/chat_template.jinja b/checkpoint-4500/chat_template.jinja
new file mode 100644
index 0000000000000000000000000000000000000000..01be9b307daa2d425f7c168c9fb145a286e0afb4
--- /dev/null
+++ b/checkpoint-4500/chat_template.jinja
@@ -0,0 +1,89 @@
+{%- if tools %}
+ {{- '<|im_start|>system\n' }}
+ {%- if messages[0].role == 'system' %}
+ {{- messages[0].content + '\n\n' }}
+ {%- endif %}
+ {{- "# Tools\n\nYou may call one or more functions to assist with the user query.\n\nYou are provided with function signatures within XML tags:\n" }}
+ {%- for tool in tools %}
+ {{- "\n" }}
+ {{- tool | tojson }}
+ {%- endfor %}
+ {{- "\n\n\nFor each function call, return a json object with function name and arguments within XML tags:\n\n{\"name\": , \"arguments\": }\n<|im_end|>\n" }}
+{%- else %}
+ {%- if messages[0].role == 'system' %}
+ {{- '<|im_start|>system\n' + messages[0].content + '<|im_end|>\n' }}
+ {%- endif %}
+{%- endif %}
+{%- set ns = namespace(multi_step_tool=true, last_query_index=messages|length - 1) %}
+{%- for message in messages[::-1] %}
+ {%- set index = (messages|length - 1) - loop.index0 %}
+ {%- if ns.multi_step_tool and message.role == "user" and message.content is string and not(message.content.startswith('') and message.content.endswith('')) %}
+ {%- set ns.multi_step_tool = false %}
+ {%- set ns.last_query_index = index %}
+ {%- endif %}
+{%- endfor %}
+{%- for message in messages %}
+ {%- if message.content is string %}
+ {%- set content = message.content %}
+ {%- else %}
+ {%- set content = '' %}
+ {%- endif %}
+ {%- if (message.role == "user") or (message.role == "system" and not loop.first) %}
+ {{- '<|im_start|>' + message.role + '\n' + content + '<|im_end|>' + '\n' }}
+ {%- elif message.role == "assistant" %}
+ {%- set reasoning_content = '' %}
+ {%- if message.reasoning_content is string %}
+ {%- set reasoning_content = message.reasoning_content %}
+ {%- else %}
+ {%- if '' in content %}
+ {%- set reasoning_content = content.split('')[0].rstrip('\n').split('')[-1].lstrip('\n') %}
+ {%- set content = content.split('')[-1].lstrip('\n') %}
+ {%- endif %}
+ {%- endif %}
+ {%- if loop.index0 > ns.last_query_index %}
+ {%- if loop.last or (not loop.last and reasoning_content) %}
+ {{- '<|im_start|>' + message.role + '\n\n' + reasoning_content.strip('\n') + '\n\n\n' + content.lstrip('\n') }}
+ {%- else %}
+ {{- '<|im_start|>' + message.role + '\n' + content }}
+ {%- endif %}
+ {%- else %}
+ {{- '<|im_start|>' + message.role + '\n' + content }}
+ {%- endif %}
+ {%- if message.tool_calls %}
+ {%- for tool_call in message.tool_calls %}
+ {%- if (loop.first and content) or (not loop.first) %}
+ {{- '\n' }}
+ {%- endif %}
+ {%- if tool_call.function %}
+ {%- set tool_call = tool_call.function %}
+ {%- endif %}
+ {{- '\n{"name": "' }}
+ {{- tool_call.name }}
+ {{- '", "arguments": ' }}
+ {%- if tool_call.arguments is string %}
+ {{- tool_call.arguments }}
+ {%- else %}
+ {{- tool_call.arguments | tojson }}
+ {%- endif %}
+ {{- '}\n' }}
+ {%- endfor %}
+ {%- endif %}
+ {{- '<|im_end|>\n' }}
+ {%- elif message.role == "tool" %}
+ {%- if loop.first or (messages[loop.index0 - 1].role != "tool") %}
+ {{- '<|im_start|>user' }}
+ {%- endif %}
+ {{- '\n\n' }}
+ {{- content }}
+ {{- '\n' }}
+ {%- if loop.last or (messages[loop.index0 + 1].role != "tool") %}
+ {{- '<|im_end|>\n' }}
+ {%- endif %}
+ {%- endif %}
+{%- endfor %}
+{%- if add_generation_prompt %}
+ {{- '<|im_start|>assistant\n' }}
+ {%- if enable_thinking is defined and enable_thinking is false %}
+ {{- '\n\n\n\n' }}
+ {%- endif %}
+{%- endif %}
\ No newline at end of file
diff --git a/checkpoint-4500/config.json b/checkpoint-4500/config.json
new file mode 100644
index 0000000000000000000000000000000000000000..1d219d6a6d288625229a71646710b85cf7ae5def
--- /dev/null
+++ b/checkpoint-4500/config.json
@@ -0,0 +1,63 @@
+{
+ "architectures": [
+ "Qwen3ForCausalLM"
+ ],
+ "attention_bias": false,
+ "attention_dropout": 0.0,
+ "bos_token_id": null,
+ "dtype": "bfloat16",
+ "eos_token_id": 151645,
+ "head_dim": 128,
+ "hidden_act": "silu",
+ "hidden_size": 2048,
+ "initializer_range": 0.02,
+ "intermediate_size": 6144,
+ "layer_types": [
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention"
+ ],
+ "max_position_embeddings": 40960,
+ "max_window_layers": 28,
+ "model_type": "qwen3",
+ "num_attention_heads": 16,
+ "num_hidden_layers": 28,
+ "num_key_value_heads": 8,
+ "pad_token_id": 151643,
+ "rms_norm_eps": 1e-06,
+ "rope_parameters": {
+ "rope_theta": 1000000,
+ "rope_type": "default"
+ },
+ "sliding_window": null,
+ "tie_word_embeddings": true,
+ "transformers_version": "5.7.0",
+ "use_cache": false,
+ "use_sliding_window": false,
+ "vocab_size": 151936
+}
diff --git a/checkpoint-4500/generation_config.json b/checkpoint-4500/generation_config.json
new file mode 100644
index 0000000000000000000000000000000000000000..5ec133d867b1427d43a5ee0ea332bd088fc12013
--- /dev/null
+++ b/checkpoint-4500/generation_config.json
@@ -0,0 +1,12 @@
+{
+ "do_sample": true,
+ "eos_token_id": [
+ 151645,
+ 151643
+ ],
+ "pad_token_id": 151643,
+ "temperature": 0.6,
+ "top_k": 20,
+ "top_p": 0.95,
+ "transformers_version": "5.7.0"
+}
diff --git a/checkpoint-4500/model.safetensors b/checkpoint-4500/model.safetensors
new file mode 100644
index 0000000000000000000000000000000000000000..9cff9c145355b24ee7e0f7584dea73dff3df6993
--- /dev/null
+++ b/checkpoint-4500/model.safetensors
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:748cbfd96fce4131cccae2cfbbf7e5943bb67a7960feaffb683370fbb897f84b
+size 3441185608
diff --git a/checkpoint-4500/optimizer.pt b/checkpoint-4500/optimizer.pt
new file mode 100644
index 0000000000000000000000000000000000000000..9e4988131c2e8a89d5a7e5f90db0526addf4cc9a
--- /dev/null
+++ b/checkpoint-4500/optimizer.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:504fdf18c65054e63b9b27fd2ef1b64e410a2d62984669334f0bbfee95f68807
+size 6882572207
diff --git a/checkpoint-4500/rng_state.pth b/checkpoint-4500/rng_state.pth
new file mode 100644
index 0000000000000000000000000000000000000000..5a8f17addf8aaee350625e9847751582bc9f86dd
--- /dev/null
+++ b/checkpoint-4500/rng_state.pth
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:61c19bab1174704a4a4441475683bf1270277af15d2e2c95e964789128e482c4
+size 14645
diff --git a/checkpoint-4500/scheduler.pt b/checkpoint-4500/scheduler.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65bfe7f6c8d4f1193cfb8976b3725ba192b4667f
--- /dev/null
+++ b/checkpoint-4500/scheduler.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f0c65d41abe4fed1f917d1493ac9e89afd04deb14307279675e66aa4431576fd
+size 1465
diff --git a/checkpoint-4500/tokenizer.json b/checkpoint-4500/tokenizer.json
new file mode 100644
index 0000000000000000000000000000000000000000..c7afbed2efcdf019f88ab0572ec29d3bf595dfe2
--- /dev/null
+++ b/checkpoint-4500/tokenizer.json
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:be75606093db2094d7cd20f3c2f385c212750648bd6ea4fb2bf507a6a4c55506
+size 11422650
diff --git a/checkpoint-4500/tokenizer_config.json b/checkpoint-4500/tokenizer_config.json
new file mode 100644
index 0000000000000000000000000000000000000000..770e41d6c92519d525eede4cbcf3ba27f6425311
--- /dev/null
+++ b/checkpoint-4500/tokenizer_config.json
@@ -0,0 +1,30 @@
+{
+ "add_prefix_space": false,
+ "backend": "tokenizers",
+ "bos_token": null,
+ "clean_up_tokenization_spaces": false,
+ "eos_token": "<|im_end|>",
+ "errors": "replace",
+ "extra_special_tokens": [
+ "<|im_start|>",
+ "<|im_end|>",
+ "<|object_ref_start|>",
+ "<|object_ref_end|>",
+ "<|box_start|>",
+ "<|box_end|>",
+ "<|quad_start|>",
+ "<|quad_end|>",
+ "<|vision_start|>",
+ "<|vision_end|>",
+ "<|vision_pad|>",
+ "<|image_pad|>",
+ "<|video_pad|>"
+ ],
+ "is_local": false,
+ "local_files_only": false,
+ "model_max_length": 131072,
+ "pad_token": "<|endoftext|>",
+ "split_special_tokens": false,
+ "tokenizer_class": "Qwen2Tokenizer",
+ "unk_token": null
+}
diff --git a/checkpoint-4500/trainer_state.json b/checkpoint-4500/trainer_state.json
new file mode 100644
index 0000000000000000000000000000000000000000..704a9da76c08a8022357a781826a6ea2cf5a0fac
--- /dev/null
+++ b/checkpoint-4500/trainer_state.json
@@ -0,0 +1,934 @@
+{
+ "best_global_step": null,
+ "best_metric": null,
+ "best_model_checkpoint": null,
+ "epoch": 0.7211249549296903,
+ "eval_steps": 500,
+ "global_step": 4500,
+ "is_hyper_param_search": false,
+ "is_local_process_zero": true,
+ "is_world_process_zero": true,
+ "log_history": [
+ {
+ "entropy": 2.223189059495926,
+ "epoch": 0.008012499499218782,
+ "grad_norm": 2.875,
+ "learning_rate": 1.9842973882390643e-05,
+ "loss": 2.3428109741210936,
+ "mean_token_accuracy": 0.4963963569700718,
+ "num_tokens": 305366.0,
+ "step": 50
+ },
+ {
+ "entropy": 2.188130117058754,
+ "epoch": 0.016024998998437564,
+ "grad_norm": 3.328125,
+ "learning_rate": 1.96827431501362e-05,
+ "loss": 2.225358734130859,
+ "mean_token_accuracy": 0.5110921229422093,
+ "num_tokens": 605042.0,
+ "step": 100
+ },
+ {
+ "entropy": 2.197347580194473,
+ "epoch": 0.024037498497656344,
+ "grad_norm": 3.453125,
+ "learning_rate": 1.952251241788175e-05,
+ "loss": 2.2299420166015627,
+ "mean_token_accuracy": 0.5093490639328957,
+ "num_tokens": 906888.0,
+ "step": 150
+ },
+ {
+ "entropy": 2.1913950395584108,
+ "epoch": 0.03204999799687513,
+ "grad_norm": 2.90625,
+ "learning_rate": 1.9362281685627303e-05,
+ "loss": 2.210995941162109,
+ "mean_token_accuracy": 0.5097909501194954,
+ "num_tokens": 1213125.0,
+ "step": 200
+ },
+ {
+ "entropy": 2.1611998677253723,
+ "epoch": 0.040062497496093905,
+ "grad_norm": 3.25,
+ "learning_rate": 1.920205095337286e-05,
+ "loss": 2.1867474365234374,
+ "mean_token_accuracy": 0.5154410660266876,
+ "num_tokens": 1509026.0,
+ "step": 250
+ },
+ {
+ "entropy": 2.168051454424858,
+ "epoch": 0.04807499699531269,
+ "grad_norm": 2.9375,
+ "learning_rate": 1.904182022111841e-05,
+ "loss": 2.190071716308594,
+ "mean_token_accuracy": 0.5156884534657001,
+ "num_tokens": 1806850.0,
+ "step": 300
+ },
+ {
+ "entropy": 2.1737154543399813,
+ "epoch": 0.05608749649453147,
+ "grad_norm": 3.421875,
+ "learning_rate": 1.8881589488863967e-05,
+ "loss": 2.199147186279297,
+ "mean_token_accuracy": 0.514045044630766,
+ "num_tokens": 2104505.0,
+ "step": 350
+ },
+ {
+ "entropy": 2.148039159178734,
+ "epoch": 0.06409999599375026,
+ "grad_norm": 3.109375,
+ "learning_rate": 1.872135875660952e-05,
+ "loss": 2.176946105957031,
+ "mean_token_accuracy": 0.5177112428843975,
+ "num_tokens": 2391088.0,
+ "step": 400
+ },
+ {
+ "entropy": 2.194890711903572,
+ "epoch": 0.07211249549296903,
+ "grad_norm": 3.28125,
+ "learning_rate": 1.856112802435507e-05,
+ "loss": 2.216288757324219,
+ "mean_token_accuracy": 0.5110335703194141,
+ "num_tokens": 2691272.0,
+ "step": 450
+ },
+ {
+ "entropy": 2.1557786524295808,
+ "epoch": 0.08012499499218781,
+ "grad_norm": 4.15625,
+ "learning_rate": 1.8400897292100627e-05,
+ "loss": 2.1812680053710936,
+ "mean_token_accuracy": 0.5176697050035,
+ "num_tokens": 2991855.0,
+ "step": 500
+ },
+ {
+ "entropy": 2.1708167713880537,
+ "epoch": 0.08813749449140659,
+ "grad_norm": 4.34375,
+ "learning_rate": 1.824066655984618e-05,
+ "loss": 2.1921002197265627,
+ "mean_token_accuracy": 0.5156507430970669,
+ "num_tokens": 3295049.0,
+ "step": 550
+ },
+ {
+ "entropy": 2.1686314886808393,
+ "epoch": 0.09614999399062538,
+ "grad_norm": 2.875,
+ "learning_rate": 1.8080435827591735e-05,
+ "loss": 2.1914617919921877,
+ "mean_token_accuracy": 0.5139613692462445,
+ "num_tokens": 3597206.0,
+ "step": 600
+ },
+ {
+ "entropy": 2.15742906332016,
+ "epoch": 0.10416249348984416,
+ "grad_norm": 3.109375,
+ "learning_rate": 1.7920205095337287e-05,
+ "loss": 2.174853973388672,
+ "mean_token_accuracy": 0.5159518288075924,
+ "num_tokens": 3905268.0,
+ "step": 650
+ },
+ {
+ "entropy": 2.162363375425339,
+ "epoch": 0.11217499298906294,
+ "grad_norm": 3.171875,
+ "learning_rate": 1.7759974363082843e-05,
+ "loss": 2.184231872558594,
+ "mean_token_accuracy": 0.516631413847208,
+ "num_tokens": 4207934.0,
+ "step": 700
+ },
+ {
+ "entropy": 2.1516289287805557,
+ "epoch": 0.12018749248828171,
+ "grad_norm": 3.609375,
+ "learning_rate": 1.7599743630828395e-05,
+ "loss": 2.167020263671875,
+ "mean_token_accuracy": 0.516358617246151,
+ "num_tokens": 4515774.0,
+ "step": 750
+ },
+ {
+ "entropy": 2.1549544781446457,
+ "epoch": 0.1281999919875005,
+ "grad_norm": 3.28125,
+ "learning_rate": 1.7439512898573947e-05,
+ "loss": 2.181917877197266,
+ "mean_token_accuracy": 0.5182220858335495,
+ "num_tokens": 4814709.0,
+ "step": 800
+ },
+ {
+ "entropy": 2.1532615756988527,
+ "epoch": 0.1362124914867193,
+ "grad_norm": 3.078125,
+ "learning_rate": 1.7279282166319503e-05,
+ "loss": 2.1659007263183594,
+ "mean_token_accuracy": 0.5180425970256328,
+ "num_tokens": 5112585.0,
+ "step": 850
+ },
+ {
+ "entropy": 2.1704017293453215,
+ "epoch": 0.14422499098593805,
+ "grad_norm": 2.921875,
+ "learning_rate": 1.7119051434065055e-05,
+ "loss": 2.1934996032714844,
+ "mean_token_accuracy": 0.5154964691400528,
+ "num_tokens": 5411369.0,
+ "step": 900
+ },
+ {
+ "entropy": 2.1349107217788696,
+ "epoch": 0.15223749048515683,
+ "grad_norm": 2.984375,
+ "learning_rate": 1.6958820701810608e-05,
+ "loss": 2.1598545837402345,
+ "mean_token_accuracy": 0.5208920259773732,
+ "num_tokens": 5707060.0,
+ "step": 950
+ },
+ {
+ "entropy": 2.1541705918312073,
+ "epoch": 0.16024998998437562,
+ "grad_norm": 3.203125,
+ "learning_rate": 1.6798589969556163e-05,
+ "loss": 2.170956573486328,
+ "mean_token_accuracy": 0.5204938448965549,
+ "num_tokens": 6004079.0,
+ "step": 1000
+ },
+ {
+ "entropy": 2.1523867017030716,
+ "epoch": 0.1682624894835944,
+ "grad_norm": 3.09375,
+ "learning_rate": 1.6638359237301716e-05,
+ "loss": 2.1695118713378907,
+ "mean_token_accuracy": 0.5177376234531402,
+ "num_tokens": 6315462.0,
+ "step": 1050
+ },
+ {
+ "entropy": 2.1356022965908052,
+ "epoch": 0.17627498898281319,
+ "grad_norm": 3.21875,
+ "learning_rate": 1.6478128505047268e-05,
+ "loss": 2.1513107299804686,
+ "mean_token_accuracy": 0.5217180585861206,
+ "num_tokens": 6613060.0,
+ "step": 1100
+ },
+ {
+ "entropy": 2.1483460652828215,
+ "epoch": 0.18428748848203197,
+ "grad_norm": 3.21875,
+ "learning_rate": 1.6317897772792824e-05,
+ "loss": 2.1671595764160156,
+ "mean_token_accuracy": 0.5199222292006016,
+ "num_tokens": 6909655.0,
+ "step": 1150
+ },
+ {
+ "entropy": 2.1074040442705155,
+ "epoch": 0.19229998798125075,
+ "grad_norm": 3.828125,
+ "learning_rate": 1.6157667040538376e-05,
+ "loss": 2.12770751953125,
+ "mean_token_accuracy": 0.5252153894305229,
+ "num_tokens": 7203263.0,
+ "step": 1200
+ },
+ {
+ "entropy": 2.1354027593135836,
+ "epoch": 0.20031248748046954,
+ "grad_norm": 2.921875,
+ "learning_rate": 1.5997436308283932e-05,
+ "loss": 2.156183166503906,
+ "mean_token_accuracy": 0.518907299786806,
+ "num_tokens": 7505411.0,
+ "step": 1250
+ },
+ {
+ "entropy": 2.142924894094467,
+ "epoch": 0.20832498697968832,
+ "grad_norm": 3.5625,
+ "learning_rate": 1.5837205576029484e-05,
+ "loss": 2.158486633300781,
+ "mean_token_accuracy": 0.5205909128487111,
+ "num_tokens": 7803562.0,
+ "step": 1300
+ },
+ {
+ "entropy": 2.1305291467905043,
+ "epoch": 0.2163374864789071,
+ "grad_norm": 3.34375,
+ "learning_rate": 1.5676974843775036e-05,
+ "loss": 2.151043701171875,
+ "mean_token_accuracy": 0.5211617235839366,
+ "num_tokens": 8108037.0,
+ "step": 1350
+ },
+ {
+ "entropy": 2.160947166085243,
+ "epoch": 0.2243499859781259,
+ "grad_norm": 3.234375,
+ "learning_rate": 1.5516744111520592e-05,
+ "loss": 2.177792663574219,
+ "mean_token_accuracy": 0.5167004990577698,
+ "num_tokens": 8408674.0,
+ "step": 1400
+ },
+ {
+ "entropy": 2.120403967499733,
+ "epoch": 0.23236248547734467,
+ "grad_norm": 3.375,
+ "learning_rate": 1.5356513379266144e-05,
+ "loss": 2.1462562561035154,
+ "mean_token_accuracy": 0.5225985661149025,
+ "num_tokens": 8707650.0,
+ "step": 1450
+ },
+ {
+ "entropy": 2.138882914185524,
+ "epoch": 0.24037498497656343,
+ "grad_norm": 3.171875,
+ "learning_rate": 1.5196282647011698e-05,
+ "loss": 2.1499905395507812,
+ "mean_token_accuracy": 0.5204668046534061,
+ "num_tokens": 9006296.0,
+ "step": 1500
+ },
+ {
+ "entropy": 2.131927245259285,
+ "epoch": 0.2483874844757822,
+ "grad_norm": 3.546875,
+ "learning_rate": 1.5036051914757252e-05,
+ "loss": 2.157076416015625,
+ "mean_token_accuracy": 0.5220408822596073,
+ "num_tokens": 9307203.0,
+ "step": 1550
+ },
+ {
+ "entropy": 2.1308567851781843,
+ "epoch": 0.256399983975001,
+ "grad_norm": 3.546875,
+ "learning_rate": 1.4875821182502805e-05,
+ "loss": 2.146601104736328,
+ "mean_token_accuracy": 0.5219539681077003,
+ "num_tokens": 9610070.0,
+ "step": 1600
+ },
+ {
+ "entropy": 2.1097783839702604,
+ "epoch": 0.2644124834742198,
+ "grad_norm": 3.84375,
+ "learning_rate": 1.4715590450248359e-05,
+ "loss": 2.1268209838867187,
+ "mean_token_accuracy": 0.5236652019619942,
+ "num_tokens": 9909117.0,
+ "step": 1650
+ },
+ {
+ "entropy": 2.123499180674553,
+ "epoch": 0.2724249829734386,
+ "grad_norm": 3.46875,
+ "learning_rate": 1.4555359717993913e-05,
+ "loss": 2.1409934997558593,
+ "mean_token_accuracy": 0.5225465354323388,
+ "num_tokens": 10209461.0,
+ "step": 1700
+ },
+ {
+ "entropy": 2.13052960395813,
+ "epoch": 0.2804374824726573,
+ "grad_norm": 2.71875,
+ "learning_rate": 1.4395128985739467e-05,
+ "loss": 2.151373291015625,
+ "mean_token_accuracy": 0.5227928113937378,
+ "num_tokens": 10509282.0,
+ "step": 1750
+ },
+ {
+ "entropy": 2.1226114094257356,
+ "epoch": 0.2884499819718761,
+ "grad_norm": 3.65625,
+ "learning_rate": 1.4234898253485019e-05,
+ "loss": 2.144451141357422,
+ "mean_token_accuracy": 0.5233827362954616,
+ "num_tokens": 10811056.0,
+ "step": 1800
+ },
+ {
+ "entropy": 2.126143154501915,
+ "epoch": 0.2964624814710949,
+ "grad_norm": 3.359375,
+ "learning_rate": 1.4074667521230575e-05,
+ "loss": 2.1457598876953123,
+ "mean_token_accuracy": 0.5221967925131321,
+ "num_tokens": 11111153.0,
+ "step": 1850
+ },
+ {
+ "entropy": 2.1048326581716537,
+ "epoch": 0.30447498097031367,
+ "grad_norm": 3.59375,
+ "learning_rate": 1.3914436788976127e-05,
+ "loss": 2.1260116577148436,
+ "mean_token_accuracy": 0.5272719909250736,
+ "num_tokens": 11405606.0,
+ "step": 1900
+ },
+ {
+ "entropy": 2.1236617982387545,
+ "epoch": 0.31248748046953245,
+ "grad_norm": 3.0,
+ "learning_rate": 1.375420605672168e-05,
+ "loss": 2.151782684326172,
+ "mean_token_accuracy": 0.5227546173334122,
+ "num_tokens": 11703943.0,
+ "step": 1950
+ },
+ {
+ "entropy": 2.121563429236412,
+ "epoch": 0.32049997996875124,
+ "grad_norm": 3.34375,
+ "learning_rate": 1.3593975324467235e-05,
+ "loss": 2.1370144653320313,
+ "mean_token_accuracy": 0.5235123547911644,
+ "num_tokens": 12002522.0,
+ "step": 2000
+ },
+ {
+ "entropy": 2.1155461168289182,
+ "epoch": 0.32851247946797,
+ "grad_norm": 3.46875,
+ "learning_rate": 1.3433744592212787e-05,
+ "loss": 2.1364064025878906,
+ "mean_token_accuracy": 0.5237135806679726,
+ "num_tokens": 12303098.0,
+ "step": 2050
+ },
+ {
+ "entropy": 2.1386052322387696,
+ "epoch": 0.3365249789671888,
+ "grad_norm": 3.484375,
+ "learning_rate": 1.327351385995834e-05,
+ "loss": 2.149888458251953,
+ "mean_token_accuracy": 0.522669050693512,
+ "num_tokens": 12605534.0,
+ "step": 2100
+ },
+ {
+ "entropy": 2.102600795030594,
+ "epoch": 0.3445374784664076,
+ "grad_norm": 3.796875,
+ "learning_rate": 1.3113283127703895e-05,
+ "loss": 2.1236566162109374,
+ "mean_token_accuracy": 0.5279092155396938,
+ "num_tokens": 12901218.0,
+ "step": 2150
+ },
+ {
+ "entropy": 2.132645187973976,
+ "epoch": 0.35254997796562637,
+ "grad_norm": 3.53125,
+ "learning_rate": 1.2953052395449447e-05,
+ "loss": 2.1523503112792968,
+ "mean_token_accuracy": 0.5198129612207413,
+ "num_tokens": 13213018.0,
+ "step": 2200
+ },
+ {
+ "entropy": 2.1204321217536926,
+ "epoch": 0.36056247746484515,
+ "grad_norm": 3.25,
+ "learning_rate": 1.2792821663195001e-05,
+ "loss": 2.1325447082519533,
+ "mean_token_accuracy": 0.5243636940419674,
+ "num_tokens": 13514408.0,
+ "step": 2250
+ },
+ {
+ "entropy": 2.102410731315613,
+ "epoch": 0.36857497696406394,
+ "grad_norm": 3.1875,
+ "learning_rate": 1.2632590930940555e-05,
+ "loss": 2.123565368652344,
+ "mean_token_accuracy": 0.5257614709436893,
+ "num_tokens": 13818272.0,
+ "step": 2300
+ },
+ {
+ "entropy": 2.0876205080747603,
+ "epoch": 0.3765874764632827,
+ "grad_norm": 3.328125,
+ "learning_rate": 1.247236019868611e-05,
+ "loss": 2.116696472167969,
+ "mean_token_accuracy": 0.5275124031305313,
+ "num_tokens": 14110702.0,
+ "step": 2350
+ },
+ {
+ "entropy": 2.1078518223762512,
+ "epoch": 0.3845999759625015,
+ "grad_norm": 3.453125,
+ "learning_rate": 1.2312129466431662e-05,
+ "loss": 2.1291993713378905,
+ "mean_token_accuracy": 0.524967464953661,
+ "num_tokens": 14407015.0,
+ "step": 2400
+ },
+ {
+ "entropy": 2.0930898267030718,
+ "epoch": 0.3926124754617203,
+ "grad_norm": 3.5,
+ "learning_rate": 1.2151898734177216e-05,
+ "loss": 2.1094123840332033,
+ "mean_token_accuracy": 0.5290587748587131,
+ "num_tokens": 14696565.0,
+ "step": 2450
+ },
+ {
+ "entropy": 2.1109757220745085,
+ "epoch": 0.4006249749609391,
+ "grad_norm": 3.5,
+ "learning_rate": 1.199166800192277e-05,
+ "loss": 2.1246888732910154,
+ "mean_token_accuracy": 0.5258957357704639,
+ "num_tokens": 14994401.0,
+ "step": 2500
+ },
+ {
+ "entropy": 2.1087811160087586,
+ "epoch": 0.40863747446015786,
+ "grad_norm": 3.1875,
+ "learning_rate": 1.1831437269668324e-05,
+ "loss": 2.12890625,
+ "mean_token_accuracy": 0.5269917643070221,
+ "num_tokens": 15292837.0,
+ "step": 2550
+ },
+ {
+ "entropy": 2.097252429127693,
+ "epoch": 0.41664997395937664,
+ "grad_norm": 3.0,
+ "learning_rate": 1.1671206537413878e-05,
+ "loss": 2.1145870971679686,
+ "mean_token_accuracy": 0.5274137698113919,
+ "num_tokens": 15590647.0,
+ "step": 2600
+ },
+ {
+ "entropy": 2.111199595332146,
+ "epoch": 0.4246624734585954,
+ "grad_norm": 3.25,
+ "learning_rate": 1.151097580515943e-05,
+ "loss": 2.1274327087402343,
+ "mean_token_accuracy": 0.5250249288976192,
+ "num_tokens": 15892772.0,
+ "step": 2650
+ },
+ {
+ "entropy": 2.1051502913236617,
+ "epoch": 0.4326749729578142,
+ "grad_norm": 3.203125,
+ "learning_rate": 1.1350745072904986e-05,
+ "loss": 2.127935333251953,
+ "mean_token_accuracy": 0.5263640700280666,
+ "num_tokens": 16191611.0,
+ "step": 2700
+ },
+ {
+ "entropy": 2.1127885603904724,
+ "epoch": 0.440687472457033,
+ "grad_norm": 3.515625,
+ "learning_rate": 1.1190514340650538e-05,
+ "loss": 2.129211883544922,
+ "mean_token_accuracy": 0.5250939278304577,
+ "num_tokens": 16483582.0,
+ "step": 2750
+ },
+ {
+ "entropy": 2.110930626988411,
+ "epoch": 0.4486999719562518,
+ "grad_norm": 3.1875,
+ "learning_rate": 1.103028360839609e-05,
+ "loss": 2.1220079040527344,
+ "mean_token_accuracy": 0.5250919266045093,
+ "num_tokens": 16787253.0,
+ "step": 2800
+ },
+ {
+ "entropy": 2.1182780677080153,
+ "epoch": 0.45671247145547056,
+ "grad_norm": 3.8125,
+ "learning_rate": 1.0870052876141646e-05,
+ "loss": 2.1358837890625,
+ "mean_token_accuracy": 0.5217854312062263,
+ "num_tokens": 17088741.0,
+ "step": 2850
+ },
+ {
+ "entropy": 2.1117833071947096,
+ "epoch": 0.46472497095468934,
+ "grad_norm": 3.171875,
+ "learning_rate": 1.0709822143887198e-05,
+ "loss": 2.125662841796875,
+ "mean_token_accuracy": 0.524649152457714,
+ "num_tokens": 17389236.0,
+ "step": 2900
+ },
+ {
+ "entropy": 2.1262730568647386,
+ "epoch": 0.47273747045390807,
+ "grad_norm": 3.625,
+ "learning_rate": 1.054959141163275e-05,
+ "loss": 2.142789611816406,
+ "mean_token_accuracy": 0.5236384868621826,
+ "num_tokens": 17693413.0,
+ "step": 2950
+ },
+ {
+ "entropy": 2.115410739183426,
+ "epoch": 0.48074996995312685,
+ "grad_norm": 3.140625,
+ "learning_rate": 1.0389360679378306e-05,
+ "loss": 2.1337106323242185,
+ "mean_token_accuracy": 0.5249222332239151,
+ "num_tokens": 17990326.0,
+ "step": 3000
+ },
+ {
+ "entropy": 2.0951381689310073,
+ "epoch": 0.48876246945234564,
+ "grad_norm": 3.234375,
+ "learning_rate": 1.0229129947123859e-05,
+ "loss": 2.108144073486328,
+ "mean_token_accuracy": 0.5269971822202205,
+ "num_tokens": 18293136.0,
+ "step": 3050
+ },
+ {
+ "entropy": 2.116977259516716,
+ "epoch": 0.4967749689515644,
+ "grad_norm": 3.421875,
+ "learning_rate": 1.0068899214869413e-05,
+ "loss": 2.1327830505371095,
+ "mean_token_accuracy": 0.5238270902633667,
+ "num_tokens": 18591481.0,
+ "step": 3100
+ },
+ {
+ "entropy": 2.08598925113678,
+ "epoch": 0.5047874684507833,
+ "grad_norm": 2.96875,
+ "learning_rate": 9.908668482614967e-06,
+ "loss": 2.1032344055175782,
+ "mean_token_accuracy": 0.5289211909472943,
+ "num_tokens": 18885221.0,
+ "step": 3150
+ },
+ {
+ "entropy": 2.097712662220001,
+ "epoch": 0.512799967950002,
+ "grad_norm": 3.40625,
+ "learning_rate": 9.74843775036052e-06,
+ "loss": 2.1150633239746095,
+ "mean_token_accuracy": 0.5288272814452648,
+ "num_tokens": 19184784.0,
+ "step": 3200
+ },
+ {
+ "entropy": 2.1063760107755662,
+ "epoch": 0.5208124674492208,
+ "grad_norm": 3.28125,
+ "learning_rate": 9.588207018106075e-06,
+ "loss": 2.1217109680175783,
+ "mean_token_accuracy": 0.526290989369154,
+ "num_tokens": 19482538.0,
+ "step": 3250
+ },
+ {
+ "entropy": 2.090182801485062,
+ "epoch": 0.5288249669484396,
+ "grad_norm": 2.875,
+ "learning_rate": 9.427976285851627e-06,
+ "loss": 2.1130311584472654,
+ "mean_token_accuracy": 0.5285439921915531,
+ "num_tokens": 19782544.0,
+ "step": 3300
+ },
+ {
+ "entropy": 2.103260806798935,
+ "epoch": 0.5368374664476584,
+ "grad_norm": 3.59375,
+ "learning_rate": 9.267745553597181e-06,
+ "loss": 2.116189727783203,
+ "mean_token_accuracy": 0.5273195558786392,
+ "num_tokens": 20086408.0,
+ "step": 3350
+ },
+ {
+ "entropy": 2.0673747235536575,
+ "epoch": 0.5448499659468772,
+ "grad_norm": 3.125,
+ "learning_rate": 9.107514821342735e-06,
+ "loss": 2.087482147216797,
+ "mean_token_accuracy": 0.532562478929758,
+ "num_tokens": 20380429.0,
+ "step": 3400
+ },
+ {
+ "entropy": 2.1030789083242416,
+ "epoch": 0.552862465446096,
+ "grad_norm": 3.296875,
+ "learning_rate": 8.947284089088289e-06,
+ "loss": 2.122591247558594,
+ "mean_token_accuracy": 0.5256703786551953,
+ "num_tokens": 20682849.0,
+ "step": 3450
+ },
+ {
+ "entropy": 2.0988716661930082,
+ "epoch": 0.5608749649453146,
+ "grad_norm": 3.546875,
+ "learning_rate": 8.787053356833841e-06,
+ "loss": 2.115867156982422,
+ "mean_token_accuracy": 0.5266730943322182,
+ "num_tokens": 20979225.0,
+ "step": 3500
+ },
+ {
+ "entropy": 2.0954072701931,
+ "epoch": 0.5688874644445334,
+ "grad_norm": 3.859375,
+ "learning_rate": 8.626822624579395e-06,
+ "loss": 2.1116868591308595,
+ "mean_token_accuracy": 0.5267495161294937,
+ "num_tokens": 21284166.0,
+ "step": 3550
+ },
+ {
+ "entropy": 2.11238471031189,
+ "epoch": 0.5768999639437522,
+ "grad_norm": 3.484375,
+ "learning_rate": 8.466591892324949e-06,
+ "loss": 2.1292352294921875,
+ "mean_token_accuracy": 0.5249243360757828,
+ "num_tokens": 21584612.0,
+ "step": 3600
+ },
+ {
+ "entropy": 2.091464368700981,
+ "epoch": 0.584912463442971,
+ "grad_norm": 3.359375,
+ "learning_rate": 8.306361160070501e-06,
+ "loss": 2.1099319458007812,
+ "mean_token_accuracy": 0.5292035435140133,
+ "num_tokens": 21879345.0,
+ "step": 3650
+ },
+ {
+ "entropy": 2.095204224586487,
+ "epoch": 0.5929249629421898,
+ "grad_norm": 2.96875,
+ "learning_rate": 8.146130427816055e-06,
+ "loss": 2.1198716735839844,
+ "mean_token_accuracy": 0.5255991271138192,
+ "num_tokens": 22183486.0,
+ "step": 3700
+ },
+ {
+ "entropy": 2.0666573679447175,
+ "epoch": 0.6009374624414086,
+ "grad_norm": 2.90625,
+ "learning_rate": 7.98589969556161e-06,
+ "loss": 2.08689697265625,
+ "mean_token_accuracy": 0.5318664258718491,
+ "num_tokens": 22476438.0,
+ "step": 3750
+ },
+ {
+ "entropy": 2.122153177857399,
+ "epoch": 0.6089499619406273,
+ "grad_norm": 3.40625,
+ "learning_rate": 7.825668963307162e-06,
+ "loss": 2.138288879394531,
+ "mean_token_accuracy": 0.5232584895193577,
+ "num_tokens": 22780385.0,
+ "step": 3800
+ },
+ {
+ "entropy": 2.1190748488903046,
+ "epoch": 0.6169624614398461,
+ "grad_norm": 3.46875,
+ "learning_rate": 7.665438231052716e-06,
+ "loss": 2.139790344238281,
+ "mean_token_accuracy": 0.5243287613987923,
+ "num_tokens": 23080847.0,
+ "step": 3850
+ },
+ {
+ "entropy": 2.1312570214271545,
+ "epoch": 0.6249749609390649,
+ "grad_norm": 3.765625,
+ "learning_rate": 7.5052074987982706e-06,
+ "loss": 2.1471080017089843,
+ "mean_token_accuracy": 0.5232046774029732,
+ "num_tokens": 23384738.0,
+ "step": 3900
+ },
+ {
+ "entropy": 2.0779008239507677,
+ "epoch": 0.6329874604382837,
+ "grad_norm": 3.171875,
+ "learning_rate": 7.344976766543823e-06,
+ "loss": 2.0974729919433592,
+ "mean_token_accuracy": 0.5305448272824287,
+ "num_tokens": 23682750.0,
+ "step": 3950
+ },
+ {
+ "entropy": 2.1121455317735673,
+ "epoch": 0.6409999599375025,
+ "grad_norm": 3.921875,
+ "learning_rate": 7.184746034289377e-06,
+ "loss": 2.127069549560547,
+ "mean_token_accuracy": 0.5252936625480652,
+ "num_tokens": 23989029.0,
+ "step": 4000
+ },
+ {
+ "entropy": 2.0982888478040693,
+ "epoch": 0.6490124594367213,
+ "grad_norm": 4.0625,
+ "learning_rate": 7.024515302034931e-06,
+ "loss": 2.112042236328125,
+ "mean_token_accuracy": 0.5281323495507241,
+ "num_tokens": 24292063.0,
+ "step": 4050
+ },
+ {
+ "entropy": 2.098363963365555,
+ "epoch": 0.65702495893594,
+ "grad_norm": 3.046875,
+ "learning_rate": 6.864284569780485e-06,
+ "loss": 2.1156489562988283,
+ "mean_token_accuracy": 0.5275120075047016,
+ "num_tokens": 24590201.0,
+ "step": 4100
+ },
+ {
+ "entropy": 2.1065632277727127,
+ "epoch": 0.6650374584351588,
+ "grad_norm": 3.375,
+ "learning_rate": 6.704053837526038e-06,
+ "loss": 2.12911376953125,
+ "mean_token_accuracy": 0.5262684133648873,
+ "num_tokens": 24892122.0,
+ "step": 4150
+ },
+ {
+ "entropy": 2.1014072370529173,
+ "epoch": 0.6730499579343776,
+ "grad_norm": 2.84375,
+ "learning_rate": 6.543823105271592e-06,
+ "loss": 2.1130335998535155,
+ "mean_token_accuracy": 0.5262750512361527,
+ "num_tokens": 25194969.0,
+ "step": 4200
+ },
+ {
+ "entropy": 2.0905236744880678,
+ "epoch": 0.6810624574335964,
+ "grad_norm": 3.03125,
+ "learning_rate": 6.383592373017146e-06,
+ "loss": 2.1153533935546873,
+ "mean_token_accuracy": 0.527137616276741,
+ "num_tokens": 25499521.0,
+ "step": 4250
+ },
+ {
+ "entropy": 2.1032627063989637,
+ "epoch": 0.6890749569328152,
+ "grad_norm": 3.125,
+ "learning_rate": 6.223361640762698e-06,
+ "loss": 2.122831115722656,
+ "mean_token_accuracy": 0.5254572249948979,
+ "num_tokens": 25798933.0,
+ "step": 4300
+ },
+ {
+ "entropy": 2.118139310479164,
+ "epoch": 0.697087456432034,
+ "grad_norm": 3.28125,
+ "learning_rate": 6.063130908508252e-06,
+ "loss": 2.1376751708984374,
+ "mean_token_accuracy": 0.5243131402134895,
+ "num_tokens": 26096619.0,
+ "step": 4350
+ },
+ {
+ "entropy": 2.082554380297661,
+ "epoch": 0.7050999559312527,
+ "grad_norm": 3.390625,
+ "learning_rate": 5.902900176253806e-06,
+ "loss": 2.0949493408203126,
+ "mean_token_accuracy": 0.5316410692036152,
+ "num_tokens": 26394395.0,
+ "step": 4400
+ },
+ {
+ "entropy": 2.09870474755764,
+ "epoch": 0.7131124554304715,
+ "grad_norm": 3.421875,
+ "learning_rate": 5.7426694439993594e-06,
+ "loss": 2.1184346008300783,
+ "mean_token_accuracy": 0.5267573061585427,
+ "num_tokens": 26692333.0,
+ "step": 4450
+ },
+ {
+ "entropy": 2.0866885662078856,
+ "epoch": 0.7211249549296903,
+ "grad_norm": 3.71875,
+ "learning_rate": 5.5824387117449134e-06,
+ "loss": 2.102904968261719,
+ "mean_token_accuracy": 0.529726449996233,
+ "num_tokens": 26989384.0,
+ "step": 4500
+ }
+ ],
+ "logging_steps": 50,
+ "max_steps": 6241,
+ "num_input_tokens_seen": 0,
+ "num_train_epochs": 1,
+ "save_steps": 500,
+ "stateful_callbacks": {
+ "TrainerControl": {
+ "args": {
+ "should_epoch_stop": false,
+ "should_evaluate": false,
+ "should_log": false,
+ "should_save": true,
+ "should_training_stop": false
+ },
+ "attributes": {}
+ }
+ },
+ "total_flos": 3.1866411651588096e+17,
+ "train_batch_size": 4,
+ "trial_name": null,
+ "trial_params": null
+}
diff --git a/checkpoint-4500/training_args.bin b/checkpoint-4500/training_args.bin
new file mode 100644
index 0000000000000000000000000000000000000000..4b409d5a147856f0d5a3d3da0f3b4922a9def7dd
--- /dev/null
+++ b/checkpoint-4500/training_args.bin
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:78622f63304ec0913ed0b4ea7a1b26c08a5621b3661a380d798c4ea426235671
+size 5713
diff --git a/checkpoint-500/chat_template.jinja b/checkpoint-500/chat_template.jinja
new file mode 100644
index 0000000000000000000000000000000000000000..01be9b307daa2d425f7c168c9fb145a286e0afb4
--- /dev/null
+++ b/checkpoint-500/chat_template.jinja
@@ -0,0 +1,89 @@
+{%- if tools %}
+ {{- '<|im_start|>system\n' }}
+ {%- if messages[0].role == 'system' %}
+ {{- messages[0].content + '\n\n' }}
+ {%- endif %}
+ {{- "# Tools\n\nYou may call one or more functions to assist with the user query.\n\nYou are provided with function signatures within XML tags:\n" }}
+ {%- for tool in tools %}
+ {{- "\n" }}
+ {{- tool | tojson }}
+ {%- endfor %}
+ {{- "\n\n\nFor each function call, return a json object with function name and arguments within XML tags:\n\n{\"name\": , \"arguments\": }\n<|im_end|>\n" }}
+{%- else %}
+ {%- if messages[0].role == 'system' %}
+ {{- '<|im_start|>system\n' + messages[0].content + '<|im_end|>\n' }}
+ {%- endif %}
+{%- endif %}
+{%- set ns = namespace(multi_step_tool=true, last_query_index=messages|length - 1) %}
+{%- for message in messages[::-1] %}
+ {%- set index = (messages|length - 1) - loop.index0 %}
+ {%- if ns.multi_step_tool and message.role == "user" and message.content is string and not(message.content.startswith('') and message.content.endswith('')) %}
+ {%- set ns.multi_step_tool = false %}
+ {%- set ns.last_query_index = index %}
+ {%- endif %}
+{%- endfor %}
+{%- for message in messages %}
+ {%- if message.content is string %}
+ {%- set content = message.content %}
+ {%- else %}
+ {%- set content = '' %}
+ {%- endif %}
+ {%- if (message.role == "user") or (message.role == "system" and not loop.first) %}
+ {{- '<|im_start|>' + message.role + '\n' + content + '<|im_end|>' + '\n' }}
+ {%- elif message.role == "assistant" %}
+ {%- set reasoning_content = '' %}
+ {%- if message.reasoning_content is string %}
+ {%- set reasoning_content = message.reasoning_content %}
+ {%- else %}
+ {%- if '' in content %}
+ {%- set reasoning_content = content.split('')[0].rstrip('\n').split('')[-1].lstrip('\n') %}
+ {%- set content = content.split('')[-1].lstrip('\n') %}
+ {%- endif %}
+ {%- endif %}
+ {%- if loop.index0 > ns.last_query_index %}
+ {%- if loop.last or (not loop.last and reasoning_content) %}
+ {{- '<|im_start|>' + message.role + '\n\n' + reasoning_content.strip('\n') + '\n\n\n' + content.lstrip('\n') }}
+ {%- else %}
+ {{- '<|im_start|>' + message.role + '\n' + content }}
+ {%- endif %}
+ {%- else %}
+ {{- '<|im_start|>' + message.role + '\n' + content }}
+ {%- endif %}
+ {%- if message.tool_calls %}
+ {%- for tool_call in message.tool_calls %}
+ {%- if (loop.first and content) or (not loop.first) %}
+ {{- '\n' }}
+ {%- endif %}
+ {%- if tool_call.function %}
+ {%- set tool_call = tool_call.function %}
+ {%- endif %}
+ {{- '\n{"name": "' }}
+ {{- tool_call.name }}
+ {{- '", "arguments": ' }}
+ {%- if tool_call.arguments is string %}
+ {{- tool_call.arguments }}
+ {%- else %}
+ {{- tool_call.arguments | tojson }}
+ {%- endif %}
+ {{- '}\n' }}
+ {%- endfor %}
+ {%- endif %}
+ {{- '<|im_end|>\n' }}
+ {%- elif message.role == "tool" %}
+ {%- if loop.first or (messages[loop.index0 - 1].role != "tool") %}
+ {{- '<|im_start|>user' }}
+ {%- endif %}
+ {{- '\n\n' }}
+ {{- content }}
+ {{- '\n' }}
+ {%- if loop.last or (messages[loop.index0 + 1].role != "tool") %}
+ {{- '<|im_end|>\n' }}
+ {%- endif %}
+ {%- endif %}
+{%- endfor %}
+{%- if add_generation_prompt %}
+ {{- '<|im_start|>assistant\n' }}
+ {%- if enable_thinking is defined and enable_thinking is false %}
+ {{- '\n\n\n\n' }}
+ {%- endif %}
+{%- endif %}
\ No newline at end of file
diff --git a/checkpoint-500/config.json b/checkpoint-500/config.json
new file mode 100644
index 0000000000000000000000000000000000000000..1d219d6a6d288625229a71646710b85cf7ae5def
--- /dev/null
+++ b/checkpoint-500/config.json
@@ -0,0 +1,63 @@
+{
+ "architectures": [
+ "Qwen3ForCausalLM"
+ ],
+ "attention_bias": false,
+ "attention_dropout": 0.0,
+ "bos_token_id": null,
+ "dtype": "bfloat16",
+ "eos_token_id": 151645,
+ "head_dim": 128,
+ "hidden_act": "silu",
+ "hidden_size": 2048,
+ "initializer_range": 0.02,
+ "intermediate_size": 6144,
+ "layer_types": [
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention"
+ ],
+ "max_position_embeddings": 40960,
+ "max_window_layers": 28,
+ "model_type": "qwen3",
+ "num_attention_heads": 16,
+ "num_hidden_layers": 28,
+ "num_key_value_heads": 8,
+ "pad_token_id": 151643,
+ "rms_norm_eps": 1e-06,
+ "rope_parameters": {
+ "rope_theta": 1000000,
+ "rope_type": "default"
+ },
+ "sliding_window": null,
+ "tie_word_embeddings": true,
+ "transformers_version": "5.7.0",
+ "use_cache": false,
+ "use_sliding_window": false,
+ "vocab_size": 151936
+}
diff --git a/checkpoint-500/generation_config.json b/checkpoint-500/generation_config.json
new file mode 100644
index 0000000000000000000000000000000000000000..5ec133d867b1427d43a5ee0ea332bd088fc12013
--- /dev/null
+++ b/checkpoint-500/generation_config.json
@@ -0,0 +1,12 @@
+{
+ "do_sample": true,
+ "eos_token_id": [
+ 151645,
+ 151643
+ ],
+ "pad_token_id": 151643,
+ "temperature": 0.6,
+ "top_k": 20,
+ "top_p": 0.95,
+ "transformers_version": "5.7.0"
+}
diff --git a/checkpoint-500/model.safetensors b/checkpoint-500/model.safetensors
new file mode 100644
index 0000000000000000000000000000000000000000..cb8297bb3e314ce55d47b6213197ffafe2a2ce07
--- /dev/null
+++ b/checkpoint-500/model.safetensors
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:510bdc2405d027342acd0e2047289831efc0b3f25febcd5c74681c16385dd5c2
+size 3441185608
diff --git a/checkpoint-500/optimizer.pt b/checkpoint-500/optimizer.pt
new file mode 100644
index 0000000000000000000000000000000000000000..0e2302727dbce2d412a67d0550e36ef0538238d0
--- /dev/null
+++ b/checkpoint-500/optimizer.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:3466f40535b1f6a6dd5a3dfad0962141622b90ac955b344d5278efbb20cd2ba1
+size 6882572207
diff --git a/checkpoint-500/rng_state.pth b/checkpoint-500/rng_state.pth
new file mode 100644
index 0000000000000000000000000000000000000000..5a8f17addf8aaee350625e9847751582bc9f86dd
--- /dev/null
+++ b/checkpoint-500/rng_state.pth
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:61c19bab1174704a4a4441475683bf1270277af15d2e2c95e964789128e482c4
+size 14645
diff --git a/checkpoint-500/scheduler.pt b/checkpoint-500/scheduler.pt
new file mode 100644
index 0000000000000000000000000000000000000000..dc233f2448c25c6d3aef7d688603929e344c0467
--- /dev/null
+++ b/checkpoint-500/scheduler.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e36e50838b6f150abbf126d12ba88d9ca9bcc1b4d6927bad9696043424255e41
+size 1465
diff --git a/checkpoint-500/tokenizer.json b/checkpoint-500/tokenizer.json
new file mode 100644
index 0000000000000000000000000000000000000000..c7afbed2efcdf019f88ab0572ec29d3bf595dfe2
--- /dev/null
+++ b/checkpoint-500/tokenizer.json
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:be75606093db2094d7cd20f3c2f385c212750648bd6ea4fb2bf507a6a4c55506
+size 11422650
diff --git a/checkpoint-500/tokenizer_config.json b/checkpoint-500/tokenizer_config.json
new file mode 100644
index 0000000000000000000000000000000000000000..770e41d6c92519d525eede4cbcf3ba27f6425311
--- /dev/null
+++ b/checkpoint-500/tokenizer_config.json
@@ -0,0 +1,30 @@
+{
+ "add_prefix_space": false,
+ "backend": "tokenizers",
+ "bos_token": null,
+ "clean_up_tokenization_spaces": false,
+ "eos_token": "<|im_end|>",
+ "errors": "replace",
+ "extra_special_tokens": [
+ "<|im_start|>",
+ "<|im_end|>",
+ "<|object_ref_start|>",
+ "<|object_ref_end|>",
+ "<|box_start|>",
+ "<|box_end|>",
+ "<|quad_start|>",
+ "<|quad_end|>",
+ "<|vision_start|>",
+ "<|vision_end|>",
+ "<|vision_pad|>",
+ "<|image_pad|>",
+ "<|video_pad|>"
+ ],
+ "is_local": false,
+ "local_files_only": false,
+ "model_max_length": 131072,
+ "pad_token": "<|endoftext|>",
+ "split_special_tokens": false,
+ "tokenizer_class": "Qwen2Tokenizer",
+ "unk_token": null
+}
diff --git a/checkpoint-500/trainer_state.json b/checkpoint-500/trainer_state.json
new file mode 100644
index 0000000000000000000000000000000000000000..47a03323f9a39d7344585be5009922b04a6cf3fa
--- /dev/null
+++ b/checkpoint-500/trainer_state.json
@@ -0,0 +1,134 @@
+{
+ "best_global_step": null,
+ "best_metric": null,
+ "best_model_checkpoint": null,
+ "epoch": 0.08012499499218781,
+ "eval_steps": 500,
+ "global_step": 500,
+ "is_hyper_param_search": false,
+ "is_local_process_zero": true,
+ "is_world_process_zero": true,
+ "log_history": [
+ {
+ "entropy": 2.223189059495926,
+ "epoch": 0.008012499499218782,
+ "grad_norm": 2.875,
+ "learning_rate": 1.9842973882390643e-05,
+ "loss": 2.3428109741210936,
+ "mean_token_accuracy": 0.4963963569700718,
+ "num_tokens": 305366.0,
+ "step": 50
+ },
+ {
+ "entropy": 2.188130117058754,
+ "epoch": 0.016024998998437564,
+ "grad_norm": 3.328125,
+ "learning_rate": 1.96827431501362e-05,
+ "loss": 2.225358734130859,
+ "mean_token_accuracy": 0.5110921229422093,
+ "num_tokens": 605042.0,
+ "step": 100
+ },
+ {
+ "entropy": 2.197347580194473,
+ "epoch": 0.024037498497656344,
+ "grad_norm": 3.453125,
+ "learning_rate": 1.952251241788175e-05,
+ "loss": 2.2299420166015627,
+ "mean_token_accuracy": 0.5093490639328957,
+ "num_tokens": 906888.0,
+ "step": 150
+ },
+ {
+ "entropy": 2.1913950395584108,
+ "epoch": 0.03204999799687513,
+ "grad_norm": 2.90625,
+ "learning_rate": 1.9362281685627303e-05,
+ "loss": 2.210995941162109,
+ "mean_token_accuracy": 0.5097909501194954,
+ "num_tokens": 1213125.0,
+ "step": 200
+ },
+ {
+ "entropy": 2.1611998677253723,
+ "epoch": 0.040062497496093905,
+ "grad_norm": 3.25,
+ "learning_rate": 1.920205095337286e-05,
+ "loss": 2.1867474365234374,
+ "mean_token_accuracy": 0.5154410660266876,
+ "num_tokens": 1509026.0,
+ "step": 250
+ },
+ {
+ "entropy": 2.168051454424858,
+ "epoch": 0.04807499699531269,
+ "grad_norm": 2.9375,
+ "learning_rate": 1.904182022111841e-05,
+ "loss": 2.190071716308594,
+ "mean_token_accuracy": 0.5156884534657001,
+ "num_tokens": 1806850.0,
+ "step": 300
+ },
+ {
+ "entropy": 2.1737154543399813,
+ "epoch": 0.05608749649453147,
+ "grad_norm": 3.421875,
+ "learning_rate": 1.8881589488863967e-05,
+ "loss": 2.199147186279297,
+ "mean_token_accuracy": 0.514045044630766,
+ "num_tokens": 2104505.0,
+ "step": 350
+ },
+ {
+ "entropy": 2.148039159178734,
+ "epoch": 0.06409999599375026,
+ "grad_norm": 3.109375,
+ "learning_rate": 1.872135875660952e-05,
+ "loss": 2.176946105957031,
+ "mean_token_accuracy": 0.5177112428843975,
+ "num_tokens": 2391088.0,
+ "step": 400
+ },
+ {
+ "entropy": 2.194890711903572,
+ "epoch": 0.07211249549296903,
+ "grad_norm": 3.28125,
+ "learning_rate": 1.856112802435507e-05,
+ "loss": 2.216288757324219,
+ "mean_token_accuracy": 0.5110335703194141,
+ "num_tokens": 2691272.0,
+ "step": 450
+ },
+ {
+ "entropy": 2.1557786524295808,
+ "epoch": 0.08012499499218781,
+ "grad_norm": 4.15625,
+ "learning_rate": 1.8400897292100627e-05,
+ "loss": 2.1812680053710936,
+ "mean_token_accuracy": 0.5176697050035,
+ "num_tokens": 2991855.0,
+ "step": 500
+ }
+ ],
+ "logging_steps": 50,
+ "max_steps": 6241,
+ "num_input_tokens_seen": 0,
+ "num_train_epochs": 1,
+ "save_steps": 500,
+ "stateful_callbacks": {
+ "TrainerControl": {
+ "args": {
+ "should_epoch_stop": false,
+ "should_evaluate": false,
+ "should_log": false,
+ "should_save": true,
+ "should_training_stop": false
+ },
+ "attributes": {}
+ }
+ },
+ "total_flos": 3.560511422243635e+16,
+ "train_batch_size": 4,
+ "trial_name": null,
+ "trial_params": null
+}
diff --git a/checkpoint-500/training_args.bin b/checkpoint-500/training_args.bin
new file mode 100644
index 0000000000000000000000000000000000000000..4b409d5a147856f0d5a3d3da0f3b4922a9def7dd
--- /dev/null
+++ b/checkpoint-500/training_args.bin
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:78622f63304ec0913ed0b4ea7a1b26c08a5621b3661a380d798c4ea426235671
+size 5713
diff --git a/checkpoint-5000/chat_template.jinja b/checkpoint-5000/chat_template.jinja
new file mode 100644
index 0000000000000000000000000000000000000000..01be9b307daa2d425f7c168c9fb145a286e0afb4
--- /dev/null
+++ b/checkpoint-5000/chat_template.jinja
@@ -0,0 +1,89 @@
+{%- if tools %}
+ {{- '<|im_start|>system\n' }}
+ {%- if messages[0].role == 'system' %}
+ {{- messages[0].content + '\n\n' }}
+ {%- endif %}
+ {{- "# Tools\n\nYou may call one or more functions to assist with the user query.\n\nYou are provided with function signatures within XML tags:\n" }}
+ {%- for tool in tools %}
+ {{- "\n" }}
+ {{- tool | tojson }}
+ {%- endfor %}
+ {{- "\n\n\nFor each function call, return a json object with function name and arguments within XML tags:\n\n{\"name\": , \"arguments\": }\n<|im_end|>\n" }}
+{%- else %}
+ {%- if messages[0].role == 'system' %}
+ {{- '<|im_start|>system\n' + messages[0].content + '<|im_end|>\n' }}
+ {%- endif %}
+{%- endif %}
+{%- set ns = namespace(multi_step_tool=true, last_query_index=messages|length - 1) %}
+{%- for message in messages[::-1] %}
+ {%- set index = (messages|length - 1) - loop.index0 %}
+ {%- if ns.multi_step_tool and message.role == "user" and message.content is string and not(message.content.startswith('') and message.content.endswith('')) %}
+ {%- set ns.multi_step_tool = false %}
+ {%- set ns.last_query_index = index %}
+ {%- endif %}
+{%- endfor %}
+{%- for message in messages %}
+ {%- if message.content is string %}
+ {%- set content = message.content %}
+ {%- else %}
+ {%- set content = '' %}
+ {%- endif %}
+ {%- if (message.role == "user") or (message.role == "system" and not loop.first) %}
+ {{- '<|im_start|>' + message.role + '\n' + content + '<|im_end|>' + '\n' }}
+ {%- elif message.role == "assistant" %}
+ {%- set reasoning_content = '' %}
+ {%- if message.reasoning_content is string %}
+ {%- set reasoning_content = message.reasoning_content %}
+ {%- else %}
+ {%- if '' in content %}
+ {%- set reasoning_content = content.split('')[0].rstrip('\n').split('')[-1].lstrip('\n') %}
+ {%- set content = content.split('')[-1].lstrip('\n') %}
+ {%- endif %}
+ {%- endif %}
+ {%- if loop.index0 > ns.last_query_index %}
+ {%- if loop.last or (not loop.last and reasoning_content) %}
+ {{- '<|im_start|>' + message.role + '\n\n' + reasoning_content.strip('\n') + '\n\n\n' + content.lstrip('\n') }}
+ {%- else %}
+ {{- '<|im_start|>' + message.role + '\n' + content }}
+ {%- endif %}
+ {%- else %}
+ {{- '<|im_start|>' + message.role + '\n' + content }}
+ {%- endif %}
+ {%- if message.tool_calls %}
+ {%- for tool_call in message.tool_calls %}
+ {%- if (loop.first and content) or (not loop.first) %}
+ {{- '\n' }}
+ {%- endif %}
+ {%- if tool_call.function %}
+ {%- set tool_call = tool_call.function %}
+ {%- endif %}
+ {{- '\n{"name": "' }}
+ {{- tool_call.name }}
+ {{- '", "arguments": ' }}
+ {%- if tool_call.arguments is string %}
+ {{- tool_call.arguments }}
+ {%- else %}
+ {{- tool_call.arguments | tojson }}
+ {%- endif %}
+ {{- '}\n' }}
+ {%- endfor %}
+ {%- endif %}
+ {{- '<|im_end|>\n' }}
+ {%- elif message.role == "tool" %}
+ {%- if loop.first or (messages[loop.index0 - 1].role != "tool") %}
+ {{- '<|im_start|>user' }}
+ {%- endif %}
+ {{- '\n\n' }}
+ {{- content }}
+ {{- '\n' }}
+ {%- if loop.last or (messages[loop.index0 + 1].role != "tool") %}
+ {{- '<|im_end|>\n' }}
+ {%- endif %}
+ {%- endif %}
+{%- endfor %}
+{%- if add_generation_prompt %}
+ {{- '<|im_start|>assistant\n' }}
+ {%- if enable_thinking is defined and enable_thinking is false %}
+ {{- '\n\n\n\n' }}
+ {%- endif %}
+{%- endif %}
\ No newline at end of file
diff --git a/checkpoint-5000/config.json b/checkpoint-5000/config.json
new file mode 100644
index 0000000000000000000000000000000000000000..1d219d6a6d288625229a71646710b85cf7ae5def
--- /dev/null
+++ b/checkpoint-5000/config.json
@@ -0,0 +1,63 @@
+{
+ "architectures": [
+ "Qwen3ForCausalLM"
+ ],
+ "attention_bias": false,
+ "attention_dropout": 0.0,
+ "bos_token_id": null,
+ "dtype": "bfloat16",
+ "eos_token_id": 151645,
+ "head_dim": 128,
+ "hidden_act": "silu",
+ "hidden_size": 2048,
+ "initializer_range": 0.02,
+ "intermediate_size": 6144,
+ "layer_types": [
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention"
+ ],
+ "max_position_embeddings": 40960,
+ "max_window_layers": 28,
+ "model_type": "qwen3",
+ "num_attention_heads": 16,
+ "num_hidden_layers": 28,
+ "num_key_value_heads": 8,
+ "pad_token_id": 151643,
+ "rms_norm_eps": 1e-06,
+ "rope_parameters": {
+ "rope_theta": 1000000,
+ "rope_type": "default"
+ },
+ "sliding_window": null,
+ "tie_word_embeddings": true,
+ "transformers_version": "5.7.0",
+ "use_cache": false,
+ "use_sliding_window": false,
+ "vocab_size": 151936
+}
diff --git a/checkpoint-5000/generation_config.json b/checkpoint-5000/generation_config.json
new file mode 100644
index 0000000000000000000000000000000000000000..5ec133d867b1427d43a5ee0ea332bd088fc12013
--- /dev/null
+++ b/checkpoint-5000/generation_config.json
@@ -0,0 +1,12 @@
+{
+ "do_sample": true,
+ "eos_token_id": [
+ 151645,
+ 151643
+ ],
+ "pad_token_id": 151643,
+ "temperature": 0.6,
+ "top_k": 20,
+ "top_p": 0.95,
+ "transformers_version": "5.7.0"
+}
diff --git a/checkpoint-5000/model.safetensors b/checkpoint-5000/model.safetensors
new file mode 100644
index 0000000000000000000000000000000000000000..591746f183e1225a2743ffe685fe73a7a1cc1980
--- /dev/null
+++ b/checkpoint-5000/model.safetensors
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9210c0f8c0427681f33f67f527b1ec49a79f01625fbc1759118f2ecc59421a63
+size 3441185608
diff --git a/checkpoint-5000/optimizer.pt b/checkpoint-5000/optimizer.pt
new file mode 100644
index 0000000000000000000000000000000000000000..8c34497b6483af5d1536955a8b837708a1afdf9c
--- /dev/null
+++ b/checkpoint-5000/optimizer.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:0640613439df62a0358096494db2b4d610bf4a9169660130ece88bebfa231026
+size 6882572207
diff --git a/checkpoint-5000/rng_state.pth b/checkpoint-5000/rng_state.pth
new file mode 100644
index 0000000000000000000000000000000000000000..5a8f17addf8aaee350625e9847751582bc9f86dd
--- /dev/null
+++ b/checkpoint-5000/rng_state.pth
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:61c19bab1174704a4a4441475683bf1270277af15d2e2c95e964789128e482c4
+size 14645
diff --git a/checkpoint-5000/scheduler.pt b/checkpoint-5000/scheduler.pt
new file mode 100644
index 0000000000000000000000000000000000000000..f10b6dc631e78c424c7dcd61336c154bc2b80d8c
--- /dev/null
+++ b/checkpoint-5000/scheduler.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:25aa8e41a989b4b3c58d4ee4aed8e685dad485c899acf803301071d8faa75034
+size 1465
diff --git a/checkpoint-5000/tokenizer.json b/checkpoint-5000/tokenizer.json
new file mode 100644
index 0000000000000000000000000000000000000000..c7afbed2efcdf019f88ab0572ec29d3bf595dfe2
--- /dev/null
+++ b/checkpoint-5000/tokenizer.json
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:be75606093db2094d7cd20f3c2f385c212750648bd6ea4fb2bf507a6a4c55506
+size 11422650
diff --git a/checkpoint-5000/tokenizer_config.json b/checkpoint-5000/tokenizer_config.json
new file mode 100644
index 0000000000000000000000000000000000000000..770e41d6c92519d525eede4cbcf3ba27f6425311
--- /dev/null
+++ b/checkpoint-5000/tokenizer_config.json
@@ -0,0 +1,30 @@
+{
+ "add_prefix_space": false,
+ "backend": "tokenizers",
+ "bos_token": null,
+ "clean_up_tokenization_spaces": false,
+ "eos_token": "<|im_end|>",
+ "errors": "replace",
+ "extra_special_tokens": [
+ "<|im_start|>",
+ "<|im_end|>",
+ "<|object_ref_start|>",
+ "<|object_ref_end|>",
+ "<|box_start|>",
+ "<|box_end|>",
+ "<|quad_start|>",
+ "<|quad_end|>",
+ "<|vision_start|>",
+ "<|vision_end|>",
+ "<|vision_pad|>",
+ "<|image_pad|>",
+ "<|video_pad|>"
+ ],
+ "is_local": false,
+ "local_files_only": false,
+ "model_max_length": 131072,
+ "pad_token": "<|endoftext|>",
+ "split_special_tokens": false,
+ "tokenizer_class": "Qwen2Tokenizer",
+ "unk_token": null
+}
diff --git a/checkpoint-5000/trainer_state.json b/checkpoint-5000/trainer_state.json
new file mode 100644
index 0000000000000000000000000000000000000000..43cbf87802692535735307d76a825f72887ed76a
--- /dev/null
+++ b/checkpoint-5000/trainer_state.json
@@ -0,0 +1,1034 @@
+{
+ "best_global_step": null,
+ "best_metric": null,
+ "best_model_checkpoint": null,
+ "epoch": 0.8012499499218781,
+ "eval_steps": 500,
+ "global_step": 5000,
+ "is_hyper_param_search": false,
+ "is_local_process_zero": true,
+ "is_world_process_zero": true,
+ "log_history": [
+ {
+ "entropy": 2.223189059495926,
+ "epoch": 0.008012499499218782,
+ "grad_norm": 2.875,
+ "learning_rate": 1.9842973882390643e-05,
+ "loss": 2.3428109741210936,
+ "mean_token_accuracy": 0.4963963569700718,
+ "num_tokens": 305366.0,
+ "step": 50
+ },
+ {
+ "entropy": 2.188130117058754,
+ "epoch": 0.016024998998437564,
+ "grad_norm": 3.328125,
+ "learning_rate": 1.96827431501362e-05,
+ "loss": 2.225358734130859,
+ "mean_token_accuracy": 0.5110921229422093,
+ "num_tokens": 605042.0,
+ "step": 100
+ },
+ {
+ "entropy": 2.197347580194473,
+ "epoch": 0.024037498497656344,
+ "grad_norm": 3.453125,
+ "learning_rate": 1.952251241788175e-05,
+ "loss": 2.2299420166015627,
+ "mean_token_accuracy": 0.5093490639328957,
+ "num_tokens": 906888.0,
+ "step": 150
+ },
+ {
+ "entropy": 2.1913950395584108,
+ "epoch": 0.03204999799687513,
+ "grad_norm": 2.90625,
+ "learning_rate": 1.9362281685627303e-05,
+ "loss": 2.210995941162109,
+ "mean_token_accuracy": 0.5097909501194954,
+ "num_tokens": 1213125.0,
+ "step": 200
+ },
+ {
+ "entropy": 2.1611998677253723,
+ "epoch": 0.040062497496093905,
+ "grad_norm": 3.25,
+ "learning_rate": 1.920205095337286e-05,
+ "loss": 2.1867474365234374,
+ "mean_token_accuracy": 0.5154410660266876,
+ "num_tokens": 1509026.0,
+ "step": 250
+ },
+ {
+ "entropy": 2.168051454424858,
+ "epoch": 0.04807499699531269,
+ "grad_norm": 2.9375,
+ "learning_rate": 1.904182022111841e-05,
+ "loss": 2.190071716308594,
+ "mean_token_accuracy": 0.5156884534657001,
+ "num_tokens": 1806850.0,
+ "step": 300
+ },
+ {
+ "entropy": 2.1737154543399813,
+ "epoch": 0.05608749649453147,
+ "grad_norm": 3.421875,
+ "learning_rate": 1.8881589488863967e-05,
+ "loss": 2.199147186279297,
+ "mean_token_accuracy": 0.514045044630766,
+ "num_tokens": 2104505.0,
+ "step": 350
+ },
+ {
+ "entropy": 2.148039159178734,
+ "epoch": 0.06409999599375026,
+ "grad_norm": 3.109375,
+ "learning_rate": 1.872135875660952e-05,
+ "loss": 2.176946105957031,
+ "mean_token_accuracy": 0.5177112428843975,
+ "num_tokens": 2391088.0,
+ "step": 400
+ },
+ {
+ "entropy": 2.194890711903572,
+ "epoch": 0.07211249549296903,
+ "grad_norm": 3.28125,
+ "learning_rate": 1.856112802435507e-05,
+ "loss": 2.216288757324219,
+ "mean_token_accuracy": 0.5110335703194141,
+ "num_tokens": 2691272.0,
+ "step": 450
+ },
+ {
+ "entropy": 2.1557786524295808,
+ "epoch": 0.08012499499218781,
+ "grad_norm": 4.15625,
+ "learning_rate": 1.8400897292100627e-05,
+ "loss": 2.1812680053710936,
+ "mean_token_accuracy": 0.5176697050035,
+ "num_tokens": 2991855.0,
+ "step": 500
+ },
+ {
+ "entropy": 2.1708167713880537,
+ "epoch": 0.08813749449140659,
+ "grad_norm": 4.34375,
+ "learning_rate": 1.824066655984618e-05,
+ "loss": 2.1921002197265627,
+ "mean_token_accuracy": 0.5156507430970669,
+ "num_tokens": 3295049.0,
+ "step": 550
+ },
+ {
+ "entropy": 2.1686314886808393,
+ "epoch": 0.09614999399062538,
+ "grad_norm": 2.875,
+ "learning_rate": 1.8080435827591735e-05,
+ "loss": 2.1914617919921877,
+ "mean_token_accuracy": 0.5139613692462445,
+ "num_tokens": 3597206.0,
+ "step": 600
+ },
+ {
+ "entropy": 2.15742906332016,
+ "epoch": 0.10416249348984416,
+ "grad_norm": 3.109375,
+ "learning_rate": 1.7920205095337287e-05,
+ "loss": 2.174853973388672,
+ "mean_token_accuracy": 0.5159518288075924,
+ "num_tokens": 3905268.0,
+ "step": 650
+ },
+ {
+ "entropy": 2.162363375425339,
+ "epoch": 0.11217499298906294,
+ "grad_norm": 3.171875,
+ "learning_rate": 1.7759974363082843e-05,
+ "loss": 2.184231872558594,
+ "mean_token_accuracy": 0.516631413847208,
+ "num_tokens": 4207934.0,
+ "step": 700
+ },
+ {
+ "entropy": 2.1516289287805557,
+ "epoch": 0.12018749248828171,
+ "grad_norm": 3.609375,
+ "learning_rate": 1.7599743630828395e-05,
+ "loss": 2.167020263671875,
+ "mean_token_accuracy": 0.516358617246151,
+ "num_tokens": 4515774.0,
+ "step": 750
+ },
+ {
+ "entropy": 2.1549544781446457,
+ "epoch": 0.1281999919875005,
+ "grad_norm": 3.28125,
+ "learning_rate": 1.7439512898573947e-05,
+ "loss": 2.181917877197266,
+ "mean_token_accuracy": 0.5182220858335495,
+ "num_tokens": 4814709.0,
+ "step": 800
+ },
+ {
+ "entropy": 2.1532615756988527,
+ "epoch": 0.1362124914867193,
+ "grad_norm": 3.078125,
+ "learning_rate": 1.7279282166319503e-05,
+ "loss": 2.1659007263183594,
+ "mean_token_accuracy": 0.5180425970256328,
+ "num_tokens": 5112585.0,
+ "step": 850
+ },
+ {
+ "entropy": 2.1704017293453215,
+ "epoch": 0.14422499098593805,
+ "grad_norm": 2.921875,
+ "learning_rate": 1.7119051434065055e-05,
+ "loss": 2.1934996032714844,
+ "mean_token_accuracy": 0.5154964691400528,
+ "num_tokens": 5411369.0,
+ "step": 900
+ },
+ {
+ "entropy": 2.1349107217788696,
+ "epoch": 0.15223749048515683,
+ "grad_norm": 2.984375,
+ "learning_rate": 1.6958820701810608e-05,
+ "loss": 2.1598545837402345,
+ "mean_token_accuracy": 0.5208920259773732,
+ "num_tokens": 5707060.0,
+ "step": 950
+ },
+ {
+ "entropy": 2.1541705918312073,
+ "epoch": 0.16024998998437562,
+ "grad_norm": 3.203125,
+ "learning_rate": 1.6798589969556163e-05,
+ "loss": 2.170956573486328,
+ "mean_token_accuracy": 0.5204938448965549,
+ "num_tokens": 6004079.0,
+ "step": 1000
+ },
+ {
+ "entropy": 2.1523867017030716,
+ "epoch": 0.1682624894835944,
+ "grad_norm": 3.09375,
+ "learning_rate": 1.6638359237301716e-05,
+ "loss": 2.1695118713378907,
+ "mean_token_accuracy": 0.5177376234531402,
+ "num_tokens": 6315462.0,
+ "step": 1050
+ },
+ {
+ "entropy": 2.1356022965908052,
+ "epoch": 0.17627498898281319,
+ "grad_norm": 3.21875,
+ "learning_rate": 1.6478128505047268e-05,
+ "loss": 2.1513107299804686,
+ "mean_token_accuracy": 0.5217180585861206,
+ "num_tokens": 6613060.0,
+ "step": 1100
+ },
+ {
+ "entropy": 2.1483460652828215,
+ "epoch": 0.18428748848203197,
+ "grad_norm": 3.21875,
+ "learning_rate": 1.6317897772792824e-05,
+ "loss": 2.1671595764160156,
+ "mean_token_accuracy": 0.5199222292006016,
+ "num_tokens": 6909655.0,
+ "step": 1150
+ },
+ {
+ "entropy": 2.1074040442705155,
+ "epoch": 0.19229998798125075,
+ "grad_norm": 3.828125,
+ "learning_rate": 1.6157667040538376e-05,
+ "loss": 2.12770751953125,
+ "mean_token_accuracy": 0.5252153894305229,
+ "num_tokens": 7203263.0,
+ "step": 1200
+ },
+ {
+ "entropy": 2.1354027593135836,
+ "epoch": 0.20031248748046954,
+ "grad_norm": 2.921875,
+ "learning_rate": 1.5997436308283932e-05,
+ "loss": 2.156183166503906,
+ "mean_token_accuracy": 0.518907299786806,
+ "num_tokens": 7505411.0,
+ "step": 1250
+ },
+ {
+ "entropy": 2.142924894094467,
+ "epoch": 0.20832498697968832,
+ "grad_norm": 3.5625,
+ "learning_rate": 1.5837205576029484e-05,
+ "loss": 2.158486633300781,
+ "mean_token_accuracy": 0.5205909128487111,
+ "num_tokens": 7803562.0,
+ "step": 1300
+ },
+ {
+ "entropy": 2.1305291467905043,
+ "epoch": 0.2163374864789071,
+ "grad_norm": 3.34375,
+ "learning_rate": 1.5676974843775036e-05,
+ "loss": 2.151043701171875,
+ "mean_token_accuracy": 0.5211617235839366,
+ "num_tokens": 8108037.0,
+ "step": 1350
+ },
+ {
+ "entropy": 2.160947166085243,
+ "epoch": 0.2243499859781259,
+ "grad_norm": 3.234375,
+ "learning_rate": 1.5516744111520592e-05,
+ "loss": 2.177792663574219,
+ "mean_token_accuracy": 0.5167004990577698,
+ "num_tokens": 8408674.0,
+ "step": 1400
+ },
+ {
+ "entropy": 2.120403967499733,
+ "epoch": 0.23236248547734467,
+ "grad_norm": 3.375,
+ "learning_rate": 1.5356513379266144e-05,
+ "loss": 2.1462562561035154,
+ "mean_token_accuracy": 0.5225985661149025,
+ "num_tokens": 8707650.0,
+ "step": 1450
+ },
+ {
+ "entropy": 2.138882914185524,
+ "epoch": 0.24037498497656343,
+ "grad_norm": 3.171875,
+ "learning_rate": 1.5196282647011698e-05,
+ "loss": 2.1499905395507812,
+ "mean_token_accuracy": 0.5204668046534061,
+ "num_tokens": 9006296.0,
+ "step": 1500
+ },
+ {
+ "entropy": 2.131927245259285,
+ "epoch": 0.2483874844757822,
+ "grad_norm": 3.546875,
+ "learning_rate": 1.5036051914757252e-05,
+ "loss": 2.157076416015625,
+ "mean_token_accuracy": 0.5220408822596073,
+ "num_tokens": 9307203.0,
+ "step": 1550
+ },
+ {
+ "entropy": 2.1308567851781843,
+ "epoch": 0.256399983975001,
+ "grad_norm": 3.546875,
+ "learning_rate": 1.4875821182502805e-05,
+ "loss": 2.146601104736328,
+ "mean_token_accuracy": 0.5219539681077003,
+ "num_tokens": 9610070.0,
+ "step": 1600
+ },
+ {
+ "entropy": 2.1097783839702604,
+ "epoch": 0.2644124834742198,
+ "grad_norm": 3.84375,
+ "learning_rate": 1.4715590450248359e-05,
+ "loss": 2.1268209838867187,
+ "mean_token_accuracy": 0.5236652019619942,
+ "num_tokens": 9909117.0,
+ "step": 1650
+ },
+ {
+ "entropy": 2.123499180674553,
+ "epoch": 0.2724249829734386,
+ "grad_norm": 3.46875,
+ "learning_rate": 1.4555359717993913e-05,
+ "loss": 2.1409934997558593,
+ "mean_token_accuracy": 0.5225465354323388,
+ "num_tokens": 10209461.0,
+ "step": 1700
+ },
+ {
+ "entropy": 2.13052960395813,
+ "epoch": 0.2804374824726573,
+ "grad_norm": 2.71875,
+ "learning_rate": 1.4395128985739467e-05,
+ "loss": 2.151373291015625,
+ "mean_token_accuracy": 0.5227928113937378,
+ "num_tokens": 10509282.0,
+ "step": 1750
+ },
+ {
+ "entropy": 2.1226114094257356,
+ "epoch": 0.2884499819718761,
+ "grad_norm": 3.65625,
+ "learning_rate": 1.4234898253485019e-05,
+ "loss": 2.144451141357422,
+ "mean_token_accuracy": 0.5233827362954616,
+ "num_tokens": 10811056.0,
+ "step": 1800
+ },
+ {
+ "entropy": 2.126143154501915,
+ "epoch": 0.2964624814710949,
+ "grad_norm": 3.359375,
+ "learning_rate": 1.4074667521230575e-05,
+ "loss": 2.1457598876953123,
+ "mean_token_accuracy": 0.5221967925131321,
+ "num_tokens": 11111153.0,
+ "step": 1850
+ },
+ {
+ "entropy": 2.1048326581716537,
+ "epoch": 0.30447498097031367,
+ "grad_norm": 3.59375,
+ "learning_rate": 1.3914436788976127e-05,
+ "loss": 2.1260116577148436,
+ "mean_token_accuracy": 0.5272719909250736,
+ "num_tokens": 11405606.0,
+ "step": 1900
+ },
+ {
+ "entropy": 2.1236617982387545,
+ "epoch": 0.31248748046953245,
+ "grad_norm": 3.0,
+ "learning_rate": 1.375420605672168e-05,
+ "loss": 2.151782684326172,
+ "mean_token_accuracy": 0.5227546173334122,
+ "num_tokens": 11703943.0,
+ "step": 1950
+ },
+ {
+ "entropy": 2.121563429236412,
+ "epoch": 0.32049997996875124,
+ "grad_norm": 3.34375,
+ "learning_rate": 1.3593975324467235e-05,
+ "loss": 2.1370144653320313,
+ "mean_token_accuracy": 0.5235123547911644,
+ "num_tokens": 12002522.0,
+ "step": 2000
+ },
+ {
+ "entropy": 2.1155461168289182,
+ "epoch": 0.32851247946797,
+ "grad_norm": 3.46875,
+ "learning_rate": 1.3433744592212787e-05,
+ "loss": 2.1364064025878906,
+ "mean_token_accuracy": 0.5237135806679726,
+ "num_tokens": 12303098.0,
+ "step": 2050
+ },
+ {
+ "entropy": 2.1386052322387696,
+ "epoch": 0.3365249789671888,
+ "grad_norm": 3.484375,
+ "learning_rate": 1.327351385995834e-05,
+ "loss": 2.149888458251953,
+ "mean_token_accuracy": 0.522669050693512,
+ "num_tokens": 12605534.0,
+ "step": 2100
+ },
+ {
+ "entropy": 2.102600795030594,
+ "epoch": 0.3445374784664076,
+ "grad_norm": 3.796875,
+ "learning_rate": 1.3113283127703895e-05,
+ "loss": 2.1236566162109374,
+ "mean_token_accuracy": 0.5279092155396938,
+ "num_tokens": 12901218.0,
+ "step": 2150
+ },
+ {
+ "entropy": 2.132645187973976,
+ "epoch": 0.35254997796562637,
+ "grad_norm": 3.53125,
+ "learning_rate": 1.2953052395449447e-05,
+ "loss": 2.1523503112792968,
+ "mean_token_accuracy": 0.5198129612207413,
+ "num_tokens": 13213018.0,
+ "step": 2200
+ },
+ {
+ "entropy": 2.1204321217536926,
+ "epoch": 0.36056247746484515,
+ "grad_norm": 3.25,
+ "learning_rate": 1.2792821663195001e-05,
+ "loss": 2.1325447082519533,
+ "mean_token_accuracy": 0.5243636940419674,
+ "num_tokens": 13514408.0,
+ "step": 2250
+ },
+ {
+ "entropy": 2.102410731315613,
+ "epoch": 0.36857497696406394,
+ "grad_norm": 3.1875,
+ "learning_rate": 1.2632590930940555e-05,
+ "loss": 2.123565368652344,
+ "mean_token_accuracy": 0.5257614709436893,
+ "num_tokens": 13818272.0,
+ "step": 2300
+ },
+ {
+ "entropy": 2.0876205080747603,
+ "epoch": 0.3765874764632827,
+ "grad_norm": 3.328125,
+ "learning_rate": 1.247236019868611e-05,
+ "loss": 2.116696472167969,
+ "mean_token_accuracy": 0.5275124031305313,
+ "num_tokens": 14110702.0,
+ "step": 2350
+ },
+ {
+ "entropy": 2.1078518223762512,
+ "epoch": 0.3845999759625015,
+ "grad_norm": 3.453125,
+ "learning_rate": 1.2312129466431662e-05,
+ "loss": 2.1291993713378905,
+ "mean_token_accuracy": 0.524967464953661,
+ "num_tokens": 14407015.0,
+ "step": 2400
+ },
+ {
+ "entropy": 2.0930898267030718,
+ "epoch": 0.3926124754617203,
+ "grad_norm": 3.5,
+ "learning_rate": 1.2151898734177216e-05,
+ "loss": 2.1094123840332033,
+ "mean_token_accuracy": 0.5290587748587131,
+ "num_tokens": 14696565.0,
+ "step": 2450
+ },
+ {
+ "entropy": 2.1109757220745085,
+ "epoch": 0.4006249749609391,
+ "grad_norm": 3.5,
+ "learning_rate": 1.199166800192277e-05,
+ "loss": 2.1246888732910154,
+ "mean_token_accuracy": 0.5258957357704639,
+ "num_tokens": 14994401.0,
+ "step": 2500
+ },
+ {
+ "entropy": 2.1087811160087586,
+ "epoch": 0.40863747446015786,
+ "grad_norm": 3.1875,
+ "learning_rate": 1.1831437269668324e-05,
+ "loss": 2.12890625,
+ "mean_token_accuracy": 0.5269917643070221,
+ "num_tokens": 15292837.0,
+ "step": 2550
+ },
+ {
+ "entropy": 2.097252429127693,
+ "epoch": 0.41664997395937664,
+ "grad_norm": 3.0,
+ "learning_rate": 1.1671206537413878e-05,
+ "loss": 2.1145870971679686,
+ "mean_token_accuracy": 0.5274137698113919,
+ "num_tokens": 15590647.0,
+ "step": 2600
+ },
+ {
+ "entropy": 2.111199595332146,
+ "epoch": 0.4246624734585954,
+ "grad_norm": 3.25,
+ "learning_rate": 1.151097580515943e-05,
+ "loss": 2.1274327087402343,
+ "mean_token_accuracy": 0.5250249288976192,
+ "num_tokens": 15892772.0,
+ "step": 2650
+ },
+ {
+ "entropy": 2.1051502913236617,
+ "epoch": 0.4326749729578142,
+ "grad_norm": 3.203125,
+ "learning_rate": 1.1350745072904986e-05,
+ "loss": 2.127935333251953,
+ "mean_token_accuracy": 0.5263640700280666,
+ "num_tokens": 16191611.0,
+ "step": 2700
+ },
+ {
+ "entropy": 2.1127885603904724,
+ "epoch": 0.440687472457033,
+ "grad_norm": 3.515625,
+ "learning_rate": 1.1190514340650538e-05,
+ "loss": 2.129211883544922,
+ "mean_token_accuracy": 0.5250939278304577,
+ "num_tokens": 16483582.0,
+ "step": 2750
+ },
+ {
+ "entropy": 2.110930626988411,
+ "epoch": 0.4486999719562518,
+ "grad_norm": 3.1875,
+ "learning_rate": 1.103028360839609e-05,
+ "loss": 2.1220079040527344,
+ "mean_token_accuracy": 0.5250919266045093,
+ "num_tokens": 16787253.0,
+ "step": 2800
+ },
+ {
+ "entropy": 2.1182780677080153,
+ "epoch": 0.45671247145547056,
+ "grad_norm": 3.8125,
+ "learning_rate": 1.0870052876141646e-05,
+ "loss": 2.1358837890625,
+ "mean_token_accuracy": 0.5217854312062263,
+ "num_tokens": 17088741.0,
+ "step": 2850
+ },
+ {
+ "entropy": 2.1117833071947096,
+ "epoch": 0.46472497095468934,
+ "grad_norm": 3.171875,
+ "learning_rate": 1.0709822143887198e-05,
+ "loss": 2.125662841796875,
+ "mean_token_accuracy": 0.524649152457714,
+ "num_tokens": 17389236.0,
+ "step": 2900
+ },
+ {
+ "entropy": 2.1262730568647386,
+ "epoch": 0.47273747045390807,
+ "grad_norm": 3.625,
+ "learning_rate": 1.054959141163275e-05,
+ "loss": 2.142789611816406,
+ "mean_token_accuracy": 0.5236384868621826,
+ "num_tokens": 17693413.0,
+ "step": 2950
+ },
+ {
+ "entropy": 2.115410739183426,
+ "epoch": 0.48074996995312685,
+ "grad_norm": 3.140625,
+ "learning_rate": 1.0389360679378306e-05,
+ "loss": 2.1337106323242185,
+ "mean_token_accuracy": 0.5249222332239151,
+ "num_tokens": 17990326.0,
+ "step": 3000
+ },
+ {
+ "entropy": 2.0951381689310073,
+ "epoch": 0.48876246945234564,
+ "grad_norm": 3.234375,
+ "learning_rate": 1.0229129947123859e-05,
+ "loss": 2.108144073486328,
+ "mean_token_accuracy": 0.5269971822202205,
+ "num_tokens": 18293136.0,
+ "step": 3050
+ },
+ {
+ "entropy": 2.116977259516716,
+ "epoch": 0.4967749689515644,
+ "grad_norm": 3.421875,
+ "learning_rate": 1.0068899214869413e-05,
+ "loss": 2.1327830505371095,
+ "mean_token_accuracy": 0.5238270902633667,
+ "num_tokens": 18591481.0,
+ "step": 3100
+ },
+ {
+ "entropy": 2.08598925113678,
+ "epoch": 0.5047874684507833,
+ "grad_norm": 2.96875,
+ "learning_rate": 9.908668482614967e-06,
+ "loss": 2.1032344055175782,
+ "mean_token_accuracy": 0.5289211909472943,
+ "num_tokens": 18885221.0,
+ "step": 3150
+ },
+ {
+ "entropy": 2.097712662220001,
+ "epoch": 0.512799967950002,
+ "grad_norm": 3.40625,
+ "learning_rate": 9.74843775036052e-06,
+ "loss": 2.1150633239746095,
+ "mean_token_accuracy": 0.5288272814452648,
+ "num_tokens": 19184784.0,
+ "step": 3200
+ },
+ {
+ "entropy": 2.1063760107755662,
+ "epoch": 0.5208124674492208,
+ "grad_norm": 3.28125,
+ "learning_rate": 9.588207018106075e-06,
+ "loss": 2.1217109680175783,
+ "mean_token_accuracy": 0.526290989369154,
+ "num_tokens": 19482538.0,
+ "step": 3250
+ },
+ {
+ "entropy": 2.090182801485062,
+ "epoch": 0.5288249669484396,
+ "grad_norm": 2.875,
+ "learning_rate": 9.427976285851627e-06,
+ "loss": 2.1130311584472654,
+ "mean_token_accuracy": 0.5285439921915531,
+ "num_tokens": 19782544.0,
+ "step": 3300
+ },
+ {
+ "entropy": 2.103260806798935,
+ "epoch": 0.5368374664476584,
+ "grad_norm": 3.59375,
+ "learning_rate": 9.267745553597181e-06,
+ "loss": 2.116189727783203,
+ "mean_token_accuracy": 0.5273195558786392,
+ "num_tokens": 20086408.0,
+ "step": 3350
+ },
+ {
+ "entropy": 2.0673747235536575,
+ "epoch": 0.5448499659468772,
+ "grad_norm": 3.125,
+ "learning_rate": 9.107514821342735e-06,
+ "loss": 2.087482147216797,
+ "mean_token_accuracy": 0.532562478929758,
+ "num_tokens": 20380429.0,
+ "step": 3400
+ },
+ {
+ "entropy": 2.1030789083242416,
+ "epoch": 0.552862465446096,
+ "grad_norm": 3.296875,
+ "learning_rate": 8.947284089088289e-06,
+ "loss": 2.122591247558594,
+ "mean_token_accuracy": 0.5256703786551953,
+ "num_tokens": 20682849.0,
+ "step": 3450
+ },
+ {
+ "entropy": 2.0988716661930082,
+ "epoch": 0.5608749649453146,
+ "grad_norm": 3.546875,
+ "learning_rate": 8.787053356833841e-06,
+ "loss": 2.115867156982422,
+ "mean_token_accuracy": 0.5266730943322182,
+ "num_tokens": 20979225.0,
+ "step": 3500
+ },
+ {
+ "entropy": 2.0954072701931,
+ "epoch": 0.5688874644445334,
+ "grad_norm": 3.859375,
+ "learning_rate": 8.626822624579395e-06,
+ "loss": 2.1116868591308595,
+ "mean_token_accuracy": 0.5267495161294937,
+ "num_tokens": 21284166.0,
+ "step": 3550
+ },
+ {
+ "entropy": 2.11238471031189,
+ "epoch": 0.5768999639437522,
+ "grad_norm": 3.484375,
+ "learning_rate": 8.466591892324949e-06,
+ "loss": 2.1292352294921875,
+ "mean_token_accuracy": 0.5249243360757828,
+ "num_tokens": 21584612.0,
+ "step": 3600
+ },
+ {
+ "entropy": 2.091464368700981,
+ "epoch": 0.584912463442971,
+ "grad_norm": 3.359375,
+ "learning_rate": 8.306361160070501e-06,
+ "loss": 2.1099319458007812,
+ "mean_token_accuracy": 0.5292035435140133,
+ "num_tokens": 21879345.0,
+ "step": 3650
+ },
+ {
+ "entropy": 2.095204224586487,
+ "epoch": 0.5929249629421898,
+ "grad_norm": 2.96875,
+ "learning_rate": 8.146130427816055e-06,
+ "loss": 2.1198716735839844,
+ "mean_token_accuracy": 0.5255991271138192,
+ "num_tokens": 22183486.0,
+ "step": 3700
+ },
+ {
+ "entropy": 2.0666573679447175,
+ "epoch": 0.6009374624414086,
+ "grad_norm": 2.90625,
+ "learning_rate": 7.98589969556161e-06,
+ "loss": 2.08689697265625,
+ "mean_token_accuracy": 0.5318664258718491,
+ "num_tokens": 22476438.0,
+ "step": 3750
+ },
+ {
+ "entropy": 2.122153177857399,
+ "epoch": 0.6089499619406273,
+ "grad_norm": 3.40625,
+ "learning_rate": 7.825668963307162e-06,
+ "loss": 2.138288879394531,
+ "mean_token_accuracy": 0.5232584895193577,
+ "num_tokens": 22780385.0,
+ "step": 3800
+ },
+ {
+ "entropy": 2.1190748488903046,
+ "epoch": 0.6169624614398461,
+ "grad_norm": 3.46875,
+ "learning_rate": 7.665438231052716e-06,
+ "loss": 2.139790344238281,
+ "mean_token_accuracy": 0.5243287613987923,
+ "num_tokens": 23080847.0,
+ "step": 3850
+ },
+ {
+ "entropy": 2.1312570214271545,
+ "epoch": 0.6249749609390649,
+ "grad_norm": 3.765625,
+ "learning_rate": 7.5052074987982706e-06,
+ "loss": 2.1471080017089843,
+ "mean_token_accuracy": 0.5232046774029732,
+ "num_tokens": 23384738.0,
+ "step": 3900
+ },
+ {
+ "entropy": 2.0779008239507677,
+ "epoch": 0.6329874604382837,
+ "grad_norm": 3.171875,
+ "learning_rate": 7.344976766543823e-06,
+ "loss": 2.0974729919433592,
+ "mean_token_accuracy": 0.5305448272824287,
+ "num_tokens": 23682750.0,
+ "step": 3950
+ },
+ {
+ "entropy": 2.1121455317735673,
+ "epoch": 0.6409999599375025,
+ "grad_norm": 3.921875,
+ "learning_rate": 7.184746034289377e-06,
+ "loss": 2.127069549560547,
+ "mean_token_accuracy": 0.5252936625480652,
+ "num_tokens": 23989029.0,
+ "step": 4000
+ },
+ {
+ "entropy": 2.0982888478040693,
+ "epoch": 0.6490124594367213,
+ "grad_norm": 4.0625,
+ "learning_rate": 7.024515302034931e-06,
+ "loss": 2.112042236328125,
+ "mean_token_accuracy": 0.5281323495507241,
+ "num_tokens": 24292063.0,
+ "step": 4050
+ },
+ {
+ "entropy": 2.098363963365555,
+ "epoch": 0.65702495893594,
+ "grad_norm": 3.046875,
+ "learning_rate": 6.864284569780485e-06,
+ "loss": 2.1156489562988283,
+ "mean_token_accuracy": 0.5275120075047016,
+ "num_tokens": 24590201.0,
+ "step": 4100
+ },
+ {
+ "entropy": 2.1065632277727127,
+ "epoch": 0.6650374584351588,
+ "grad_norm": 3.375,
+ "learning_rate": 6.704053837526038e-06,
+ "loss": 2.12911376953125,
+ "mean_token_accuracy": 0.5262684133648873,
+ "num_tokens": 24892122.0,
+ "step": 4150
+ },
+ {
+ "entropy": 2.1014072370529173,
+ "epoch": 0.6730499579343776,
+ "grad_norm": 2.84375,
+ "learning_rate": 6.543823105271592e-06,
+ "loss": 2.1130335998535155,
+ "mean_token_accuracy": 0.5262750512361527,
+ "num_tokens": 25194969.0,
+ "step": 4200
+ },
+ {
+ "entropy": 2.0905236744880678,
+ "epoch": 0.6810624574335964,
+ "grad_norm": 3.03125,
+ "learning_rate": 6.383592373017146e-06,
+ "loss": 2.1153533935546873,
+ "mean_token_accuracy": 0.527137616276741,
+ "num_tokens": 25499521.0,
+ "step": 4250
+ },
+ {
+ "entropy": 2.1032627063989637,
+ "epoch": 0.6890749569328152,
+ "grad_norm": 3.125,
+ "learning_rate": 6.223361640762698e-06,
+ "loss": 2.122831115722656,
+ "mean_token_accuracy": 0.5254572249948979,
+ "num_tokens": 25798933.0,
+ "step": 4300
+ },
+ {
+ "entropy": 2.118139310479164,
+ "epoch": 0.697087456432034,
+ "grad_norm": 3.28125,
+ "learning_rate": 6.063130908508252e-06,
+ "loss": 2.1376751708984374,
+ "mean_token_accuracy": 0.5243131402134895,
+ "num_tokens": 26096619.0,
+ "step": 4350
+ },
+ {
+ "entropy": 2.082554380297661,
+ "epoch": 0.7050999559312527,
+ "grad_norm": 3.390625,
+ "learning_rate": 5.902900176253806e-06,
+ "loss": 2.0949493408203126,
+ "mean_token_accuracy": 0.5316410692036152,
+ "num_tokens": 26394395.0,
+ "step": 4400
+ },
+ {
+ "entropy": 2.09870474755764,
+ "epoch": 0.7131124554304715,
+ "grad_norm": 3.421875,
+ "learning_rate": 5.7426694439993594e-06,
+ "loss": 2.1184346008300783,
+ "mean_token_accuracy": 0.5267573061585427,
+ "num_tokens": 26692333.0,
+ "step": 4450
+ },
+ {
+ "entropy": 2.0866885662078856,
+ "epoch": 0.7211249549296903,
+ "grad_norm": 3.71875,
+ "learning_rate": 5.5824387117449134e-06,
+ "loss": 2.102904968261719,
+ "mean_token_accuracy": 0.529726449996233,
+ "num_tokens": 26989384.0,
+ "step": 4500
+ },
+ {
+ "entropy": 2.114828346967697,
+ "epoch": 0.7291374544289091,
+ "grad_norm": 3.46875,
+ "learning_rate": 5.4222079794904666e-06,
+ "loss": 2.128611602783203,
+ "mean_token_accuracy": 0.5247649565339089,
+ "num_tokens": 27299927.0,
+ "step": 4550
+ },
+ {
+ "entropy": 2.09617226600647,
+ "epoch": 0.7371499539281279,
+ "grad_norm": 3.015625,
+ "learning_rate": 5.26197724723602e-06,
+ "loss": 2.1166041564941405,
+ "mean_token_accuracy": 0.5273530712723732,
+ "num_tokens": 27599011.0,
+ "step": 4600
+ },
+ {
+ "entropy": 2.0975282275676728,
+ "epoch": 0.7451624534273467,
+ "grad_norm": 3.3125,
+ "learning_rate": 5.101746514981574e-06,
+ "loss": 2.112121124267578,
+ "mean_token_accuracy": 0.5281847220659256,
+ "num_tokens": 27892355.0,
+ "step": 4650
+ },
+ {
+ "entropy": 2.095783769488335,
+ "epoch": 0.7531749529265654,
+ "grad_norm": 3.78125,
+ "learning_rate": 4.941515782727128e-06,
+ "loss": 2.110701904296875,
+ "mean_token_accuracy": 0.5272985179722309,
+ "num_tokens": 28187615.0,
+ "step": 4700
+ },
+ {
+ "entropy": 2.0964103603363036,
+ "epoch": 0.7611874524257842,
+ "grad_norm": 3.1875,
+ "learning_rate": 4.781285050472681e-06,
+ "loss": 2.11992919921875,
+ "mean_token_accuracy": 0.5271916574239731,
+ "num_tokens": 28485419.0,
+ "step": 4750
+ },
+ {
+ "entropy": 2.0820577853918074,
+ "epoch": 0.769199951925003,
+ "grad_norm": 3.5,
+ "learning_rate": 4.621054318218234e-06,
+ "loss": 2.095073394775391,
+ "mean_token_accuracy": 0.5297581899166107,
+ "num_tokens": 28780158.0,
+ "step": 4800
+ },
+ {
+ "entropy": 2.089591929912567,
+ "epoch": 0.7772124514242218,
+ "grad_norm": 3.125,
+ "learning_rate": 4.460823585963788e-06,
+ "loss": 2.109657897949219,
+ "mean_token_accuracy": 0.528961935043335,
+ "num_tokens": 29081052.0,
+ "step": 4850
+ },
+ {
+ "entropy": 2.092900555729866,
+ "epoch": 0.7852249509234406,
+ "grad_norm": 3.46875,
+ "learning_rate": 4.300592853709342e-06,
+ "loss": 2.106436920166016,
+ "mean_token_accuracy": 0.5284854583442211,
+ "num_tokens": 29376324.0,
+ "step": 4900
+ },
+ {
+ "entropy": 2.0630873823165894,
+ "epoch": 0.7932374504226594,
+ "grad_norm": 3.28125,
+ "learning_rate": 4.140362121454896e-06,
+ "loss": 2.088908233642578,
+ "mean_token_accuracy": 0.532561699450016,
+ "num_tokens": 29675269.0,
+ "step": 4950
+ },
+ {
+ "entropy": 2.0753238475322724,
+ "epoch": 0.8012499499218781,
+ "grad_norm": 3.21875,
+ "learning_rate": 3.980131389200449e-06,
+ "loss": 2.0961412048339843,
+ "mean_token_accuracy": 0.5304154875874519,
+ "num_tokens": 29974838.0,
+ "step": 5000
+ }
+ ],
+ "logging_steps": 50,
+ "max_steps": 6241,
+ "num_input_tokens_seen": 0,
+ "num_train_epochs": 1,
+ "save_steps": 500,
+ "stateful_callbacks": {
+ "TrainerControl": {
+ "args": {
+ "should_epoch_stop": false,
+ "should_evaluate": false,
+ "should_log": false,
+ "should_save": true,
+ "should_training_stop": false
+ },
+ "attributes": {}
+ }
+ },
+ "total_flos": 3.540934716676915e+17,
+ "train_batch_size": 4,
+ "trial_name": null,
+ "trial_params": null
+}
diff --git a/checkpoint-5000/training_args.bin b/checkpoint-5000/training_args.bin
new file mode 100644
index 0000000000000000000000000000000000000000..4b409d5a147856f0d5a3d3da0f3b4922a9def7dd
--- /dev/null
+++ b/checkpoint-5000/training_args.bin
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:78622f63304ec0913ed0b4ea7a1b26c08a5621b3661a380d798c4ea426235671
+size 5713
diff --git a/checkpoint-5500/chat_template.jinja b/checkpoint-5500/chat_template.jinja
new file mode 100644
index 0000000000000000000000000000000000000000..01be9b307daa2d425f7c168c9fb145a286e0afb4
--- /dev/null
+++ b/checkpoint-5500/chat_template.jinja
@@ -0,0 +1,89 @@
+{%- if tools %}
+ {{- '<|im_start|>system\n' }}
+ {%- if messages[0].role == 'system' %}
+ {{- messages[0].content + '\n\n' }}
+ {%- endif %}
+ {{- "# Tools\n\nYou may call one or more functions to assist with the user query.\n\nYou are provided with function signatures within XML tags:\n" }}
+ {%- for tool in tools %}
+ {{- "\n" }}
+ {{- tool | tojson }}
+ {%- endfor %}
+ {{- "\n\n\nFor each function call, return a json object with function name and arguments within XML tags:\n\n{\"name\": , \"arguments\": }\n<|im_end|>\n" }}
+{%- else %}
+ {%- if messages[0].role == 'system' %}
+ {{- '<|im_start|>system\n' + messages[0].content + '<|im_end|>\n' }}
+ {%- endif %}
+{%- endif %}
+{%- set ns = namespace(multi_step_tool=true, last_query_index=messages|length - 1) %}
+{%- for message in messages[::-1] %}
+ {%- set index = (messages|length - 1) - loop.index0 %}
+ {%- if ns.multi_step_tool and message.role == "user" and message.content is string and not(message.content.startswith('') and message.content.endswith('')) %}
+ {%- set ns.multi_step_tool = false %}
+ {%- set ns.last_query_index = index %}
+ {%- endif %}
+{%- endfor %}
+{%- for message in messages %}
+ {%- if message.content is string %}
+ {%- set content = message.content %}
+ {%- else %}
+ {%- set content = '' %}
+ {%- endif %}
+ {%- if (message.role == "user") or (message.role == "system" and not loop.first) %}
+ {{- '<|im_start|>' + message.role + '\n' + content + '<|im_end|>' + '\n' }}
+ {%- elif message.role == "assistant" %}
+ {%- set reasoning_content = '' %}
+ {%- if message.reasoning_content is string %}
+ {%- set reasoning_content = message.reasoning_content %}
+ {%- else %}
+ {%- if '' in content %}
+ {%- set reasoning_content = content.split('')[0].rstrip('\n').split('')[-1].lstrip('\n') %}
+ {%- set content = content.split('')[-1].lstrip('\n') %}
+ {%- endif %}
+ {%- endif %}
+ {%- if loop.index0 > ns.last_query_index %}
+ {%- if loop.last or (not loop.last and reasoning_content) %}
+ {{- '<|im_start|>' + message.role + '\n\n' + reasoning_content.strip('\n') + '\n\n\n' + content.lstrip('\n') }}
+ {%- else %}
+ {{- '<|im_start|>' + message.role + '\n' + content }}
+ {%- endif %}
+ {%- else %}
+ {{- '<|im_start|>' + message.role + '\n' + content }}
+ {%- endif %}
+ {%- if message.tool_calls %}
+ {%- for tool_call in message.tool_calls %}
+ {%- if (loop.first and content) or (not loop.first) %}
+ {{- '\n' }}
+ {%- endif %}
+ {%- if tool_call.function %}
+ {%- set tool_call = tool_call.function %}
+ {%- endif %}
+ {{- '\n{"name": "' }}
+ {{- tool_call.name }}
+ {{- '", "arguments": ' }}
+ {%- if tool_call.arguments is string %}
+ {{- tool_call.arguments }}
+ {%- else %}
+ {{- tool_call.arguments | tojson }}
+ {%- endif %}
+ {{- '}\n' }}
+ {%- endfor %}
+ {%- endif %}
+ {{- '<|im_end|>\n' }}
+ {%- elif message.role == "tool" %}
+ {%- if loop.first or (messages[loop.index0 - 1].role != "tool") %}
+ {{- '<|im_start|>user' }}
+ {%- endif %}
+ {{- '\n\n' }}
+ {{- content }}
+ {{- '\n' }}
+ {%- if loop.last or (messages[loop.index0 + 1].role != "tool") %}
+ {{- '<|im_end|>\n' }}
+ {%- endif %}
+ {%- endif %}
+{%- endfor %}
+{%- if add_generation_prompt %}
+ {{- '<|im_start|>assistant\n' }}
+ {%- if enable_thinking is defined and enable_thinking is false %}
+ {{- '\n\n\n\n' }}
+ {%- endif %}
+{%- endif %}
\ No newline at end of file
diff --git a/checkpoint-5500/config.json b/checkpoint-5500/config.json
new file mode 100644
index 0000000000000000000000000000000000000000..1d219d6a6d288625229a71646710b85cf7ae5def
--- /dev/null
+++ b/checkpoint-5500/config.json
@@ -0,0 +1,63 @@
+{
+ "architectures": [
+ "Qwen3ForCausalLM"
+ ],
+ "attention_bias": false,
+ "attention_dropout": 0.0,
+ "bos_token_id": null,
+ "dtype": "bfloat16",
+ "eos_token_id": 151645,
+ "head_dim": 128,
+ "hidden_act": "silu",
+ "hidden_size": 2048,
+ "initializer_range": 0.02,
+ "intermediate_size": 6144,
+ "layer_types": [
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention"
+ ],
+ "max_position_embeddings": 40960,
+ "max_window_layers": 28,
+ "model_type": "qwen3",
+ "num_attention_heads": 16,
+ "num_hidden_layers": 28,
+ "num_key_value_heads": 8,
+ "pad_token_id": 151643,
+ "rms_norm_eps": 1e-06,
+ "rope_parameters": {
+ "rope_theta": 1000000,
+ "rope_type": "default"
+ },
+ "sliding_window": null,
+ "tie_word_embeddings": true,
+ "transformers_version": "5.7.0",
+ "use_cache": false,
+ "use_sliding_window": false,
+ "vocab_size": 151936
+}
diff --git a/checkpoint-5500/generation_config.json b/checkpoint-5500/generation_config.json
new file mode 100644
index 0000000000000000000000000000000000000000..5ec133d867b1427d43a5ee0ea332bd088fc12013
--- /dev/null
+++ b/checkpoint-5500/generation_config.json
@@ -0,0 +1,12 @@
+{
+ "do_sample": true,
+ "eos_token_id": [
+ 151645,
+ 151643
+ ],
+ "pad_token_id": 151643,
+ "temperature": 0.6,
+ "top_k": 20,
+ "top_p": 0.95,
+ "transformers_version": "5.7.0"
+}
diff --git a/checkpoint-5500/model.safetensors b/checkpoint-5500/model.safetensors
new file mode 100644
index 0000000000000000000000000000000000000000..364fd7bcb3e72bb7f984d8971d924db6146a0fbd
--- /dev/null
+++ b/checkpoint-5500/model.safetensors
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d2ea18394260e36468ba955f0edf3aed5e86e05cfddad9dd67132e0c90d1659f
+size 3441185608
diff --git a/checkpoint-5500/optimizer.pt b/checkpoint-5500/optimizer.pt
new file mode 100644
index 0000000000000000000000000000000000000000..67abe68ee2f38a0e6f92a4ce863e48cc3a11c01e
--- /dev/null
+++ b/checkpoint-5500/optimizer.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:adf82561ff68c22f9de236f8f7a595bd320f78bca6143d43d6339be6cf6d5f3e
+size 6882572207
diff --git a/checkpoint-5500/rng_state.pth b/checkpoint-5500/rng_state.pth
new file mode 100644
index 0000000000000000000000000000000000000000..5a8f17addf8aaee350625e9847751582bc9f86dd
--- /dev/null
+++ b/checkpoint-5500/rng_state.pth
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:61c19bab1174704a4a4441475683bf1270277af15d2e2c95e964789128e482c4
+size 14645
diff --git a/checkpoint-5500/scheduler.pt b/checkpoint-5500/scheduler.pt
new file mode 100644
index 0000000000000000000000000000000000000000..e1440b67341d529aaa3526f3b5b15881fd4b5c8b
--- /dev/null
+++ b/checkpoint-5500/scheduler.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:8240268d8934f8e0ca422959548ab4fb30a8e1118fb691e38334f307db3923e8
+size 1465
diff --git a/checkpoint-5500/tokenizer.json b/checkpoint-5500/tokenizer.json
new file mode 100644
index 0000000000000000000000000000000000000000..c7afbed2efcdf019f88ab0572ec29d3bf595dfe2
--- /dev/null
+++ b/checkpoint-5500/tokenizer.json
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:be75606093db2094d7cd20f3c2f385c212750648bd6ea4fb2bf507a6a4c55506
+size 11422650
diff --git a/checkpoint-5500/tokenizer_config.json b/checkpoint-5500/tokenizer_config.json
new file mode 100644
index 0000000000000000000000000000000000000000..770e41d6c92519d525eede4cbcf3ba27f6425311
--- /dev/null
+++ b/checkpoint-5500/tokenizer_config.json
@@ -0,0 +1,30 @@
+{
+ "add_prefix_space": false,
+ "backend": "tokenizers",
+ "bos_token": null,
+ "clean_up_tokenization_spaces": false,
+ "eos_token": "<|im_end|>",
+ "errors": "replace",
+ "extra_special_tokens": [
+ "<|im_start|>",
+ "<|im_end|>",
+ "<|object_ref_start|>",
+ "<|object_ref_end|>",
+ "<|box_start|>",
+ "<|box_end|>",
+ "<|quad_start|>",
+ "<|quad_end|>",
+ "<|vision_start|>",
+ "<|vision_end|>",
+ "<|vision_pad|>",
+ "<|image_pad|>",
+ "<|video_pad|>"
+ ],
+ "is_local": false,
+ "local_files_only": false,
+ "model_max_length": 131072,
+ "pad_token": "<|endoftext|>",
+ "split_special_tokens": false,
+ "tokenizer_class": "Qwen2Tokenizer",
+ "unk_token": null
+}
diff --git a/checkpoint-5500/trainer_state.json b/checkpoint-5500/trainer_state.json
new file mode 100644
index 0000000000000000000000000000000000000000..3c73eab0b5c5e9c0bd8df423a4a4c6b538d6108b
--- /dev/null
+++ b/checkpoint-5500/trainer_state.json
@@ -0,0 +1,1134 @@
+{
+ "best_global_step": null,
+ "best_metric": null,
+ "best_model_checkpoint": null,
+ "epoch": 0.881374944914066,
+ "eval_steps": 500,
+ "global_step": 5500,
+ "is_hyper_param_search": false,
+ "is_local_process_zero": true,
+ "is_world_process_zero": true,
+ "log_history": [
+ {
+ "entropy": 2.223189059495926,
+ "epoch": 0.008012499499218782,
+ "grad_norm": 2.875,
+ "learning_rate": 1.9842973882390643e-05,
+ "loss": 2.3428109741210936,
+ "mean_token_accuracy": 0.4963963569700718,
+ "num_tokens": 305366.0,
+ "step": 50
+ },
+ {
+ "entropy": 2.188130117058754,
+ "epoch": 0.016024998998437564,
+ "grad_norm": 3.328125,
+ "learning_rate": 1.96827431501362e-05,
+ "loss": 2.225358734130859,
+ "mean_token_accuracy": 0.5110921229422093,
+ "num_tokens": 605042.0,
+ "step": 100
+ },
+ {
+ "entropy": 2.197347580194473,
+ "epoch": 0.024037498497656344,
+ "grad_norm": 3.453125,
+ "learning_rate": 1.952251241788175e-05,
+ "loss": 2.2299420166015627,
+ "mean_token_accuracy": 0.5093490639328957,
+ "num_tokens": 906888.0,
+ "step": 150
+ },
+ {
+ "entropy": 2.1913950395584108,
+ "epoch": 0.03204999799687513,
+ "grad_norm": 2.90625,
+ "learning_rate": 1.9362281685627303e-05,
+ "loss": 2.210995941162109,
+ "mean_token_accuracy": 0.5097909501194954,
+ "num_tokens": 1213125.0,
+ "step": 200
+ },
+ {
+ "entropy": 2.1611998677253723,
+ "epoch": 0.040062497496093905,
+ "grad_norm": 3.25,
+ "learning_rate": 1.920205095337286e-05,
+ "loss": 2.1867474365234374,
+ "mean_token_accuracy": 0.5154410660266876,
+ "num_tokens": 1509026.0,
+ "step": 250
+ },
+ {
+ "entropy": 2.168051454424858,
+ "epoch": 0.04807499699531269,
+ "grad_norm": 2.9375,
+ "learning_rate": 1.904182022111841e-05,
+ "loss": 2.190071716308594,
+ "mean_token_accuracy": 0.5156884534657001,
+ "num_tokens": 1806850.0,
+ "step": 300
+ },
+ {
+ "entropy": 2.1737154543399813,
+ "epoch": 0.05608749649453147,
+ "grad_norm": 3.421875,
+ "learning_rate": 1.8881589488863967e-05,
+ "loss": 2.199147186279297,
+ "mean_token_accuracy": 0.514045044630766,
+ "num_tokens": 2104505.0,
+ "step": 350
+ },
+ {
+ "entropy": 2.148039159178734,
+ "epoch": 0.06409999599375026,
+ "grad_norm": 3.109375,
+ "learning_rate": 1.872135875660952e-05,
+ "loss": 2.176946105957031,
+ "mean_token_accuracy": 0.5177112428843975,
+ "num_tokens": 2391088.0,
+ "step": 400
+ },
+ {
+ "entropy": 2.194890711903572,
+ "epoch": 0.07211249549296903,
+ "grad_norm": 3.28125,
+ "learning_rate": 1.856112802435507e-05,
+ "loss": 2.216288757324219,
+ "mean_token_accuracy": 0.5110335703194141,
+ "num_tokens": 2691272.0,
+ "step": 450
+ },
+ {
+ "entropy": 2.1557786524295808,
+ "epoch": 0.08012499499218781,
+ "grad_norm": 4.15625,
+ "learning_rate": 1.8400897292100627e-05,
+ "loss": 2.1812680053710936,
+ "mean_token_accuracy": 0.5176697050035,
+ "num_tokens": 2991855.0,
+ "step": 500
+ },
+ {
+ "entropy": 2.1708167713880537,
+ "epoch": 0.08813749449140659,
+ "grad_norm": 4.34375,
+ "learning_rate": 1.824066655984618e-05,
+ "loss": 2.1921002197265627,
+ "mean_token_accuracy": 0.5156507430970669,
+ "num_tokens": 3295049.0,
+ "step": 550
+ },
+ {
+ "entropy": 2.1686314886808393,
+ "epoch": 0.09614999399062538,
+ "grad_norm": 2.875,
+ "learning_rate": 1.8080435827591735e-05,
+ "loss": 2.1914617919921877,
+ "mean_token_accuracy": 0.5139613692462445,
+ "num_tokens": 3597206.0,
+ "step": 600
+ },
+ {
+ "entropy": 2.15742906332016,
+ "epoch": 0.10416249348984416,
+ "grad_norm": 3.109375,
+ "learning_rate": 1.7920205095337287e-05,
+ "loss": 2.174853973388672,
+ "mean_token_accuracy": 0.5159518288075924,
+ "num_tokens": 3905268.0,
+ "step": 650
+ },
+ {
+ "entropy": 2.162363375425339,
+ "epoch": 0.11217499298906294,
+ "grad_norm": 3.171875,
+ "learning_rate": 1.7759974363082843e-05,
+ "loss": 2.184231872558594,
+ "mean_token_accuracy": 0.516631413847208,
+ "num_tokens": 4207934.0,
+ "step": 700
+ },
+ {
+ "entropy": 2.1516289287805557,
+ "epoch": 0.12018749248828171,
+ "grad_norm": 3.609375,
+ "learning_rate": 1.7599743630828395e-05,
+ "loss": 2.167020263671875,
+ "mean_token_accuracy": 0.516358617246151,
+ "num_tokens": 4515774.0,
+ "step": 750
+ },
+ {
+ "entropy": 2.1549544781446457,
+ "epoch": 0.1281999919875005,
+ "grad_norm": 3.28125,
+ "learning_rate": 1.7439512898573947e-05,
+ "loss": 2.181917877197266,
+ "mean_token_accuracy": 0.5182220858335495,
+ "num_tokens": 4814709.0,
+ "step": 800
+ },
+ {
+ "entropy": 2.1532615756988527,
+ "epoch": 0.1362124914867193,
+ "grad_norm": 3.078125,
+ "learning_rate": 1.7279282166319503e-05,
+ "loss": 2.1659007263183594,
+ "mean_token_accuracy": 0.5180425970256328,
+ "num_tokens": 5112585.0,
+ "step": 850
+ },
+ {
+ "entropy": 2.1704017293453215,
+ "epoch": 0.14422499098593805,
+ "grad_norm": 2.921875,
+ "learning_rate": 1.7119051434065055e-05,
+ "loss": 2.1934996032714844,
+ "mean_token_accuracy": 0.5154964691400528,
+ "num_tokens": 5411369.0,
+ "step": 900
+ },
+ {
+ "entropy": 2.1349107217788696,
+ "epoch": 0.15223749048515683,
+ "grad_norm": 2.984375,
+ "learning_rate": 1.6958820701810608e-05,
+ "loss": 2.1598545837402345,
+ "mean_token_accuracy": 0.5208920259773732,
+ "num_tokens": 5707060.0,
+ "step": 950
+ },
+ {
+ "entropy": 2.1541705918312073,
+ "epoch": 0.16024998998437562,
+ "grad_norm": 3.203125,
+ "learning_rate": 1.6798589969556163e-05,
+ "loss": 2.170956573486328,
+ "mean_token_accuracy": 0.5204938448965549,
+ "num_tokens": 6004079.0,
+ "step": 1000
+ },
+ {
+ "entropy": 2.1523867017030716,
+ "epoch": 0.1682624894835944,
+ "grad_norm": 3.09375,
+ "learning_rate": 1.6638359237301716e-05,
+ "loss": 2.1695118713378907,
+ "mean_token_accuracy": 0.5177376234531402,
+ "num_tokens": 6315462.0,
+ "step": 1050
+ },
+ {
+ "entropy": 2.1356022965908052,
+ "epoch": 0.17627498898281319,
+ "grad_norm": 3.21875,
+ "learning_rate": 1.6478128505047268e-05,
+ "loss": 2.1513107299804686,
+ "mean_token_accuracy": 0.5217180585861206,
+ "num_tokens": 6613060.0,
+ "step": 1100
+ },
+ {
+ "entropy": 2.1483460652828215,
+ "epoch": 0.18428748848203197,
+ "grad_norm": 3.21875,
+ "learning_rate": 1.6317897772792824e-05,
+ "loss": 2.1671595764160156,
+ "mean_token_accuracy": 0.5199222292006016,
+ "num_tokens": 6909655.0,
+ "step": 1150
+ },
+ {
+ "entropy": 2.1074040442705155,
+ "epoch": 0.19229998798125075,
+ "grad_norm": 3.828125,
+ "learning_rate": 1.6157667040538376e-05,
+ "loss": 2.12770751953125,
+ "mean_token_accuracy": 0.5252153894305229,
+ "num_tokens": 7203263.0,
+ "step": 1200
+ },
+ {
+ "entropy": 2.1354027593135836,
+ "epoch": 0.20031248748046954,
+ "grad_norm": 2.921875,
+ "learning_rate": 1.5997436308283932e-05,
+ "loss": 2.156183166503906,
+ "mean_token_accuracy": 0.518907299786806,
+ "num_tokens": 7505411.0,
+ "step": 1250
+ },
+ {
+ "entropy": 2.142924894094467,
+ "epoch": 0.20832498697968832,
+ "grad_norm": 3.5625,
+ "learning_rate": 1.5837205576029484e-05,
+ "loss": 2.158486633300781,
+ "mean_token_accuracy": 0.5205909128487111,
+ "num_tokens": 7803562.0,
+ "step": 1300
+ },
+ {
+ "entropy": 2.1305291467905043,
+ "epoch": 0.2163374864789071,
+ "grad_norm": 3.34375,
+ "learning_rate": 1.5676974843775036e-05,
+ "loss": 2.151043701171875,
+ "mean_token_accuracy": 0.5211617235839366,
+ "num_tokens": 8108037.0,
+ "step": 1350
+ },
+ {
+ "entropy": 2.160947166085243,
+ "epoch": 0.2243499859781259,
+ "grad_norm": 3.234375,
+ "learning_rate": 1.5516744111520592e-05,
+ "loss": 2.177792663574219,
+ "mean_token_accuracy": 0.5167004990577698,
+ "num_tokens": 8408674.0,
+ "step": 1400
+ },
+ {
+ "entropy": 2.120403967499733,
+ "epoch": 0.23236248547734467,
+ "grad_norm": 3.375,
+ "learning_rate": 1.5356513379266144e-05,
+ "loss": 2.1462562561035154,
+ "mean_token_accuracy": 0.5225985661149025,
+ "num_tokens": 8707650.0,
+ "step": 1450
+ },
+ {
+ "entropy": 2.138882914185524,
+ "epoch": 0.24037498497656343,
+ "grad_norm": 3.171875,
+ "learning_rate": 1.5196282647011698e-05,
+ "loss": 2.1499905395507812,
+ "mean_token_accuracy": 0.5204668046534061,
+ "num_tokens": 9006296.0,
+ "step": 1500
+ },
+ {
+ "entropy": 2.131927245259285,
+ "epoch": 0.2483874844757822,
+ "grad_norm": 3.546875,
+ "learning_rate": 1.5036051914757252e-05,
+ "loss": 2.157076416015625,
+ "mean_token_accuracy": 0.5220408822596073,
+ "num_tokens": 9307203.0,
+ "step": 1550
+ },
+ {
+ "entropy": 2.1308567851781843,
+ "epoch": 0.256399983975001,
+ "grad_norm": 3.546875,
+ "learning_rate": 1.4875821182502805e-05,
+ "loss": 2.146601104736328,
+ "mean_token_accuracy": 0.5219539681077003,
+ "num_tokens": 9610070.0,
+ "step": 1600
+ },
+ {
+ "entropy": 2.1097783839702604,
+ "epoch": 0.2644124834742198,
+ "grad_norm": 3.84375,
+ "learning_rate": 1.4715590450248359e-05,
+ "loss": 2.1268209838867187,
+ "mean_token_accuracy": 0.5236652019619942,
+ "num_tokens": 9909117.0,
+ "step": 1650
+ },
+ {
+ "entropy": 2.123499180674553,
+ "epoch": 0.2724249829734386,
+ "grad_norm": 3.46875,
+ "learning_rate": 1.4555359717993913e-05,
+ "loss": 2.1409934997558593,
+ "mean_token_accuracy": 0.5225465354323388,
+ "num_tokens": 10209461.0,
+ "step": 1700
+ },
+ {
+ "entropy": 2.13052960395813,
+ "epoch": 0.2804374824726573,
+ "grad_norm": 2.71875,
+ "learning_rate": 1.4395128985739467e-05,
+ "loss": 2.151373291015625,
+ "mean_token_accuracy": 0.5227928113937378,
+ "num_tokens": 10509282.0,
+ "step": 1750
+ },
+ {
+ "entropy": 2.1226114094257356,
+ "epoch": 0.2884499819718761,
+ "grad_norm": 3.65625,
+ "learning_rate": 1.4234898253485019e-05,
+ "loss": 2.144451141357422,
+ "mean_token_accuracy": 0.5233827362954616,
+ "num_tokens": 10811056.0,
+ "step": 1800
+ },
+ {
+ "entropy": 2.126143154501915,
+ "epoch": 0.2964624814710949,
+ "grad_norm": 3.359375,
+ "learning_rate": 1.4074667521230575e-05,
+ "loss": 2.1457598876953123,
+ "mean_token_accuracy": 0.5221967925131321,
+ "num_tokens": 11111153.0,
+ "step": 1850
+ },
+ {
+ "entropy": 2.1048326581716537,
+ "epoch": 0.30447498097031367,
+ "grad_norm": 3.59375,
+ "learning_rate": 1.3914436788976127e-05,
+ "loss": 2.1260116577148436,
+ "mean_token_accuracy": 0.5272719909250736,
+ "num_tokens": 11405606.0,
+ "step": 1900
+ },
+ {
+ "entropy": 2.1236617982387545,
+ "epoch": 0.31248748046953245,
+ "grad_norm": 3.0,
+ "learning_rate": 1.375420605672168e-05,
+ "loss": 2.151782684326172,
+ "mean_token_accuracy": 0.5227546173334122,
+ "num_tokens": 11703943.0,
+ "step": 1950
+ },
+ {
+ "entropy": 2.121563429236412,
+ "epoch": 0.32049997996875124,
+ "grad_norm": 3.34375,
+ "learning_rate": 1.3593975324467235e-05,
+ "loss": 2.1370144653320313,
+ "mean_token_accuracy": 0.5235123547911644,
+ "num_tokens": 12002522.0,
+ "step": 2000
+ },
+ {
+ "entropy": 2.1155461168289182,
+ "epoch": 0.32851247946797,
+ "grad_norm": 3.46875,
+ "learning_rate": 1.3433744592212787e-05,
+ "loss": 2.1364064025878906,
+ "mean_token_accuracy": 0.5237135806679726,
+ "num_tokens": 12303098.0,
+ "step": 2050
+ },
+ {
+ "entropy": 2.1386052322387696,
+ "epoch": 0.3365249789671888,
+ "grad_norm": 3.484375,
+ "learning_rate": 1.327351385995834e-05,
+ "loss": 2.149888458251953,
+ "mean_token_accuracy": 0.522669050693512,
+ "num_tokens": 12605534.0,
+ "step": 2100
+ },
+ {
+ "entropy": 2.102600795030594,
+ "epoch": 0.3445374784664076,
+ "grad_norm": 3.796875,
+ "learning_rate": 1.3113283127703895e-05,
+ "loss": 2.1236566162109374,
+ "mean_token_accuracy": 0.5279092155396938,
+ "num_tokens": 12901218.0,
+ "step": 2150
+ },
+ {
+ "entropy": 2.132645187973976,
+ "epoch": 0.35254997796562637,
+ "grad_norm": 3.53125,
+ "learning_rate": 1.2953052395449447e-05,
+ "loss": 2.1523503112792968,
+ "mean_token_accuracy": 0.5198129612207413,
+ "num_tokens": 13213018.0,
+ "step": 2200
+ },
+ {
+ "entropy": 2.1204321217536926,
+ "epoch": 0.36056247746484515,
+ "grad_norm": 3.25,
+ "learning_rate": 1.2792821663195001e-05,
+ "loss": 2.1325447082519533,
+ "mean_token_accuracy": 0.5243636940419674,
+ "num_tokens": 13514408.0,
+ "step": 2250
+ },
+ {
+ "entropy": 2.102410731315613,
+ "epoch": 0.36857497696406394,
+ "grad_norm": 3.1875,
+ "learning_rate": 1.2632590930940555e-05,
+ "loss": 2.123565368652344,
+ "mean_token_accuracy": 0.5257614709436893,
+ "num_tokens": 13818272.0,
+ "step": 2300
+ },
+ {
+ "entropy": 2.0876205080747603,
+ "epoch": 0.3765874764632827,
+ "grad_norm": 3.328125,
+ "learning_rate": 1.247236019868611e-05,
+ "loss": 2.116696472167969,
+ "mean_token_accuracy": 0.5275124031305313,
+ "num_tokens": 14110702.0,
+ "step": 2350
+ },
+ {
+ "entropy": 2.1078518223762512,
+ "epoch": 0.3845999759625015,
+ "grad_norm": 3.453125,
+ "learning_rate": 1.2312129466431662e-05,
+ "loss": 2.1291993713378905,
+ "mean_token_accuracy": 0.524967464953661,
+ "num_tokens": 14407015.0,
+ "step": 2400
+ },
+ {
+ "entropy": 2.0930898267030718,
+ "epoch": 0.3926124754617203,
+ "grad_norm": 3.5,
+ "learning_rate": 1.2151898734177216e-05,
+ "loss": 2.1094123840332033,
+ "mean_token_accuracy": 0.5290587748587131,
+ "num_tokens": 14696565.0,
+ "step": 2450
+ },
+ {
+ "entropy": 2.1109757220745085,
+ "epoch": 0.4006249749609391,
+ "grad_norm": 3.5,
+ "learning_rate": 1.199166800192277e-05,
+ "loss": 2.1246888732910154,
+ "mean_token_accuracy": 0.5258957357704639,
+ "num_tokens": 14994401.0,
+ "step": 2500
+ },
+ {
+ "entropy": 2.1087811160087586,
+ "epoch": 0.40863747446015786,
+ "grad_norm": 3.1875,
+ "learning_rate": 1.1831437269668324e-05,
+ "loss": 2.12890625,
+ "mean_token_accuracy": 0.5269917643070221,
+ "num_tokens": 15292837.0,
+ "step": 2550
+ },
+ {
+ "entropy": 2.097252429127693,
+ "epoch": 0.41664997395937664,
+ "grad_norm": 3.0,
+ "learning_rate": 1.1671206537413878e-05,
+ "loss": 2.1145870971679686,
+ "mean_token_accuracy": 0.5274137698113919,
+ "num_tokens": 15590647.0,
+ "step": 2600
+ },
+ {
+ "entropy": 2.111199595332146,
+ "epoch": 0.4246624734585954,
+ "grad_norm": 3.25,
+ "learning_rate": 1.151097580515943e-05,
+ "loss": 2.1274327087402343,
+ "mean_token_accuracy": 0.5250249288976192,
+ "num_tokens": 15892772.0,
+ "step": 2650
+ },
+ {
+ "entropy": 2.1051502913236617,
+ "epoch": 0.4326749729578142,
+ "grad_norm": 3.203125,
+ "learning_rate": 1.1350745072904986e-05,
+ "loss": 2.127935333251953,
+ "mean_token_accuracy": 0.5263640700280666,
+ "num_tokens": 16191611.0,
+ "step": 2700
+ },
+ {
+ "entropy": 2.1127885603904724,
+ "epoch": 0.440687472457033,
+ "grad_norm": 3.515625,
+ "learning_rate": 1.1190514340650538e-05,
+ "loss": 2.129211883544922,
+ "mean_token_accuracy": 0.5250939278304577,
+ "num_tokens": 16483582.0,
+ "step": 2750
+ },
+ {
+ "entropy": 2.110930626988411,
+ "epoch": 0.4486999719562518,
+ "grad_norm": 3.1875,
+ "learning_rate": 1.103028360839609e-05,
+ "loss": 2.1220079040527344,
+ "mean_token_accuracy": 0.5250919266045093,
+ "num_tokens": 16787253.0,
+ "step": 2800
+ },
+ {
+ "entropy": 2.1182780677080153,
+ "epoch": 0.45671247145547056,
+ "grad_norm": 3.8125,
+ "learning_rate": 1.0870052876141646e-05,
+ "loss": 2.1358837890625,
+ "mean_token_accuracy": 0.5217854312062263,
+ "num_tokens": 17088741.0,
+ "step": 2850
+ },
+ {
+ "entropy": 2.1117833071947096,
+ "epoch": 0.46472497095468934,
+ "grad_norm": 3.171875,
+ "learning_rate": 1.0709822143887198e-05,
+ "loss": 2.125662841796875,
+ "mean_token_accuracy": 0.524649152457714,
+ "num_tokens": 17389236.0,
+ "step": 2900
+ },
+ {
+ "entropy": 2.1262730568647386,
+ "epoch": 0.47273747045390807,
+ "grad_norm": 3.625,
+ "learning_rate": 1.054959141163275e-05,
+ "loss": 2.142789611816406,
+ "mean_token_accuracy": 0.5236384868621826,
+ "num_tokens": 17693413.0,
+ "step": 2950
+ },
+ {
+ "entropy": 2.115410739183426,
+ "epoch": 0.48074996995312685,
+ "grad_norm": 3.140625,
+ "learning_rate": 1.0389360679378306e-05,
+ "loss": 2.1337106323242185,
+ "mean_token_accuracy": 0.5249222332239151,
+ "num_tokens": 17990326.0,
+ "step": 3000
+ },
+ {
+ "entropy": 2.0951381689310073,
+ "epoch": 0.48876246945234564,
+ "grad_norm": 3.234375,
+ "learning_rate": 1.0229129947123859e-05,
+ "loss": 2.108144073486328,
+ "mean_token_accuracy": 0.5269971822202205,
+ "num_tokens": 18293136.0,
+ "step": 3050
+ },
+ {
+ "entropy": 2.116977259516716,
+ "epoch": 0.4967749689515644,
+ "grad_norm": 3.421875,
+ "learning_rate": 1.0068899214869413e-05,
+ "loss": 2.1327830505371095,
+ "mean_token_accuracy": 0.5238270902633667,
+ "num_tokens": 18591481.0,
+ "step": 3100
+ },
+ {
+ "entropy": 2.08598925113678,
+ "epoch": 0.5047874684507833,
+ "grad_norm": 2.96875,
+ "learning_rate": 9.908668482614967e-06,
+ "loss": 2.1032344055175782,
+ "mean_token_accuracy": 0.5289211909472943,
+ "num_tokens": 18885221.0,
+ "step": 3150
+ },
+ {
+ "entropy": 2.097712662220001,
+ "epoch": 0.512799967950002,
+ "grad_norm": 3.40625,
+ "learning_rate": 9.74843775036052e-06,
+ "loss": 2.1150633239746095,
+ "mean_token_accuracy": 0.5288272814452648,
+ "num_tokens": 19184784.0,
+ "step": 3200
+ },
+ {
+ "entropy": 2.1063760107755662,
+ "epoch": 0.5208124674492208,
+ "grad_norm": 3.28125,
+ "learning_rate": 9.588207018106075e-06,
+ "loss": 2.1217109680175783,
+ "mean_token_accuracy": 0.526290989369154,
+ "num_tokens": 19482538.0,
+ "step": 3250
+ },
+ {
+ "entropy": 2.090182801485062,
+ "epoch": 0.5288249669484396,
+ "grad_norm": 2.875,
+ "learning_rate": 9.427976285851627e-06,
+ "loss": 2.1130311584472654,
+ "mean_token_accuracy": 0.5285439921915531,
+ "num_tokens": 19782544.0,
+ "step": 3300
+ },
+ {
+ "entropy": 2.103260806798935,
+ "epoch": 0.5368374664476584,
+ "grad_norm": 3.59375,
+ "learning_rate": 9.267745553597181e-06,
+ "loss": 2.116189727783203,
+ "mean_token_accuracy": 0.5273195558786392,
+ "num_tokens": 20086408.0,
+ "step": 3350
+ },
+ {
+ "entropy": 2.0673747235536575,
+ "epoch": 0.5448499659468772,
+ "grad_norm": 3.125,
+ "learning_rate": 9.107514821342735e-06,
+ "loss": 2.087482147216797,
+ "mean_token_accuracy": 0.532562478929758,
+ "num_tokens": 20380429.0,
+ "step": 3400
+ },
+ {
+ "entropy": 2.1030789083242416,
+ "epoch": 0.552862465446096,
+ "grad_norm": 3.296875,
+ "learning_rate": 8.947284089088289e-06,
+ "loss": 2.122591247558594,
+ "mean_token_accuracy": 0.5256703786551953,
+ "num_tokens": 20682849.0,
+ "step": 3450
+ },
+ {
+ "entropy": 2.0988716661930082,
+ "epoch": 0.5608749649453146,
+ "grad_norm": 3.546875,
+ "learning_rate": 8.787053356833841e-06,
+ "loss": 2.115867156982422,
+ "mean_token_accuracy": 0.5266730943322182,
+ "num_tokens": 20979225.0,
+ "step": 3500
+ },
+ {
+ "entropy": 2.0954072701931,
+ "epoch": 0.5688874644445334,
+ "grad_norm": 3.859375,
+ "learning_rate": 8.626822624579395e-06,
+ "loss": 2.1116868591308595,
+ "mean_token_accuracy": 0.5267495161294937,
+ "num_tokens": 21284166.0,
+ "step": 3550
+ },
+ {
+ "entropy": 2.11238471031189,
+ "epoch": 0.5768999639437522,
+ "grad_norm": 3.484375,
+ "learning_rate": 8.466591892324949e-06,
+ "loss": 2.1292352294921875,
+ "mean_token_accuracy": 0.5249243360757828,
+ "num_tokens": 21584612.0,
+ "step": 3600
+ },
+ {
+ "entropy": 2.091464368700981,
+ "epoch": 0.584912463442971,
+ "grad_norm": 3.359375,
+ "learning_rate": 8.306361160070501e-06,
+ "loss": 2.1099319458007812,
+ "mean_token_accuracy": 0.5292035435140133,
+ "num_tokens": 21879345.0,
+ "step": 3650
+ },
+ {
+ "entropy": 2.095204224586487,
+ "epoch": 0.5929249629421898,
+ "grad_norm": 2.96875,
+ "learning_rate": 8.146130427816055e-06,
+ "loss": 2.1198716735839844,
+ "mean_token_accuracy": 0.5255991271138192,
+ "num_tokens": 22183486.0,
+ "step": 3700
+ },
+ {
+ "entropy": 2.0666573679447175,
+ "epoch": 0.6009374624414086,
+ "grad_norm": 2.90625,
+ "learning_rate": 7.98589969556161e-06,
+ "loss": 2.08689697265625,
+ "mean_token_accuracy": 0.5318664258718491,
+ "num_tokens": 22476438.0,
+ "step": 3750
+ },
+ {
+ "entropy": 2.122153177857399,
+ "epoch": 0.6089499619406273,
+ "grad_norm": 3.40625,
+ "learning_rate": 7.825668963307162e-06,
+ "loss": 2.138288879394531,
+ "mean_token_accuracy": 0.5232584895193577,
+ "num_tokens": 22780385.0,
+ "step": 3800
+ },
+ {
+ "entropy": 2.1190748488903046,
+ "epoch": 0.6169624614398461,
+ "grad_norm": 3.46875,
+ "learning_rate": 7.665438231052716e-06,
+ "loss": 2.139790344238281,
+ "mean_token_accuracy": 0.5243287613987923,
+ "num_tokens": 23080847.0,
+ "step": 3850
+ },
+ {
+ "entropy": 2.1312570214271545,
+ "epoch": 0.6249749609390649,
+ "grad_norm": 3.765625,
+ "learning_rate": 7.5052074987982706e-06,
+ "loss": 2.1471080017089843,
+ "mean_token_accuracy": 0.5232046774029732,
+ "num_tokens": 23384738.0,
+ "step": 3900
+ },
+ {
+ "entropy": 2.0779008239507677,
+ "epoch": 0.6329874604382837,
+ "grad_norm": 3.171875,
+ "learning_rate": 7.344976766543823e-06,
+ "loss": 2.0974729919433592,
+ "mean_token_accuracy": 0.5305448272824287,
+ "num_tokens": 23682750.0,
+ "step": 3950
+ },
+ {
+ "entropy": 2.1121455317735673,
+ "epoch": 0.6409999599375025,
+ "grad_norm": 3.921875,
+ "learning_rate": 7.184746034289377e-06,
+ "loss": 2.127069549560547,
+ "mean_token_accuracy": 0.5252936625480652,
+ "num_tokens": 23989029.0,
+ "step": 4000
+ },
+ {
+ "entropy": 2.0982888478040693,
+ "epoch": 0.6490124594367213,
+ "grad_norm": 4.0625,
+ "learning_rate": 7.024515302034931e-06,
+ "loss": 2.112042236328125,
+ "mean_token_accuracy": 0.5281323495507241,
+ "num_tokens": 24292063.0,
+ "step": 4050
+ },
+ {
+ "entropy": 2.098363963365555,
+ "epoch": 0.65702495893594,
+ "grad_norm": 3.046875,
+ "learning_rate": 6.864284569780485e-06,
+ "loss": 2.1156489562988283,
+ "mean_token_accuracy": 0.5275120075047016,
+ "num_tokens": 24590201.0,
+ "step": 4100
+ },
+ {
+ "entropy": 2.1065632277727127,
+ "epoch": 0.6650374584351588,
+ "grad_norm": 3.375,
+ "learning_rate": 6.704053837526038e-06,
+ "loss": 2.12911376953125,
+ "mean_token_accuracy": 0.5262684133648873,
+ "num_tokens": 24892122.0,
+ "step": 4150
+ },
+ {
+ "entropy": 2.1014072370529173,
+ "epoch": 0.6730499579343776,
+ "grad_norm": 2.84375,
+ "learning_rate": 6.543823105271592e-06,
+ "loss": 2.1130335998535155,
+ "mean_token_accuracy": 0.5262750512361527,
+ "num_tokens": 25194969.0,
+ "step": 4200
+ },
+ {
+ "entropy": 2.0905236744880678,
+ "epoch": 0.6810624574335964,
+ "grad_norm": 3.03125,
+ "learning_rate": 6.383592373017146e-06,
+ "loss": 2.1153533935546873,
+ "mean_token_accuracy": 0.527137616276741,
+ "num_tokens": 25499521.0,
+ "step": 4250
+ },
+ {
+ "entropy": 2.1032627063989637,
+ "epoch": 0.6890749569328152,
+ "grad_norm": 3.125,
+ "learning_rate": 6.223361640762698e-06,
+ "loss": 2.122831115722656,
+ "mean_token_accuracy": 0.5254572249948979,
+ "num_tokens": 25798933.0,
+ "step": 4300
+ },
+ {
+ "entropy": 2.118139310479164,
+ "epoch": 0.697087456432034,
+ "grad_norm": 3.28125,
+ "learning_rate": 6.063130908508252e-06,
+ "loss": 2.1376751708984374,
+ "mean_token_accuracy": 0.5243131402134895,
+ "num_tokens": 26096619.0,
+ "step": 4350
+ },
+ {
+ "entropy": 2.082554380297661,
+ "epoch": 0.7050999559312527,
+ "grad_norm": 3.390625,
+ "learning_rate": 5.902900176253806e-06,
+ "loss": 2.0949493408203126,
+ "mean_token_accuracy": 0.5316410692036152,
+ "num_tokens": 26394395.0,
+ "step": 4400
+ },
+ {
+ "entropy": 2.09870474755764,
+ "epoch": 0.7131124554304715,
+ "grad_norm": 3.421875,
+ "learning_rate": 5.7426694439993594e-06,
+ "loss": 2.1184346008300783,
+ "mean_token_accuracy": 0.5267573061585427,
+ "num_tokens": 26692333.0,
+ "step": 4450
+ },
+ {
+ "entropy": 2.0866885662078856,
+ "epoch": 0.7211249549296903,
+ "grad_norm": 3.71875,
+ "learning_rate": 5.5824387117449134e-06,
+ "loss": 2.102904968261719,
+ "mean_token_accuracy": 0.529726449996233,
+ "num_tokens": 26989384.0,
+ "step": 4500
+ },
+ {
+ "entropy": 2.114828346967697,
+ "epoch": 0.7291374544289091,
+ "grad_norm": 3.46875,
+ "learning_rate": 5.4222079794904666e-06,
+ "loss": 2.128611602783203,
+ "mean_token_accuracy": 0.5247649565339089,
+ "num_tokens": 27299927.0,
+ "step": 4550
+ },
+ {
+ "entropy": 2.09617226600647,
+ "epoch": 0.7371499539281279,
+ "grad_norm": 3.015625,
+ "learning_rate": 5.26197724723602e-06,
+ "loss": 2.1166041564941405,
+ "mean_token_accuracy": 0.5273530712723732,
+ "num_tokens": 27599011.0,
+ "step": 4600
+ },
+ {
+ "entropy": 2.0975282275676728,
+ "epoch": 0.7451624534273467,
+ "grad_norm": 3.3125,
+ "learning_rate": 5.101746514981574e-06,
+ "loss": 2.112121124267578,
+ "mean_token_accuracy": 0.5281847220659256,
+ "num_tokens": 27892355.0,
+ "step": 4650
+ },
+ {
+ "entropy": 2.095783769488335,
+ "epoch": 0.7531749529265654,
+ "grad_norm": 3.78125,
+ "learning_rate": 4.941515782727128e-06,
+ "loss": 2.110701904296875,
+ "mean_token_accuracy": 0.5272985179722309,
+ "num_tokens": 28187615.0,
+ "step": 4700
+ },
+ {
+ "entropy": 2.0964103603363036,
+ "epoch": 0.7611874524257842,
+ "grad_norm": 3.1875,
+ "learning_rate": 4.781285050472681e-06,
+ "loss": 2.11992919921875,
+ "mean_token_accuracy": 0.5271916574239731,
+ "num_tokens": 28485419.0,
+ "step": 4750
+ },
+ {
+ "entropy": 2.0820577853918074,
+ "epoch": 0.769199951925003,
+ "grad_norm": 3.5,
+ "learning_rate": 4.621054318218234e-06,
+ "loss": 2.095073394775391,
+ "mean_token_accuracy": 0.5297581899166107,
+ "num_tokens": 28780158.0,
+ "step": 4800
+ },
+ {
+ "entropy": 2.089591929912567,
+ "epoch": 0.7772124514242218,
+ "grad_norm": 3.125,
+ "learning_rate": 4.460823585963788e-06,
+ "loss": 2.109657897949219,
+ "mean_token_accuracy": 0.528961935043335,
+ "num_tokens": 29081052.0,
+ "step": 4850
+ },
+ {
+ "entropy": 2.092900555729866,
+ "epoch": 0.7852249509234406,
+ "grad_norm": 3.46875,
+ "learning_rate": 4.300592853709342e-06,
+ "loss": 2.106436920166016,
+ "mean_token_accuracy": 0.5284854583442211,
+ "num_tokens": 29376324.0,
+ "step": 4900
+ },
+ {
+ "entropy": 2.0630873823165894,
+ "epoch": 0.7932374504226594,
+ "grad_norm": 3.28125,
+ "learning_rate": 4.140362121454896e-06,
+ "loss": 2.088908233642578,
+ "mean_token_accuracy": 0.532561699450016,
+ "num_tokens": 29675269.0,
+ "step": 4950
+ },
+ {
+ "entropy": 2.0753238475322724,
+ "epoch": 0.8012499499218781,
+ "grad_norm": 3.21875,
+ "learning_rate": 3.980131389200449e-06,
+ "loss": 2.0961412048339843,
+ "mean_token_accuracy": 0.5304154875874519,
+ "num_tokens": 29974838.0,
+ "step": 5000
+ },
+ {
+ "entropy": 2.123110404610634,
+ "epoch": 0.8092624494210969,
+ "grad_norm": 3.28125,
+ "learning_rate": 3.819900656946002e-06,
+ "loss": 2.1538424682617188,
+ "mean_token_accuracy": 0.5222072719037533,
+ "num_tokens": 30286201.0,
+ "step": 5050
+ },
+ {
+ "entropy": 2.123967355489731,
+ "epoch": 0.8172749489203157,
+ "grad_norm": 3.046875,
+ "learning_rate": 3.6596699246915563e-06,
+ "loss": 2.1417193603515625,
+ "mean_token_accuracy": 0.5213662980496884,
+ "num_tokens": 30592837.0,
+ "step": 5100
+ },
+ {
+ "entropy": 2.080365374684334,
+ "epoch": 0.8252874484195345,
+ "grad_norm": 3.375,
+ "learning_rate": 3.4994391924371094e-06,
+ "loss": 2.0970494079589845,
+ "mean_token_accuracy": 0.5296585746109486,
+ "num_tokens": 30890674.0,
+ "step": 5150
+ },
+ {
+ "entropy": 2.1183922028541566,
+ "epoch": 0.8332999479187533,
+ "grad_norm": 3.359375,
+ "learning_rate": 3.3392084601826634e-06,
+ "loss": 2.132384948730469,
+ "mean_token_accuracy": 0.5249108146131038,
+ "num_tokens": 31195601.0,
+ "step": 5200
+ },
+ {
+ "entropy": 2.1079335528612138,
+ "epoch": 0.8413124474179721,
+ "grad_norm": 2.890625,
+ "learning_rate": 3.178977727928217e-06,
+ "loss": 2.1310113525390624,
+ "mean_token_accuracy": 0.5255681444704533,
+ "num_tokens": 31498768.0,
+ "step": 5250
+ },
+ {
+ "entropy": 2.1180070728063582,
+ "epoch": 0.8493249469171908,
+ "grad_norm": 3.78125,
+ "learning_rate": 3.01874699567377e-06,
+ "loss": 2.1365049743652342,
+ "mean_token_accuracy": 0.5235744562745094,
+ "num_tokens": 31801891.0,
+ "step": 5300
+ },
+ {
+ "entropy": 2.0909912359714506,
+ "epoch": 0.8573374464164096,
+ "grad_norm": 3.078125,
+ "learning_rate": 2.858516263419324e-06,
+ "loss": 2.108803405761719,
+ "mean_token_accuracy": 0.5267959542572498,
+ "num_tokens": 32102627.0,
+ "step": 5350
+ },
+ {
+ "entropy": 2.1209339940547944,
+ "epoch": 0.8653499459156284,
+ "grad_norm": 3.140625,
+ "learning_rate": 2.6982855311648777e-06,
+ "loss": 2.143946228027344,
+ "mean_token_accuracy": 0.5227122528851033,
+ "num_tokens": 32405500.0,
+ "step": 5400
+ },
+ {
+ "entropy": 2.1137486273050308,
+ "epoch": 0.8733624454148472,
+ "grad_norm": 3.03125,
+ "learning_rate": 2.5380547989104317e-06,
+ "loss": 2.127569122314453,
+ "mean_token_accuracy": 0.5245758745074273,
+ "num_tokens": 32712521.0,
+ "step": 5450
+ },
+ {
+ "entropy": 2.0872016245126725,
+ "epoch": 0.881374944914066,
+ "grad_norm": 3.171875,
+ "learning_rate": 2.377824066655985e-06,
+ "loss": 2.100517120361328,
+ "mean_token_accuracy": 0.5295753349363804,
+ "num_tokens": 33016210.0,
+ "step": 5500
+ }
+ ],
+ "logging_steps": 50,
+ "max_steps": 6241,
+ "num_input_tokens_seen": 0,
+ "num_train_epochs": 1,
+ "save_steps": 500,
+ "stateful_callbacks": {
+ "TrainerControl": {
+ "args": {
+ "should_epoch_stop": false,
+ "should_evaluate": false,
+ "should_log": false,
+ "should_save": true,
+ "should_training_stop": false
+ },
+ "attributes": {}
+ }
+ },
+ "total_flos": 3.898195132722463e+17,
+ "train_batch_size": 4,
+ "trial_name": null,
+ "trial_params": null
+}
diff --git a/checkpoint-5500/training_args.bin b/checkpoint-5500/training_args.bin
new file mode 100644
index 0000000000000000000000000000000000000000..4b409d5a147856f0d5a3d3da0f3b4922a9def7dd
--- /dev/null
+++ b/checkpoint-5500/training_args.bin
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:78622f63304ec0913ed0b4ea7a1b26c08a5621b3661a380d798c4ea426235671
+size 5713
diff --git a/checkpoint-6000/chat_template.jinja b/checkpoint-6000/chat_template.jinja
new file mode 100644
index 0000000000000000000000000000000000000000..01be9b307daa2d425f7c168c9fb145a286e0afb4
--- /dev/null
+++ b/checkpoint-6000/chat_template.jinja
@@ -0,0 +1,89 @@
+{%- if tools %}
+ {{- '<|im_start|>system\n' }}
+ {%- if messages[0].role == 'system' %}
+ {{- messages[0].content + '\n\n' }}
+ {%- endif %}
+ {{- "# Tools\n\nYou may call one or more functions to assist with the user query.\n\nYou are provided with function signatures within XML tags:\n" }}
+ {%- for tool in tools %}
+ {{- "\n" }}
+ {{- tool | tojson }}
+ {%- endfor %}
+ {{- "\n\n\nFor each function call, return a json object with function name and arguments within XML tags:\n\n{\"name\": , \"arguments\": }\n<|im_end|>\n" }}
+{%- else %}
+ {%- if messages[0].role == 'system' %}
+ {{- '<|im_start|>system\n' + messages[0].content + '<|im_end|>\n' }}
+ {%- endif %}
+{%- endif %}
+{%- set ns = namespace(multi_step_tool=true, last_query_index=messages|length - 1) %}
+{%- for message in messages[::-1] %}
+ {%- set index = (messages|length - 1) - loop.index0 %}
+ {%- if ns.multi_step_tool and message.role == "user" and message.content is string and not(message.content.startswith('') and message.content.endswith('')) %}
+ {%- set ns.multi_step_tool = false %}
+ {%- set ns.last_query_index = index %}
+ {%- endif %}
+{%- endfor %}
+{%- for message in messages %}
+ {%- if message.content is string %}
+ {%- set content = message.content %}
+ {%- else %}
+ {%- set content = '' %}
+ {%- endif %}
+ {%- if (message.role == "user") or (message.role == "system" and not loop.first) %}
+ {{- '<|im_start|>' + message.role + '\n' + content + '<|im_end|>' + '\n' }}
+ {%- elif message.role == "assistant" %}
+ {%- set reasoning_content = '' %}
+ {%- if message.reasoning_content is string %}
+ {%- set reasoning_content = message.reasoning_content %}
+ {%- else %}
+ {%- if '' in content %}
+ {%- set reasoning_content = content.split('')[0].rstrip('\n').split('')[-1].lstrip('\n') %}
+ {%- set content = content.split('')[-1].lstrip('\n') %}
+ {%- endif %}
+ {%- endif %}
+ {%- if loop.index0 > ns.last_query_index %}
+ {%- if loop.last or (not loop.last and reasoning_content) %}
+ {{- '<|im_start|>' + message.role + '\n\n' + reasoning_content.strip('\n') + '\n\n\n' + content.lstrip('\n') }}
+ {%- else %}
+ {{- '<|im_start|>' + message.role + '\n' + content }}
+ {%- endif %}
+ {%- else %}
+ {{- '<|im_start|>' + message.role + '\n' + content }}
+ {%- endif %}
+ {%- if message.tool_calls %}
+ {%- for tool_call in message.tool_calls %}
+ {%- if (loop.first and content) or (not loop.first) %}
+ {{- '\n' }}
+ {%- endif %}
+ {%- if tool_call.function %}
+ {%- set tool_call = tool_call.function %}
+ {%- endif %}
+ {{- '\n{"name": "' }}
+ {{- tool_call.name }}
+ {{- '", "arguments": ' }}
+ {%- if tool_call.arguments is string %}
+ {{- tool_call.arguments }}
+ {%- else %}
+ {{- tool_call.arguments | tojson }}
+ {%- endif %}
+ {{- '}\n' }}
+ {%- endfor %}
+ {%- endif %}
+ {{- '<|im_end|>\n' }}
+ {%- elif message.role == "tool" %}
+ {%- if loop.first or (messages[loop.index0 - 1].role != "tool") %}
+ {{- '<|im_start|>user' }}
+ {%- endif %}
+ {{- '\n\n' }}
+ {{- content }}
+ {{- '\n' }}
+ {%- if loop.last or (messages[loop.index0 + 1].role != "tool") %}
+ {{- '<|im_end|>\n' }}
+ {%- endif %}
+ {%- endif %}
+{%- endfor %}
+{%- if add_generation_prompt %}
+ {{- '<|im_start|>assistant\n' }}
+ {%- if enable_thinking is defined and enable_thinking is false %}
+ {{- '\n\n\n\n' }}
+ {%- endif %}
+{%- endif %}
\ No newline at end of file
diff --git a/checkpoint-6000/config.json b/checkpoint-6000/config.json
new file mode 100644
index 0000000000000000000000000000000000000000..1d219d6a6d288625229a71646710b85cf7ae5def
--- /dev/null
+++ b/checkpoint-6000/config.json
@@ -0,0 +1,63 @@
+{
+ "architectures": [
+ "Qwen3ForCausalLM"
+ ],
+ "attention_bias": false,
+ "attention_dropout": 0.0,
+ "bos_token_id": null,
+ "dtype": "bfloat16",
+ "eos_token_id": 151645,
+ "head_dim": 128,
+ "hidden_act": "silu",
+ "hidden_size": 2048,
+ "initializer_range": 0.02,
+ "intermediate_size": 6144,
+ "layer_types": [
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention"
+ ],
+ "max_position_embeddings": 40960,
+ "max_window_layers": 28,
+ "model_type": "qwen3",
+ "num_attention_heads": 16,
+ "num_hidden_layers": 28,
+ "num_key_value_heads": 8,
+ "pad_token_id": 151643,
+ "rms_norm_eps": 1e-06,
+ "rope_parameters": {
+ "rope_theta": 1000000,
+ "rope_type": "default"
+ },
+ "sliding_window": null,
+ "tie_word_embeddings": true,
+ "transformers_version": "5.7.0",
+ "use_cache": false,
+ "use_sliding_window": false,
+ "vocab_size": 151936
+}
diff --git a/checkpoint-6000/generation_config.json b/checkpoint-6000/generation_config.json
new file mode 100644
index 0000000000000000000000000000000000000000..5ec133d867b1427d43a5ee0ea332bd088fc12013
--- /dev/null
+++ b/checkpoint-6000/generation_config.json
@@ -0,0 +1,12 @@
+{
+ "do_sample": true,
+ "eos_token_id": [
+ 151645,
+ 151643
+ ],
+ "pad_token_id": 151643,
+ "temperature": 0.6,
+ "top_k": 20,
+ "top_p": 0.95,
+ "transformers_version": "5.7.0"
+}
diff --git a/checkpoint-6000/model.safetensors b/checkpoint-6000/model.safetensors
new file mode 100644
index 0000000000000000000000000000000000000000..9414bab80c4b86a6ea8dae29a5082690e0d082a4
--- /dev/null
+++ b/checkpoint-6000/model.safetensors
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:aaa2f97519d1276e44439361d1c0c60b06a32d06573589fa8d33e65b2a46fd18
+size 3441185608
diff --git a/checkpoint-6000/optimizer.pt b/checkpoint-6000/optimizer.pt
new file mode 100644
index 0000000000000000000000000000000000000000..cde8e283e9b0cffca993e8f6dd6fd964afd746ef
--- /dev/null
+++ b/checkpoint-6000/optimizer.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:70cc931a62d58dd3242a68c2eb093d0a8602f0100fe0c366a73d0430de508143
+size 6882572207
diff --git a/checkpoint-6000/rng_state.pth b/checkpoint-6000/rng_state.pth
new file mode 100644
index 0000000000000000000000000000000000000000..5a8f17addf8aaee350625e9847751582bc9f86dd
--- /dev/null
+++ b/checkpoint-6000/rng_state.pth
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:61c19bab1174704a4a4441475683bf1270277af15d2e2c95e964789128e482c4
+size 14645
diff --git a/checkpoint-6000/scheduler.pt b/checkpoint-6000/scheduler.pt
new file mode 100644
index 0000000000000000000000000000000000000000..1b79d05bc5c72241be9f5568b02641010890b53b
--- /dev/null
+++ b/checkpoint-6000/scheduler.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:1ebaa185fb474ec5ec68068be2b7cdb67749a5f183dd0899caddc37f33c89035
+size 1465
diff --git a/checkpoint-6000/tokenizer.json b/checkpoint-6000/tokenizer.json
new file mode 100644
index 0000000000000000000000000000000000000000..c7afbed2efcdf019f88ab0572ec29d3bf595dfe2
--- /dev/null
+++ b/checkpoint-6000/tokenizer.json
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:be75606093db2094d7cd20f3c2f385c212750648bd6ea4fb2bf507a6a4c55506
+size 11422650
diff --git a/checkpoint-6000/tokenizer_config.json b/checkpoint-6000/tokenizer_config.json
new file mode 100644
index 0000000000000000000000000000000000000000..770e41d6c92519d525eede4cbcf3ba27f6425311
--- /dev/null
+++ b/checkpoint-6000/tokenizer_config.json
@@ -0,0 +1,30 @@
+{
+ "add_prefix_space": false,
+ "backend": "tokenizers",
+ "bos_token": null,
+ "clean_up_tokenization_spaces": false,
+ "eos_token": "<|im_end|>",
+ "errors": "replace",
+ "extra_special_tokens": [
+ "<|im_start|>",
+ "<|im_end|>",
+ "<|object_ref_start|>",
+ "<|object_ref_end|>",
+ "<|box_start|>",
+ "<|box_end|>",
+ "<|quad_start|>",
+ "<|quad_end|>",
+ "<|vision_start|>",
+ "<|vision_end|>",
+ "<|vision_pad|>",
+ "<|image_pad|>",
+ "<|video_pad|>"
+ ],
+ "is_local": false,
+ "local_files_only": false,
+ "model_max_length": 131072,
+ "pad_token": "<|endoftext|>",
+ "split_special_tokens": false,
+ "tokenizer_class": "Qwen2Tokenizer",
+ "unk_token": null
+}
diff --git a/checkpoint-6000/trainer_state.json b/checkpoint-6000/trainer_state.json
new file mode 100644
index 0000000000000000000000000000000000000000..cacd1b16811f7cebba452d577c801468db885a17
--- /dev/null
+++ b/checkpoint-6000/trainer_state.json
@@ -0,0 +1,1234 @@
+{
+ "best_global_step": null,
+ "best_metric": null,
+ "best_model_checkpoint": null,
+ "epoch": 0.9614999399062537,
+ "eval_steps": 500,
+ "global_step": 6000,
+ "is_hyper_param_search": false,
+ "is_local_process_zero": true,
+ "is_world_process_zero": true,
+ "log_history": [
+ {
+ "entropy": 2.223189059495926,
+ "epoch": 0.008012499499218782,
+ "grad_norm": 2.875,
+ "learning_rate": 1.9842973882390643e-05,
+ "loss": 2.3428109741210936,
+ "mean_token_accuracy": 0.4963963569700718,
+ "num_tokens": 305366.0,
+ "step": 50
+ },
+ {
+ "entropy": 2.188130117058754,
+ "epoch": 0.016024998998437564,
+ "grad_norm": 3.328125,
+ "learning_rate": 1.96827431501362e-05,
+ "loss": 2.225358734130859,
+ "mean_token_accuracy": 0.5110921229422093,
+ "num_tokens": 605042.0,
+ "step": 100
+ },
+ {
+ "entropy": 2.197347580194473,
+ "epoch": 0.024037498497656344,
+ "grad_norm": 3.453125,
+ "learning_rate": 1.952251241788175e-05,
+ "loss": 2.2299420166015627,
+ "mean_token_accuracy": 0.5093490639328957,
+ "num_tokens": 906888.0,
+ "step": 150
+ },
+ {
+ "entropy": 2.1913950395584108,
+ "epoch": 0.03204999799687513,
+ "grad_norm": 2.90625,
+ "learning_rate": 1.9362281685627303e-05,
+ "loss": 2.210995941162109,
+ "mean_token_accuracy": 0.5097909501194954,
+ "num_tokens": 1213125.0,
+ "step": 200
+ },
+ {
+ "entropy": 2.1611998677253723,
+ "epoch": 0.040062497496093905,
+ "grad_norm": 3.25,
+ "learning_rate": 1.920205095337286e-05,
+ "loss": 2.1867474365234374,
+ "mean_token_accuracy": 0.5154410660266876,
+ "num_tokens": 1509026.0,
+ "step": 250
+ },
+ {
+ "entropy": 2.168051454424858,
+ "epoch": 0.04807499699531269,
+ "grad_norm": 2.9375,
+ "learning_rate": 1.904182022111841e-05,
+ "loss": 2.190071716308594,
+ "mean_token_accuracy": 0.5156884534657001,
+ "num_tokens": 1806850.0,
+ "step": 300
+ },
+ {
+ "entropy": 2.1737154543399813,
+ "epoch": 0.05608749649453147,
+ "grad_norm": 3.421875,
+ "learning_rate": 1.8881589488863967e-05,
+ "loss": 2.199147186279297,
+ "mean_token_accuracy": 0.514045044630766,
+ "num_tokens": 2104505.0,
+ "step": 350
+ },
+ {
+ "entropy": 2.148039159178734,
+ "epoch": 0.06409999599375026,
+ "grad_norm": 3.109375,
+ "learning_rate": 1.872135875660952e-05,
+ "loss": 2.176946105957031,
+ "mean_token_accuracy": 0.5177112428843975,
+ "num_tokens": 2391088.0,
+ "step": 400
+ },
+ {
+ "entropy": 2.194890711903572,
+ "epoch": 0.07211249549296903,
+ "grad_norm": 3.28125,
+ "learning_rate": 1.856112802435507e-05,
+ "loss": 2.216288757324219,
+ "mean_token_accuracy": 0.5110335703194141,
+ "num_tokens": 2691272.0,
+ "step": 450
+ },
+ {
+ "entropy": 2.1557786524295808,
+ "epoch": 0.08012499499218781,
+ "grad_norm": 4.15625,
+ "learning_rate": 1.8400897292100627e-05,
+ "loss": 2.1812680053710936,
+ "mean_token_accuracy": 0.5176697050035,
+ "num_tokens": 2991855.0,
+ "step": 500
+ },
+ {
+ "entropy": 2.1708167713880537,
+ "epoch": 0.08813749449140659,
+ "grad_norm": 4.34375,
+ "learning_rate": 1.824066655984618e-05,
+ "loss": 2.1921002197265627,
+ "mean_token_accuracy": 0.5156507430970669,
+ "num_tokens": 3295049.0,
+ "step": 550
+ },
+ {
+ "entropy": 2.1686314886808393,
+ "epoch": 0.09614999399062538,
+ "grad_norm": 2.875,
+ "learning_rate": 1.8080435827591735e-05,
+ "loss": 2.1914617919921877,
+ "mean_token_accuracy": 0.5139613692462445,
+ "num_tokens": 3597206.0,
+ "step": 600
+ },
+ {
+ "entropy": 2.15742906332016,
+ "epoch": 0.10416249348984416,
+ "grad_norm": 3.109375,
+ "learning_rate": 1.7920205095337287e-05,
+ "loss": 2.174853973388672,
+ "mean_token_accuracy": 0.5159518288075924,
+ "num_tokens": 3905268.0,
+ "step": 650
+ },
+ {
+ "entropy": 2.162363375425339,
+ "epoch": 0.11217499298906294,
+ "grad_norm": 3.171875,
+ "learning_rate": 1.7759974363082843e-05,
+ "loss": 2.184231872558594,
+ "mean_token_accuracy": 0.516631413847208,
+ "num_tokens": 4207934.0,
+ "step": 700
+ },
+ {
+ "entropy": 2.1516289287805557,
+ "epoch": 0.12018749248828171,
+ "grad_norm": 3.609375,
+ "learning_rate": 1.7599743630828395e-05,
+ "loss": 2.167020263671875,
+ "mean_token_accuracy": 0.516358617246151,
+ "num_tokens": 4515774.0,
+ "step": 750
+ },
+ {
+ "entropy": 2.1549544781446457,
+ "epoch": 0.1281999919875005,
+ "grad_norm": 3.28125,
+ "learning_rate": 1.7439512898573947e-05,
+ "loss": 2.181917877197266,
+ "mean_token_accuracy": 0.5182220858335495,
+ "num_tokens": 4814709.0,
+ "step": 800
+ },
+ {
+ "entropy": 2.1532615756988527,
+ "epoch": 0.1362124914867193,
+ "grad_norm": 3.078125,
+ "learning_rate": 1.7279282166319503e-05,
+ "loss": 2.1659007263183594,
+ "mean_token_accuracy": 0.5180425970256328,
+ "num_tokens": 5112585.0,
+ "step": 850
+ },
+ {
+ "entropy": 2.1704017293453215,
+ "epoch": 0.14422499098593805,
+ "grad_norm": 2.921875,
+ "learning_rate": 1.7119051434065055e-05,
+ "loss": 2.1934996032714844,
+ "mean_token_accuracy": 0.5154964691400528,
+ "num_tokens": 5411369.0,
+ "step": 900
+ },
+ {
+ "entropy": 2.1349107217788696,
+ "epoch": 0.15223749048515683,
+ "grad_norm": 2.984375,
+ "learning_rate": 1.6958820701810608e-05,
+ "loss": 2.1598545837402345,
+ "mean_token_accuracy": 0.5208920259773732,
+ "num_tokens": 5707060.0,
+ "step": 950
+ },
+ {
+ "entropy": 2.1541705918312073,
+ "epoch": 0.16024998998437562,
+ "grad_norm": 3.203125,
+ "learning_rate": 1.6798589969556163e-05,
+ "loss": 2.170956573486328,
+ "mean_token_accuracy": 0.5204938448965549,
+ "num_tokens": 6004079.0,
+ "step": 1000
+ },
+ {
+ "entropy": 2.1523867017030716,
+ "epoch": 0.1682624894835944,
+ "grad_norm": 3.09375,
+ "learning_rate": 1.6638359237301716e-05,
+ "loss": 2.1695118713378907,
+ "mean_token_accuracy": 0.5177376234531402,
+ "num_tokens": 6315462.0,
+ "step": 1050
+ },
+ {
+ "entropy": 2.1356022965908052,
+ "epoch": 0.17627498898281319,
+ "grad_norm": 3.21875,
+ "learning_rate": 1.6478128505047268e-05,
+ "loss": 2.1513107299804686,
+ "mean_token_accuracy": 0.5217180585861206,
+ "num_tokens": 6613060.0,
+ "step": 1100
+ },
+ {
+ "entropy": 2.1483460652828215,
+ "epoch": 0.18428748848203197,
+ "grad_norm": 3.21875,
+ "learning_rate": 1.6317897772792824e-05,
+ "loss": 2.1671595764160156,
+ "mean_token_accuracy": 0.5199222292006016,
+ "num_tokens": 6909655.0,
+ "step": 1150
+ },
+ {
+ "entropy": 2.1074040442705155,
+ "epoch": 0.19229998798125075,
+ "grad_norm": 3.828125,
+ "learning_rate": 1.6157667040538376e-05,
+ "loss": 2.12770751953125,
+ "mean_token_accuracy": 0.5252153894305229,
+ "num_tokens": 7203263.0,
+ "step": 1200
+ },
+ {
+ "entropy": 2.1354027593135836,
+ "epoch": 0.20031248748046954,
+ "grad_norm": 2.921875,
+ "learning_rate": 1.5997436308283932e-05,
+ "loss": 2.156183166503906,
+ "mean_token_accuracy": 0.518907299786806,
+ "num_tokens": 7505411.0,
+ "step": 1250
+ },
+ {
+ "entropy": 2.142924894094467,
+ "epoch": 0.20832498697968832,
+ "grad_norm": 3.5625,
+ "learning_rate": 1.5837205576029484e-05,
+ "loss": 2.158486633300781,
+ "mean_token_accuracy": 0.5205909128487111,
+ "num_tokens": 7803562.0,
+ "step": 1300
+ },
+ {
+ "entropy": 2.1305291467905043,
+ "epoch": 0.2163374864789071,
+ "grad_norm": 3.34375,
+ "learning_rate": 1.5676974843775036e-05,
+ "loss": 2.151043701171875,
+ "mean_token_accuracy": 0.5211617235839366,
+ "num_tokens": 8108037.0,
+ "step": 1350
+ },
+ {
+ "entropy": 2.160947166085243,
+ "epoch": 0.2243499859781259,
+ "grad_norm": 3.234375,
+ "learning_rate": 1.5516744111520592e-05,
+ "loss": 2.177792663574219,
+ "mean_token_accuracy": 0.5167004990577698,
+ "num_tokens": 8408674.0,
+ "step": 1400
+ },
+ {
+ "entropy": 2.120403967499733,
+ "epoch": 0.23236248547734467,
+ "grad_norm": 3.375,
+ "learning_rate": 1.5356513379266144e-05,
+ "loss": 2.1462562561035154,
+ "mean_token_accuracy": 0.5225985661149025,
+ "num_tokens": 8707650.0,
+ "step": 1450
+ },
+ {
+ "entropy": 2.138882914185524,
+ "epoch": 0.24037498497656343,
+ "grad_norm": 3.171875,
+ "learning_rate": 1.5196282647011698e-05,
+ "loss": 2.1499905395507812,
+ "mean_token_accuracy": 0.5204668046534061,
+ "num_tokens": 9006296.0,
+ "step": 1500
+ },
+ {
+ "entropy": 2.131927245259285,
+ "epoch": 0.2483874844757822,
+ "grad_norm": 3.546875,
+ "learning_rate": 1.5036051914757252e-05,
+ "loss": 2.157076416015625,
+ "mean_token_accuracy": 0.5220408822596073,
+ "num_tokens": 9307203.0,
+ "step": 1550
+ },
+ {
+ "entropy": 2.1308567851781843,
+ "epoch": 0.256399983975001,
+ "grad_norm": 3.546875,
+ "learning_rate": 1.4875821182502805e-05,
+ "loss": 2.146601104736328,
+ "mean_token_accuracy": 0.5219539681077003,
+ "num_tokens": 9610070.0,
+ "step": 1600
+ },
+ {
+ "entropy": 2.1097783839702604,
+ "epoch": 0.2644124834742198,
+ "grad_norm": 3.84375,
+ "learning_rate": 1.4715590450248359e-05,
+ "loss": 2.1268209838867187,
+ "mean_token_accuracy": 0.5236652019619942,
+ "num_tokens": 9909117.0,
+ "step": 1650
+ },
+ {
+ "entropy": 2.123499180674553,
+ "epoch": 0.2724249829734386,
+ "grad_norm": 3.46875,
+ "learning_rate": 1.4555359717993913e-05,
+ "loss": 2.1409934997558593,
+ "mean_token_accuracy": 0.5225465354323388,
+ "num_tokens": 10209461.0,
+ "step": 1700
+ },
+ {
+ "entropy": 2.13052960395813,
+ "epoch": 0.2804374824726573,
+ "grad_norm": 2.71875,
+ "learning_rate": 1.4395128985739467e-05,
+ "loss": 2.151373291015625,
+ "mean_token_accuracy": 0.5227928113937378,
+ "num_tokens": 10509282.0,
+ "step": 1750
+ },
+ {
+ "entropy": 2.1226114094257356,
+ "epoch": 0.2884499819718761,
+ "grad_norm": 3.65625,
+ "learning_rate": 1.4234898253485019e-05,
+ "loss": 2.144451141357422,
+ "mean_token_accuracy": 0.5233827362954616,
+ "num_tokens": 10811056.0,
+ "step": 1800
+ },
+ {
+ "entropy": 2.126143154501915,
+ "epoch": 0.2964624814710949,
+ "grad_norm": 3.359375,
+ "learning_rate": 1.4074667521230575e-05,
+ "loss": 2.1457598876953123,
+ "mean_token_accuracy": 0.5221967925131321,
+ "num_tokens": 11111153.0,
+ "step": 1850
+ },
+ {
+ "entropy": 2.1048326581716537,
+ "epoch": 0.30447498097031367,
+ "grad_norm": 3.59375,
+ "learning_rate": 1.3914436788976127e-05,
+ "loss": 2.1260116577148436,
+ "mean_token_accuracy": 0.5272719909250736,
+ "num_tokens": 11405606.0,
+ "step": 1900
+ },
+ {
+ "entropy": 2.1236617982387545,
+ "epoch": 0.31248748046953245,
+ "grad_norm": 3.0,
+ "learning_rate": 1.375420605672168e-05,
+ "loss": 2.151782684326172,
+ "mean_token_accuracy": 0.5227546173334122,
+ "num_tokens": 11703943.0,
+ "step": 1950
+ },
+ {
+ "entropy": 2.121563429236412,
+ "epoch": 0.32049997996875124,
+ "grad_norm": 3.34375,
+ "learning_rate": 1.3593975324467235e-05,
+ "loss": 2.1370144653320313,
+ "mean_token_accuracy": 0.5235123547911644,
+ "num_tokens": 12002522.0,
+ "step": 2000
+ },
+ {
+ "entropy": 2.1155461168289182,
+ "epoch": 0.32851247946797,
+ "grad_norm": 3.46875,
+ "learning_rate": 1.3433744592212787e-05,
+ "loss": 2.1364064025878906,
+ "mean_token_accuracy": 0.5237135806679726,
+ "num_tokens": 12303098.0,
+ "step": 2050
+ },
+ {
+ "entropy": 2.1386052322387696,
+ "epoch": 0.3365249789671888,
+ "grad_norm": 3.484375,
+ "learning_rate": 1.327351385995834e-05,
+ "loss": 2.149888458251953,
+ "mean_token_accuracy": 0.522669050693512,
+ "num_tokens": 12605534.0,
+ "step": 2100
+ },
+ {
+ "entropy": 2.102600795030594,
+ "epoch": 0.3445374784664076,
+ "grad_norm": 3.796875,
+ "learning_rate": 1.3113283127703895e-05,
+ "loss": 2.1236566162109374,
+ "mean_token_accuracy": 0.5279092155396938,
+ "num_tokens": 12901218.0,
+ "step": 2150
+ },
+ {
+ "entropy": 2.132645187973976,
+ "epoch": 0.35254997796562637,
+ "grad_norm": 3.53125,
+ "learning_rate": 1.2953052395449447e-05,
+ "loss": 2.1523503112792968,
+ "mean_token_accuracy": 0.5198129612207413,
+ "num_tokens": 13213018.0,
+ "step": 2200
+ },
+ {
+ "entropy": 2.1204321217536926,
+ "epoch": 0.36056247746484515,
+ "grad_norm": 3.25,
+ "learning_rate": 1.2792821663195001e-05,
+ "loss": 2.1325447082519533,
+ "mean_token_accuracy": 0.5243636940419674,
+ "num_tokens": 13514408.0,
+ "step": 2250
+ },
+ {
+ "entropy": 2.102410731315613,
+ "epoch": 0.36857497696406394,
+ "grad_norm": 3.1875,
+ "learning_rate": 1.2632590930940555e-05,
+ "loss": 2.123565368652344,
+ "mean_token_accuracy": 0.5257614709436893,
+ "num_tokens": 13818272.0,
+ "step": 2300
+ },
+ {
+ "entropy": 2.0876205080747603,
+ "epoch": 0.3765874764632827,
+ "grad_norm": 3.328125,
+ "learning_rate": 1.247236019868611e-05,
+ "loss": 2.116696472167969,
+ "mean_token_accuracy": 0.5275124031305313,
+ "num_tokens": 14110702.0,
+ "step": 2350
+ },
+ {
+ "entropy": 2.1078518223762512,
+ "epoch": 0.3845999759625015,
+ "grad_norm": 3.453125,
+ "learning_rate": 1.2312129466431662e-05,
+ "loss": 2.1291993713378905,
+ "mean_token_accuracy": 0.524967464953661,
+ "num_tokens": 14407015.0,
+ "step": 2400
+ },
+ {
+ "entropy": 2.0930898267030718,
+ "epoch": 0.3926124754617203,
+ "grad_norm": 3.5,
+ "learning_rate": 1.2151898734177216e-05,
+ "loss": 2.1094123840332033,
+ "mean_token_accuracy": 0.5290587748587131,
+ "num_tokens": 14696565.0,
+ "step": 2450
+ },
+ {
+ "entropy": 2.1109757220745085,
+ "epoch": 0.4006249749609391,
+ "grad_norm": 3.5,
+ "learning_rate": 1.199166800192277e-05,
+ "loss": 2.1246888732910154,
+ "mean_token_accuracy": 0.5258957357704639,
+ "num_tokens": 14994401.0,
+ "step": 2500
+ },
+ {
+ "entropy": 2.1087811160087586,
+ "epoch": 0.40863747446015786,
+ "grad_norm": 3.1875,
+ "learning_rate": 1.1831437269668324e-05,
+ "loss": 2.12890625,
+ "mean_token_accuracy": 0.5269917643070221,
+ "num_tokens": 15292837.0,
+ "step": 2550
+ },
+ {
+ "entropy": 2.097252429127693,
+ "epoch": 0.41664997395937664,
+ "grad_norm": 3.0,
+ "learning_rate": 1.1671206537413878e-05,
+ "loss": 2.1145870971679686,
+ "mean_token_accuracy": 0.5274137698113919,
+ "num_tokens": 15590647.0,
+ "step": 2600
+ },
+ {
+ "entropy": 2.111199595332146,
+ "epoch": 0.4246624734585954,
+ "grad_norm": 3.25,
+ "learning_rate": 1.151097580515943e-05,
+ "loss": 2.1274327087402343,
+ "mean_token_accuracy": 0.5250249288976192,
+ "num_tokens": 15892772.0,
+ "step": 2650
+ },
+ {
+ "entropy": 2.1051502913236617,
+ "epoch": 0.4326749729578142,
+ "grad_norm": 3.203125,
+ "learning_rate": 1.1350745072904986e-05,
+ "loss": 2.127935333251953,
+ "mean_token_accuracy": 0.5263640700280666,
+ "num_tokens": 16191611.0,
+ "step": 2700
+ },
+ {
+ "entropy": 2.1127885603904724,
+ "epoch": 0.440687472457033,
+ "grad_norm": 3.515625,
+ "learning_rate": 1.1190514340650538e-05,
+ "loss": 2.129211883544922,
+ "mean_token_accuracy": 0.5250939278304577,
+ "num_tokens": 16483582.0,
+ "step": 2750
+ },
+ {
+ "entropy": 2.110930626988411,
+ "epoch": 0.4486999719562518,
+ "grad_norm": 3.1875,
+ "learning_rate": 1.103028360839609e-05,
+ "loss": 2.1220079040527344,
+ "mean_token_accuracy": 0.5250919266045093,
+ "num_tokens": 16787253.0,
+ "step": 2800
+ },
+ {
+ "entropy": 2.1182780677080153,
+ "epoch": 0.45671247145547056,
+ "grad_norm": 3.8125,
+ "learning_rate": 1.0870052876141646e-05,
+ "loss": 2.1358837890625,
+ "mean_token_accuracy": 0.5217854312062263,
+ "num_tokens": 17088741.0,
+ "step": 2850
+ },
+ {
+ "entropy": 2.1117833071947096,
+ "epoch": 0.46472497095468934,
+ "grad_norm": 3.171875,
+ "learning_rate": 1.0709822143887198e-05,
+ "loss": 2.125662841796875,
+ "mean_token_accuracy": 0.524649152457714,
+ "num_tokens": 17389236.0,
+ "step": 2900
+ },
+ {
+ "entropy": 2.1262730568647386,
+ "epoch": 0.47273747045390807,
+ "grad_norm": 3.625,
+ "learning_rate": 1.054959141163275e-05,
+ "loss": 2.142789611816406,
+ "mean_token_accuracy": 0.5236384868621826,
+ "num_tokens": 17693413.0,
+ "step": 2950
+ },
+ {
+ "entropy": 2.115410739183426,
+ "epoch": 0.48074996995312685,
+ "grad_norm": 3.140625,
+ "learning_rate": 1.0389360679378306e-05,
+ "loss": 2.1337106323242185,
+ "mean_token_accuracy": 0.5249222332239151,
+ "num_tokens": 17990326.0,
+ "step": 3000
+ },
+ {
+ "entropy": 2.0951381689310073,
+ "epoch": 0.48876246945234564,
+ "grad_norm": 3.234375,
+ "learning_rate": 1.0229129947123859e-05,
+ "loss": 2.108144073486328,
+ "mean_token_accuracy": 0.5269971822202205,
+ "num_tokens": 18293136.0,
+ "step": 3050
+ },
+ {
+ "entropy": 2.116977259516716,
+ "epoch": 0.4967749689515644,
+ "grad_norm": 3.421875,
+ "learning_rate": 1.0068899214869413e-05,
+ "loss": 2.1327830505371095,
+ "mean_token_accuracy": 0.5238270902633667,
+ "num_tokens": 18591481.0,
+ "step": 3100
+ },
+ {
+ "entropy": 2.08598925113678,
+ "epoch": 0.5047874684507833,
+ "grad_norm": 2.96875,
+ "learning_rate": 9.908668482614967e-06,
+ "loss": 2.1032344055175782,
+ "mean_token_accuracy": 0.5289211909472943,
+ "num_tokens": 18885221.0,
+ "step": 3150
+ },
+ {
+ "entropy": 2.097712662220001,
+ "epoch": 0.512799967950002,
+ "grad_norm": 3.40625,
+ "learning_rate": 9.74843775036052e-06,
+ "loss": 2.1150633239746095,
+ "mean_token_accuracy": 0.5288272814452648,
+ "num_tokens": 19184784.0,
+ "step": 3200
+ },
+ {
+ "entropy": 2.1063760107755662,
+ "epoch": 0.5208124674492208,
+ "grad_norm": 3.28125,
+ "learning_rate": 9.588207018106075e-06,
+ "loss": 2.1217109680175783,
+ "mean_token_accuracy": 0.526290989369154,
+ "num_tokens": 19482538.0,
+ "step": 3250
+ },
+ {
+ "entropy": 2.090182801485062,
+ "epoch": 0.5288249669484396,
+ "grad_norm": 2.875,
+ "learning_rate": 9.427976285851627e-06,
+ "loss": 2.1130311584472654,
+ "mean_token_accuracy": 0.5285439921915531,
+ "num_tokens": 19782544.0,
+ "step": 3300
+ },
+ {
+ "entropy": 2.103260806798935,
+ "epoch": 0.5368374664476584,
+ "grad_norm": 3.59375,
+ "learning_rate": 9.267745553597181e-06,
+ "loss": 2.116189727783203,
+ "mean_token_accuracy": 0.5273195558786392,
+ "num_tokens": 20086408.0,
+ "step": 3350
+ },
+ {
+ "entropy": 2.0673747235536575,
+ "epoch": 0.5448499659468772,
+ "grad_norm": 3.125,
+ "learning_rate": 9.107514821342735e-06,
+ "loss": 2.087482147216797,
+ "mean_token_accuracy": 0.532562478929758,
+ "num_tokens": 20380429.0,
+ "step": 3400
+ },
+ {
+ "entropy": 2.1030789083242416,
+ "epoch": 0.552862465446096,
+ "grad_norm": 3.296875,
+ "learning_rate": 8.947284089088289e-06,
+ "loss": 2.122591247558594,
+ "mean_token_accuracy": 0.5256703786551953,
+ "num_tokens": 20682849.0,
+ "step": 3450
+ },
+ {
+ "entropy": 2.0988716661930082,
+ "epoch": 0.5608749649453146,
+ "grad_norm": 3.546875,
+ "learning_rate": 8.787053356833841e-06,
+ "loss": 2.115867156982422,
+ "mean_token_accuracy": 0.5266730943322182,
+ "num_tokens": 20979225.0,
+ "step": 3500
+ },
+ {
+ "entropy": 2.0954072701931,
+ "epoch": 0.5688874644445334,
+ "grad_norm": 3.859375,
+ "learning_rate": 8.626822624579395e-06,
+ "loss": 2.1116868591308595,
+ "mean_token_accuracy": 0.5267495161294937,
+ "num_tokens": 21284166.0,
+ "step": 3550
+ },
+ {
+ "entropy": 2.11238471031189,
+ "epoch": 0.5768999639437522,
+ "grad_norm": 3.484375,
+ "learning_rate": 8.466591892324949e-06,
+ "loss": 2.1292352294921875,
+ "mean_token_accuracy": 0.5249243360757828,
+ "num_tokens": 21584612.0,
+ "step": 3600
+ },
+ {
+ "entropy": 2.091464368700981,
+ "epoch": 0.584912463442971,
+ "grad_norm": 3.359375,
+ "learning_rate": 8.306361160070501e-06,
+ "loss": 2.1099319458007812,
+ "mean_token_accuracy": 0.5292035435140133,
+ "num_tokens": 21879345.0,
+ "step": 3650
+ },
+ {
+ "entropy": 2.095204224586487,
+ "epoch": 0.5929249629421898,
+ "grad_norm": 2.96875,
+ "learning_rate": 8.146130427816055e-06,
+ "loss": 2.1198716735839844,
+ "mean_token_accuracy": 0.5255991271138192,
+ "num_tokens": 22183486.0,
+ "step": 3700
+ },
+ {
+ "entropy": 2.0666573679447175,
+ "epoch": 0.6009374624414086,
+ "grad_norm": 2.90625,
+ "learning_rate": 7.98589969556161e-06,
+ "loss": 2.08689697265625,
+ "mean_token_accuracy": 0.5318664258718491,
+ "num_tokens": 22476438.0,
+ "step": 3750
+ },
+ {
+ "entropy": 2.122153177857399,
+ "epoch": 0.6089499619406273,
+ "grad_norm": 3.40625,
+ "learning_rate": 7.825668963307162e-06,
+ "loss": 2.138288879394531,
+ "mean_token_accuracy": 0.5232584895193577,
+ "num_tokens": 22780385.0,
+ "step": 3800
+ },
+ {
+ "entropy": 2.1190748488903046,
+ "epoch": 0.6169624614398461,
+ "grad_norm": 3.46875,
+ "learning_rate": 7.665438231052716e-06,
+ "loss": 2.139790344238281,
+ "mean_token_accuracy": 0.5243287613987923,
+ "num_tokens": 23080847.0,
+ "step": 3850
+ },
+ {
+ "entropy": 2.1312570214271545,
+ "epoch": 0.6249749609390649,
+ "grad_norm": 3.765625,
+ "learning_rate": 7.5052074987982706e-06,
+ "loss": 2.1471080017089843,
+ "mean_token_accuracy": 0.5232046774029732,
+ "num_tokens": 23384738.0,
+ "step": 3900
+ },
+ {
+ "entropy": 2.0779008239507677,
+ "epoch": 0.6329874604382837,
+ "grad_norm": 3.171875,
+ "learning_rate": 7.344976766543823e-06,
+ "loss": 2.0974729919433592,
+ "mean_token_accuracy": 0.5305448272824287,
+ "num_tokens": 23682750.0,
+ "step": 3950
+ },
+ {
+ "entropy": 2.1121455317735673,
+ "epoch": 0.6409999599375025,
+ "grad_norm": 3.921875,
+ "learning_rate": 7.184746034289377e-06,
+ "loss": 2.127069549560547,
+ "mean_token_accuracy": 0.5252936625480652,
+ "num_tokens": 23989029.0,
+ "step": 4000
+ },
+ {
+ "entropy": 2.0982888478040693,
+ "epoch": 0.6490124594367213,
+ "grad_norm": 4.0625,
+ "learning_rate": 7.024515302034931e-06,
+ "loss": 2.112042236328125,
+ "mean_token_accuracy": 0.5281323495507241,
+ "num_tokens": 24292063.0,
+ "step": 4050
+ },
+ {
+ "entropy": 2.098363963365555,
+ "epoch": 0.65702495893594,
+ "grad_norm": 3.046875,
+ "learning_rate": 6.864284569780485e-06,
+ "loss": 2.1156489562988283,
+ "mean_token_accuracy": 0.5275120075047016,
+ "num_tokens": 24590201.0,
+ "step": 4100
+ },
+ {
+ "entropy": 2.1065632277727127,
+ "epoch": 0.6650374584351588,
+ "grad_norm": 3.375,
+ "learning_rate": 6.704053837526038e-06,
+ "loss": 2.12911376953125,
+ "mean_token_accuracy": 0.5262684133648873,
+ "num_tokens": 24892122.0,
+ "step": 4150
+ },
+ {
+ "entropy": 2.1014072370529173,
+ "epoch": 0.6730499579343776,
+ "grad_norm": 2.84375,
+ "learning_rate": 6.543823105271592e-06,
+ "loss": 2.1130335998535155,
+ "mean_token_accuracy": 0.5262750512361527,
+ "num_tokens": 25194969.0,
+ "step": 4200
+ },
+ {
+ "entropy": 2.0905236744880678,
+ "epoch": 0.6810624574335964,
+ "grad_norm": 3.03125,
+ "learning_rate": 6.383592373017146e-06,
+ "loss": 2.1153533935546873,
+ "mean_token_accuracy": 0.527137616276741,
+ "num_tokens": 25499521.0,
+ "step": 4250
+ },
+ {
+ "entropy": 2.1032627063989637,
+ "epoch": 0.6890749569328152,
+ "grad_norm": 3.125,
+ "learning_rate": 6.223361640762698e-06,
+ "loss": 2.122831115722656,
+ "mean_token_accuracy": 0.5254572249948979,
+ "num_tokens": 25798933.0,
+ "step": 4300
+ },
+ {
+ "entropy": 2.118139310479164,
+ "epoch": 0.697087456432034,
+ "grad_norm": 3.28125,
+ "learning_rate": 6.063130908508252e-06,
+ "loss": 2.1376751708984374,
+ "mean_token_accuracy": 0.5243131402134895,
+ "num_tokens": 26096619.0,
+ "step": 4350
+ },
+ {
+ "entropy": 2.082554380297661,
+ "epoch": 0.7050999559312527,
+ "grad_norm": 3.390625,
+ "learning_rate": 5.902900176253806e-06,
+ "loss": 2.0949493408203126,
+ "mean_token_accuracy": 0.5316410692036152,
+ "num_tokens": 26394395.0,
+ "step": 4400
+ },
+ {
+ "entropy": 2.09870474755764,
+ "epoch": 0.7131124554304715,
+ "grad_norm": 3.421875,
+ "learning_rate": 5.7426694439993594e-06,
+ "loss": 2.1184346008300783,
+ "mean_token_accuracy": 0.5267573061585427,
+ "num_tokens": 26692333.0,
+ "step": 4450
+ },
+ {
+ "entropy": 2.0866885662078856,
+ "epoch": 0.7211249549296903,
+ "grad_norm": 3.71875,
+ "learning_rate": 5.5824387117449134e-06,
+ "loss": 2.102904968261719,
+ "mean_token_accuracy": 0.529726449996233,
+ "num_tokens": 26989384.0,
+ "step": 4500
+ },
+ {
+ "entropy": 2.114828346967697,
+ "epoch": 0.7291374544289091,
+ "grad_norm": 3.46875,
+ "learning_rate": 5.4222079794904666e-06,
+ "loss": 2.128611602783203,
+ "mean_token_accuracy": 0.5247649565339089,
+ "num_tokens": 27299927.0,
+ "step": 4550
+ },
+ {
+ "entropy": 2.09617226600647,
+ "epoch": 0.7371499539281279,
+ "grad_norm": 3.015625,
+ "learning_rate": 5.26197724723602e-06,
+ "loss": 2.1166041564941405,
+ "mean_token_accuracy": 0.5273530712723732,
+ "num_tokens": 27599011.0,
+ "step": 4600
+ },
+ {
+ "entropy": 2.0975282275676728,
+ "epoch": 0.7451624534273467,
+ "grad_norm": 3.3125,
+ "learning_rate": 5.101746514981574e-06,
+ "loss": 2.112121124267578,
+ "mean_token_accuracy": 0.5281847220659256,
+ "num_tokens": 27892355.0,
+ "step": 4650
+ },
+ {
+ "entropy": 2.095783769488335,
+ "epoch": 0.7531749529265654,
+ "grad_norm": 3.78125,
+ "learning_rate": 4.941515782727128e-06,
+ "loss": 2.110701904296875,
+ "mean_token_accuracy": 0.5272985179722309,
+ "num_tokens": 28187615.0,
+ "step": 4700
+ },
+ {
+ "entropy": 2.0964103603363036,
+ "epoch": 0.7611874524257842,
+ "grad_norm": 3.1875,
+ "learning_rate": 4.781285050472681e-06,
+ "loss": 2.11992919921875,
+ "mean_token_accuracy": 0.5271916574239731,
+ "num_tokens": 28485419.0,
+ "step": 4750
+ },
+ {
+ "entropy": 2.0820577853918074,
+ "epoch": 0.769199951925003,
+ "grad_norm": 3.5,
+ "learning_rate": 4.621054318218234e-06,
+ "loss": 2.095073394775391,
+ "mean_token_accuracy": 0.5297581899166107,
+ "num_tokens": 28780158.0,
+ "step": 4800
+ },
+ {
+ "entropy": 2.089591929912567,
+ "epoch": 0.7772124514242218,
+ "grad_norm": 3.125,
+ "learning_rate": 4.460823585963788e-06,
+ "loss": 2.109657897949219,
+ "mean_token_accuracy": 0.528961935043335,
+ "num_tokens": 29081052.0,
+ "step": 4850
+ },
+ {
+ "entropy": 2.092900555729866,
+ "epoch": 0.7852249509234406,
+ "grad_norm": 3.46875,
+ "learning_rate": 4.300592853709342e-06,
+ "loss": 2.106436920166016,
+ "mean_token_accuracy": 0.5284854583442211,
+ "num_tokens": 29376324.0,
+ "step": 4900
+ },
+ {
+ "entropy": 2.0630873823165894,
+ "epoch": 0.7932374504226594,
+ "grad_norm": 3.28125,
+ "learning_rate": 4.140362121454896e-06,
+ "loss": 2.088908233642578,
+ "mean_token_accuracy": 0.532561699450016,
+ "num_tokens": 29675269.0,
+ "step": 4950
+ },
+ {
+ "entropy": 2.0753238475322724,
+ "epoch": 0.8012499499218781,
+ "grad_norm": 3.21875,
+ "learning_rate": 3.980131389200449e-06,
+ "loss": 2.0961412048339843,
+ "mean_token_accuracy": 0.5304154875874519,
+ "num_tokens": 29974838.0,
+ "step": 5000
+ },
+ {
+ "entropy": 2.123110404610634,
+ "epoch": 0.8092624494210969,
+ "grad_norm": 3.28125,
+ "learning_rate": 3.819900656946002e-06,
+ "loss": 2.1538424682617188,
+ "mean_token_accuracy": 0.5222072719037533,
+ "num_tokens": 30286201.0,
+ "step": 5050
+ },
+ {
+ "entropy": 2.123967355489731,
+ "epoch": 0.8172749489203157,
+ "grad_norm": 3.046875,
+ "learning_rate": 3.6596699246915563e-06,
+ "loss": 2.1417193603515625,
+ "mean_token_accuracy": 0.5213662980496884,
+ "num_tokens": 30592837.0,
+ "step": 5100
+ },
+ {
+ "entropy": 2.080365374684334,
+ "epoch": 0.8252874484195345,
+ "grad_norm": 3.375,
+ "learning_rate": 3.4994391924371094e-06,
+ "loss": 2.0970494079589845,
+ "mean_token_accuracy": 0.5296585746109486,
+ "num_tokens": 30890674.0,
+ "step": 5150
+ },
+ {
+ "entropy": 2.1183922028541566,
+ "epoch": 0.8332999479187533,
+ "grad_norm": 3.359375,
+ "learning_rate": 3.3392084601826634e-06,
+ "loss": 2.132384948730469,
+ "mean_token_accuracy": 0.5249108146131038,
+ "num_tokens": 31195601.0,
+ "step": 5200
+ },
+ {
+ "entropy": 2.1079335528612138,
+ "epoch": 0.8413124474179721,
+ "grad_norm": 2.890625,
+ "learning_rate": 3.178977727928217e-06,
+ "loss": 2.1310113525390624,
+ "mean_token_accuracy": 0.5255681444704533,
+ "num_tokens": 31498768.0,
+ "step": 5250
+ },
+ {
+ "entropy": 2.1180070728063582,
+ "epoch": 0.8493249469171908,
+ "grad_norm": 3.78125,
+ "learning_rate": 3.01874699567377e-06,
+ "loss": 2.1365049743652342,
+ "mean_token_accuracy": 0.5235744562745094,
+ "num_tokens": 31801891.0,
+ "step": 5300
+ },
+ {
+ "entropy": 2.0909912359714506,
+ "epoch": 0.8573374464164096,
+ "grad_norm": 3.078125,
+ "learning_rate": 2.858516263419324e-06,
+ "loss": 2.108803405761719,
+ "mean_token_accuracy": 0.5267959542572498,
+ "num_tokens": 32102627.0,
+ "step": 5350
+ },
+ {
+ "entropy": 2.1209339940547944,
+ "epoch": 0.8653499459156284,
+ "grad_norm": 3.140625,
+ "learning_rate": 2.6982855311648777e-06,
+ "loss": 2.143946228027344,
+ "mean_token_accuracy": 0.5227122528851033,
+ "num_tokens": 32405500.0,
+ "step": 5400
+ },
+ {
+ "entropy": 2.1137486273050308,
+ "epoch": 0.8733624454148472,
+ "grad_norm": 3.03125,
+ "learning_rate": 2.5380547989104317e-06,
+ "loss": 2.127569122314453,
+ "mean_token_accuracy": 0.5245758745074273,
+ "num_tokens": 32712521.0,
+ "step": 5450
+ },
+ {
+ "entropy": 2.0872016245126725,
+ "epoch": 0.881374944914066,
+ "grad_norm": 3.171875,
+ "learning_rate": 2.377824066655985e-06,
+ "loss": 2.100517120361328,
+ "mean_token_accuracy": 0.5295753349363804,
+ "num_tokens": 33016210.0,
+ "step": 5500
+ },
+ {
+ "entropy": 2.0836504286527635,
+ "epoch": 0.8893874444132848,
+ "grad_norm": 3.328125,
+ "learning_rate": 2.2175933344015384e-06,
+ "loss": 2.1047386169433593,
+ "mean_token_accuracy": 0.5296267972886562,
+ "num_tokens": 33311705.0,
+ "step": 5550
+ },
+ {
+ "entropy": 2.0777365696430206,
+ "epoch": 0.8973999439125036,
+ "grad_norm": 3.59375,
+ "learning_rate": 2.057362602147092e-06,
+ "loss": 2.0985890197753907,
+ "mean_token_accuracy": 0.5303736463189125,
+ "num_tokens": 33603231.0,
+ "step": 5600
+ },
+ {
+ "entropy": 2.095007498860359,
+ "epoch": 0.9054124434117223,
+ "grad_norm": 3.59375,
+ "learning_rate": 1.8971318698926456e-06,
+ "loss": 2.106617431640625,
+ "mean_token_accuracy": 0.527913925498724,
+ "num_tokens": 33902978.0,
+ "step": 5650
+ },
+ {
+ "entropy": 2.106599664092064,
+ "epoch": 0.9134249429109411,
+ "grad_norm": 3.234375,
+ "learning_rate": 1.7369011376381991e-06,
+ "loss": 2.1318267822265624,
+ "mean_token_accuracy": 0.525322500616312,
+ "num_tokens": 34206935.0,
+ "step": 5700
+ },
+ {
+ "entropy": 2.094651755094528,
+ "epoch": 0.9214374424101599,
+ "grad_norm": 3.65625,
+ "learning_rate": 1.576670405383753e-06,
+ "loss": 2.110823211669922,
+ "mean_token_accuracy": 0.5286212807893753,
+ "num_tokens": 34499610.0,
+ "step": 5750
+ },
+ {
+ "entropy": 2.119523124098778,
+ "epoch": 0.9294499419093787,
+ "grad_norm": 3.375,
+ "learning_rate": 1.4164396731293063e-06,
+ "loss": 2.13890625,
+ "mean_token_accuracy": 0.5239527927339077,
+ "num_tokens": 34806213.0,
+ "step": 5800
+ },
+ {
+ "entropy": 2.0914100110530853,
+ "epoch": 0.9374624414085974,
+ "grad_norm": 3.46875,
+ "learning_rate": 1.2562089408748598e-06,
+ "loss": 2.113725280761719,
+ "mean_token_accuracy": 0.5276746636629105,
+ "num_tokens": 35106711.0,
+ "step": 5850
+ },
+ {
+ "entropy": 2.0877316004037856,
+ "epoch": 0.9454749409078161,
+ "grad_norm": 3.125,
+ "learning_rate": 1.0959782086204134e-06,
+ "loss": 2.097077178955078,
+ "mean_token_accuracy": 0.5296605123579502,
+ "num_tokens": 35398548.0,
+ "step": 5900
+ },
+ {
+ "entropy": 2.099357485175133,
+ "epoch": 0.9534874404070349,
+ "grad_norm": 3.234375,
+ "learning_rate": 9.357474763659671e-07,
+ "loss": 2.115228729248047,
+ "mean_token_accuracy": 0.5284087285399437,
+ "num_tokens": 35689259.0,
+ "step": 5950
+ },
+ {
+ "entropy": 2.0721785259246825,
+ "epoch": 0.9614999399062537,
+ "grad_norm": 3.84375,
+ "learning_rate": 7.755167441115207e-07,
+ "loss": 2.098111114501953,
+ "mean_token_accuracy": 0.5322318106889725,
+ "num_tokens": 35989481.0,
+ "step": 6000
+ }
+ ],
+ "logging_steps": 50,
+ "max_steps": 6241,
+ "num_input_tokens_seen": 0,
+ "num_train_epochs": 1,
+ "save_steps": 500,
+ "stateful_callbacks": {
+ "TrainerControl": {
+ "args": {
+ "should_epoch_stop": false,
+ "should_evaluate": false,
+ "should_log": false,
+ "should_save": true,
+ "should_training_stop": false
+ },
+ "attributes": {}
+ }
+ },
+ "total_flos": 4.250307255744676e+17,
+ "train_batch_size": 4,
+ "trial_name": null,
+ "trial_params": null
+}
diff --git a/checkpoint-6000/training_args.bin b/checkpoint-6000/training_args.bin
new file mode 100644
index 0000000000000000000000000000000000000000..4b409d5a147856f0d5a3d3da0f3b4922a9def7dd
--- /dev/null
+++ b/checkpoint-6000/training_args.bin
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:78622f63304ec0913ed0b4ea7a1b26c08a5621b3661a380d798c4ea426235671
+size 5713
diff --git a/checkpoint-6241/chat_template.jinja b/checkpoint-6241/chat_template.jinja
new file mode 100644
index 0000000000000000000000000000000000000000..01be9b307daa2d425f7c168c9fb145a286e0afb4
--- /dev/null
+++ b/checkpoint-6241/chat_template.jinja
@@ -0,0 +1,89 @@
+{%- if tools %}
+ {{- '<|im_start|>system\n' }}
+ {%- if messages[0].role == 'system' %}
+ {{- messages[0].content + '\n\n' }}
+ {%- endif %}
+ {{- "# Tools\n\nYou may call one or more functions to assist with the user query.\n\nYou are provided with function signatures within XML tags:\n" }}
+ {%- for tool in tools %}
+ {{- "\n" }}
+ {{- tool | tojson }}
+ {%- endfor %}
+ {{- "\n\n\nFor each function call, return a json object with function name and arguments within XML tags:\n\n{\"name\": , \"arguments\": }\n<|im_end|>\n" }}
+{%- else %}
+ {%- if messages[0].role == 'system' %}
+ {{- '<|im_start|>system\n' + messages[0].content + '<|im_end|>\n' }}
+ {%- endif %}
+{%- endif %}
+{%- set ns = namespace(multi_step_tool=true, last_query_index=messages|length - 1) %}
+{%- for message in messages[::-1] %}
+ {%- set index = (messages|length - 1) - loop.index0 %}
+ {%- if ns.multi_step_tool and message.role == "user" and message.content is string and not(message.content.startswith('') and message.content.endswith('')) %}
+ {%- set ns.multi_step_tool = false %}
+ {%- set ns.last_query_index = index %}
+ {%- endif %}
+{%- endfor %}
+{%- for message in messages %}
+ {%- if message.content is string %}
+ {%- set content = message.content %}
+ {%- else %}
+ {%- set content = '' %}
+ {%- endif %}
+ {%- if (message.role == "user") or (message.role == "system" and not loop.first) %}
+ {{- '<|im_start|>' + message.role + '\n' + content + '<|im_end|>' + '\n' }}
+ {%- elif message.role == "assistant" %}
+ {%- set reasoning_content = '' %}
+ {%- if message.reasoning_content is string %}
+ {%- set reasoning_content = message.reasoning_content %}
+ {%- else %}
+ {%- if '' in content %}
+ {%- set reasoning_content = content.split('')[0].rstrip('\n').split('')[-1].lstrip('\n') %}
+ {%- set content = content.split('')[-1].lstrip('\n') %}
+ {%- endif %}
+ {%- endif %}
+ {%- if loop.index0 > ns.last_query_index %}
+ {%- if loop.last or (not loop.last and reasoning_content) %}
+ {{- '<|im_start|>' + message.role + '\n\n' + reasoning_content.strip('\n') + '\n\n\n' + content.lstrip('\n') }}
+ {%- else %}
+ {{- '<|im_start|>' + message.role + '\n' + content }}
+ {%- endif %}
+ {%- else %}
+ {{- '<|im_start|>' + message.role + '\n' + content }}
+ {%- endif %}
+ {%- if message.tool_calls %}
+ {%- for tool_call in message.tool_calls %}
+ {%- if (loop.first and content) or (not loop.first) %}
+ {{- '\n' }}
+ {%- endif %}
+ {%- if tool_call.function %}
+ {%- set tool_call = tool_call.function %}
+ {%- endif %}
+ {{- '\n{"name": "' }}
+ {{- tool_call.name }}
+ {{- '", "arguments": ' }}
+ {%- if tool_call.arguments is string %}
+ {{- tool_call.arguments }}
+ {%- else %}
+ {{- tool_call.arguments | tojson }}
+ {%- endif %}
+ {{- '}\n' }}
+ {%- endfor %}
+ {%- endif %}
+ {{- '<|im_end|>\n' }}
+ {%- elif message.role == "tool" %}
+ {%- if loop.first or (messages[loop.index0 - 1].role != "tool") %}
+ {{- '<|im_start|>user' }}
+ {%- endif %}
+ {{- '\n\n' }}
+ {{- content }}
+ {{- '\n' }}
+ {%- if loop.last or (messages[loop.index0 + 1].role != "tool") %}
+ {{- '<|im_end|>\n' }}
+ {%- endif %}
+ {%- endif %}
+{%- endfor %}
+{%- if add_generation_prompt %}
+ {{- '<|im_start|>assistant\n' }}
+ {%- if enable_thinking is defined and enable_thinking is false %}
+ {{- '\n\n\n\n' }}
+ {%- endif %}
+{%- endif %}
\ No newline at end of file
diff --git a/checkpoint-6241/config.json b/checkpoint-6241/config.json
new file mode 100644
index 0000000000000000000000000000000000000000..1d219d6a6d288625229a71646710b85cf7ae5def
--- /dev/null
+++ b/checkpoint-6241/config.json
@@ -0,0 +1,63 @@
+{
+ "architectures": [
+ "Qwen3ForCausalLM"
+ ],
+ "attention_bias": false,
+ "attention_dropout": 0.0,
+ "bos_token_id": null,
+ "dtype": "bfloat16",
+ "eos_token_id": 151645,
+ "head_dim": 128,
+ "hidden_act": "silu",
+ "hidden_size": 2048,
+ "initializer_range": 0.02,
+ "intermediate_size": 6144,
+ "layer_types": [
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention"
+ ],
+ "max_position_embeddings": 40960,
+ "max_window_layers": 28,
+ "model_type": "qwen3",
+ "num_attention_heads": 16,
+ "num_hidden_layers": 28,
+ "num_key_value_heads": 8,
+ "pad_token_id": 151643,
+ "rms_norm_eps": 1e-06,
+ "rope_parameters": {
+ "rope_theta": 1000000,
+ "rope_type": "default"
+ },
+ "sliding_window": null,
+ "tie_word_embeddings": true,
+ "transformers_version": "5.7.0",
+ "use_cache": false,
+ "use_sliding_window": false,
+ "vocab_size": 151936
+}
diff --git a/checkpoint-6241/generation_config.json b/checkpoint-6241/generation_config.json
new file mode 100644
index 0000000000000000000000000000000000000000..5ec133d867b1427d43a5ee0ea332bd088fc12013
--- /dev/null
+++ b/checkpoint-6241/generation_config.json
@@ -0,0 +1,12 @@
+{
+ "do_sample": true,
+ "eos_token_id": [
+ 151645,
+ 151643
+ ],
+ "pad_token_id": 151643,
+ "temperature": 0.6,
+ "top_k": 20,
+ "top_p": 0.95,
+ "transformers_version": "5.7.0"
+}
diff --git a/checkpoint-6241/model.safetensors b/checkpoint-6241/model.safetensors
new file mode 100644
index 0000000000000000000000000000000000000000..c6cea834c2762ddba2e2783f704d2e14442c343e
--- /dev/null
+++ b/checkpoint-6241/model.safetensors
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:3140e22b016936d08d1a9957161ddab89478b00c1cab50daaa2ad5dc5d16a4fc
+size 3441185608
diff --git a/checkpoint-6241/optimizer.pt b/checkpoint-6241/optimizer.pt
new file mode 100644
index 0000000000000000000000000000000000000000..a94c4529e71686249eed17f60fbb6ca70bf2c627
--- /dev/null
+++ b/checkpoint-6241/optimizer.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:87995b276edaded0776df2b192a978dbc6826df9aa65c79653b7296b87a4fac0
+size 6882572207
diff --git a/checkpoint-6241/rng_state.pth b/checkpoint-6241/rng_state.pth
new file mode 100644
index 0000000000000000000000000000000000000000..5a8f17addf8aaee350625e9847751582bc9f86dd
--- /dev/null
+++ b/checkpoint-6241/rng_state.pth
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:61c19bab1174704a4a4441475683bf1270277af15d2e2c95e964789128e482c4
+size 14645
diff --git a/checkpoint-6241/scheduler.pt b/checkpoint-6241/scheduler.pt
new file mode 100644
index 0000000000000000000000000000000000000000..32660c082c113d6fe39a0cce6af95f0509daf506
--- /dev/null
+++ b/checkpoint-6241/scheduler.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ed1f7c61e35590392ce30e26956732331fee27e786c2a4cf9b1aadf0ef625f40
+size 1465
diff --git a/checkpoint-6241/tokenizer.json b/checkpoint-6241/tokenizer.json
new file mode 100644
index 0000000000000000000000000000000000000000..c7afbed2efcdf019f88ab0572ec29d3bf595dfe2
--- /dev/null
+++ b/checkpoint-6241/tokenizer.json
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:be75606093db2094d7cd20f3c2f385c212750648bd6ea4fb2bf507a6a4c55506
+size 11422650
diff --git a/checkpoint-6241/tokenizer_config.json b/checkpoint-6241/tokenizer_config.json
new file mode 100644
index 0000000000000000000000000000000000000000..770e41d6c92519d525eede4cbcf3ba27f6425311
--- /dev/null
+++ b/checkpoint-6241/tokenizer_config.json
@@ -0,0 +1,30 @@
+{
+ "add_prefix_space": false,
+ "backend": "tokenizers",
+ "bos_token": null,
+ "clean_up_tokenization_spaces": false,
+ "eos_token": "<|im_end|>",
+ "errors": "replace",
+ "extra_special_tokens": [
+ "<|im_start|>",
+ "<|im_end|>",
+ "<|object_ref_start|>",
+ "<|object_ref_end|>",
+ "<|box_start|>",
+ "<|box_end|>",
+ "<|quad_start|>",
+ "<|quad_end|>",
+ "<|vision_start|>",
+ "<|vision_end|>",
+ "<|vision_pad|>",
+ "<|image_pad|>",
+ "<|video_pad|>"
+ ],
+ "is_local": false,
+ "local_files_only": false,
+ "model_max_length": 131072,
+ "pad_token": "<|endoftext|>",
+ "split_special_tokens": false,
+ "tokenizer_class": "Qwen2Tokenizer",
+ "unk_token": null
+}
diff --git a/checkpoint-6241/trainer_state.json b/checkpoint-6241/trainer_state.json
new file mode 100644
index 0000000000000000000000000000000000000000..99c534ed33c5a03f86ec9fa72e8eb8b157b38982
--- /dev/null
+++ b/checkpoint-6241/trainer_state.json
@@ -0,0 +1,1274 @@
+{
+ "best_global_step": null,
+ "best_metric": null,
+ "best_model_checkpoint": null,
+ "epoch": 1.0,
+ "eval_steps": 500,
+ "global_step": 6241,
+ "is_hyper_param_search": false,
+ "is_local_process_zero": true,
+ "is_world_process_zero": true,
+ "log_history": [
+ {
+ "entropy": 2.223189059495926,
+ "epoch": 0.008012499499218782,
+ "grad_norm": 2.875,
+ "learning_rate": 1.9842973882390643e-05,
+ "loss": 2.3428109741210936,
+ "mean_token_accuracy": 0.4963963569700718,
+ "num_tokens": 305366.0,
+ "step": 50
+ },
+ {
+ "entropy": 2.188130117058754,
+ "epoch": 0.016024998998437564,
+ "grad_norm": 3.328125,
+ "learning_rate": 1.96827431501362e-05,
+ "loss": 2.225358734130859,
+ "mean_token_accuracy": 0.5110921229422093,
+ "num_tokens": 605042.0,
+ "step": 100
+ },
+ {
+ "entropy": 2.197347580194473,
+ "epoch": 0.024037498497656344,
+ "grad_norm": 3.453125,
+ "learning_rate": 1.952251241788175e-05,
+ "loss": 2.2299420166015627,
+ "mean_token_accuracy": 0.5093490639328957,
+ "num_tokens": 906888.0,
+ "step": 150
+ },
+ {
+ "entropy": 2.1913950395584108,
+ "epoch": 0.03204999799687513,
+ "grad_norm": 2.90625,
+ "learning_rate": 1.9362281685627303e-05,
+ "loss": 2.210995941162109,
+ "mean_token_accuracy": 0.5097909501194954,
+ "num_tokens": 1213125.0,
+ "step": 200
+ },
+ {
+ "entropy": 2.1611998677253723,
+ "epoch": 0.040062497496093905,
+ "grad_norm": 3.25,
+ "learning_rate": 1.920205095337286e-05,
+ "loss": 2.1867474365234374,
+ "mean_token_accuracy": 0.5154410660266876,
+ "num_tokens": 1509026.0,
+ "step": 250
+ },
+ {
+ "entropy": 2.168051454424858,
+ "epoch": 0.04807499699531269,
+ "grad_norm": 2.9375,
+ "learning_rate": 1.904182022111841e-05,
+ "loss": 2.190071716308594,
+ "mean_token_accuracy": 0.5156884534657001,
+ "num_tokens": 1806850.0,
+ "step": 300
+ },
+ {
+ "entropy": 2.1737154543399813,
+ "epoch": 0.05608749649453147,
+ "grad_norm": 3.421875,
+ "learning_rate": 1.8881589488863967e-05,
+ "loss": 2.199147186279297,
+ "mean_token_accuracy": 0.514045044630766,
+ "num_tokens": 2104505.0,
+ "step": 350
+ },
+ {
+ "entropy": 2.148039159178734,
+ "epoch": 0.06409999599375026,
+ "grad_norm": 3.109375,
+ "learning_rate": 1.872135875660952e-05,
+ "loss": 2.176946105957031,
+ "mean_token_accuracy": 0.5177112428843975,
+ "num_tokens": 2391088.0,
+ "step": 400
+ },
+ {
+ "entropy": 2.194890711903572,
+ "epoch": 0.07211249549296903,
+ "grad_norm": 3.28125,
+ "learning_rate": 1.856112802435507e-05,
+ "loss": 2.216288757324219,
+ "mean_token_accuracy": 0.5110335703194141,
+ "num_tokens": 2691272.0,
+ "step": 450
+ },
+ {
+ "entropy": 2.1557786524295808,
+ "epoch": 0.08012499499218781,
+ "grad_norm": 4.15625,
+ "learning_rate": 1.8400897292100627e-05,
+ "loss": 2.1812680053710936,
+ "mean_token_accuracy": 0.5176697050035,
+ "num_tokens": 2991855.0,
+ "step": 500
+ },
+ {
+ "entropy": 2.1708167713880537,
+ "epoch": 0.08813749449140659,
+ "grad_norm": 4.34375,
+ "learning_rate": 1.824066655984618e-05,
+ "loss": 2.1921002197265627,
+ "mean_token_accuracy": 0.5156507430970669,
+ "num_tokens": 3295049.0,
+ "step": 550
+ },
+ {
+ "entropy": 2.1686314886808393,
+ "epoch": 0.09614999399062538,
+ "grad_norm": 2.875,
+ "learning_rate": 1.8080435827591735e-05,
+ "loss": 2.1914617919921877,
+ "mean_token_accuracy": 0.5139613692462445,
+ "num_tokens": 3597206.0,
+ "step": 600
+ },
+ {
+ "entropy": 2.15742906332016,
+ "epoch": 0.10416249348984416,
+ "grad_norm": 3.109375,
+ "learning_rate": 1.7920205095337287e-05,
+ "loss": 2.174853973388672,
+ "mean_token_accuracy": 0.5159518288075924,
+ "num_tokens": 3905268.0,
+ "step": 650
+ },
+ {
+ "entropy": 2.162363375425339,
+ "epoch": 0.11217499298906294,
+ "grad_norm": 3.171875,
+ "learning_rate": 1.7759974363082843e-05,
+ "loss": 2.184231872558594,
+ "mean_token_accuracy": 0.516631413847208,
+ "num_tokens": 4207934.0,
+ "step": 700
+ },
+ {
+ "entropy": 2.1516289287805557,
+ "epoch": 0.12018749248828171,
+ "grad_norm": 3.609375,
+ "learning_rate": 1.7599743630828395e-05,
+ "loss": 2.167020263671875,
+ "mean_token_accuracy": 0.516358617246151,
+ "num_tokens": 4515774.0,
+ "step": 750
+ },
+ {
+ "entropy": 2.1549544781446457,
+ "epoch": 0.1281999919875005,
+ "grad_norm": 3.28125,
+ "learning_rate": 1.7439512898573947e-05,
+ "loss": 2.181917877197266,
+ "mean_token_accuracy": 0.5182220858335495,
+ "num_tokens": 4814709.0,
+ "step": 800
+ },
+ {
+ "entropy": 2.1532615756988527,
+ "epoch": 0.1362124914867193,
+ "grad_norm": 3.078125,
+ "learning_rate": 1.7279282166319503e-05,
+ "loss": 2.1659007263183594,
+ "mean_token_accuracy": 0.5180425970256328,
+ "num_tokens": 5112585.0,
+ "step": 850
+ },
+ {
+ "entropy": 2.1704017293453215,
+ "epoch": 0.14422499098593805,
+ "grad_norm": 2.921875,
+ "learning_rate": 1.7119051434065055e-05,
+ "loss": 2.1934996032714844,
+ "mean_token_accuracy": 0.5154964691400528,
+ "num_tokens": 5411369.0,
+ "step": 900
+ },
+ {
+ "entropy": 2.1349107217788696,
+ "epoch": 0.15223749048515683,
+ "grad_norm": 2.984375,
+ "learning_rate": 1.6958820701810608e-05,
+ "loss": 2.1598545837402345,
+ "mean_token_accuracy": 0.5208920259773732,
+ "num_tokens": 5707060.0,
+ "step": 950
+ },
+ {
+ "entropy": 2.1541705918312073,
+ "epoch": 0.16024998998437562,
+ "grad_norm": 3.203125,
+ "learning_rate": 1.6798589969556163e-05,
+ "loss": 2.170956573486328,
+ "mean_token_accuracy": 0.5204938448965549,
+ "num_tokens": 6004079.0,
+ "step": 1000
+ },
+ {
+ "entropy": 2.1523867017030716,
+ "epoch": 0.1682624894835944,
+ "grad_norm": 3.09375,
+ "learning_rate": 1.6638359237301716e-05,
+ "loss": 2.1695118713378907,
+ "mean_token_accuracy": 0.5177376234531402,
+ "num_tokens": 6315462.0,
+ "step": 1050
+ },
+ {
+ "entropy": 2.1356022965908052,
+ "epoch": 0.17627498898281319,
+ "grad_norm": 3.21875,
+ "learning_rate": 1.6478128505047268e-05,
+ "loss": 2.1513107299804686,
+ "mean_token_accuracy": 0.5217180585861206,
+ "num_tokens": 6613060.0,
+ "step": 1100
+ },
+ {
+ "entropy": 2.1483460652828215,
+ "epoch": 0.18428748848203197,
+ "grad_norm": 3.21875,
+ "learning_rate": 1.6317897772792824e-05,
+ "loss": 2.1671595764160156,
+ "mean_token_accuracy": 0.5199222292006016,
+ "num_tokens": 6909655.0,
+ "step": 1150
+ },
+ {
+ "entropy": 2.1074040442705155,
+ "epoch": 0.19229998798125075,
+ "grad_norm": 3.828125,
+ "learning_rate": 1.6157667040538376e-05,
+ "loss": 2.12770751953125,
+ "mean_token_accuracy": 0.5252153894305229,
+ "num_tokens": 7203263.0,
+ "step": 1200
+ },
+ {
+ "entropy": 2.1354027593135836,
+ "epoch": 0.20031248748046954,
+ "grad_norm": 2.921875,
+ "learning_rate": 1.5997436308283932e-05,
+ "loss": 2.156183166503906,
+ "mean_token_accuracy": 0.518907299786806,
+ "num_tokens": 7505411.0,
+ "step": 1250
+ },
+ {
+ "entropy": 2.142924894094467,
+ "epoch": 0.20832498697968832,
+ "grad_norm": 3.5625,
+ "learning_rate": 1.5837205576029484e-05,
+ "loss": 2.158486633300781,
+ "mean_token_accuracy": 0.5205909128487111,
+ "num_tokens": 7803562.0,
+ "step": 1300
+ },
+ {
+ "entropy": 2.1305291467905043,
+ "epoch": 0.2163374864789071,
+ "grad_norm": 3.34375,
+ "learning_rate": 1.5676974843775036e-05,
+ "loss": 2.151043701171875,
+ "mean_token_accuracy": 0.5211617235839366,
+ "num_tokens": 8108037.0,
+ "step": 1350
+ },
+ {
+ "entropy": 2.160947166085243,
+ "epoch": 0.2243499859781259,
+ "grad_norm": 3.234375,
+ "learning_rate": 1.5516744111520592e-05,
+ "loss": 2.177792663574219,
+ "mean_token_accuracy": 0.5167004990577698,
+ "num_tokens": 8408674.0,
+ "step": 1400
+ },
+ {
+ "entropy": 2.120403967499733,
+ "epoch": 0.23236248547734467,
+ "grad_norm": 3.375,
+ "learning_rate": 1.5356513379266144e-05,
+ "loss": 2.1462562561035154,
+ "mean_token_accuracy": 0.5225985661149025,
+ "num_tokens": 8707650.0,
+ "step": 1450
+ },
+ {
+ "entropy": 2.138882914185524,
+ "epoch": 0.24037498497656343,
+ "grad_norm": 3.171875,
+ "learning_rate": 1.5196282647011698e-05,
+ "loss": 2.1499905395507812,
+ "mean_token_accuracy": 0.5204668046534061,
+ "num_tokens": 9006296.0,
+ "step": 1500
+ },
+ {
+ "entropy": 2.131927245259285,
+ "epoch": 0.2483874844757822,
+ "grad_norm": 3.546875,
+ "learning_rate": 1.5036051914757252e-05,
+ "loss": 2.157076416015625,
+ "mean_token_accuracy": 0.5220408822596073,
+ "num_tokens": 9307203.0,
+ "step": 1550
+ },
+ {
+ "entropy": 2.1308567851781843,
+ "epoch": 0.256399983975001,
+ "grad_norm": 3.546875,
+ "learning_rate": 1.4875821182502805e-05,
+ "loss": 2.146601104736328,
+ "mean_token_accuracy": 0.5219539681077003,
+ "num_tokens": 9610070.0,
+ "step": 1600
+ },
+ {
+ "entropy": 2.1097783839702604,
+ "epoch": 0.2644124834742198,
+ "grad_norm": 3.84375,
+ "learning_rate": 1.4715590450248359e-05,
+ "loss": 2.1268209838867187,
+ "mean_token_accuracy": 0.5236652019619942,
+ "num_tokens": 9909117.0,
+ "step": 1650
+ },
+ {
+ "entropy": 2.123499180674553,
+ "epoch": 0.2724249829734386,
+ "grad_norm": 3.46875,
+ "learning_rate": 1.4555359717993913e-05,
+ "loss": 2.1409934997558593,
+ "mean_token_accuracy": 0.5225465354323388,
+ "num_tokens": 10209461.0,
+ "step": 1700
+ },
+ {
+ "entropy": 2.13052960395813,
+ "epoch": 0.2804374824726573,
+ "grad_norm": 2.71875,
+ "learning_rate": 1.4395128985739467e-05,
+ "loss": 2.151373291015625,
+ "mean_token_accuracy": 0.5227928113937378,
+ "num_tokens": 10509282.0,
+ "step": 1750
+ },
+ {
+ "entropy": 2.1226114094257356,
+ "epoch": 0.2884499819718761,
+ "grad_norm": 3.65625,
+ "learning_rate": 1.4234898253485019e-05,
+ "loss": 2.144451141357422,
+ "mean_token_accuracy": 0.5233827362954616,
+ "num_tokens": 10811056.0,
+ "step": 1800
+ },
+ {
+ "entropy": 2.126143154501915,
+ "epoch": 0.2964624814710949,
+ "grad_norm": 3.359375,
+ "learning_rate": 1.4074667521230575e-05,
+ "loss": 2.1457598876953123,
+ "mean_token_accuracy": 0.5221967925131321,
+ "num_tokens": 11111153.0,
+ "step": 1850
+ },
+ {
+ "entropy": 2.1048326581716537,
+ "epoch": 0.30447498097031367,
+ "grad_norm": 3.59375,
+ "learning_rate": 1.3914436788976127e-05,
+ "loss": 2.1260116577148436,
+ "mean_token_accuracy": 0.5272719909250736,
+ "num_tokens": 11405606.0,
+ "step": 1900
+ },
+ {
+ "entropy": 2.1236617982387545,
+ "epoch": 0.31248748046953245,
+ "grad_norm": 3.0,
+ "learning_rate": 1.375420605672168e-05,
+ "loss": 2.151782684326172,
+ "mean_token_accuracy": 0.5227546173334122,
+ "num_tokens": 11703943.0,
+ "step": 1950
+ },
+ {
+ "entropy": 2.121563429236412,
+ "epoch": 0.32049997996875124,
+ "grad_norm": 3.34375,
+ "learning_rate": 1.3593975324467235e-05,
+ "loss": 2.1370144653320313,
+ "mean_token_accuracy": 0.5235123547911644,
+ "num_tokens": 12002522.0,
+ "step": 2000
+ },
+ {
+ "entropy": 2.1155461168289182,
+ "epoch": 0.32851247946797,
+ "grad_norm": 3.46875,
+ "learning_rate": 1.3433744592212787e-05,
+ "loss": 2.1364064025878906,
+ "mean_token_accuracy": 0.5237135806679726,
+ "num_tokens": 12303098.0,
+ "step": 2050
+ },
+ {
+ "entropy": 2.1386052322387696,
+ "epoch": 0.3365249789671888,
+ "grad_norm": 3.484375,
+ "learning_rate": 1.327351385995834e-05,
+ "loss": 2.149888458251953,
+ "mean_token_accuracy": 0.522669050693512,
+ "num_tokens": 12605534.0,
+ "step": 2100
+ },
+ {
+ "entropy": 2.102600795030594,
+ "epoch": 0.3445374784664076,
+ "grad_norm": 3.796875,
+ "learning_rate": 1.3113283127703895e-05,
+ "loss": 2.1236566162109374,
+ "mean_token_accuracy": 0.5279092155396938,
+ "num_tokens": 12901218.0,
+ "step": 2150
+ },
+ {
+ "entropy": 2.132645187973976,
+ "epoch": 0.35254997796562637,
+ "grad_norm": 3.53125,
+ "learning_rate": 1.2953052395449447e-05,
+ "loss": 2.1523503112792968,
+ "mean_token_accuracy": 0.5198129612207413,
+ "num_tokens": 13213018.0,
+ "step": 2200
+ },
+ {
+ "entropy": 2.1204321217536926,
+ "epoch": 0.36056247746484515,
+ "grad_norm": 3.25,
+ "learning_rate": 1.2792821663195001e-05,
+ "loss": 2.1325447082519533,
+ "mean_token_accuracy": 0.5243636940419674,
+ "num_tokens": 13514408.0,
+ "step": 2250
+ },
+ {
+ "entropy": 2.102410731315613,
+ "epoch": 0.36857497696406394,
+ "grad_norm": 3.1875,
+ "learning_rate": 1.2632590930940555e-05,
+ "loss": 2.123565368652344,
+ "mean_token_accuracy": 0.5257614709436893,
+ "num_tokens": 13818272.0,
+ "step": 2300
+ },
+ {
+ "entropy": 2.0876205080747603,
+ "epoch": 0.3765874764632827,
+ "grad_norm": 3.328125,
+ "learning_rate": 1.247236019868611e-05,
+ "loss": 2.116696472167969,
+ "mean_token_accuracy": 0.5275124031305313,
+ "num_tokens": 14110702.0,
+ "step": 2350
+ },
+ {
+ "entropy": 2.1078518223762512,
+ "epoch": 0.3845999759625015,
+ "grad_norm": 3.453125,
+ "learning_rate": 1.2312129466431662e-05,
+ "loss": 2.1291993713378905,
+ "mean_token_accuracy": 0.524967464953661,
+ "num_tokens": 14407015.0,
+ "step": 2400
+ },
+ {
+ "entropy": 2.0930898267030718,
+ "epoch": 0.3926124754617203,
+ "grad_norm": 3.5,
+ "learning_rate": 1.2151898734177216e-05,
+ "loss": 2.1094123840332033,
+ "mean_token_accuracy": 0.5290587748587131,
+ "num_tokens": 14696565.0,
+ "step": 2450
+ },
+ {
+ "entropy": 2.1109757220745085,
+ "epoch": 0.4006249749609391,
+ "grad_norm": 3.5,
+ "learning_rate": 1.199166800192277e-05,
+ "loss": 2.1246888732910154,
+ "mean_token_accuracy": 0.5258957357704639,
+ "num_tokens": 14994401.0,
+ "step": 2500
+ },
+ {
+ "entropy": 2.1087811160087586,
+ "epoch": 0.40863747446015786,
+ "grad_norm": 3.1875,
+ "learning_rate": 1.1831437269668324e-05,
+ "loss": 2.12890625,
+ "mean_token_accuracy": 0.5269917643070221,
+ "num_tokens": 15292837.0,
+ "step": 2550
+ },
+ {
+ "entropy": 2.097252429127693,
+ "epoch": 0.41664997395937664,
+ "grad_norm": 3.0,
+ "learning_rate": 1.1671206537413878e-05,
+ "loss": 2.1145870971679686,
+ "mean_token_accuracy": 0.5274137698113919,
+ "num_tokens": 15590647.0,
+ "step": 2600
+ },
+ {
+ "entropy": 2.111199595332146,
+ "epoch": 0.4246624734585954,
+ "grad_norm": 3.25,
+ "learning_rate": 1.151097580515943e-05,
+ "loss": 2.1274327087402343,
+ "mean_token_accuracy": 0.5250249288976192,
+ "num_tokens": 15892772.0,
+ "step": 2650
+ },
+ {
+ "entropy": 2.1051502913236617,
+ "epoch": 0.4326749729578142,
+ "grad_norm": 3.203125,
+ "learning_rate": 1.1350745072904986e-05,
+ "loss": 2.127935333251953,
+ "mean_token_accuracy": 0.5263640700280666,
+ "num_tokens": 16191611.0,
+ "step": 2700
+ },
+ {
+ "entropy": 2.1127885603904724,
+ "epoch": 0.440687472457033,
+ "grad_norm": 3.515625,
+ "learning_rate": 1.1190514340650538e-05,
+ "loss": 2.129211883544922,
+ "mean_token_accuracy": 0.5250939278304577,
+ "num_tokens": 16483582.0,
+ "step": 2750
+ },
+ {
+ "entropy": 2.110930626988411,
+ "epoch": 0.4486999719562518,
+ "grad_norm": 3.1875,
+ "learning_rate": 1.103028360839609e-05,
+ "loss": 2.1220079040527344,
+ "mean_token_accuracy": 0.5250919266045093,
+ "num_tokens": 16787253.0,
+ "step": 2800
+ },
+ {
+ "entropy": 2.1182780677080153,
+ "epoch": 0.45671247145547056,
+ "grad_norm": 3.8125,
+ "learning_rate": 1.0870052876141646e-05,
+ "loss": 2.1358837890625,
+ "mean_token_accuracy": 0.5217854312062263,
+ "num_tokens": 17088741.0,
+ "step": 2850
+ },
+ {
+ "entropy": 2.1117833071947096,
+ "epoch": 0.46472497095468934,
+ "grad_norm": 3.171875,
+ "learning_rate": 1.0709822143887198e-05,
+ "loss": 2.125662841796875,
+ "mean_token_accuracy": 0.524649152457714,
+ "num_tokens": 17389236.0,
+ "step": 2900
+ },
+ {
+ "entropy": 2.1262730568647386,
+ "epoch": 0.47273747045390807,
+ "grad_norm": 3.625,
+ "learning_rate": 1.054959141163275e-05,
+ "loss": 2.142789611816406,
+ "mean_token_accuracy": 0.5236384868621826,
+ "num_tokens": 17693413.0,
+ "step": 2950
+ },
+ {
+ "entropy": 2.115410739183426,
+ "epoch": 0.48074996995312685,
+ "grad_norm": 3.140625,
+ "learning_rate": 1.0389360679378306e-05,
+ "loss": 2.1337106323242185,
+ "mean_token_accuracy": 0.5249222332239151,
+ "num_tokens": 17990326.0,
+ "step": 3000
+ },
+ {
+ "entropy": 2.0951381689310073,
+ "epoch": 0.48876246945234564,
+ "grad_norm": 3.234375,
+ "learning_rate": 1.0229129947123859e-05,
+ "loss": 2.108144073486328,
+ "mean_token_accuracy": 0.5269971822202205,
+ "num_tokens": 18293136.0,
+ "step": 3050
+ },
+ {
+ "entropy": 2.116977259516716,
+ "epoch": 0.4967749689515644,
+ "grad_norm": 3.421875,
+ "learning_rate": 1.0068899214869413e-05,
+ "loss": 2.1327830505371095,
+ "mean_token_accuracy": 0.5238270902633667,
+ "num_tokens": 18591481.0,
+ "step": 3100
+ },
+ {
+ "entropy": 2.08598925113678,
+ "epoch": 0.5047874684507833,
+ "grad_norm": 2.96875,
+ "learning_rate": 9.908668482614967e-06,
+ "loss": 2.1032344055175782,
+ "mean_token_accuracy": 0.5289211909472943,
+ "num_tokens": 18885221.0,
+ "step": 3150
+ },
+ {
+ "entropy": 2.097712662220001,
+ "epoch": 0.512799967950002,
+ "grad_norm": 3.40625,
+ "learning_rate": 9.74843775036052e-06,
+ "loss": 2.1150633239746095,
+ "mean_token_accuracy": 0.5288272814452648,
+ "num_tokens": 19184784.0,
+ "step": 3200
+ },
+ {
+ "entropy": 2.1063760107755662,
+ "epoch": 0.5208124674492208,
+ "grad_norm": 3.28125,
+ "learning_rate": 9.588207018106075e-06,
+ "loss": 2.1217109680175783,
+ "mean_token_accuracy": 0.526290989369154,
+ "num_tokens": 19482538.0,
+ "step": 3250
+ },
+ {
+ "entropy": 2.090182801485062,
+ "epoch": 0.5288249669484396,
+ "grad_norm": 2.875,
+ "learning_rate": 9.427976285851627e-06,
+ "loss": 2.1130311584472654,
+ "mean_token_accuracy": 0.5285439921915531,
+ "num_tokens": 19782544.0,
+ "step": 3300
+ },
+ {
+ "entropy": 2.103260806798935,
+ "epoch": 0.5368374664476584,
+ "grad_norm": 3.59375,
+ "learning_rate": 9.267745553597181e-06,
+ "loss": 2.116189727783203,
+ "mean_token_accuracy": 0.5273195558786392,
+ "num_tokens": 20086408.0,
+ "step": 3350
+ },
+ {
+ "entropy": 2.0673747235536575,
+ "epoch": 0.5448499659468772,
+ "grad_norm": 3.125,
+ "learning_rate": 9.107514821342735e-06,
+ "loss": 2.087482147216797,
+ "mean_token_accuracy": 0.532562478929758,
+ "num_tokens": 20380429.0,
+ "step": 3400
+ },
+ {
+ "entropy": 2.1030789083242416,
+ "epoch": 0.552862465446096,
+ "grad_norm": 3.296875,
+ "learning_rate": 8.947284089088289e-06,
+ "loss": 2.122591247558594,
+ "mean_token_accuracy": 0.5256703786551953,
+ "num_tokens": 20682849.0,
+ "step": 3450
+ },
+ {
+ "entropy": 2.0988716661930082,
+ "epoch": 0.5608749649453146,
+ "grad_norm": 3.546875,
+ "learning_rate": 8.787053356833841e-06,
+ "loss": 2.115867156982422,
+ "mean_token_accuracy": 0.5266730943322182,
+ "num_tokens": 20979225.0,
+ "step": 3500
+ },
+ {
+ "entropy": 2.0954072701931,
+ "epoch": 0.5688874644445334,
+ "grad_norm": 3.859375,
+ "learning_rate": 8.626822624579395e-06,
+ "loss": 2.1116868591308595,
+ "mean_token_accuracy": 0.5267495161294937,
+ "num_tokens": 21284166.0,
+ "step": 3550
+ },
+ {
+ "entropy": 2.11238471031189,
+ "epoch": 0.5768999639437522,
+ "grad_norm": 3.484375,
+ "learning_rate": 8.466591892324949e-06,
+ "loss": 2.1292352294921875,
+ "mean_token_accuracy": 0.5249243360757828,
+ "num_tokens": 21584612.0,
+ "step": 3600
+ },
+ {
+ "entropy": 2.091464368700981,
+ "epoch": 0.584912463442971,
+ "grad_norm": 3.359375,
+ "learning_rate": 8.306361160070501e-06,
+ "loss": 2.1099319458007812,
+ "mean_token_accuracy": 0.5292035435140133,
+ "num_tokens": 21879345.0,
+ "step": 3650
+ },
+ {
+ "entropy": 2.095204224586487,
+ "epoch": 0.5929249629421898,
+ "grad_norm": 2.96875,
+ "learning_rate": 8.146130427816055e-06,
+ "loss": 2.1198716735839844,
+ "mean_token_accuracy": 0.5255991271138192,
+ "num_tokens": 22183486.0,
+ "step": 3700
+ },
+ {
+ "entropy": 2.0666573679447175,
+ "epoch": 0.6009374624414086,
+ "grad_norm": 2.90625,
+ "learning_rate": 7.98589969556161e-06,
+ "loss": 2.08689697265625,
+ "mean_token_accuracy": 0.5318664258718491,
+ "num_tokens": 22476438.0,
+ "step": 3750
+ },
+ {
+ "entropy": 2.122153177857399,
+ "epoch": 0.6089499619406273,
+ "grad_norm": 3.40625,
+ "learning_rate": 7.825668963307162e-06,
+ "loss": 2.138288879394531,
+ "mean_token_accuracy": 0.5232584895193577,
+ "num_tokens": 22780385.0,
+ "step": 3800
+ },
+ {
+ "entropy": 2.1190748488903046,
+ "epoch": 0.6169624614398461,
+ "grad_norm": 3.46875,
+ "learning_rate": 7.665438231052716e-06,
+ "loss": 2.139790344238281,
+ "mean_token_accuracy": 0.5243287613987923,
+ "num_tokens": 23080847.0,
+ "step": 3850
+ },
+ {
+ "entropy": 2.1312570214271545,
+ "epoch": 0.6249749609390649,
+ "grad_norm": 3.765625,
+ "learning_rate": 7.5052074987982706e-06,
+ "loss": 2.1471080017089843,
+ "mean_token_accuracy": 0.5232046774029732,
+ "num_tokens": 23384738.0,
+ "step": 3900
+ },
+ {
+ "entropy": 2.0779008239507677,
+ "epoch": 0.6329874604382837,
+ "grad_norm": 3.171875,
+ "learning_rate": 7.344976766543823e-06,
+ "loss": 2.0974729919433592,
+ "mean_token_accuracy": 0.5305448272824287,
+ "num_tokens": 23682750.0,
+ "step": 3950
+ },
+ {
+ "entropy": 2.1121455317735673,
+ "epoch": 0.6409999599375025,
+ "grad_norm": 3.921875,
+ "learning_rate": 7.184746034289377e-06,
+ "loss": 2.127069549560547,
+ "mean_token_accuracy": 0.5252936625480652,
+ "num_tokens": 23989029.0,
+ "step": 4000
+ },
+ {
+ "entropy": 2.0982888478040693,
+ "epoch": 0.6490124594367213,
+ "grad_norm": 4.0625,
+ "learning_rate": 7.024515302034931e-06,
+ "loss": 2.112042236328125,
+ "mean_token_accuracy": 0.5281323495507241,
+ "num_tokens": 24292063.0,
+ "step": 4050
+ },
+ {
+ "entropy": 2.098363963365555,
+ "epoch": 0.65702495893594,
+ "grad_norm": 3.046875,
+ "learning_rate": 6.864284569780485e-06,
+ "loss": 2.1156489562988283,
+ "mean_token_accuracy": 0.5275120075047016,
+ "num_tokens": 24590201.0,
+ "step": 4100
+ },
+ {
+ "entropy": 2.1065632277727127,
+ "epoch": 0.6650374584351588,
+ "grad_norm": 3.375,
+ "learning_rate": 6.704053837526038e-06,
+ "loss": 2.12911376953125,
+ "mean_token_accuracy": 0.5262684133648873,
+ "num_tokens": 24892122.0,
+ "step": 4150
+ },
+ {
+ "entropy": 2.1014072370529173,
+ "epoch": 0.6730499579343776,
+ "grad_norm": 2.84375,
+ "learning_rate": 6.543823105271592e-06,
+ "loss": 2.1130335998535155,
+ "mean_token_accuracy": 0.5262750512361527,
+ "num_tokens": 25194969.0,
+ "step": 4200
+ },
+ {
+ "entropy": 2.0905236744880678,
+ "epoch": 0.6810624574335964,
+ "grad_norm": 3.03125,
+ "learning_rate": 6.383592373017146e-06,
+ "loss": 2.1153533935546873,
+ "mean_token_accuracy": 0.527137616276741,
+ "num_tokens": 25499521.0,
+ "step": 4250
+ },
+ {
+ "entropy": 2.1032627063989637,
+ "epoch": 0.6890749569328152,
+ "grad_norm": 3.125,
+ "learning_rate": 6.223361640762698e-06,
+ "loss": 2.122831115722656,
+ "mean_token_accuracy": 0.5254572249948979,
+ "num_tokens": 25798933.0,
+ "step": 4300
+ },
+ {
+ "entropy": 2.118139310479164,
+ "epoch": 0.697087456432034,
+ "grad_norm": 3.28125,
+ "learning_rate": 6.063130908508252e-06,
+ "loss": 2.1376751708984374,
+ "mean_token_accuracy": 0.5243131402134895,
+ "num_tokens": 26096619.0,
+ "step": 4350
+ },
+ {
+ "entropy": 2.082554380297661,
+ "epoch": 0.7050999559312527,
+ "grad_norm": 3.390625,
+ "learning_rate": 5.902900176253806e-06,
+ "loss": 2.0949493408203126,
+ "mean_token_accuracy": 0.5316410692036152,
+ "num_tokens": 26394395.0,
+ "step": 4400
+ },
+ {
+ "entropy": 2.09870474755764,
+ "epoch": 0.7131124554304715,
+ "grad_norm": 3.421875,
+ "learning_rate": 5.7426694439993594e-06,
+ "loss": 2.1184346008300783,
+ "mean_token_accuracy": 0.5267573061585427,
+ "num_tokens": 26692333.0,
+ "step": 4450
+ },
+ {
+ "entropy": 2.0866885662078856,
+ "epoch": 0.7211249549296903,
+ "grad_norm": 3.71875,
+ "learning_rate": 5.5824387117449134e-06,
+ "loss": 2.102904968261719,
+ "mean_token_accuracy": 0.529726449996233,
+ "num_tokens": 26989384.0,
+ "step": 4500
+ },
+ {
+ "entropy": 2.114828346967697,
+ "epoch": 0.7291374544289091,
+ "grad_norm": 3.46875,
+ "learning_rate": 5.4222079794904666e-06,
+ "loss": 2.128611602783203,
+ "mean_token_accuracy": 0.5247649565339089,
+ "num_tokens": 27299927.0,
+ "step": 4550
+ },
+ {
+ "entropy": 2.09617226600647,
+ "epoch": 0.7371499539281279,
+ "grad_norm": 3.015625,
+ "learning_rate": 5.26197724723602e-06,
+ "loss": 2.1166041564941405,
+ "mean_token_accuracy": 0.5273530712723732,
+ "num_tokens": 27599011.0,
+ "step": 4600
+ },
+ {
+ "entropy": 2.0975282275676728,
+ "epoch": 0.7451624534273467,
+ "grad_norm": 3.3125,
+ "learning_rate": 5.101746514981574e-06,
+ "loss": 2.112121124267578,
+ "mean_token_accuracy": 0.5281847220659256,
+ "num_tokens": 27892355.0,
+ "step": 4650
+ },
+ {
+ "entropy": 2.095783769488335,
+ "epoch": 0.7531749529265654,
+ "grad_norm": 3.78125,
+ "learning_rate": 4.941515782727128e-06,
+ "loss": 2.110701904296875,
+ "mean_token_accuracy": 0.5272985179722309,
+ "num_tokens": 28187615.0,
+ "step": 4700
+ },
+ {
+ "entropy": 2.0964103603363036,
+ "epoch": 0.7611874524257842,
+ "grad_norm": 3.1875,
+ "learning_rate": 4.781285050472681e-06,
+ "loss": 2.11992919921875,
+ "mean_token_accuracy": 0.5271916574239731,
+ "num_tokens": 28485419.0,
+ "step": 4750
+ },
+ {
+ "entropy": 2.0820577853918074,
+ "epoch": 0.769199951925003,
+ "grad_norm": 3.5,
+ "learning_rate": 4.621054318218234e-06,
+ "loss": 2.095073394775391,
+ "mean_token_accuracy": 0.5297581899166107,
+ "num_tokens": 28780158.0,
+ "step": 4800
+ },
+ {
+ "entropy": 2.089591929912567,
+ "epoch": 0.7772124514242218,
+ "grad_norm": 3.125,
+ "learning_rate": 4.460823585963788e-06,
+ "loss": 2.109657897949219,
+ "mean_token_accuracy": 0.528961935043335,
+ "num_tokens": 29081052.0,
+ "step": 4850
+ },
+ {
+ "entropy": 2.092900555729866,
+ "epoch": 0.7852249509234406,
+ "grad_norm": 3.46875,
+ "learning_rate": 4.300592853709342e-06,
+ "loss": 2.106436920166016,
+ "mean_token_accuracy": 0.5284854583442211,
+ "num_tokens": 29376324.0,
+ "step": 4900
+ },
+ {
+ "entropy": 2.0630873823165894,
+ "epoch": 0.7932374504226594,
+ "grad_norm": 3.28125,
+ "learning_rate": 4.140362121454896e-06,
+ "loss": 2.088908233642578,
+ "mean_token_accuracy": 0.532561699450016,
+ "num_tokens": 29675269.0,
+ "step": 4950
+ },
+ {
+ "entropy": 2.0753238475322724,
+ "epoch": 0.8012499499218781,
+ "grad_norm": 3.21875,
+ "learning_rate": 3.980131389200449e-06,
+ "loss": 2.0961412048339843,
+ "mean_token_accuracy": 0.5304154875874519,
+ "num_tokens": 29974838.0,
+ "step": 5000
+ },
+ {
+ "entropy": 2.123110404610634,
+ "epoch": 0.8092624494210969,
+ "grad_norm": 3.28125,
+ "learning_rate": 3.819900656946002e-06,
+ "loss": 2.1538424682617188,
+ "mean_token_accuracy": 0.5222072719037533,
+ "num_tokens": 30286201.0,
+ "step": 5050
+ },
+ {
+ "entropy": 2.123967355489731,
+ "epoch": 0.8172749489203157,
+ "grad_norm": 3.046875,
+ "learning_rate": 3.6596699246915563e-06,
+ "loss": 2.1417193603515625,
+ "mean_token_accuracy": 0.5213662980496884,
+ "num_tokens": 30592837.0,
+ "step": 5100
+ },
+ {
+ "entropy": 2.080365374684334,
+ "epoch": 0.8252874484195345,
+ "grad_norm": 3.375,
+ "learning_rate": 3.4994391924371094e-06,
+ "loss": 2.0970494079589845,
+ "mean_token_accuracy": 0.5296585746109486,
+ "num_tokens": 30890674.0,
+ "step": 5150
+ },
+ {
+ "entropy": 2.1183922028541566,
+ "epoch": 0.8332999479187533,
+ "grad_norm": 3.359375,
+ "learning_rate": 3.3392084601826634e-06,
+ "loss": 2.132384948730469,
+ "mean_token_accuracy": 0.5249108146131038,
+ "num_tokens": 31195601.0,
+ "step": 5200
+ },
+ {
+ "entropy": 2.1079335528612138,
+ "epoch": 0.8413124474179721,
+ "grad_norm": 2.890625,
+ "learning_rate": 3.178977727928217e-06,
+ "loss": 2.1310113525390624,
+ "mean_token_accuracy": 0.5255681444704533,
+ "num_tokens": 31498768.0,
+ "step": 5250
+ },
+ {
+ "entropy": 2.1180070728063582,
+ "epoch": 0.8493249469171908,
+ "grad_norm": 3.78125,
+ "learning_rate": 3.01874699567377e-06,
+ "loss": 2.1365049743652342,
+ "mean_token_accuracy": 0.5235744562745094,
+ "num_tokens": 31801891.0,
+ "step": 5300
+ },
+ {
+ "entropy": 2.0909912359714506,
+ "epoch": 0.8573374464164096,
+ "grad_norm": 3.078125,
+ "learning_rate": 2.858516263419324e-06,
+ "loss": 2.108803405761719,
+ "mean_token_accuracy": 0.5267959542572498,
+ "num_tokens": 32102627.0,
+ "step": 5350
+ },
+ {
+ "entropy": 2.1209339940547944,
+ "epoch": 0.8653499459156284,
+ "grad_norm": 3.140625,
+ "learning_rate": 2.6982855311648777e-06,
+ "loss": 2.143946228027344,
+ "mean_token_accuracy": 0.5227122528851033,
+ "num_tokens": 32405500.0,
+ "step": 5400
+ },
+ {
+ "entropy": 2.1137486273050308,
+ "epoch": 0.8733624454148472,
+ "grad_norm": 3.03125,
+ "learning_rate": 2.5380547989104317e-06,
+ "loss": 2.127569122314453,
+ "mean_token_accuracy": 0.5245758745074273,
+ "num_tokens": 32712521.0,
+ "step": 5450
+ },
+ {
+ "entropy": 2.0872016245126725,
+ "epoch": 0.881374944914066,
+ "grad_norm": 3.171875,
+ "learning_rate": 2.377824066655985e-06,
+ "loss": 2.100517120361328,
+ "mean_token_accuracy": 0.5295753349363804,
+ "num_tokens": 33016210.0,
+ "step": 5500
+ },
+ {
+ "entropy": 2.0836504286527635,
+ "epoch": 0.8893874444132848,
+ "grad_norm": 3.328125,
+ "learning_rate": 2.2175933344015384e-06,
+ "loss": 2.1047386169433593,
+ "mean_token_accuracy": 0.5296267972886562,
+ "num_tokens": 33311705.0,
+ "step": 5550
+ },
+ {
+ "entropy": 2.0777365696430206,
+ "epoch": 0.8973999439125036,
+ "grad_norm": 3.59375,
+ "learning_rate": 2.057362602147092e-06,
+ "loss": 2.0985890197753907,
+ "mean_token_accuracy": 0.5303736463189125,
+ "num_tokens": 33603231.0,
+ "step": 5600
+ },
+ {
+ "entropy": 2.095007498860359,
+ "epoch": 0.9054124434117223,
+ "grad_norm": 3.59375,
+ "learning_rate": 1.8971318698926456e-06,
+ "loss": 2.106617431640625,
+ "mean_token_accuracy": 0.527913925498724,
+ "num_tokens": 33902978.0,
+ "step": 5650
+ },
+ {
+ "entropy": 2.106599664092064,
+ "epoch": 0.9134249429109411,
+ "grad_norm": 3.234375,
+ "learning_rate": 1.7369011376381991e-06,
+ "loss": 2.1318267822265624,
+ "mean_token_accuracy": 0.525322500616312,
+ "num_tokens": 34206935.0,
+ "step": 5700
+ },
+ {
+ "entropy": 2.094651755094528,
+ "epoch": 0.9214374424101599,
+ "grad_norm": 3.65625,
+ "learning_rate": 1.576670405383753e-06,
+ "loss": 2.110823211669922,
+ "mean_token_accuracy": 0.5286212807893753,
+ "num_tokens": 34499610.0,
+ "step": 5750
+ },
+ {
+ "entropy": 2.119523124098778,
+ "epoch": 0.9294499419093787,
+ "grad_norm": 3.375,
+ "learning_rate": 1.4164396731293063e-06,
+ "loss": 2.13890625,
+ "mean_token_accuracy": 0.5239527927339077,
+ "num_tokens": 34806213.0,
+ "step": 5800
+ },
+ {
+ "entropy": 2.0914100110530853,
+ "epoch": 0.9374624414085974,
+ "grad_norm": 3.46875,
+ "learning_rate": 1.2562089408748598e-06,
+ "loss": 2.113725280761719,
+ "mean_token_accuracy": 0.5276746636629105,
+ "num_tokens": 35106711.0,
+ "step": 5850
+ },
+ {
+ "entropy": 2.0877316004037856,
+ "epoch": 0.9454749409078161,
+ "grad_norm": 3.125,
+ "learning_rate": 1.0959782086204134e-06,
+ "loss": 2.097077178955078,
+ "mean_token_accuracy": 0.5296605123579502,
+ "num_tokens": 35398548.0,
+ "step": 5900
+ },
+ {
+ "entropy": 2.099357485175133,
+ "epoch": 0.9534874404070349,
+ "grad_norm": 3.234375,
+ "learning_rate": 9.357474763659671e-07,
+ "loss": 2.115228729248047,
+ "mean_token_accuracy": 0.5284087285399437,
+ "num_tokens": 35689259.0,
+ "step": 5950
+ },
+ {
+ "entropy": 2.0721785259246825,
+ "epoch": 0.9614999399062537,
+ "grad_norm": 3.84375,
+ "learning_rate": 7.755167441115207e-07,
+ "loss": 2.098111114501953,
+ "mean_token_accuracy": 0.5322318106889725,
+ "num_tokens": 35989481.0,
+ "step": 6000
+ },
+ {
+ "entropy": 2.0931629753112793,
+ "epoch": 0.9695124394054725,
+ "grad_norm": 3.15625,
+ "learning_rate": 6.152860118570742e-07,
+ "loss": 2.1118145751953126,
+ "mean_token_accuracy": 0.5284056887030602,
+ "num_tokens": 36287566.0,
+ "step": 6050
+ },
+ {
+ "entropy": 2.1098696780204773,
+ "epoch": 0.9775249389046913,
+ "grad_norm": 3.421875,
+ "learning_rate": 4.550552796026278e-07,
+ "loss": 2.135518341064453,
+ "mean_token_accuracy": 0.5236021775007248,
+ "num_tokens": 36586868.0,
+ "step": 6100
+ },
+ {
+ "entropy": 2.074869705438614,
+ "epoch": 0.9855374384039101,
+ "grad_norm": 3.625,
+ "learning_rate": 2.9482454734818144e-07,
+ "loss": 2.0890965270996094,
+ "mean_token_accuracy": 0.5310191959142685,
+ "num_tokens": 36882175.0,
+ "step": 6150
+ },
+ {
+ "entropy": 2.1156716483831404,
+ "epoch": 0.9935499379031288,
+ "grad_norm": 3.75,
+ "learning_rate": 1.3459381509373498e-07,
+ "loss": 2.136824951171875,
+ "mean_token_accuracy": 0.5256458315253257,
+ "num_tokens": 37180928.0,
+ "step": 6200
+ }
+ ],
+ "logging_steps": 50,
+ "max_steps": 6241,
+ "num_input_tokens_seen": 0,
+ "num_train_epochs": 1,
+ "save_steps": 500,
+ "stateful_callbacks": {
+ "TrainerControl": {
+ "args": {
+ "should_epoch_stop": false,
+ "should_evaluate": false,
+ "should_log": false,
+ "should_save": true,
+ "should_training_stop": true
+ },
+ "attributes": {}
+ }
+ },
+ "total_flos": 4.419576669616251e+17,
+ "train_batch_size": 4,
+ "trial_name": null,
+ "trial_params": null
+}
diff --git a/checkpoint-6241/training_args.bin b/checkpoint-6241/training_args.bin
new file mode 100644
index 0000000000000000000000000000000000000000..4b409d5a147856f0d5a3d3da0f3b4922a9def7dd
--- /dev/null
+++ b/checkpoint-6241/training_args.bin
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:78622f63304ec0913ed0b4ea7a1b26c08a5621b3661a380d798c4ea426235671
+size 5713
diff --git a/config.json b/config.json
new file mode 100644
index 0000000000000000000000000000000000000000..1d219d6a6d288625229a71646710b85cf7ae5def
--- /dev/null
+++ b/config.json
@@ -0,0 +1,63 @@
+{
+ "architectures": [
+ "Qwen3ForCausalLM"
+ ],
+ "attention_bias": false,
+ "attention_dropout": 0.0,
+ "bos_token_id": null,
+ "dtype": "bfloat16",
+ "eos_token_id": 151645,
+ "head_dim": 128,
+ "hidden_act": "silu",
+ "hidden_size": 2048,
+ "initializer_range": 0.02,
+ "intermediate_size": 6144,
+ "layer_types": [
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention"
+ ],
+ "max_position_embeddings": 40960,
+ "max_window_layers": 28,
+ "model_type": "qwen3",
+ "num_attention_heads": 16,
+ "num_hidden_layers": 28,
+ "num_key_value_heads": 8,
+ "pad_token_id": 151643,
+ "rms_norm_eps": 1e-06,
+ "rope_parameters": {
+ "rope_theta": 1000000,
+ "rope_type": "default"
+ },
+ "sliding_window": null,
+ "tie_word_embeddings": true,
+ "transformers_version": "5.7.0",
+ "use_cache": false,
+ "use_sliding_window": false,
+ "vocab_size": 151936
+}
diff --git a/generation_config.json b/generation_config.json
new file mode 100644
index 0000000000000000000000000000000000000000..5ec133d867b1427d43a5ee0ea332bd088fc12013
--- /dev/null
+++ b/generation_config.json
@@ -0,0 +1,12 @@
+{
+ "do_sample": true,
+ "eos_token_id": [
+ 151645,
+ 151643
+ ],
+ "pad_token_id": 151643,
+ "temperature": 0.6,
+ "top_k": 20,
+ "top_p": 0.95,
+ "transformers_version": "5.7.0"
+}
diff --git a/model.safetensors b/model.safetensors
new file mode 100644
index 0000000000000000000000000000000000000000..c6cea834c2762ddba2e2783f704d2e14442c343e
--- /dev/null
+++ b/model.safetensors
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:3140e22b016936d08d1a9957161ddab89478b00c1cab50daaa2ad5dc5d16a4fc
+size 3441185608
diff --git a/tokenizer.json b/tokenizer.json
new file mode 100644
index 0000000000000000000000000000000000000000..c7afbed2efcdf019f88ab0572ec29d3bf595dfe2
--- /dev/null
+++ b/tokenizer.json
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:be75606093db2094d7cd20f3c2f385c212750648bd6ea4fb2bf507a6a4c55506
+size 11422650
diff --git a/tokenizer_config.json b/tokenizer_config.json
new file mode 100644
index 0000000000000000000000000000000000000000..770e41d6c92519d525eede4cbcf3ba27f6425311
--- /dev/null
+++ b/tokenizer_config.json
@@ -0,0 +1,30 @@
+{
+ "add_prefix_space": false,
+ "backend": "tokenizers",
+ "bos_token": null,
+ "clean_up_tokenization_spaces": false,
+ "eos_token": "<|im_end|>",
+ "errors": "replace",
+ "extra_special_tokens": [
+ "<|im_start|>",
+ "<|im_end|>",
+ "<|object_ref_start|>",
+ "<|object_ref_end|>",
+ "<|box_start|>",
+ "<|box_end|>",
+ "<|quad_start|>",
+ "<|quad_end|>",
+ "<|vision_start|>",
+ "<|vision_end|>",
+ "<|vision_pad|>",
+ "<|image_pad|>",
+ "<|video_pad|>"
+ ],
+ "is_local": false,
+ "local_files_only": false,
+ "model_max_length": 131072,
+ "pad_token": "<|endoftext|>",
+ "split_special_tokens": false,
+ "tokenizer_class": "Qwen2Tokenizer",
+ "unk_token": null
+}
diff --git a/training_args.bin b/training_args.bin
new file mode 100644
index 0000000000000000000000000000000000000000..4b409d5a147856f0d5a3d3da0f3b4922a9def7dd
--- /dev/null
+++ b/training_args.bin
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:78622f63304ec0913ed0b4ea7a1b26c08a5621b3661a380d798c4ea426235671
+size 5713