diff --git a/.gitattributes b/.gitattributes
index a6344aac8c09253b3b630fb776ae94478aa0275b..1da1495d0e44a18a356820680f3cf850d368d2e2 100644
--- a/.gitattributes
+++ b/.gitattributes
@@ -33,3 +33,12 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
*.zip filter=lfs diff=lfs merge=lfs -text
*.zst filter=lfs diff=lfs merge=lfs -text
*tfevents* filter=lfs diff=lfs merge=lfs -text
+qwen3-1.7b-teutonic-2e5/checkpoint-1000/tokenizer.json filter=lfs diff=lfs merge=lfs -text
+qwen3-1.7b-teutonic-2e5/checkpoint-1485/tokenizer.json filter=lfs diff=lfs merge=lfs -text
+qwen3-1.7b-teutonic-2e5/checkpoint-500/tokenizer.json filter=lfs diff=lfs merge=lfs -text
+qwen3-1.7b-teutonic-2e5/tokenizer.json filter=lfs diff=lfs merge=lfs -text
+qwen3-1.7b-teutonic-5e6/checkpoint-1000/tokenizer.json filter=lfs diff=lfs merge=lfs -text
+qwen3-1.7b-teutonic-5e6/checkpoint-1500/tokenizer.json filter=lfs diff=lfs merge=lfs -text
+qwen3-1.7b-teutonic-5e6/checkpoint-2000/tokenizer.json filter=lfs diff=lfs merge=lfs -text
+qwen3-1.7b-teutonic-5e6/checkpoint-2475/tokenizer.json filter=lfs diff=lfs merge=lfs -text
+qwen3-1.7b-teutonic-5e6/checkpoint-500/tokenizer.json filter=lfs diff=lfs merge=lfs -text
diff --git a/README.md b/README.md
new file mode 100644
index 0000000000000000000000000000000000000000..a503414d7c85e57a38b16572136a5a8bb102f5cb
--- /dev/null
+++ b/README.md
@@ -0,0 +1,58 @@
+---
+base_model: Qwen/Qwen3-1.7B
+library_name: transformers
+model_name: checkpoints
+tags:
+- generated_from_trainer
+- trl
+- sft
+licence: license
+---
+
+# Model Card for checkpoints
+
+This model is a fine-tuned version of [Qwen/Qwen3-1.7B](https://huggingface.co/Qwen/Qwen3-1.7B).
+It has been trained using [TRL](https://github.com/huggingface/trl).
+
+## Quick start
+
+```python
+from transformers import pipeline
+
+question = "If you had a time machine, but could only go to the past or the future once and never return, which would you choose and why?"
+generator = pipeline("text-generation", model="None", device_map="auto")
+output = generator([{"role": "user", "content": question}], max_new_tokens=128, return_full_text=False)[0]
+print(output["generated_text"])
+```
+
+## Training procedure
+
+
+
+
+
+This model was trained with SFT.
+
+### Framework versions
+
+- TRL: 1.14.0
+- Transformers: 5.17.0
+- Pytorch: 2.14.0
+- Datasets: 5.0.1
+- Tokenizers: 0.23.2
+
+## Citations
+
+
+
+Cite TRL as:
+
+```bibtex
+@software{vonwerra2020trl,
+ title = {{TRL: Transformers Reinforcement Learning}},
+ author = {von Werra, Leandro and Belkada, Younes and Tunstall, Lewis and Beeching, Edward and Thrush, Tristan and Lambert, Nathan and Huang, Shengyi and Rasul, Kashif and Gallouédec, Quentin},
+ license = {Apache-2.0},
+ url = {https://github.com/huggingface/trl},
+ year = {2020}
+}
+```
\ No newline at end of file
diff --git a/qwen3-1.7b-teutonic-2e5/README.md b/qwen3-1.7b-teutonic-2e5/README.md
new file mode 100644
index 0000000000000000000000000000000000000000..f8d3965b4a9dcebbc42d1cb24a520aeac449423b
--- /dev/null
+++ b/qwen3-1.7b-teutonic-2e5/README.md
@@ -0,0 +1,58 @@
+---
+base_model: Qwen/Qwen3-1.7B
+library_name: transformers
+model_name: qwen3-1.7b-teutonic
+tags:
+- generated_from_trainer
+- sft
+- trl
+licence: license
+---
+
+# Model Card for qwen3-1.7b-teutonic
+
+This model is a fine-tuned version of [Qwen/Qwen3-1.7B](https://huggingface.co/Qwen/Qwen3-1.7B).
+It has been trained using [TRL](https://github.com/huggingface/trl).
+
+## Quick start
+
+```python
+from transformers import pipeline
+
+question = "If you had a time machine, but could only go to the past or the future once and never return, which would you choose and why?"
+generator = pipeline("text-generation", model="None", device_map="auto")
+output = generator([{"role": "user", "content": question}], max_new_tokens=128, return_full_text=False)[0]
+print(output["generated_text"])
+```
+
+## Training procedure
+
+[
](https://wandb.ai/digitallibrary849-adobe/tiny-teutonic-sn3/runs/p68ygpe9)
+
+
+
+This model was trained with SFT.
+
+### Framework versions
+
+- TRL: 1.14.0
+- Transformers: 5.17.0
+- Pytorch: 2.14.0
+- Datasets: 5.0.1
+- Tokenizers: 0.23.2
+
+## Citations
+
+
+
+Cite TRL as:
+
+```bibtex
+@software{vonwerra2020trl,
+ title = {{TRL: Transformers Reinforcement Learning}},
+ author = {von Werra, Leandro and Belkada, Younes and Tunstall, Lewis and Beeching, Edward and Thrush, Tristan and Lambert, Nathan and Huang, Shengyi and Rasul, Kashif and Gallouédec, Quentin},
+ license = {Apache-2.0},
+ url = {https://github.com/huggingface/trl},
+ year = {2020}
+}
+```
\ No newline at end of file
diff --git a/qwen3-1.7b-teutonic-2e5/chat_template.jinja b/qwen3-1.7b-teutonic-2e5/chat_template.jinja
new file mode 100644
index 0000000000000000000000000000000000000000..01be9b307daa2d425f7c168c9fb145a286e0afb4
--- /dev/null
+++ b/qwen3-1.7b-teutonic-2e5/chat_template.jinja
@@ -0,0 +1,89 @@
+{%- if tools %}
+ {{- '<|im_start|>system\n' }}
+ {%- if messages[0].role == 'system' %}
+ {{- messages[0].content + '\n\n' }}
+ {%- endif %}
+ {{- "# Tools\n\nYou may call one or more functions to assist with the user query.\n\nYou are provided with function signatures within XML tags:\n" }}
+ {%- for tool in tools %}
+ {{- "\n" }}
+ {{- tool | tojson }}
+ {%- endfor %}
+ {{- "\n\n\nFor each function call, return a json object with function name and arguments within XML tags:\n\n{\"name\": , \"arguments\": }\n<|im_end|>\n" }}
+{%- else %}
+ {%- if messages[0].role == 'system' %}
+ {{- '<|im_start|>system\n' + messages[0].content + '<|im_end|>\n' }}
+ {%- endif %}
+{%- endif %}
+{%- set ns = namespace(multi_step_tool=true, last_query_index=messages|length - 1) %}
+{%- for message in messages[::-1] %}
+ {%- set index = (messages|length - 1) - loop.index0 %}
+ {%- if ns.multi_step_tool and message.role == "user" and message.content is string and not(message.content.startswith('') and message.content.endswith('')) %}
+ {%- set ns.multi_step_tool = false %}
+ {%- set ns.last_query_index = index %}
+ {%- endif %}
+{%- endfor %}
+{%- for message in messages %}
+ {%- if message.content is string %}
+ {%- set content = message.content %}
+ {%- else %}
+ {%- set content = '' %}
+ {%- endif %}
+ {%- if (message.role == "user") or (message.role == "system" and not loop.first) %}
+ {{- '<|im_start|>' + message.role + '\n' + content + '<|im_end|>' + '\n' }}
+ {%- elif message.role == "assistant" %}
+ {%- set reasoning_content = '' %}
+ {%- if message.reasoning_content is string %}
+ {%- set reasoning_content = message.reasoning_content %}
+ {%- else %}
+ {%- if '' in content %}
+ {%- set reasoning_content = content.split('')[0].rstrip('\n').split('')[-1].lstrip('\n') %}
+ {%- set content = content.split('')[-1].lstrip('\n') %}
+ {%- endif %}
+ {%- endif %}
+ {%- if loop.index0 > ns.last_query_index %}
+ {%- if loop.last or (not loop.last and reasoning_content) %}
+ {{- '<|im_start|>' + message.role + '\n\n' + reasoning_content.strip('\n') + '\n\n\n' + content.lstrip('\n') }}
+ {%- else %}
+ {{- '<|im_start|>' + message.role + '\n' + content }}
+ {%- endif %}
+ {%- else %}
+ {{- '<|im_start|>' + message.role + '\n' + content }}
+ {%- endif %}
+ {%- if message.tool_calls %}
+ {%- for tool_call in message.tool_calls %}
+ {%- if (loop.first and content) or (not loop.first) %}
+ {{- '\n' }}
+ {%- endif %}
+ {%- if tool_call.function %}
+ {%- set tool_call = tool_call.function %}
+ {%- endif %}
+ {{- '\n{"name": "' }}
+ {{- tool_call.name }}
+ {{- '", "arguments": ' }}
+ {%- if tool_call.arguments is string %}
+ {{- tool_call.arguments }}
+ {%- else %}
+ {{- tool_call.arguments | tojson }}
+ {%- endif %}
+ {{- '}\n' }}
+ {%- endfor %}
+ {%- endif %}
+ {{- '<|im_end|>\n' }}
+ {%- elif message.role == "tool" %}
+ {%- if loop.first or (messages[loop.index0 - 1].role != "tool") %}
+ {{- '<|im_start|>user' }}
+ {%- endif %}
+ {{- '\n\n' }}
+ {{- content }}
+ {{- '\n' }}
+ {%- if loop.last or (messages[loop.index0 + 1].role != "tool") %}
+ {{- '<|im_end|>\n' }}
+ {%- endif %}
+ {%- endif %}
+{%- endfor %}
+{%- if add_generation_prompt %}
+ {{- '<|im_start|>assistant\n' }}
+ {%- if enable_thinking is defined and enable_thinking is false %}
+ {{- '\n\n\n\n' }}
+ {%- endif %}
+{%- endif %}
\ No newline at end of file
diff --git a/qwen3-1.7b-teutonic-2e5/checkpoint-1000/chat_template.jinja b/qwen3-1.7b-teutonic-2e5/checkpoint-1000/chat_template.jinja
new file mode 100644
index 0000000000000000000000000000000000000000..01be9b307daa2d425f7c168c9fb145a286e0afb4
--- /dev/null
+++ b/qwen3-1.7b-teutonic-2e5/checkpoint-1000/chat_template.jinja
@@ -0,0 +1,89 @@
+{%- if tools %}
+ {{- '<|im_start|>system\n' }}
+ {%- if messages[0].role == 'system' %}
+ {{- messages[0].content + '\n\n' }}
+ {%- endif %}
+ {{- "# Tools\n\nYou may call one or more functions to assist with the user query.\n\nYou are provided with function signatures within XML tags:\n" }}
+ {%- for tool in tools %}
+ {{- "\n" }}
+ {{- tool | tojson }}
+ {%- endfor %}
+ {{- "\n\n\nFor each function call, return a json object with function name and arguments within XML tags:\n\n{\"name\": , \"arguments\": }\n<|im_end|>\n" }}
+{%- else %}
+ {%- if messages[0].role == 'system' %}
+ {{- '<|im_start|>system\n' + messages[0].content + '<|im_end|>\n' }}
+ {%- endif %}
+{%- endif %}
+{%- set ns = namespace(multi_step_tool=true, last_query_index=messages|length - 1) %}
+{%- for message in messages[::-1] %}
+ {%- set index = (messages|length - 1) - loop.index0 %}
+ {%- if ns.multi_step_tool and message.role == "user" and message.content is string and not(message.content.startswith('') and message.content.endswith('')) %}
+ {%- set ns.multi_step_tool = false %}
+ {%- set ns.last_query_index = index %}
+ {%- endif %}
+{%- endfor %}
+{%- for message in messages %}
+ {%- if message.content is string %}
+ {%- set content = message.content %}
+ {%- else %}
+ {%- set content = '' %}
+ {%- endif %}
+ {%- if (message.role == "user") or (message.role == "system" and not loop.first) %}
+ {{- '<|im_start|>' + message.role + '\n' + content + '<|im_end|>' + '\n' }}
+ {%- elif message.role == "assistant" %}
+ {%- set reasoning_content = '' %}
+ {%- if message.reasoning_content is string %}
+ {%- set reasoning_content = message.reasoning_content %}
+ {%- else %}
+ {%- if '' in content %}
+ {%- set reasoning_content = content.split('')[0].rstrip('\n').split('')[-1].lstrip('\n') %}
+ {%- set content = content.split('')[-1].lstrip('\n') %}
+ {%- endif %}
+ {%- endif %}
+ {%- if loop.index0 > ns.last_query_index %}
+ {%- if loop.last or (not loop.last and reasoning_content) %}
+ {{- '<|im_start|>' + message.role + '\n\n' + reasoning_content.strip('\n') + '\n\n\n' + content.lstrip('\n') }}
+ {%- else %}
+ {{- '<|im_start|>' + message.role + '\n' + content }}
+ {%- endif %}
+ {%- else %}
+ {{- '<|im_start|>' + message.role + '\n' + content }}
+ {%- endif %}
+ {%- if message.tool_calls %}
+ {%- for tool_call in message.tool_calls %}
+ {%- if (loop.first and content) or (not loop.first) %}
+ {{- '\n' }}
+ {%- endif %}
+ {%- if tool_call.function %}
+ {%- set tool_call = tool_call.function %}
+ {%- endif %}
+ {{- '\n{"name": "' }}
+ {{- tool_call.name }}
+ {{- '", "arguments": ' }}
+ {%- if tool_call.arguments is string %}
+ {{- tool_call.arguments }}
+ {%- else %}
+ {{- tool_call.arguments | tojson }}
+ {%- endif %}
+ {{- '}\n' }}
+ {%- endfor %}
+ {%- endif %}
+ {{- '<|im_end|>\n' }}
+ {%- elif message.role == "tool" %}
+ {%- if loop.first or (messages[loop.index0 - 1].role != "tool") %}
+ {{- '<|im_start|>user' }}
+ {%- endif %}
+ {{- '\n\n' }}
+ {{- content }}
+ {{- '\n' }}
+ {%- if loop.last or (messages[loop.index0 + 1].role != "tool") %}
+ {{- '<|im_end|>\n' }}
+ {%- endif %}
+ {%- endif %}
+{%- endfor %}
+{%- if add_generation_prompt %}
+ {{- '<|im_start|>assistant\n' }}
+ {%- if enable_thinking is defined and enable_thinking is false %}
+ {{- '\n\n\n\n' }}
+ {%- endif %}
+{%- endif %}
\ No newline at end of file
diff --git a/qwen3-1.7b-teutonic-2e5/checkpoint-1000/config.json b/qwen3-1.7b-teutonic-2e5/checkpoint-1000/config.json
new file mode 100644
index 0000000000000000000000000000000000000000..5d9cef07396baadff5390e4508eb33025967a029
--- /dev/null
+++ b/qwen3-1.7b-teutonic-2e5/checkpoint-1000/config.json
@@ -0,0 +1,63 @@
+{
+ "architectures": [
+ "Qwen3ForCausalLM"
+ ],
+ "attention_bias": false,
+ "attention_dropout": 0.0,
+ "bos_token_id": null,
+ "dtype": "bfloat16",
+ "eos_token_id": 151645,
+ "head_dim": 128,
+ "hidden_act": "silu",
+ "hidden_size": 2048,
+ "initializer_range": 0.02,
+ "intermediate_size": 6144,
+ "layer_types": [
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention"
+ ],
+ "max_position_embeddings": 40960,
+ "max_window_layers": 28,
+ "model_type": "qwen3",
+ "num_attention_heads": 16,
+ "num_hidden_layers": 28,
+ "num_key_value_heads": 8,
+ "pad_token_id": 151643,
+ "rms_norm_eps": 1e-06,
+ "rope_parameters": {
+ "rope_theta": 1000000,
+ "rope_type": "default"
+ },
+ "sliding_window": null,
+ "tie_word_embeddings": true,
+ "transformers_version": "5.17.0",
+ "use_cache": false,
+ "use_sliding_window": false,
+ "vocab_size": 151936
+}
diff --git a/qwen3-1.7b-teutonic-2e5/checkpoint-1000/generation_config.json b/qwen3-1.7b-teutonic-2e5/checkpoint-1000/generation_config.json
new file mode 100644
index 0000000000000000000000000000000000000000..e6941fe4b04f26113d6eef6255d005c4d7090bda
--- /dev/null
+++ b/qwen3-1.7b-teutonic-2e5/checkpoint-1000/generation_config.json
@@ -0,0 +1,12 @@
+{
+ "do_sample": true,
+ "eos_token_id": [
+ 151645,
+ 151643
+ ],
+ "pad_token_id": 151643,
+ "temperature": 0.6,
+ "top_k": 20,
+ "top_p": 0.95,
+ "transformers_version": "5.17.0"
+}
diff --git a/qwen3-1.7b-teutonic-2e5/checkpoint-1000/global_step1000/bf16_zero_pp_rank_0_mp_rank_00_optim_states.pt b/qwen3-1.7b-teutonic-2e5/checkpoint-1000/global_step1000/bf16_zero_pp_rank_0_mp_rank_00_optim_states.pt
new file mode 100644
index 0000000000000000000000000000000000000000..cd89106459e84c169fdaa896734f96b7ce33f3e3
--- /dev/null
+++ b/qwen3-1.7b-teutonic-2e5/checkpoint-1000/global_step1000/bf16_zero_pp_rank_0_mp_rank_00_optim_states.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:96cfd9e21801ef6218fbab21a5c0f8a89417a68b23f7f172cc103b7b9096465c
+size 10323466465
diff --git a/qwen3-1.7b-teutonic-2e5/checkpoint-1000/global_step1000/bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt b/qwen3-1.7b-teutonic-2e5/checkpoint-1000/global_step1000/bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt
new file mode 100644
index 0000000000000000000000000000000000000000..0032cf1c9344ee4129931242b4c746f02a823fd7
--- /dev/null
+++ b/qwen3-1.7b-teutonic-2e5/checkpoint-1000/global_step1000/bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:8631f97305801b329e34c86ed9f95e0df053ebc887476664878777bae4a6655d
+size 10323469601
diff --git a/qwen3-1.7b-teutonic-2e5/checkpoint-1000/global_step1000/mp_rank_00_model_states.pt b/qwen3-1.7b-teutonic-2e5/checkpoint-1000/global_step1000/mp_rank_00_model_states.pt
new file mode 100644
index 0000000000000000000000000000000000000000..8c5fc4812ba7b709230b32fe3a3bb690d9ccda34
--- /dev/null
+++ b/qwen3-1.7b-teutonic-2e5/checkpoint-1000/global_step1000/mp_rank_00_model_states.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6fdea8231d72131f1af5a5eeb83f486070f5229920fd247e7c86efb5e7c2da49
+size 3441239653
diff --git a/qwen3-1.7b-teutonic-2e5/checkpoint-1000/latest b/qwen3-1.7b-teutonic-2e5/checkpoint-1000/latest
new file mode 100644
index 0000000000000000000000000000000000000000..e2d3435fb1acf8913e6bd6c51b01adfc69b11ac6
--- /dev/null
+++ b/qwen3-1.7b-teutonic-2e5/checkpoint-1000/latest
@@ -0,0 +1 @@
+global_step1000
\ No newline at end of file
diff --git a/qwen3-1.7b-teutonic-2e5/checkpoint-1000/model.safetensors b/qwen3-1.7b-teutonic-2e5/checkpoint-1000/model.safetensors
new file mode 100644
index 0000000000000000000000000000000000000000..00a4cd5d7131484cc092e3e4e7cc0dad8268209b
--- /dev/null
+++ b/qwen3-1.7b-teutonic-2e5/checkpoint-1000/model.safetensors
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:7a4034a382d3c7ce1b02a0e0d59a33235e272d8268672220ec4e5778d9d82f57
+size 4063515640
diff --git a/qwen3-1.7b-teutonic-2e5/checkpoint-1000/rng_state_0.pth b/qwen3-1.7b-teutonic-2e5/checkpoint-1000/rng_state_0.pth
new file mode 100644
index 0000000000000000000000000000000000000000..6b0b60f9361dbf4c1a1ddbad81bb2c3bcac48e02
--- /dev/null
+++ b/qwen3-1.7b-teutonic-2e5/checkpoint-1000/rng_state_0.pth
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e1557eea53882574b3f18c13a4ba48b553f2cf8514c888f71896ffe922a42d70
+size 14917
diff --git a/qwen3-1.7b-teutonic-2e5/checkpoint-1000/rng_state_1.pth b/qwen3-1.7b-teutonic-2e5/checkpoint-1000/rng_state_1.pth
new file mode 100644
index 0000000000000000000000000000000000000000..0d5432e6306f80173a86daf24553be9dcb3cbec6
--- /dev/null
+++ b/qwen3-1.7b-teutonic-2e5/checkpoint-1000/rng_state_1.pth
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ed239779e6c9f5466cc9b1bc2adf2c4f487688a6bf675c36513a13bc18b548da
+size 14917
diff --git a/qwen3-1.7b-teutonic-2e5/checkpoint-1000/scheduler.pt b/qwen3-1.7b-teutonic-2e5/checkpoint-1000/scheduler.pt
new file mode 100644
index 0000000000000000000000000000000000000000..3487f82820b735c83fedd88fd7c280069599cadd
--- /dev/null
+++ b/qwen3-1.7b-teutonic-2e5/checkpoint-1000/scheduler.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:08d162c3c2b666c2182c3e6d9eac8ea5b3fa1bb5f4607fd0a76b169eb562c178
+size 1465
diff --git a/qwen3-1.7b-teutonic-2e5/checkpoint-1000/tokenizer.json b/qwen3-1.7b-teutonic-2e5/checkpoint-1000/tokenizer.json
new file mode 100644
index 0000000000000000000000000000000000000000..c7afbed2efcdf019f88ab0572ec29d3bf595dfe2
--- /dev/null
+++ b/qwen3-1.7b-teutonic-2e5/checkpoint-1000/tokenizer.json
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:be75606093db2094d7cd20f3c2f385c212750648bd6ea4fb2bf507a6a4c55506
+size 11422650
diff --git a/qwen3-1.7b-teutonic-2e5/checkpoint-1000/tokenizer_config.json b/qwen3-1.7b-teutonic-2e5/checkpoint-1000/tokenizer_config.json
new file mode 100644
index 0000000000000000000000000000000000000000..770e41d6c92519d525eede4cbcf3ba27f6425311
--- /dev/null
+++ b/qwen3-1.7b-teutonic-2e5/checkpoint-1000/tokenizer_config.json
@@ -0,0 +1,30 @@
+{
+ "add_prefix_space": false,
+ "backend": "tokenizers",
+ "bos_token": null,
+ "clean_up_tokenization_spaces": false,
+ "eos_token": "<|im_end|>",
+ "errors": "replace",
+ "extra_special_tokens": [
+ "<|im_start|>",
+ "<|im_end|>",
+ "<|object_ref_start|>",
+ "<|object_ref_end|>",
+ "<|box_start|>",
+ "<|box_end|>",
+ "<|quad_start|>",
+ "<|quad_end|>",
+ "<|vision_start|>",
+ "<|vision_end|>",
+ "<|vision_pad|>",
+ "<|image_pad|>",
+ "<|video_pad|>"
+ ],
+ "is_local": false,
+ "local_files_only": false,
+ "model_max_length": 131072,
+ "pad_token": "<|endoftext|>",
+ "split_special_tokens": false,
+ "tokenizer_class": "Qwen2Tokenizer",
+ "unk_token": null
+}
diff --git a/qwen3-1.7b-teutonic-2e5/checkpoint-1000/trainer_state.json b/qwen3-1.7b-teutonic-2e5/checkpoint-1000/trainer_state.json
new file mode 100644
index 0000000000000000000000000000000000000000..400c59c8538c94719ec9931952a304554f2922d0
--- /dev/null
+++ b/qwen3-1.7b-teutonic-2e5/checkpoint-1000/trainer_state.json
@@ -0,0 +1,10144 @@
+{
+ "best_global_step": 400,
+ "best_metric": 1.5199862718582153,
+ "best_model_checkpoint": null,
+ "epoch": 2.0202020202020203,
+ "eval_steps": 100,
+ "global_step": 1000,
+ "is_hyper_param_search": false,
+ "is_local_process_zero": true,
+ "is_world_process_zero": true,
+ "log_history": [
+ {
+ "entropy": 0.8295683860778809,
+ "epoch": 0.00202020202020202,
+ "grad_norm": 10.885663032531738,
+ "learning_rate": 2e-05,
+ "loss": 1.5185801982879639,
+ "mean_token_accuracy": 0.672874927520752,
+ "num_tokens": 16384.0,
+ "step": 1
+ },
+ {
+ "entropy": 1.3341505527496338,
+ "epoch": 0.00404040404040404,
+ "grad_norm": 9.276060104370117,
+ "learning_rate": 1.9986531986531986e-05,
+ "loss": 2.0802578926086426,
+ "mean_token_accuracy": 0.5727283954620361,
+ "num_tokens": 32768.0,
+ "step": 2
+ },
+ {
+ "entropy": 1.159377932548523,
+ "epoch": 0.006060606060606061,
+ "grad_norm": 3.999565839767456,
+ "learning_rate": 1.9973063973063975e-05,
+ "loss": 1.4128143787384033,
+ "mean_token_accuracy": 0.6849657893180847,
+ "num_tokens": 49152.0,
+ "step": 3
+ },
+ {
+ "entropy": 1.504081130027771,
+ "epoch": 0.00808080808080808,
+ "grad_norm": 3.018310546875,
+ "learning_rate": 1.995959595959596e-05,
+ "loss": 1.5479036569595337,
+ "mean_token_accuracy": 0.647777259349823,
+ "num_tokens": 65536.0,
+ "step": 4
+ },
+ {
+ "entropy": 2.1211423873901367,
+ "epoch": 0.010101010101010102,
+ "grad_norm": 4.352237701416016,
+ "learning_rate": 1.9946127946127948e-05,
+ "loss": 1.7965757846832275,
+ "mean_token_accuracy": 0.5992916226387024,
+ "num_tokens": 81920.0,
+ "step": 5
+ },
+ {
+ "entropy": 1.6345494985580444,
+ "epoch": 0.012121212121212121,
+ "grad_norm": 3.2360711097717285,
+ "learning_rate": 1.9932659932659936e-05,
+ "loss": 1.4024577140808105,
+ "mean_token_accuracy": 0.6747679710388184,
+ "num_tokens": 98304.0,
+ "step": 6
+ },
+ {
+ "entropy": 2.105585813522339,
+ "epoch": 0.014141414141414142,
+ "grad_norm": 2.84971284866333,
+ "learning_rate": 1.991919191919192e-05,
+ "loss": 1.9915739297866821,
+ "mean_token_accuracy": 0.5882999300956726,
+ "num_tokens": 114688.0,
+ "step": 7
+ },
+ {
+ "entropy": 1.7230671644210815,
+ "epoch": 0.01616161616161616,
+ "grad_norm": 2.686546802520752,
+ "learning_rate": 1.990572390572391e-05,
+ "loss": 1.630468726158142,
+ "mean_token_accuracy": 0.6269540786743164,
+ "num_tokens": 131072.0,
+ "step": 8
+ },
+ {
+ "entropy": 2.048288106918335,
+ "epoch": 0.01818181818181818,
+ "grad_norm": 2.6931567192077637,
+ "learning_rate": 1.9892255892255894e-05,
+ "loss": 2.212648391723633,
+ "mean_token_accuracy": 0.5294333100318909,
+ "num_tokens": 147456.0,
+ "step": 9
+ },
+ {
+ "entropy": 1.3021576404571533,
+ "epoch": 0.020202020202020204,
+ "grad_norm": 2.5286052227020264,
+ "learning_rate": 1.9878787878787878e-05,
+ "loss": 1.4357099533081055,
+ "mean_token_accuracy": 0.6633487939834595,
+ "num_tokens": 163840.0,
+ "step": 10
+ },
+ {
+ "entropy": 1.6998594999313354,
+ "epoch": 0.022222222222222223,
+ "grad_norm": 2.673692226409912,
+ "learning_rate": 1.9865319865319866e-05,
+ "loss": 1.8616552352905273,
+ "mean_token_accuracy": 0.6129091382026672,
+ "num_tokens": 180224.0,
+ "step": 11
+ },
+ {
+ "entropy": 1.5349891185760498,
+ "epoch": 0.024242424242424242,
+ "grad_norm": 2.882964611053467,
+ "learning_rate": 1.9851851851851855e-05,
+ "loss": 1.7540476322174072,
+ "mean_token_accuracy": 0.6111993193626404,
+ "num_tokens": 196608.0,
+ "step": 12
+ },
+ {
+ "entropy": 1.4202098846435547,
+ "epoch": 0.026262626262626262,
+ "grad_norm": 2.3892722129821777,
+ "learning_rate": 1.983838383838384e-05,
+ "loss": 1.5638885498046875,
+ "mean_token_accuracy": 0.642892062664032,
+ "num_tokens": 212992.0,
+ "step": 13
+ },
+ {
+ "entropy": 1.462074875831604,
+ "epoch": 0.028282828282828285,
+ "grad_norm": 2.357353925704956,
+ "learning_rate": 1.9824915824915828e-05,
+ "loss": 1.5339899063110352,
+ "mean_token_accuracy": 0.643991231918335,
+ "num_tokens": 229376.0,
+ "step": 14
+ },
+ {
+ "entropy": 1.6386674642562866,
+ "epoch": 0.030303030303030304,
+ "grad_norm": 2.1600515842437744,
+ "learning_rate": 1.9811447811447812e-05,
+ "loss": 1.6242434978485107,
+ "mean_token_accuracy": 0.6472887396812439,
+ "num_tokens": 245760.0,
+ "step": 15
+ },
+ {
+ "entropy": 1.882306456565857,
+ "epoch": 0.03232323232323232,
+ "grad_norm": 2.478746175765991,
+ "learning_rate": 1.97979797979798e-05,
+ "loss": 1.83274245262146,
+ "mean_token_accuracy": 0.5993527173995972,
+ "num_tokens": 262144.0,
+ "step": 16
+ },
+ {
+ "entropy": 1.5836528539657593,
+ "epoch": 0.03434343434343434,
+ "grad_norm": 2.5016613006591797,
+ "learning_rate": 1.9784511784511785e-05,
+ "loss": 1.5364878177642822,
+ "mean_token_accuracy": 0.6441743969917297,
+ "num_tokens": 278528.0,
+ "step": 17
+ },
+ {
+ "entropy": 1.5695534944534302,
+ "epoch": 0.03636363636363636,
+ "grad_norm": 2.2971248626708984,
+ "learning_rate": 1.9771043771043774e-05,
+ "loss": 1.5179738998413086,
+ "mean_token_accuracy": 0.6437469720840454,
+ "num_tokens": 294912.0,
+ "step": 18
+ },
+ {
+ "entropy": 1.3835400342941284,
+ "epoch": 0.03838383838383838,
+ "grad_norm": 2.1497576236724854,
+ "learning_rate": 1.975757575757576e-05,
+ "loss": 1.3505406379699707,
+ "mean_token_accuracy": 0.6729970574378967,
+ "num_tokens": 311296.0,
+ "step": 19
+ },
+ {
+ "entropy": 1.4660464525222778,
+ "epoch": 0.04040404040404041,
+ "grad_norm": 2.048318862915039,
+ "learning_rate": 1.9744107744107747e-05,
+ "loss": 1.3898100852966309,
+ "mean_token_accuracy": 0.6654250025749207,
+ "num_tokens": 327680.0,
+ "step": 20
+ },
+ {
+ "entropy": 1.959633469581604,
+ "epoch": 0.04242424242424243,
+ "grad_norm": 2.2084245681762695,
+ "learning_rate": 1.973063973063973e-05,
+ "loss": 1.9230711460113525,
+ "mean_token_accuracy": 0.5726062655448914,
+ "num_tokens": 344064.0,
+ "step": 21
+ },
+ {
+ "entropy": 1.4872093200683594,
+ "epoch": 0.044444444444444446,
+ "grad_norm": 2.116982936859131,
+ "learning_rate": 1.971717171717172e-05,
+ "loss": 1.514385461807251,
+ "mean_token_accuracy": 0.6615168452262878,
+ "num_tokens": 360448.0,
+ "step": 22
+ },
+ {
+ "entropy": 1.845888614654541,
+ "epoch": 0.046464646464646465,
+ "grad_norm": 2.4377331733703613,
+ "learning_rate": 1.9703703703703704e-05,
+ "loss": 1.8750195503234863,
+ "mean_token_accuracy": 0.5962994694709778,
+ "num_tokens": 376832.0,
+ "step": 23
+ },
+ {
+ "entropy": 1.5549575090408325,
+ "epoch": 0.048484848484848485,
+ "grad_norm": 2.44534969329834,
+ "learning_rate": 1.9690235690235692e-05,
+ "loss": 1.627150058746338,
+ "mean_token_accuracy": 0.6215803623199463,
+ "num_tokens": 393216.0,
+ "step": 24
+ },
+ {
+ "entropy": 1.4516701698303223,
+ "epoch": 0.050505050505050504,
+ "grad_norm": 2.2948875427246094,
+ "learning_rate": 1.9676767676767677e-05,
+ "loss": 1.4912647008895874,
+ "mean_token_accuracy": 0.6431363224983215,
+ "num_tokens": 409600.0,
+ "step": 25
+ },
+ {
+ "entropy": 1.3493973016738892,
+ "epoch": 0.052525252525252523,
+ "grad_norm": 2.1061222553253174,
+ "learning_rate": 1.9663299663299665e-05,
+ "loss": 1.44138765335083,
+ "mean_token_accuracy": 0.6621885895729065,
+ "num_tokens": 425984.0,
+ "step": 26
+ },
+ {
+ "entropy": 1.7119438648223877,
+ "epoch": 0.05454545454545454,
+ "grad_norm": 2.5149970054626465,
+ "learning_rate": 1.9649831649831654e-05,
+ "loss": 1.776426076889038,
+ "mean_token_accuracy": 0.5944675207138062,
+ "num_tokens": 442368.0,
+ "step": 27
+ },
+ {
+ "entropy": 1.361685872077942,
+ "epoch": 0.05656565656565657,
+ "grad_norm": 2.343843460083008,
+ "learning_rate": 1.963636363636364e-05,
+ "loss": 1.404232144355774,
+ "mean_token_accuracy": 0.6639594435691833,
+ "num_tokens": 458752.0,
+ "step": 28
+ },
+ {
+ "entropy": 1.2213996648788452,
+ "epoch": 0.05858585858585859,
+ "grad_norm": 1.9002994298934937,
+ "learning_rate": 1.9622895622895623e-05,
+ "loss": 1.249969244003296,
+ "mean_token_accuracy": 0.6933927536010742,
+ "num_tokens": 475136.0,
+ "step": 29
+ },
+ {
+ "entropy": 1.7157055139541626,
+ "epoch": 0.06060606060606061,
+ "grad_norm": 2.21567702293396,
+ "learning_rate": 1.960942760942761e-05,
+ "loss": 1.7103240489959717,
+ "mean_token_accuracy": 0.6218856573104858,
+ "num_tokens": 491520.0,
+ "step": 30
+ },
+ {
+ "entropy": 1.8233915567398071,
+ "epoch": 0.06262626262626263,
+ "grad_norm": 2.0189297199249268,
+ "learning_rate": 1.9595959595959596e-05,
+ "loss": 1.820570707321167,
+ "mean_token_accuracy": 0.6126648783683777,
+ "num_tokens": 507904.0,
+ "step": 31
+ },
+ {
+ "entropy": 1.82878839969635,
+ "epoch": 0.06464646464646465,
+ "grad_norm": 2.556058883666992,
+ "learning_rate": 1.9582491582491584e-05,
+ "loss": 1.773336410522461,
+ "mean_token_accuracy": 0.5889716744422913,
+ "num_tokens": 524288.0,
+ "step": 32
+ },
+ {
+ "entropy": 1.4608416557312012,
+ "epoch": 0.06666666666666667,
+ "grad_norm": 2.0254712104797363,
+ "learning_rate": 1.956902356902357e-05,
+ "loss": 1.4261068105697632,
+ "mean_token_accuracy": 0.6561431288719177,
+ "num_tokens": 540672.0,
+ "step": 33
+ },
+ {
+ "entropy": 1.4706993103027344,
+ "epoch": 0.06868686868686869,
+ "grad_norm": 2.1344153881073,
+ "learning_rate": 1.9555555555555557e-05,
+ "loss": 1.4610451459884644,
+ "mean_token_accuracy": 0.6472276449203491,
+ "num_tokens": 557056.0,
+ "step": 34
+ },
+ {
+ "entropy": 1.9629759788513184,
+ "epoch": 0.0707070707070707,
+ "grad_norm": 2.209073066711426,
+ "learning_rate": 1.9542087542087545e-05,
+ "loss": 1.9688678979873657,
+ "mean_token_accuracy": 0.5702857971191406,
+ "num_tokens": 573440.0,
+ "step": 35
+ },
+ {
+ "entropy": 1.5821412801742554,
+ "epoch": 0.07272727272727272,
+ "grad_norm": 1.9711852073669434,
+ "learning_rate": 1.952861952861953e-05,
+ "loss": 1.5614874362945557,
+ "mean_token_accuracy": 0.639167070388794,
+ "num_tokens": 589824.0,
+ "step": 36
+ },
+ {
+ "entropy": 1.4696686267852783,
+ "epoch": 0.07474747474747474,
+ "grad_norm": 2.07849383354187,
+ "learning_rate": 1.9515151515151515e-05,
+ "loss": 1.497229814529419,
+ "mean_token_accuracy": 0.6555324792861938,
+ "num_tokens": 606208.0,
+ "step": 37
+ },
+ {
+ "entropy": 1.481307864189148,
+ "epoch": 0.07676767676767676,
+ "grad_norm": 2.028839588165283,
+ "learning_rate": 1.9501683501683503e-05,
+ "loss": 1.495100975036621,
+ "mean_token_accuracy": 0.6472276449203491,
+ "num_tokens": 622592.0,
+ "step": 38
+ },
+ {
+ "entropy": 1.515073537826538,
+ "epoch": 0.07878787878787878,
+ "grad_norm": 2.2417402267456055,
+ "learning_rate": 1.9488215488215488e-05,
+ "loss": 1.6139262914657593,
+ "mean_token_accuracy": 0.6312286257743835,
+ "num_tokens": 638976.0,
+ "step": 39
+ },
+ {
+ "entropy": 1.9048941135406494,
+ "epoch": 0.08080808080808081,
+ "grad_norm": 2.232630968093872,
+ "learning_rate": 1.9474747474747476e-05,
+ "loss": 1.9573429822921753,
+ "mean_token_accuracy": 0.5796287059783936,
+ "num_tokens": 655360.0,
+ "step": 40
+ },
+ {
+ "entropy": 1.7393985986709595,
+ "epoch": 0.08282828282828283,
+ "grad_norm": 2.1077799797058105,
+ "learning_rate": 1.9461279461279464e-05,
+ "loss": 1.825897216796875,
+ "mean_token_accuracy": 0.5946506857872009,
+ "num_tokens": 671744.0,
+ "step": 41
+ },
+ {
+ "entropy": 1.6294668912887573,
+ "epoch": 0.08484848484848485,
+ "grad_norm": 2.299520492553711,
+ "learning_rate": 1.944781144781145e-05,
+ "loss": 1.6667912006378174,
+ "mean_token_accuracy": 0.6225574016571045,
+ "num_tokens": 688128.0,
+ "step": 42
+ },
+ {
+ "entropy": 1.4176290035247803,
+ "epoch": 0.08686868686868687,
+ "grad_norm": 1.9385138750076294,
+ "learning_rate": 1.9434343434343437e-05,
+ "loss": 1.422924280166626,
+ "mean_token_accuracy": 0.6719589829444885,
+ "num_tokens": 704512.0,
+ "step": 43
+ },
+ {
+ "entropy": 1.6570991277694702,
+ "epoch": 0.08888888888888889,
+ "grad_norm": 2.1650922298431396,
+ "learning_rate": 1.9420875420875422e-05,
+ "loss": 1.6351218223571777,
+ "mean_token_accuracy": 0.6342818737030029,
+ "num_tokens": 720896.0,
+ "step": 44
+ },
+ {
+ "entropy": 1.520472764968872,
+ "epoch": 0.09090909090909091,
+ "grad_norm": 2.116955280303955,
+ "learning_rate": 1.9407407407407407e-05,
+ "loss": 1.4788711071014404,
+ "mean_token_accuracy": 0.6557767391204834,
+ "num_tokens": 737280.0,
+ "step": 45
+ },
+ {
+ "entropy": 1.3090498447418213,
+ "epoch": 0.09292929292929293,
+ "grad_norm": 2.0493314266204834,
+ "learning_rate": 1.9393939393939395e-05,
+ "loss": 1.3372726440429688,
+ "mean_token_accuracy": 0.6762945652008057,
+ "num_tokens": 753664.0,
+ "step": 46
+ },
+ {
+ "entropy": 1.6449471712112427,
+ "epoch": 0.09494949494949495,
+ "grad_norm": 2.2450437545776367,
+ "learning_rate": 1.9380471380471383e-05,
+ "loss": 1.6683790683746338,
+ "mean_token_accuracy": 0.6266487836837769,
+ "num_tokens": 770048.0,
+ "step": 47
+ },
+ {
+ "entropy": 1.2516838312149048,
+ "epoch": 0.09696969696969697,
+ "grad_norm": 2.168203592300415,
+ "learning_rate": 1.9367003367003368e-05,
+ "loss": 1.2885866165161133,
+ "mean_token_accuracy": 0.689667820930481,
+ "num_tokens": 786432.0,
+ "step": 48
+ },
+ {
+ "entropy": 1.7858967781066895,
+ "epoch": 0.09898989898989899,
+ "grad_norm": 2.0490918159484863,
+ "learning_rate": 1.9353535353535356e-05,
+ "loss": 1.8125935792922974,
+ "mean_token_accuracy": 0.5957498550415039,
+ "num_tokens": 802816.0,
+ "step": 49
+ },
+ {
+ "entropy": 1.6833406686782837,
+ "epoch": 0.10101010101010101,
+ "grad_norm": 1.9622541666030884,
+ "learning_rate": 1.934006734006734e-05,
+ "loss": 1.715425729751587,
+ "mean_token_accuracy": 0.606008768081665,
+ "num_tokens": 819200.0,
+ "step": 50
+ },
+ {
+ "entropy": 1.4103842973709106,
+ "epoch": 0.10303030303030303,
+ "grad_norm": 2.015503406524658,
+ "learning_rate": 1.932659932659933e-05,
+ "loss": 1.4374600648880005,
+ "mean_token_accuracy": 0.6615168452262878,
+ "num_tokens": 835584.0,
+ "step": 51
+ },
+ {
+ "entropy": 1.6805938482284546,
+ "epoch": 0.10505050505050505,
+ "grad_norm": 2.139333963394165,
+ "learning_rate": 1.9313131313131314e-05,
+ "loss": 1.684693455696106,
+ "mean_token_accuracy": 0.6270151734352112,
+ "num_tokens": 851968.0,
+ "step": 52
+ },
+ {
+ "entropy": 1.770232915878296,
+ "epoch": 0.10707070707070707,
+ "grad_norm": 2.3331124782562256,
+ "learning_rate": 1.92996632996633e-05,
+ "loss": 1.781606674194336,
+ "mean_token_accuracy": 0.6006350517272949,
+ "num_tokens": 868352.0,
+ "step": 53
+ },
+ {
+ "entropy": 1.676267147064209,
+ "epoch": 0.10909090909090909,
+ "grad_norm": 2.1863765716552734,
+ "learning_rate": 1.9286195286195287e-05,
+ "loss": 1.6873016357421875,
+ "mean_token_accuracy": 0.6182828545570374,
+ "num_tokens": 884736.0,
+ "step": 54
+ },
+ {
+ "entropy": 1.3133333921432495,
+ "epoch": 0.1111111111111111,
+ "grad_norm": 2.455390691757202,
+ "learning_rate": 1.9272727272727275e-05,
+ "loss": 1.336955189704895,
+ "mean_token_accuracy": 0.682584285736084,
+ "num_tokens": 901120.0,
+ "step": 55
+ },
+ {
+ "entropy": 1.8064881563186646,
+ "epoch": 0.11313131313131314,
+ "grad_norm": 2.218869686126709,
+ "learning_rate": 1.925925925925926e-05,
+ "loss": 1.8163663148880005,
+ "mean_token_accuracy": 0.5947728157043457,
+ "num_tokens": 917504.0,
+ "step": 56
+ },
+ {
+ "entropy": 1.495352864265442,
+ "epoch": 0.11515151515151516,
+ "grad_norm": 2.040064573287964,
+ "learning_rate": 1.9245791245791248e-05,
+ "loss": 1.4638073444366455,
+ "mean_token_accuracy": 0.6512579321861267,
+ "num_tokens": 933888.0,
+ "step": 57
+ },
+ {
+ "entropy": 1.5757899284362793,
+ "epoch": 0.11717171717171718,
+ "grad_norm": 1.8449796438217163,
+ "learning_rate": 1.9232323232323233e-05,
+ "loss": 1.5526244640350342,
+ "mean_token_accuracy": 0.6346482634544373,
+ "num_tokens": 950272.0,
+ "step": 58
+ },
+ {
+ "entropy": 1.6834555864334106,
+ "epoch": 0.1191919191919192,
+ "grad_norm": 2.0705320835113525,
+ "learning_rate": 1.921885521885522e-05,
+ "loss": 1.753828763961792,
+ "mean_token_accuracy": 0.6237176060676575,
+ "num_tokens": 966656.0,
+ "step": 59
+ },
+ {
+ "entropy": 1.5231870412826538,
+ "epoch": 0.12121212121212122,
+ "grad_norm": 2.1398122310638428,
+ "learning_rate": 1.9205387205387206e-05,
+ "loss": 1.515265941619873,
+ "mean_token_accuracy": 0.6452125310897827,
+ "num_tokens": 983040.0,
+ "step": 60
+ },
+ {
+ "entropy": 1.6915754079818726,
+ "epoch": 0.12323232323232323,
+ "grad_norm": 2.179158926010132,
+ "learning_rate": 1.9191919191919194e-05,
+ "loss": 1.6949875354766846,
+ "mean_token_accuracy": 0.6109550595283508,
+ "num_tokens": 999424.0,
+ "step": 61
+ },
+ {
+ "entropy": 1.580087661743164,
+ "epoch": 0.12525252525252525,
+ "grad_norm": 2.8643219470977783,
+ "learning_rate": 1.9178451178451182e-05,
+ "loss": 1.6217050552368164,
+ "mean_token_accuracy": 0.6498534679412842,
+ "num_tokens": 1015808.0,
+ "step": 62
+ },
+ {
+ "entropy": 1.7331494092941284,
+ "epoch": 0.12727272727272726,
+ "grad_norm": 2.0613038539886475,
+ "learning_rate": 1.9164983164983167e-05,
+ "loss": 1.7527930736541748,
+ "mean_token_accuracy": 0.59513920545578,
+ "num_tokens": 1032192.0,
+ "step": 63
+ },
+ {
+ "entropy": 1.3311079740524292,
+ "epoch": 0.1292929292929293,
+ "grad_norm": 1.996009111404419,
+ "learning_rate": 1.9151515151515152e-05,
+ "loss": 1.3299833536148071,
+ "mean_token_accuracy": 0.6760503053665161,
+ "num_tokens": 1048576.0,
+ "step": 64
+ },
+ {
+ "entropy": 1.1875317096710205,
+ "epoch": 0.13131313131313133,
+ "grad_norm": 1.9458681344985962,
+ "learning_rate": 1.913804713804714e-05,
+ "loss": 1.1885290145874023,
+ "mean_token_accuracy": 0.6996213793754578,
+ "num_tokens": 1064960.0,
+ "step": 65
+ },
+ {
+ "entropy": 1.0825527906417847,
+ "epoch": 0.13333333333333333,
+ "grad_norm": 1.8164746761322021,
+ "learning_rate": 1.9124579124579125e-05,
+ "loss": 1.0918856859207153,
+ "mean_token_accuracy": 0.7477405667304993,
+ "num_tokens": 1081344.0,
+ "step": 66
+ },
+ {
+ "entropy": 1.5940357446670532,
+ "epoch": 0.13535353535353536,
+ "grad_norm": 1.9322525262832642,
+ "learning_rate": 1.9111111111111113e-05,
+ "loss": 1.6157622337341309,
+ "mean_token_accuracy": 0.6340376138687134,
+ "num_tokens": 1097728.0,
+ "step": 67
+ },
+ {
+ "entropy": 1.683976173400879,
+ "epoch": 0.13737373737373737,
+ "grad_norm": 2.1792430877685547,
+ "learning_rate": 1.9097643097643098e-05,
+ "loss": 1.7643585205078125,
+ "mean_token_accuracy": 0.5998412370681763,
+ "num_tokens": 1114112.0,
+ "step": 68
+ },
+ {
+ "entropy": 1.5753607749938965,
+ "epoch": 0.1393939393939394,
+ "grad_norm": 2.033538818359375,
+ "learning_rate": 1.9084175084175086e-05,
+ "loss": 1.628584861755371,
+ "mean_token_accuracy": 0.6401441097259521,
+ "num_tokens": 1130496.0,
+ "step": 69
+ },
+ {
+ "entropy": 1.4061329364776611,
+ "epoch": 0.1414141414141414,
+ "grad_norm": 1.9239107370376587,
+ "learning_rate": 1.9070707070707074e-05,
+ "loss": 1.4355204105377197,
+ "mean_token_accuracy": 0.6727527976036072,
+ "num_tokens": 1146880.0,
+ "step": 70
+ },
+ {
+ "entropy": 1.1722302436828613,
+ "epoch": 0.14343434343434344,
+ "grad_norm": 2.0911052227020264,
+ "learning_rate": 1.905723905723906e-05,
+ "loss": 1.2008472681045532,
+ "mean_token_accuracy": 0.7064606547355652,
+ "num_tokens": 1163264.0,
+ "step": 71
+ },
+ {
+ "entropy": 1.5291107892990112,
+ "epoch": 0.14545454545454545,
+ "grad_norm": 2.1054158210754395,
+ "learning_rate": 1.9043771043771044e-05,
+ "loss": 1.574873924255371,
+ "mean_token_accuracy": 0.6356253027915955,
+ "num_tokens": 1179648.0,
+ "step": 72
+ },
+ {
+ "entropy": 1.3679379224777222,
+ "epoch": 0.14747474747474748,
+ "grad_norm": 1.928666353225708,
+ "learning_rate": 1.9030303030303032e-05,
+ "loss": 1.3473236560821533,
+ "mean_token_accuracy": 0.679347813129425,
+ "num_tokens": 1196032.0,
+ "step": 73
+ },
+ {
+ "entropy": 1.7593064308166504,
+ "epoch": 0.1494949494949495,
+ "grad_norm": 2.3172972202301025,
+ "learning_rate": 1.9016835016835017e-05,
+ "loss": 1.7953016757965088,
+ "mean_token_accuracy": 0.6107107996940613,
+ "num_tokens": 1212416.0,
+ "step": 74
+ },
+ {
+ "entropy": 1.3841301202774048,
+ "epoch": 0.15151515151515152,
+ "grad_norm": 1.9938782453536987,
+ "learning_rate": 1.9003367003367005e-05,
+ "loss": 1.3923728466033936,
+ "mean_token_accuracy": 0.6686614751815796,
+ "num_tokens": 1228800.0,
+ "step": 75
+ },
+ {
+ "entropy": 1.6030631065368652,
+ "epoch": 0.15353535353535352,
+ "grad_norm": 2.0514845848083496,
+ "learning_rate": 1.8989898989898993e-05,
+ "loss": 1.5902166366577148,
+ "mean_token_accuracy": 0.6376404762268066,
+ "num_tokens": 1245184.0,
+ "step": 76
+ },
+ {
+ "entropy": 1.5902619361877441,
+ "epoch": 0.15555555555555556,
+ "grad_norm": 2.115469217300415,
+ "learning_rate": 1.8976430976430978e-05,
+ "loss": 1.5841395854949951,
+ "mean_token_accuracy": 0.6361138224601746,
+ "num_tokens": 1261568.0,
+ "step": 77
+ },
+ {
+ "entropy": 1.7821043729782104,
+ "epoch": 0.15757575757575756,
+ "grad_norm": 2.008962869644165,
+ "learning_rate": 1.8962962962962966e-05,
+ "loss": 1.7826645374298096,
+ "mean_token_accuracy": 0.6053370833396912,
+ "num_tokens": 1277952.0,
+ "step": 78
+ },
+ {
+ "entropy": 1.833274006843567,
+ "epoch": 0.1595959595959596,
+ "grad_norm": 2.0258517265319824,
+ "learning_rate": 1.894949494949495e-05,
+ "loss": 1.8566474914550781,
+ "mean_token_accuracy": 0.5800561904907227,
+ "num_tokens": 1294336.0,
+ "step": 79
+ },
+ {
+ "entropy": 1.785520315170288,
+ "epoch": 0.16161616161616163,
+ "grad_norm": 2.0918498039245605,
+ "learning_rate": 1.8936026936026936e-05,
+ "loss": 1.775907278060913,
+ "mean_token_accuracy": 0.6063141226768494,
+ "num_tokens": 1310720.0,
+ "step": 80
+ },
+ {
+ "entropy": 1.9341778755187988,
+ "epoch": 0.16363636363636364,
+ "grad_norm": 2.0399975776672363,
+ "learning_rate": 1.8922558922558924e-05,
+ "loss": 1.9579840898513794,
+ "mean_token_accuracy": 0.5619809627532959,
+ "num_tokens": 1327104.0,
+ "step": 81
+ },
+ {
+ "entropy": 1.4945461750030518,
+ "epoch": 0.16565656565656567,
+ "grad_norm": 1.9975999593734741,
+ "learning_rate": 1.8909090909090912e-05,
+ "loss": 1.5436582565307617,
+ "mean_token_accuracy": 0.6423424482345581,
+ "num_tokens": 1343488.0,
+ "step": 82
+ },
+ {
+ "entropy": 1.6639432907104492,
+ "epoch": 0.16767676767676767,
+ "grad_norm": 2.0375208854675293,
+ "learning_rate": 1.8895622895622897e-05,
+ "loss": 1.679243564605713,
+ "mean_token_accuracy": 0.6226184368133545,
+ "num_tokens": 1359872.0,
+ "step": 83
+ },
+ {
+ "entropy": 1.4505668878555298,
+ "epoch": 0.1696969696969697,
+ "grad_norm": 1.8894681930541992,
+ "learning_rate": 1.8882154882154885e-05,
+ "loss": 1.4778342247009277,
+ "mean_token_accuracy": 0.6637151837348938,
+ "num_tokens": 1376256.0,
+ "step": 84
+ },
+ {
+ "entropy": 1.3845125436782837,
+ "epoch": 0.1717171717171717,
+ "grad_norm": 2.1110754013061523,
+ "learning_rate": 1.886868686868687e-05,
+ "loss": 1.4153435230255127,
+ "mean_token_accuracy": 0.6737298369407654,
+ "num_tokens": 1392640.0,
+ "step": 85
+ },
+ {
+ "entropy": 1.8785539865493774,
+ "epoch": 0.17373737373737375,
+ "grad_norm": 2.0894815921783447,
+ "learning_rate": 1.8855218855218858e-05,
+ "loss": 1.8658334016799927,
+ "mean_token_accuracy": 0.5846360325813293,
+ "num_tokens": 1409024.0,
+ "step": 86
+ },
+ {
+ "entropy": 1.6006972789764404,
+ "epoch": 0.17575757575757575,
+ "grad_norm": 2.088477849960327,
+ "learning_rate": 1.8841750841750843e-05,
+ "loss": 1.6257164478302002,
+ "mean_token_accuracy": 0.623900830745697,
+ "num_tokens": 1425408.0,
+ "step": 87
+ },
+ {
+ "entropy": 1.1098109483718872,
+ "epoch": 0.17777777777777778,
+ "grad_norm": 1.934228539466858,
+ "learning_rate": 1.8828282828282827e-05,
+ "loss": 1.0948461294174194,
+ "mean_token_accuracy": 0.7248412370681763,
+ "num_tokens": 1441792.0,
+ "step": 88
+ },
+ {
+ "entropy": 1.4807215929031372,
+ "epoch": 0.1797979797979798,
+ "grad_norm": 2.06062388420105,
+ "learning_rate": 1.8814814814814816e-05,
+ "loss": 1.5115268230438232,
+ "mean_token_accuracy": 0.6463117003440857,
+ "num_tokens": 1458176.0,
+ "step": 89
+ },
+ {
+ "entropy": 1.07956862449646,
+ "epoch": 0.18181818181818182,
+ "grad_norm": 1.8913685083389282,
+ "learning_rate": 1.8801346801346804e-05,
+ "loss": 1.0902754068374634,
+ "mean_token_accuracy": 0.7308256030082703,
+ "num_tokens": 1474560.0,
+ "step": 90
+ },
+ {
+ "entropy": 1.4571733474731445,
+ "epoch": 0.18383838383838383,
+ "grad_norm": 2.2199525833129883,
+ "learning_rate": 1.8787878787878792e-05,
+ "loss": 1.4686330556869507,
+ "mean_token_accuracy": 0.6566927433013916,
+ "num_tokens": 1490944.0,
+ "step": 91
+ },
+ {
+ "entropy": 1.4484740495681763,
+ "epoch": 0.18585858585858586,
+ "grad_norm": 2.0670580863952637,
+ "learning_rate": 1.8774410774410777e-05,
+ "loss": 1.4791338443756104,
+ "mean_token_accuracy": 0.6493649482727051,
+ "num_tokens": 1507328.0,
+ "step": 92
+ },
+ {
+ "entropy": 1.498313546180725,
+ "epoch": 0.18787878787878787,
+ "grad_norm": 2.024329423904419,
+ "learning_rate": 1.876094276094276e-05,
+ "loss": 1.4964686632156372,
+ "mean_token_accuracy": 0.6457620859146118,
+ "num_tokens": 1523712.0,
+ "step": 93
+ },
+ {
+ "entropy": 1.4806159734725952,
+ "epoch": 0.1898989898989899,
+ "grad_norm": 2.1369152069091797,
+ "learning_rate": 1.874747474747475e-05,
+ "loss": 1.4587206840515137,
+ "mean_token_accuracy": 0.6521739363670349,
+ "num_tokens": 1540096.0,
+ "step": 94
+ },
+ {
+ "entropy": 1.3746109008789062,
+ "epoch": 0.1919191919191919,
+ "grad_norm": 2.216958522796631,
+ "learning_rate": 1.8734006734006735e-05,
+ "loss": 1.3600832223892212,
+ "mean_token_accuracy": 0.671775758266449,
+ "num_tokens": 1556480.0,
+ "step": 95
+ },
+ {
+ "entropy": 1.3492207527160645,
+ "epoch": 0.19393939393939394,
+ "grad_norm": 2.093956708908081,
+ "learning_rate": 1.8720538720538723e-05,
+ "loss": 1.3867243528366089,
+ "mean_token_accuracy": 0.6625549793243408,
+ "num_tokens": 1572864.0,
+ "step": 96
+ },
+ {
+ "entropy": 1.560738205909729,
+ "epoch": 0.19595959595959597,
+ "grad_norm": 2.0691707134246826,
+ "learning_rate": 1.8707070707070707e-05,
+ "loss": 1.5875897407531738,
+ "mean_token_accuracy": 0.6345872282981873,
+ "num_tokens": 1589248.0,
+ "step": 97
+ },
+ {
+ "entropy": 1.46450674533844,
+ "epoch": 0.19797979797979798,
+ "grad_norm": 2.064547538757324,
+ "learning_rate": 1.8693602693602696e-05,
+ "loss": 1.4920742511749268,
+ "mean_token_accuracy": 0.6411822438240051,
+ "num_tokens": 1605632.0,
+ "step": 98
+ },
+ {
+ "entropy": 1.5433454513549805,
+ "epoch": 0.2,
+ "grad_norm": 2.014409303665161,
+ "learning_rate": 1.868013468013468e-05,
+ "loss": 1.5737671852111816,
+ "mean_token_accuracy": 0.6349536180496216,
+ "num_tokens": 1622016.0,
+ "step": 99
+ },
+ {
+ "entropy": 1.358772873878479,
+ "epoch": 0.20202020202020202,
+ "grad_norm": 1.9765456914901733,
+ "learning_rate": 1.866666666666667e-05,
+ "loss": 1.3583091497421265,
+ "mean_token_accuracy": 0.680446982383728,
+ "num_tokens": 1638400.0,
+ "step": 100
+ },
+ {
+ "epoch": 0.20202020202020202,
+ "eval_entropy": 1.5664977690865916,
+ "eval_loss": 1.5804312229156494,
+ "eval_mean_token_accuracy": 0.63647972239602,
+ "eval_num_tokens": 1638400.0,
+ "eval_runtime": 44.1541,
+ "eval_samples_per_second": 22.421,
+ "eval_steps_per_second": 2.808,
+ "step": 100
+ },
+ {
+ "entropy": 1.5879693031311035,
+ "epoch": 0.20404040404040405,
+ "grad_norm": 1.8491015434265137,
+ "learning_rate": 1.8653198653198653e-05,
+ "loss": 1.5892077684402466,
+ "mean_token_accuracy": 0.6665852665901184,
+ "num_tokens": 1654784.0,
+ "step": 101
+ },
+ {
+ "entropy": 1.5789942741394043,
+ "epoch": 0.20606060606060606,
+ "grad_norm": 1.9096908569335938,
+ "learning_rate": 1.863973063973064e-05,
+ "loss": 1.594193458557129,
+ "mean_token_accuracy": 0.6293966770172119,
+ "num_tokens": 1671168.0,
+ "step": 102
+ },
+ {
+ "entropy": 1.5245429277420044,
+ "epoch": 0.2080808080808081,
+ "grad_norm": 1.986904501914978,
+ "learning_rate": 1.8626262626262626e-05,
+ "loss": 1.5463616847991943,
+ "mean_token_accuracy": 0.6354421377182007,
+ "num_tokens": 1687552.0,
+ "step": 103
+ },
+ {
+ "entropy": 2.015364408493042,
+ "epoch": 0.2101010101010101,
+ "grad_norm": 2.3297250270843506,
+ "learning_rate": 1.8612794612794615e-05,
+ "loss": 1.987194538116455,
+ "mean_token_accuracy": 0.5604543089866638,
+ "num_tokens": 1703936.0,
+ "step": 104
+ },
+ {
+ "entropy": 1.770432472229004,
+ "epoch": 0.21212121212121213,
+ "grad_norm": 2.3580551147460938,
+ "learning_rate": 1.8599326599326603e-05,
+ "loss": 1.7715529203414917,
+ "mean_token_accuracy": 0.5786516666412354,
+ "num_tokens": 1720320.0,
+ "step": 105
+ },
+ {
+ "entropy": 1.3456813097000122,
+ "epoch": 0.21414141414141413,
+ "grad_norm": 1.9782105684280396,
+ "learning_rate": 1.8585858585858588e-05,
+ "loss": 1.342590570449829,
+ "mean_token_accuracy": 0.6779433488845825,
+ "num_tokens": 1736704.0,
+ "step": 106
+ },
+ {
+ "entropy": 1.4614694118499756,
+ "epoch": 0.21616161616161617,
+ "grad_norm": 1.9486815929412842,
+ "learning_rate": 1.8572390572390572e-05,
+ "loss": 1.4493391513824463,
+ "mean_token_accuracy": 0.6618832349777222,
+ "num_tokens": 1753088.0,
+ "step": 107
+ },
+ {
+ "entropy": 2.007932186126709,
+ "epoch": 0.21818181818181817,
+ "grad_norm": 2.0571465492248535,
+ "learning_rate": 1.855892255892256e-05,
+ "loss": 2.033851385116577,
+ "mean_token_accuracy": 0.5746213793754578,
+ "num_tokens": 1769472.0,
+ "step": 108
+ },
+ {
+ "entropy": 1.506022572517395,
+ "epoch": 0.2202020202020202,
+ "grad_norm": 1.9980531930923462,
+ "learning_rate": 1.8545454545454545e-05,
+ "loss": 1.5305016040802002,
+ "mean_token_accuracy": 0.6416096687316895,
+ "num_tokens": 1785856.0,
+ "step": 109
+ },
+ {
+ "entropy": 1.479506492614746,
+ "epoch": 0.2222222222222222,
+ "grad_norm": 2.411241292953491,
+ "learning_rate": 1.8531986531986533e-05,
+ "loss": 1.5501817464828491,
+ "mean_token_accuracy": 0.6364802122116089,
+ "num_tokens": 1802240.0,
+ "step": 110
+ },
+ {
+ "entropy": 1.7731081247329712,
+ "epoch": 0.22424242424242424,
+ "grad_norm": 2.1088449954986572,
+ "learning_rate": 1.851851851851852e-05,
+ "loss": 1.7582051753997803,
+ "mean_token_accuracy": 0.5985588431358337,
+ "num_tokens": 1818624.0,
+ "step": 111
+ },
+ {
+ "entropy": 1.7963190078735352,
+ "epoch": 0.22626262626262628,
+ "grad_norm": 2.180325508117676,
+ "learning_rate": 1.8505050505050506e-05,
+ "loss": 1.8436882495880127,
+ "mean_token_accuracy": 0.5850635170936584,
+ "num_tokens": 1835008.0,
+ "step": 112
+ },
+ {
+ "entropy": 1.513211727142334,
+ "epoch": 0.22828282828282828,
+ "grad_norm": 2.1563429832458496,
+ "learning_rate": 1.8491582491582495e-05,
+ "loss": 1.547263741493225,
+ "mean_token_accuracy": 0.6399609446525574,
+ "num_tokens": 1851392.0,
+ "step": 113
+ },
+ {
+ "entropy": 1.8904774188995361,
+ "epoch": 0.23030303030303031,
+ "grad_norm": 2.2191243171691895,
+ "learning_rate": 1.847811447811448e-05,
+ "loss": 1.9237418174743652,
+ "mean_token_accuracy": 0.5852466821670532,
+ "num_tokens": 1867776.0,
+ "step": 114
+ },
+ {
+ "entropy": 1.706113338470459,
+ "epoch": 0.23232323232323232,
+ "grad_norm": 2.1411335468292236,
+ "learning_rate": 1.8464646464646464e-05,
+ "loss": 1.7145740985870361,
+ "mean_token_accuracy": 0.6138861775398254,
+ "num_tokens": 1884160.0,
+ "step": 115
+ },
+ {
+ "entropy": 1.726441502571106,
+ "epoch": 0.23434343434343435,
+ "grad_norm": 2.212477922439575,
+ "learning_rate": 1.8451178451178452e-05,
+ "loss": 1.7427434921264648,
+ "mean_token_accuracy": 0.6087567210197449,
+ "num_tokens": 1900544.0,
+ "step": 116
+ },
+ {
+ "entropy": 1.49800443649292,
+ "epoch": 0.23636363636363636,
+ "grad_norm": 2.003248929977417,
+ "learning_rate": 1.8437710437710437e-05,
+ "loss": 1.4677059650421143,
+ "mean_token_accuracy": 0.6617000699043274,
+ "num_tokens": 1916928.0,
+ "step": 117
+ },
+ {
+ "entropy": 1.5079678297042847,
+ "epoch": 0.2383838383838384,
+ "grad_norm": 1.9702556133270264,
+ "learning_rate": 1.8424242424242425e-05,
+ "loss": 1.5194823741912842,
+ "mean_token_accuracy": 0.6593796014785767,
+ "num_tokens": 1933312.0,
+ "step": 118
+ },
+ {
+ "entropy": 1.760438323020935,
+ "epoch": 0.2404040404040404,
+ "grad_norm": 1.9484418630599976,
+ "learning_rate": 1.8410774410774414e-05,
+ "loss": 1.756869912147522,
+ "mean_token_accuracy": 0.6193209290504456,
+ "num_tokens": 1949696.0,
+ "step": 119
+ },
+ {
+ "entropy": 1.3706170320510864,
+ "epoch": 0.24242424242424243,
+ "grad_norm": 2.078357458114624,
+ "learning_rate": 1.83973063973064e-05,
+ "loss": 1.3532414436340332,
+ "mean_token_accuracy": 0.6680508255958557,
+ "num_tokens": 1966080.0,
+ "step": 120
+ },
+ {
+ "entropy": 1.2994399070739746,
+ "epoch": 0.24444444444444444,
+ "grad_norm": 1.9178324937820435,
+ "learning_rate": 1.8383838383838387e-05,
+ "loss": 1.298889398574829,
+ "mean_token_accuracy": 0.6831949353218079,
+ "num_tokens": 1982464.0,
+ "step": 121
+ },
+ {
+ "entropy": 1.6806142330169678,
+ "epoch": 0.24646464646464647,
+ "grad_norm": 2.287158966064453,
+ "learning_rate": 1.837037037037037e-05,
+ "loss": 1.728592038154602,
+ "mean_token_accuracy": 0.6161455512046814,
+ "num_tokens": 1998848.0,
+ "step": 122
+ },
+ {
+ "entropy": 1.2894387245178223,
+ "epoch": 0.24848484848484848,
+ "grad_norm": 1.9998525381088257,
+ "learning_rate": 1.8356902356902356e-05,
+ "loss": 1.3326170444488525,
+ "mean_token_accuracy": 0.6693331599235535,
+ "num_tokens": 2015232.0,
+ "step": 123
+ },
+ {
+ "entropy": 1.5466209650039673,
+ "epoch": 0.2505050505050505,
+ "grad_norm": 1.8439521789550781,
+ "learning_rate": 1.8343434343434344e-05,
+ "loss": 1.5606887340545654,
+ "mean_token_accuracy": 0.6571201682090759,
+ "num_tokens": 2031616.0,
+ "step": 124
+ },
+ {
+ "entropy": 1.2692567110061646,
+ "epoch": 0.25252525252525254,
+ "grad_norm": 1.9568793773651123,
+ "learning_rate": 1.8329966329966332e-05,
+ "loss": 1.308868408203125,
+ "mean_token_accuracy": 0.6954689621925354,
+ "num_tokens": 2048000.0,
+ "step": 125
+ },
+ {
+ "entropy": 1.5780351161956787,
+ "epoch": 0.2545454545454545,
+ "grad_norm": 2.151250123977661,
+ "learning_rate": 1.831649831649832e-05,
+ "loss": 1.60304856300354,
+ "mean_token_accuracy": 0.6258549094200134,
+ "num_tokens": 2064384.0,
+ "step": 126
+ },
+ {
+ "entropy": 1.487567663192749,
+ "epoch": 0.25656565656565655,
+ "grad_norm": 1.9134294986724854,
+ "learning_rate": 1.8303030303030305e-05,
+ "loss": 1.514064073562622,
+ "mean_token_accuracy": 0.6348925232887268,
+ "num_tokens": 2080768.0,
+ "step": 127
+ },
+ {
+ "entropy": 1.6072919368743896,
+ "epoch": 0.2585858585858586,
+ "grad_norm": 2.119290351867676,
+ "learning_rate": 1.828956228956229e-05,
+ "loss": 1.6062684059143066,
+ "mean_token_accuracy": 0.6332437992095947,
+ "num_tokens": 2097152.0,
+ "step": 128
+ },
+ {
+ "entropy": 1.1168174743652344,
+ "epoch": 0.2606060606060606,
+ "grad_norm": 1.8520931005477905,
+ "learning_rate": 1.827609427609428e-05,
+ "loss": 1.1421526670455933,
+ "mean_token_accuracy": 0.7078651785850525,
+ "num_tokens": 2113536.0,
+ "step": 129
+ },
+ {
+ "entropy": 1.349792718887329,
+ "epoch": 0.26262626262626265,
+ "grad_norm": 1.8463630676269531,
+ "learning_rate": 1.8262626262626263e-05,
+ "loss": 1.3712577819824219,
+ "mean_token_accuracy": 0.6736077070236206,
+ "num_tokens": 2129920.0,
+ "step": 130
+ },
+ {
+ "entropy": 1.8531655073165894,
+ "epoch": 0.26464646464646463,
+ "grad_norm": 2.0503969192504883,
+ "learning_rate": 1.824915824915825e-05,
+ "loss": 1.8971941471099854,
+ "mean_token_accuracy": 0.5845749974250793,
+ "num_tokens": 2146304.0,
+ "step": 131
+ },
+ {
+ "entropy": 1.436646819114685,
+ "epoch": 0.26666666666666666,
+ "grad_norm": 1.8517117500305176,
+ "learning_rate": 1.8235690235690236e-05,
+ "loss": 1.4255880117416382,
+ "mean_token_accuracy": 0.6653639674186707,
+ "num_tokens": 2162688.0,
+ "step": 132
+ },
+ {
+ "entropy": 1.9219673871994019,
+ "epoch": 0.2686868686868687,
+ "grad_norm": 2.1424307823181152,
+ "learning_rate": 1.8222222222222224e-05,
+ "loss": 1.9088413715362549,
+ "mean_token_accuracy": 0.5868954658508301,
+ "num_tokens": 2179072.0,
+ "step": 133
+ },
+ {
+ "entropy": 1.4034194946289062,
+ "epoch": 0.27070707070707073,
+ "grad_norm": 1.9987515211105347,
+ "learning_rate": 1.8208754208754212e-05,
+ "loss": 1.3612333536148071,
+ "mean_token_accuracy": 0.6721421480178833,
+ "num_tokens": 2195456.0,
+ "step": 134
+ },
+ {
+ "entropy": 2.064425230026245,
+ "epoch": 0.2727272727272727,
+ "grad_norm": 2.0800976753234863,
+ "learning_rate": 1.8195286195286197e-05,
+ "loss": 2.0595853328704834,
+ "mean_token_accuracy": 0.5694919228553772,
+ "num_tokens": 2211840.0,
+ "step": 135
+ },
+ {
+ "entropy": 1.2883177995681763,
+ "epoch": 0.27474747474747474,
+ "grad_norm": 1.9042892456054688,
+ "learning_rate": 1.8181818181818182e-05,
+ "loss": 1.3220078945159912,
+ "mean_token_accuracy": 0.6805080771446228,
+ "num_tokens": 2228224.0,
+ "step": 136
+ },
+ {
+ "entropy": 1.101396918296814,
+ "epoch": 0.2767676767676768,
+ "grad_norm": 1.857073187828064,
+ "learning_rate": 1.816835016835017e-05,
+ "loss": 1.1156296730041504,
+ "mean_token_accuracy": 0.7214826345443726,
+ "num_tokens": 2244608.0,
+ "step": 137
+ },
+ {
+ "entropy": 1.376597285270691,
+ "epoch": 0.2787878787878788,
+ "grad_norm": 1.9103689193725586,
+ "learning_rate": 1.8154882154882155e-05,
+ "loss": 1.3915824890136719,
+ "mean_token_accuracy": 0.6824010610580444,
+ "num_tokens": 2260992.0,
+ "step": 138
+ },
+ {
+ "entropy": 1.4654942750930786,
+ "epoch": 0.2808080808080808,
+ "grad_norm": 1.9513856172561646,
+ "learning_rate": 1.8141414141414143e-05,
+ "loss": 1.5000076293945312,
+ "mean_token_accuracy": 0.6457620859146118,
+ "num_tokens": 2277376.0,
+ "step": 139
+ },
+ {
+ "entropy": 1.1999047994613647,
+ "epoch": 0.2828282828282828,
+ "grad_norm": 1.824987530708313,
+ "learning_rate": 1.812794612794613e-05,
+ "loss": 1.2217535972595215,
+ "mean_token_accuracy": 0.7034685015678406,
+ "num_tokens": 2293760.0,
+ "step": 140
+ },
+ {
+ "entropy": 1.2822942733764648,
+ "epoch": 0.28484848484848485,
+ "grad_norm": 2.0979108810424805,
+ "learning_rate": 1.8114478114478116e-05,
+ "loss": 1.309725046157837,
+ "mean_token_accuracy": 0.6855764389038086,
+ "num_tokens": 2310144.0,
+ "step": 141
+ },
+ {
+ "entropy": 1.5589981079101562,
+ "epoch": 0.2868686868686869,
+ "grad_norm": 2.030648708343506,
+ "learning_rate": 1.8101010101010104e-05,
+ "loss": 1.5918951034545898,
+ "mean_token_accuracy": 0.6286638975143433,
+ "num_tokens": 2326528.0,
+ "step": 142
+ },
+ {
+ "entropy": 1.5600453615188599,
+ "epoch": 0.28888888888888886,
+ "grad_norm": 2.023409605026245,
+ "learning_rate": 1.808754208754209e-05,
+ "loss": 1.5725011825561523,
+ "mean_token_accuracy": 0.6384342908859253,
+ "num_tokens": 2342912.0,
+ "step": 143
+ },
+ {
+ "entropy": 1.5066077709197998,
+ "epoch": 0.2909090909090909,
+ "grad_norm": 1.9963997602462769,
+ "learning_rate": 1.8074074074074074e-05,
+ "loss": 1.5066845417022705,
+ "mean_token_accuracy": 0.6474108695983887,
+ "num_tokens": 2359296.0,
+ "step": 144
+ },
+ {
+ "entropy": 1.8313122987747192,
+ "epoch": 0.29292929292929293,
+ "grad_norm": 2.0791213512420654,
+ "learning_rate": 1.8060606060606062e-05,
+ "loss": 1.8148126602172852,
+ "mean_token_accuracy": 0.5820102691650391,
+ "num_tokens": 2375680.0,
+ "step": 145
+ },
+ {
+ "entropy": 1.8828215599060059,
+ "epoch": 0.29494949494949496,
+ "grad_norm": 2.1346335411071777,
+ "learning_rate": 1.804713804713805e-05,
+ "loss": 1.905554175376892,
+ "mean_token_accuracy": 0.6041768193244934,
+ "num_tokens": 2392064.0,
+ "step": 146
+ },
+ {
+ "entropy": 1.1984127759933472,
+ "epoch": 0.296969696969697,
+ "grad_norm": 1.6585179567337036,
+ "learning_rate": 1.8033670033670035e-05,
+ "loss": 1.2255632877349854,
+ "mean_token_accuracy": 0.712506115436554,
+ "num_tokens": 2408448.0,
+ "step": 147
+ },
+ {
+ "entropy": 1.4248595237731934,
+ "epoch": 0.298989898989899,
+ "grad_norm": 2.468965530395508,
+ "learning_rate": 1.8020202020202023e-05,
+ "loss": 1.4362621307373047,
+ "mean_token_accuracy": 0.6464948654174805,
+ "num_tokens": 2424832.0,
+ "step": 148
+ },
+ {
+ "entropy": 1.7295368909835815,
+ "epoch": 0.301010101010101,
+ "grad_norm": 1.819359302520752,
+ "learning_rate": 1.8006734006734008e-05,
+ "loss": 1.7290053367614746,
+ "mean_token_accuracy": 0.6312897205352783,
+ "num_tokens": 2441216.0,
+ "step": 149
+ },
+ {
+ "entropy": 1.7334001064300537,
+ "epoch": 0.30303030303030304,
+ "grad_norm": 2.149543285369873,
+ "learning_rate": 1.7993265993265993e-05,
+ "loss": 1.7014907598495483,
+ "mean_token_accuracy": 0.5981314182281494,
+ "num_tokens": 2457600.0,
+ "step": 150
+ },
+ {
+ "entropy": 2.0550050735473633,
+ "epoch": 0.30505050505050507,
+ "grad_norm": 1.9880189895629883,
+ "learning_rate": 1.797979797979798e-05,
+ "loss": 2.001784324645996,
+ "mean_token_accuracy": 0.5552638173103333,
+ "num_tokens": 2473984.0,
+ "step": 151
+ },
+ {
+ "entropy": 1.5352036952972412,
+ "epoch": 0.30707070707070705,
+ "grad_norm": 2.031998634338379,
+ "learning_rate": 1.7966329966329966e-05,
+ "loss": 1.5262815952301025,
+ "mean_token_accuracy": 0.6328163146972656,
+ "num_tokens": 2490368.0,
+ "step": 152
+ },
+ {
+ "entropy": 1.4278677701950073,
+ "epoch": 0.3090909090909091,
+ "grad_norm": 2.101398229598999,
+ "learning_rate": 1.7952861952861954e-05,
+ "loss": 1.4288041591644287,
+ "mean_token_accuracy": 0.6647533178329468,
+ "num_tokens": 2506752.0,
+ "step": 153
+ },
+ {
+ "entropy": 1.7469983100891113,
+ "epoch": 0.3111111111111111,
+ "grad_norm": 2.0824313163757324,
+ "learning_rate": 1.7939393939393942e-05,
+ "loss": 1.770402193069458,
+ "mean_token_accuracy": 0.5997191071510315,
+ "num_tokens": 2523136.0,
+ "step": 154
+ },
+ {
+ "entropy": 1.6565595865249634,
+ "epoch": 0.31313131313131315,
+ "grad_norm": 2.0815961360931396,
+ "learning_rate": 1.7925925925925927e-05,
+ "loss": 1.7024259567260742,
+ "mean_token_accuracy": 0.6109550595283508,
+ "num_tokens": 2539520.0,
+ "step": 155
+ },
+ {
+ "entropy": 1.4104127883911133,
+ "epoch": 0.3151515151515151,
+ "grad_norm": 2.129694700241089,
+ "learning_rate": 1.7912457912457915e-05,
+ "loss": 1.4378812313079834,
+ "mean_token_accuracy": 0.6489985585212708,
+ "num_tokens": 2555904.0,
+ "step": 156
+ },
+ {
+ "entropy": 1.5129127502441406,
+ "epoch": 0.31717171717171716,
+ "grad_norm": 2.180267095565796,
+ "learning_rate": 1.78989898989899e-05,
+ "loss": 1.5564348697662354,
+ "mean_token_accuracy": 0.6327552795410156,
+ "num_tokens": 2572288.0,
+ "step": 157
+ },
+ {
+ "entropy": 1.5240321159362793,
+ "epoch": 0.3191919191919192,
+ "grad_norm": 1.968405842781067,
+ "learning_rate": 1.7885521885521885e-05,
+ "loss": 1.585756778717041,
+ "mean_token_accuracy": 0.629885196685791,
+ "num_tokens": 2588672.0,
+ "step": 158
+ },
+ {
+ "entropy": 1.3169562816619873,
+ "epoch": 0.3212121212121212,
+ "grad_norm": 2.0452873706817627,
+ "learning_rate": 1.7872053872053873e-05,
+ "loss": 1.3569687604904175,
+ "mean_token_accuracy": 0.671775758266449,
+ "num_tokens": 2605056.0,
+ "step": 159
+ },
+ {
+ "entropy": 1.3630796670913696,
+ "epoch": 0.32323232323232326,
+ "grad_norm": 1.9721522331237793,
+ "learning_rate": 1.785858585858586e-05,
+ "loss": 1.3965554237365723,
+ "mean_token_accuracy": 0.6626160144805908,
+ "num_tokens": 2621440.0,
+ "step": 160
+ },
+ {
+ "entropy": 1.178250789642334,
+ "epoch": 0.32525252525252524,
+ "grad_norm": 2.0231528282165527,
+ "learning_rate": 1.7845117845117846e-05,
+ "loss": 1.2126985788345337,
+ "mean_token_accuracy": 0.6977894306182861,
+ "num_tokens": 2637824.0,
+ "step": 161
+ },
+ {
+ "entropy": 1.8399475812911987,
+ "epoch": 0.32727272727272727,
+ "grad_norm": 2.0615484714508057,
+ "learning_rate": 1.7831649831649834e-05,
+ "loss": 1.868000864982605,
+ "mean_token_accuracy": 0.5822545289993286,
+ "num_tokens": 2654208.0,
+ "step": 162
+ },
+ {
+ "entropy": 1.6604139804840088,
+ "epoch": 0.3292929292929293,
+ "grad_norm": 1.9838379621505737,
+ "learning_rate": 1.781818181818182e-05,
+ "loss": 1.6677067279815674,
+ "mean_token_accuracy": 0.621275007724762,
+ "num_tokens": 2670592.0,
+ "step": 163
+ },
+ {
+ "entropy": 1.3761085271835327,
+ "epoch": 0.33131313131313134,
+ "grad_norm": 2.0093271732330322,
+ "learning_rate": 1.7804713804713807e-05,
+ "loss": 1.3829190731048584,
+ "mean_token_accuracy": 0.6655471324920654,
+ "num_tokens": 2686976.0,
+ "step": 164
+ },
+ {
+ "entropy": 1.578093409538269,
+ "epoch": 0.3333333333333333,
+ "grad_norm": 1.9710773229599,
+ "learning_rate": 1.7791245791245792e-05,
+ "loss": 1.560583472251892,
+ "mean_token_accuracy": 0.6344650983810425,
+ "num_tokens": 2703360.0,
+ "step": 165
+ },
+ {
+ "entropy": 1.0447157621383667,
+ "epoch": 0.33535353535353535,
+ "grad_norm": 1.68338942527771,
+ "learning_rate": 1.7777777777777777e-05,
+ "loss": 1.0116058588027954,
+ "mean_token_accuracy": 0.7449315786361694,
+ "num_tokens": 2719744.0,
+ "step": 166
+ },
+ {
+ "entropy": 1.3687469959259033,
+ "epoch": 0.3373737373737374,
+ "grad_norm": 2.062744617462158,
+ "learning_rate": 1.7764309764309765e-05,
+ "loss": 1.3179712295532227,
+ "mean_token_accuracy": 0.6761724352836609,
+ "num_tokens": 2736128.0,
+ "step": 167
+ },
+ {
+ "entropy": 1.8639556169509888,
+ "epoch": 0.3393939393939394,
+ "grad_norm": 2.041898250579834,
+ "learning_rate": 1.7750841750841753e-05,
+ "loss": 1.8531380891799927,
+ "mean_token_accuracy": 0.5920859575271606,
+ "num_tokens": 2752512.0,
+ "step": 168
+ },
+ {
+ "entropy": 1.6900824308395386,
+ "epoch": 0.3414141414141414,
+ "grad_norm": 1.9403504133224487,
+ "learning_rate": 1.773737373737374e-05,
+ "loss": 1.701782464981079,
+ "mean_token_accuracy": 0.6102222800254822,
+ "num_tokens": 2768896.0,
+ "step": 169
+ },
+ {
+ "entropy": 1.6202301979064941,
+ "epoch": 0.3434343434343434,
+ "grad_norm": 2.027329683303833,
+ "learning_rate": 1.7723905723905726e-05,
+ "loss": 1.6471121311187744,
+ "mean_token_accuracy": 0.6310454607009888,
+ "num_tokens": 2785280.0,
+ "step": 170
+ },
+ {
+ "entropy": 1.2701244354248047,
+ "epoch": 0.34545454545454546,
+ "grad_norm": 1.8625534772872925,
+ "learning_rate": 1.771043771043771e-05,
+ "loss": 1.291702151298523,
+ "mean_token_accuracy": 0.67659991979599,
+ "num_tokens": 2801664.0,
+ "step": 171
+ },
+ {
+ "entropy": 1.5759357213974,
+ "epoch": 0.3474747474747475,
+ "grad_norm": 1.945141077041626,
+ "learning_rate": 1.76969696969697e-05,
+ "loss": 1.6189839839935303,
+ "mean_token_accuracy": 0.6225574016571045,
+ "num_tokens": 2818048.0,
+ "step": 172
+ },
+ {
+ "entropy": 1.3319392204284668,
+ "epoch": 0.34949494949494947,
+ "grad_norm": 1.8485770225524902,
+ "learning_rate": 1.7683501683501684e-05,
+ "loss": 1.3553425073623657,
+ "mean_token_accuracy": 0.6759281754493713,
+ "num_tokens": 2834432.0,
+ "step": 173
+ },
+ {
+ "entropy": 1.3251745700836182,
+ "epoch": 0.3515151515151515,
+ "grad_norm": 2.0564401149749756,
+ "learning_rate": 1.7670033670033672e-05,
+ "loss": 1.359522819519043,
+ "mean_token_accuracy": 0.6678065657615662,
+ "num_tokens": 2850816.0,
+ "step": 174
+ },
+ {
+ "entropy": 1.2764136791229248,
+ "epoch": 0.35353535353535354,
+ "grad_norm": 2.1749041080474854,
+ "learning_rate": 1.765656565656566e-05,
+ "loss": 1.3260173797607422,
+ "mean_token_accuracy": 0.6710429787635803,
+ "num_tokens": 2867200.0,
+ "step": 175
+ },
+ {
+ "entropy": 1.6382335424423218,
+ "epoch": 0.35555555555555557,
+ "grad_norm": 1.923366904258728,
+ "learning_rate": 1.7643097643097645e-05,
+ "loss": 1.6868517398834229,
+ "mean_token_accuracy": 0.6268930435180664,
+ "num_tokens": 2883584.0,
+ "step": 176
+ },
+ {
+ "entropy": 1.2399169206619263,
+ "epoch": 0.3575757575757576,
+ "grad_norm": 1.795649766921997,
+ "learning_rate": 1.7629629629629633e-05,
+ "loss": 1.2266355752944946,
+ "mean_token_accuracy": 0.689667820930481,
+ "num_tokens": 2899968.0,
+ "step": 177
+ },
+ {
+ "entropy": 1.5761929750442505,
+ "epoch": 0.3595959595959596,
+ "grad_norm": 2.0536038875579834,
+ "learning_rate": 1.7616161616161618e-05,
+ "loss": 1.5698940753936768,
+ "mean_token_accuracy": 0.6263434290885925,
+ "num_tokens": 2916352.0,
+ "step": 178
+ },
+ {
+ "entropy": 1.3612396717071533,
+ "epoch": 0.3616161616161616,
+ "grad_norm": 2.003840684890747,
+ "learning_rate": 1.7602693602693603e-05,
+ "loss": 1.359879732131958,
+ "mean_token_accuracy": 0.6798974275588989,
+ "num_tokens": 2932736.0,
+ "step": 179
+ },
+ {
+ "entropy": 1.0548239946365356,
+ "epoch": 0.36363636363636365,
+ "grad_norm": 1.6967436075210571,
+ "learning_rate": 1.758922558922559e-05,
+ "loss": 1.040539264678955,
+ "mean_token_accuracy": 0.7433438897132874,
+ "num_tokens": 2949120.0,
+ "step": 180
+ },
+ {
+ "entropy": 1.16105055809021,
+ "epoch": 0.3656565656565657,
+ "grad_norm": 1.9793111085891724,
+ "learning_rate": 1.7575757575757576e-05,
+ "loss": 1.1532450914382935,
+ "mean_token_accuracy": 0.7135441899299622,
+ "num_tokens": 2965504.0,
+ "step": 181
+ },
+ {
+ "entropy": 1.4421862363815308,
+ "epoch": 0.36767676767676766,
+ "grad_norm": 1.8337509632110596,
+ "learning_rate": 1.7562289562289564e-05,
+ "loss": 1.419133186340332,
+ "mean_token_accuracy": 0.659807026386261,
+ "num_tokens": 2981888.0,
+ "step": 182
+ },
+ {
+ "entropy": 1.6186654567718506,
+ "epoch": 0.3696969696969697,
+ "grad_norm": 2.0585362911224365,
+ "learning_rate": 1.7548821548821552e-05,
+ "loss": 1.5880110263824463,
+ "mean_token_accuracy": 0.6322056651115417,
+ "num_tokens": 2998272.0,
+ "step": 183
+ },
+ {
+ "entropy": 1.76968252658844,
+ "epoch": 0.3717171717171717,
+ "grad_norm": 2.139742374420166,
+ "learning_rate": 1.7535353535353537e-05,
+ "loss": 1.797312617301941,
+ "mean_token_accuracy": 0.6096116304397583,
+ "num_tokens": 3014656.0,
+ "step": 184
+ },
+ {
+ "entropy": 1.0575193166732788,
+ "epoch": 0.37373737373737376,
+ "grad_norm": 1.8547794818878174,
+ "learning_rate": 1.7521885521885525e-05,
+ "loss": 1.0575863122940063,
+ "mean_token_accuracy": 0.7325354218482971,
+ "num_tokens": 3031040.0,
+ "step": 185
+ },
+ {
+ "entropy": 1.2937911748886108,
+ "epoch": 0.37575757575757573,
+ "grad_norm": 2.3985140323638916,
+ "learning_rate": 1.750841750841751e-05,
+ "loss": 1.3277016878128052,
+ "mean_token_accuracy": 0.6654860973358154,
+ "num_tokens": 3047424.0,
+ "step": 186
+ },
+ {
+ "entropy": 1.1829862594604492,
+ "epoch": 0.37777777777777777,
+ "grad_norm": 1.948241114616394,
+ "learning_rate": 1.7494949494949494e-05,
+ "loss": 1.2146751880645752,
+ "mean_token_accuracy": 0.6964460015296936,
+ "num_tokens": 3063808.0,
+ "step": 187
+ },
+ {
+ "entropy": 1.4858520030975342,
+ "epoch": 0.3797979797979798,
+ "grad_norm": 2.2712836265563965,
+ "learning_rate": 1.7481481481481483e-05,
+ "loss": 1.5523631572723389,
+ "mean_token_accuracy": 0.6392281651496887,
+ "num_tokens": 3080192.0,
+ "step": 188
+ },
+ {
+ "entropy": 1.7905362844467163,
+ "epoch": 0.38181818181818183,
+ "grad_norm": 1.890699863433838,
+ "learning_rate": 1.746801346801347e-05,
+ "loss": 1.8139519691467285,
+ "mean_token_accuracy": 0.6015510559082031,
+ "num_tokens": 3096576.0,
+ "step": 189
+ },
+ {
+ "entropy": 1.3264763355255127,
+ "epoch": 0.3838383838383838,
+ "grad_norm": 2.1567318439483643,
+ "learning_rate": 1.7454545454545456e-05,
+ "loss": 1.3780615329742432,
+ "mean_token_accuracy": 0.6636541485786438,
+ "num_tokens": 3112960.0,
+ "step": 190
+ },
+ {
+ "entropy": 1.9898782968521118,
+ "epoch": 0.38585858585858585,
+ "grad_norm": 2.008547306060791,
+ "learning_rate": 1.7441077441077444e-05,
+ "loss": 2.0208630561828613,
+ "mean_token_accuracy": 0.5498290061950684,
+ "num_tokens": 3129344.0,
+ "step": 191
+ },
+ {
+ "entropy": 1.2579785585403442,
+ "epoch": 0.3878787878787879,
+ "grad_norm": 1.921314001083374,
+ "learning_rate": 1.742760942760943e-05,
+ "loss": 1.271723985671997,
+ "mean_token_accuracy": 0.6882632970809937,
+ "num_tokens": 3145728.0,
+ "step": 192
+ },
+ {
+ "entropy": 1.3366814851760864,
+ "epoch": 0.3898989898989899,
+ "grad_norm": 1.9525961875915527,
+ "learning_rate": 1.7414141414141413e-05,
+ "loss": 1.3561700582504272,
+ "mean_token_accuracy": 0.6822789311408997,
+ "num_tokens": 3162112.0,
+ "step": 193
+ },
+ {
+ "entropy": 1.5505253076553345,
+ "epoch": 0.39191919191919194,
+ "grad_norm": 2.044635057449341,
+ "learning_rate": 1.74006734006734e-05,
+ "loss": 1.5198928117752075,
+ "mean_token_accuracy": 0.6326331496238708,
+ "num_tokens": 3178496.0,
+ "step": 194
+ },
+ {
+ "entropy": 1.4198534488677979,
+ "epoch": 0.3939393939393939,
+ "grad_norm": 1.9955503940582275,
+ "learning_rate": 1.738720538720539e-05,
+ "loss": 1.4072260856628418,
+ "mean_token_accuracy": 0.6631045341491699,
+ "num_tokens": 3194880.0,
+ "step": 195
+ },
+ {
+ "entropy": 2.0099895000457764,
+ "epoch": 0.39595959595959596,
+ "grad_norm": 2.110247850418091,
+ "learning_rate": 1.7373737373737375e-05,
+ "loss": 2.0012335777282715,
+ "mean_token_accuracy": 0.5668661594390869,
+ "num_tokens": 3211264.0,
+ "step": 196
+ },
+ {
+ "entropy": 2.1619439125061035,
+ "epoch": 0.397979797979798,
+ "grad_norm": 2.0869383811950684,
+ "learning_rate": 1.7360269360269363e-05,
+ "loss": 2.1793670654296875,
+ "mean_token_accuracy": 0.5569125413894653,
+ "num_tokens": 3227648.0,
+ "step": 197
+ },
+ {
+ "entropy": 1.6383775472640991,
+ "epoch": 0.4,
+ "grad_norm": 1.7467572689056396,
+ "learning_rate": 1.7346801346801347e-05,
+ "loss": 1.656137228012085,
+ "mean_token_accuracy": 0.6395334601402283,
+ "num_tokens": 3244032.0,
+ "step": 198
+ },
+ {
+ "entropy": 1.5911450386047363,
+ "epoch": 0.402020202020202,
+ "grad_norm": 1.9362678527832031,
+ "learning_rate": 1.7333333333333336e-05,
+ "loss": 1.5865838527679443,
+ "mean_token_accuracy": 0.621275007724762,
+ "num_tokens": 3260416.0,
+ "step": 199
+ },
+ {
+ "entropy": 1.3983819484710693,
+ "epoch": 0.40404040404040403,
+ "grad_norm": 1.925863265991211,
+ "learning_rate": 1.731986531986532e-05,
+ "loss": 1.3835841417312622,
+ "mean_token_accuracy": 0.6618221998214722,
+ "num_tokens": 3276800.0,
+ "step": 200
+ },
+ {
+ "epoch": 0.40404040404040403,
+ "eval_entropy": 1.5366105552642577,
+ "eval_loss": 1.5502140522003174,
+ "eval_mean_token_accuracy": 0.6411546406246,
+ "eval_num_tokens": 3276800.0,
+ "eval_runtime": 43.8218,
+ "eval_samples_per_second": 22.592,
+ "eval_steps_per_second": 2.83,
+ "step": 200
+ },
+ {
+ "entropy": 1.291393756866455,
+ "epoch": 0.40606060606060607,
+ "grad_norm": 1.9057531356811523,
+ "learning_rate": 1.7306397306397305e-05,
+ "loss": 1.2847933769226074,
+ "mean_token_accuracy": 0.6901563405990601,
+ "num_tokens": 3293184.0,
+ "step": 201
+ },
+ {
+ "entropy": 1.1511558294296265,
+ "epoch": 0.4080808080808081,
+ "grad_norm": 1.849436640739441,
+ "learning_rate": 1.7292929292929293e-05,
+ "loss": 1.1661030054092407,
+ "mean_token_accuracy": 0.7095139026641846,
+ "num_tokens": 3309568.0,
+ "step": 202
+ },
+ {
+ "entropy": 1.5591504573822021,
+ "epoch": 0.4101010101010101,
+ "grad_norm": 1.9354233741760254,
+ "learning_rate": 1.727946127946128e-05,
+ "loss": 1.6045410633087158,
+ "mean_token_accuracy": 0.6226795315742493,
+ "num_tokens": 3325952.0,
+ "step": 203
+ },
+ {
+ "entropy": 1.7634649276733398,
+ "epoch": 0.4121212121212121,
+ "grad_norm": 2.414632797241211,
+ "learning_rate": 1.726599326599327e-05,
+ "loss": 1.8021953105926514,
+ "mean_token_accuracy": 0.6099780201911926,
+ "num_tokens": 3342336.0,
+ "step": 204
+ },
+ {
+ "entropy": 1.6080671548843384,
+ "epoch": 0.41414141414141414,
+ "grad_norm": 1.9936028718948364,
+ "learning_rate": 1.7252525252525255e-05,
+ "loss": 1.630000352859497,
+ "mean_token_accuracy": 0.6256106495857239,
+ "num_tokens": 3358720.0,
+ "step": 205
+ },
+ {
+ "entropy": 1.6845343112945557,
+ "epoch": 0.4161616161616162,
+ "grad_norm": 1.9131947755813599,
+ "learning_rate": 1.723905723905724e-05,
+ "loss": 1.6912329196929932,
+ "mean_token_accuracy": 0.6263434290885925,
+ "num_tokens": 3375104.0,
+ "step": 206
+ },
+ {
+ "entropy": 1.5946446657180786,
+ "epoch": 0.41818181818181815,
+ "grad_norm": 2.0423905849456787,
+ "learning_rate": 1.7225589225589228e-05,
+ "loss": 1.647271752357483,
+ "mean_token_accuracy": 0.6296409368515015,
+ "num_tokens": 3391488.0,
+ "step": 207
+ },
+ {
+ "entropy": 1.3124741315841675,
+ "epoch": 0.4202020202020202,
+ "grad_norm": 1.8896982669830322,
+ "learning_rate": 1.7212121212121212e-05,
+ "loss": 1.3216124773025513,
+ "mean_token_accuracy": 0.687469482421875,
+ "num_tokens": 3407872.0,
+ "step": 208
+ },
+ {
+ "entropy": 1.3931281566619873,
+ "epoch": 0.4222222222222222,
+ "grad_norm": 1.7883074283599854,
+ "learning_rate": 1.71986531986532e-05,
+ "loss": 1.4264600276947021,
+ "mean_token_accuracy": 0.6568148732185364,
+ "num_tokens": 3424256.0,
+ "step": 209
+ },
+ {
+ "entropy": 1.1427687406539917,
+ "epoch": 0.42424242424242425,
+ "grad_norm": 1.9367003440856934,
+ "learning_rate": 1.7185185185185185e-05,
+ "loss": 1.1611723899841309,
+ "mean_token_accuracy": 0.7058500051498413,
+ "num_tokens": 3440640.0,
+ "step": 210
+ },
+ {
+ "entropy": 1.7226167917251587,
+ "epoch": 0.4262626262626263,
+ "grad_norm": 2.175487518310547,
+ "learning_rate": 1.7171717171717173e-05,
+ "loss": 1.7383192777633667,
+ "mean_token_accuracy": 0.6075964570045471,
+ "num_tokens": 3457024.0,
+ "step": 211
+ },
+ {
+ "entropy": 1.3780062198638916,
+ "epoch": 0.42828282828282827,
+ "grad_norm": 1.9817614555358887,
+ "learning_rate": 1.715824915824916e-05,
+ "loss": 1.3755927085876465,
+ "mean_token_accuracy": 0.6750732660293579,
+ "num_tokens": 3473408.0,
+ "step": 212
+ },
+ {
+ "entropy": 1.637842059135437,
+ "epoch": 0.4303030303030303,
+ "grad_norm": 2.2326242923736572,
+ "learning_rate": 1.7144781144781146e-05,
+ "loss": 1.635596752166748,
+ "mean_token_accuracy": 0.6110771894454956,
+ "num_tokens": 3489792.0,
+ "step": 213
+ },
+ {
+ "entropy": 1.9970118999481201,
+ "epoch": 0.43232323232323233,
+ "grad_norm": 2.1303606033325195,
+ "learning_rate": 1.713131313131313e-05,
+ "loss": 2.0099592208862305,
+ "mean_token_accuracy": 0.5781631469726562,
+ "num_tokens": 3506176.0,
+ "step": 214
+ },
+ {
+ "entropy": 1.4759869575500488,
+ "epoch": 0.43434343434343436,
+ "grad_norm": 2.043727397918701,
+ "learning_rate": 1.711784511784512e-05,
+ "loss": 1.4756600856781006,
+ "mean_token_accuracy": 0.6483879089355469,
+ "num_tokens": 3522560.0,
+ "step": 215
+ },
+ {
+ "entropy": 1.2833726406097412,
+ "epoch": 0.43636363636363634,
+ "grad_norm": 1.9260340929031372,
+ "learning_rate": 1.7104377104377104e-05,
+ "loss": 1.2887630462646484,
+ "mean_token_accuracy": 0.6900342106819153,
+ "num_tokens": 3538944.0,
+ "step": 216
+ },
+ {
+ "entropy": 1.3936302661895752,
+ "epoch": 0.4383838383838384,
+ "grad_norm": 1.8590774536132812,
+ "learning_rate": 1.7090909090909092e-05,
+ "loss": 1.3784823417663574,
+ "mean_token_accuracy": 0.6736077070236206,
+ "num_tokens": 3555328.0,
+ "step": 217
+ },
+ {
+ "entropy": 1.2413936853408813,
+ "epoch": 0.4404040404040404,
+ "grad_norm": 2.1111643314361572,
+ "learning_rate": 1.707744107744108e-05,
+ "loss": 1.2740635871887207,
+ "mean_token_accuracy": 0.6883854269981384,
+ "num_tokens": 3571712.0,
+ "step": 218
+ },
+ {
+ "entropy": 1.4879416227340698,
+ "epoch": 0.44242424242424244,
+ "grad_norm": 2.127614736557007,
+ "learning_rate": 1.7063973063973065e-05,
+ "loss": 1.48406183719635,
+ "mean_token_accuracy": 0.649975597858429,
+ "num_tokens": 3588096.0,
+ "step": 219
+ },
+ {
+ "entropy": 1.4226471185684204,
+ "epoch": 0.4444444444444444,
+ "grad_norm": 2.164325475692749,
+ "learning_rate": 1.7050505050505054e-05,
+ "loss": 1.466269850730896,
+ "mean_token_accuracy": 0.6502808928489685,
+ "num_tokens": 3604480.0,
+ "step": 220
+ },
+ {
+ "entropy": 1.6183370351791382,
+ "epoch": 0.44646464646464645,
+ "grad_norm": 2.018846273422241,
+ "learning_rate": 1.7037037037037038e-05,
+ "loss": 1.6348307132720947,
+ "mean_token_accuracy": 0.627198338508606,
+ "num_tokens": 3620864.0,
+ "step": 221
+ },
+ {
+ "entropy": 1.6979204416275024,
+ "epoch": 0.4484848484848485,
+ "grad_norm": 2.1574854850769043,
+ "learning_rate": 1.7023569023569023e-05,
+ "loss": 1.7019602060317993,
+ "mean_token_accuracy": 0.6157181262969971,
+ "num_tokens": 3637248.0,
+ "step": 222
+ },
+ {
+ "entropy": 1.753490686416626,
+ "epoch": 0.4505050505050505,
+ "grad_norm": 2.0253705978393555,
+ "learning_rate": 1.701010101010101e-05,
+ "loss": 1.7466539144515991,
+ "mean_token_accuracy": 0.6008793115615845,
+ "num_tokens": 3653632.0,
+ "step": 223
+ },
+ {
+ "entropy": 1.717929720878601,
+ "epoch": 0.45252525252525255,
+ "grad_norm": 1.9840974807739258,
+ "learning_rate": 1.6996632996633e-05,
+ "loss": 1.703988790512085,
+ "mean_token_accuracy": 0.6099780201911926,
+ "num_tokens": 3670016.0,
+ "step": 224
+ },
+ {
+ "entropy": 1.377287745475769,
+ "epoch": 0.45454545454545453,
+ "grad_norm": 2.1027839183807373,
+ "learning_rate": 1.6983164983164984e-05,
+ "loss": 1.3902864456176758,
+ "mean_token_accuracy": 0.660906195640564,
+ "num_tokens": 3686400.0,
+ "step": 225
+ },
+ {
+ "entropy": 1.237000584602356,
+ "epoch": 0.45656565656565656,
+ "grad_norm": 1.9116249084472656,
+ "learning_rate": 1.6969696969696972e-05,
+ "loss": 1.221006155014038,
+ "mean_token_accuracy": 0.7045676708221436,
+ "num_tokens": 3702784.0,
+ "step": 226
+ },
+ {
+ "entropy": 1.4198882579803467,
+ "epoch": 0.4585858585858586,
+ "grad_norm": 2.099358558654785,
+ "learning_rate": 1.6956228956228957e-05,
+ "loss": 1.4211962223052979,
+ "mean_token_accuracy": 0.6549829244613647,
+ "num_tokens": 3719168.0,
+ "step": 227
+ },
+ {
+ "entropy": 1.4647245407104492,
+ "epoch": 0.46060606060606063,
+ "grad_norm": 1.8873655796051025,
+ "learning_rate": 1.6942760942760945e-05,
+ "loss": 1.4848006963729858,
+ "mean_token_accuracy": 0.6534562706947327,
+ "num_tokens": 3735552.0,
+ "step": 228
+ },
+ {
+ "entropy": 1.348371982574463,
+ "epoch": 0.4626262626262626,
+ "grad_norm": 2.0487565994262695,
+ "learning_rate": 1.692929292929293e-05,
+ "loss": 1.355329155921936,
+ "mean_token_accuracy": 0.6709208488464355,
+ "num_tokens": 3751936.0,
+ "step": 229
+ },
+ {
+ "entropy": 1.6201715469360352,
+ "epoch": 0.46464646464646464,
+ "grad_norm": 1.9640510082244873,
+ "learning_rate": 1.6915824915824915e-05,
+ "loss": 1.6176421642303467,
+ "mean_token_accuracy": 0.6221910119056702,
+ "num_tokens": 3768320.0,
+ "step": 230
+ },
+ {
+ "entropy": 1.461037039756775,
+ "epoch": 0.4666666666666667,
+ "grad_norm": 1.9843394756317139,
+ "learning_rate": 1.6902356902356903e-05,
+ "loss": 1.4502949714660645,
+ "mean_token_accuracy": 0.6463727355003357,
+ "num_tokens": 3784704.0,
+ "step": 231
+ },
+ {
+ "entropy": 1.4964231252670288,
+ "epoch": 0.4686868686868687,
+ "grad_norm": 2.0087051391601562,
+ "learning_rate": 1.688888888888889e-05,
+ "loss": 1.5107879638671875,
+ "mean_token_accuracy": 0.6536394953727722,
+ "num_tokens": 3801088.0,
+ "step": 232
+ },
+ {
+ "entropy": 1.3013520240783691,
+ "epoch": 0.4707070707070707,
+ "grad_norm": 2.1042368412017822,
+ "learning_rate": 1.6875420875420876e-05,
+ "loss": 1.312164306640625,
+ "mean_token_accuracy": 0.673363447189331,
+ "num_tokens": 3817472.0,
+ "step": 233
+ },
+ {
+ "entropy": 1.6242352724075317,
+ "epoch": 0.4727272727272727,
+ "grad_norm": 1.8541103601455688,
+ "learning_rate": 1.6861952861952864e-05,
+ "loss": 1.6327502727508545,
+ "mean_token_accuracy": 0.6376404762268066,
+ "num_tokens": 3833856.0,
+ "step": 234
+ },
+ {
+ "entropy": 1.4227592945098877,
+ "epoch": 0.47474747474747475,
+ "grad_norm": 1.8749339580535889,
+ "learning_rate": 1.684848484848485e-05,
+ "loss": 1.436142921447754,
+ "mean_token_accuracy": 0.6705544590950012,
+ "num_tokens": 3850240.0,
+ "step": 235
+ },
+ {
+ "entropy": 1.4807487726211548,
+ "epoch": 0.4767676767676768,
+ "grad_norm": 2.1475114822387695,
+ "learning_rate": 1.6835016835016837e-05,
+ "loss": 1.4670445919036865,
+ "mean_token_accuracy": 0.6385564208030701,
+ "num_tokens": 3866624.0,
+ "step": 236
+ },
+ {
+ "entropy": 2.0623066425323486,
+ "epoch": 0.47878787878787876,
+ "grad_norm": 2.1698546409606934,
+ "learning_rate": 1.6821548821548822e-05,
+ "loss": 2.08683180809021,
+ "mean_token_accuracy": 0.5531876087188721,
+ "num_tokens": 3883008.0,
+ "step": 237
+ },
+ {
+ "entropy": 1.6894460916519165,
+ "epoch": 0.4808080808080808,
+ "grad_norm": 2.158062219619751,
+ "learning_rate": 1.680808080808081e-05,
+ "loss": 1.7244186401367188,
+ "mean_token_accuracy": 0.6055813431739807,
+ "num_tokens": 3899392.0,
+ "step": 238
+ },
+ {
+ "entropy": 1.2741345167160034,
+ "epoch": 0.48282828282828283,
+ "grad_norm": 1.918306827545166,
+ "learning_rate": 1.67946127946128e-05,
+ "loss": 1.2937202453613281,
+ "mean_token_accuracy": 0.6878969073295593,
+ "num_tokens": 3915776.0,
+ "step": 239
+ },
+ {
+ "entropy": 1.556649088859558,
+ "epoch": 0.48484848484848486,
+ "grad_norm": 2.0560667514801025,
+ "learning_rate": 1.6781144781144783e-05,
+ "loss": 1.5751030445098877,
+ "mean_token_accuracy": 0.6324499249458313,
+ "num_tokens": 3932160.0,
+ "step": 240
+ },
+ {
+ "entropy": 1.7632875442504883,
+ "epoch": 0.4868686868686869,
+ "grad_norm": 1.8408738374710083,
+ "learning_rate": 1.6767676767676768e-05,
+ "loss": 1.7745842933654785,
+ "mean_token_accuracy": 0.6091231107711792,
+ "num_tokens": 3948544.0,
+ "step": 241
+ },
+ {
+ "entropy": 1.4793967008590698,
+ "epoch": 0.4888888888888889,
+ "grad_norm": 2.0134575366973877,
+ "learning_rate": 1.6754208754208756e-05,
+ "loss": 1.514477014541626,
+ "mean_token_accuracy": 0.6414875388145447,
+ "num_tokens": 3964928.0,
+ "step": 242
+ },
+ {
+ "entropy": 1.4762561321258545,
+ "epoch": 0.4909090909090909,
+ "grad_norm": 2.087501287460327,
+ "learning_rate": 1.674074074074074e-05,
+ "loss": 1.5195722579956055,
+ "mean_token_accuracy": 0.6289081573486328,
+ "num_tokens": 3981312.0,
+ "step": 243
+ },
+ {
+ "entropy": 1.6629496812820435,
+ "epoch": 0.49292929292929294,
+ "grad_norm": 1.7386047840118408,
+ "learning_rate": 1.672727272727273e-05,
+ "loss": 1.6932936906814575,
+ "mean_token_accuracy": 0.6289692521095276,
+ "num_tokens": 3997696.0,
+ "step": 244
+ },
+ {
+ "entropy": 1.4506888389587402,
+ "epoch": 0.494949494949495,
+ "grad_norm": 1.8268823623657227,
+ "learning_rate": 1.6713804713804714e-05,
+ "loss": 1.4257543087005615,
+ "mean_token_accuracy": 0.6678065657615662,
+ "num_tokens": 4014080.0,
+ "step": 245
+ },
+ {
+ "entropy": 1.6132714748382568,
+ "epoch": 0.49696969696969695,
+ "grad_norm": 2.168206214904785,
+ "learning_rate": 1.6700336700336702e-05,
+ "loss": 1.602414608001709,
+ "mean_token_accuracy": 0.6202979683876038,
+ "num_tokens": 4030464.0,
+ "step": 246
+ },
+ {
+ "entropy": 1.2860945463180542,
+ "epoch": 0.498989898989899,
+ "grad_norm": 1.830817699432373,
+ "learning_rate": 1.668686868686869e-05,
+ "loss": 1.284934163093567,
+ "mean_token_accuracy": 0.6854543089866638,
+ "num_tokens": 4046848.0,
+ "step": 247
+ },
+ {
+ "entropy": 1.4948641061782837,
+ "epoch": 0.501010101010101,
+ "grad_norm": 1.9775508642196655,
+ "learning_rate": 1.6673400673400675e-05,
+ "loss": 1.497206449508667,
+ "mean_token_accuracy": 0.6471055150032043,
+ "num_tokens": 4063232.0,
+ "step": 248
+ },
+ {
+ "entropy": 1.490235447883606,
+ "epoch": 0.503030303030303,
+ "grad_norm": 2.207184076309204,
+ "learning_rate": 1.665993265993266e-05,
+ "loss": 1.5091016292572021,
+ "mean_token_accuracy": 0.6482657790184021,
+ "num_tokens": 4079616.0,
+ "step": 249
+ },
+ {
+ "entropy": 1.5752832889556885,
+ "epoch": 0.5050505050505051,
+ "grad_norm": 2.0718111991882324,
+ "learning_rate": 1.6646464646464648e-05,
+ "loss": 1.576171875,
+ "mean_token_accuracy": 0.6375793814659119,
+ "num_tokens": 4096000.0,
+ "step": 250
+ },
+ {
+ "entropy": 1.3935530185699463,
+ "epoch": 0.5070707070707071,
+ "grad_norm": 1.9379569292068481,
+ "learning_rate": 1.6632996632996633e-05,
+ "loss": 1.4019020795822144,
+ "mean_token_accuracy": 0.6742794513702393,
+ "num_tokens": 4112384.0,
+ "step": 251
+ },
+ {
+ "entropy": 1.3622194528579712,
+ "epoch": 0.509090909090909,
+ "grad_norm": 1.9684133529663086,
+ "learning_rate": 1.661952861952862e-05,
+ "loss": 1.3581968545913696,
+ "mean_token_accuracy": 0.6631045341491699,
+ "num_tokens": 4128768.0,
+ "step": 252
+ },
+ {
+ "entropy": 2.1161346435546875,
+ "epoch": 0.5111111111111111,
+ "grad_norm": 1.9295378923416138,
+ "learning_rate": 1.660606060606061e-05,
+ "loss": 2.1316990852355957,
+ "mean_token_accuracy": 0.5643624663352966,
+ "num_tokens": 4145152.0,
+ "step": 253
+ },
+ {
+ "entropy": 1.4322566986083984,
+ "epoch": 0.5131313131313131,
+ "grad_norm": 2.0198557376861572,
+ "learning_rate": 1.6592592592592594e-05,
+ "loss": 1.4390883445739746,
+ "mean_token_accuracy": 0.6648143529891968,
+ "num_tokens": 4161536.0,
+ "step": 254
+ },
+ {
+ "entropy": 1.5892291069030762,
+ "epoch": 0.5151515151515151,
+ "grad_norm": 2.0127851963043213,
+ "learning_rate": 1.6579124579124582e-05,
+ "loss": 1.6437480449676514,
+ "mean_token_accuracy": 0.6223131418228149,
+ "num_tokens": 4177920.0,
+ "step": 255
+ },
+ {
+ "entropy": 1.445371150970459,
+ "epoch": 0.5171717171717172,
+ "grad_norm": 1.9758318662643433,
+ "learning_rate": 1.6565656565656567e-05,
+ "loss": 1.4749128818511963,
+ "mean_token_accuracy": 0.6457010507583618,
+ "num_tokens": 4194304.0,
+ "step": 256
+ },
+ {
+ "entropy": 1.5587478876113892,
+ "epoch": 0.5191919191919192,
+ "grad_norm": 1.918809413909912,
+ "learning_rate": 1.6552188552188552e-05,
+ "loss": 1.5739175081253052,
+ "mean_token_accuracy": 0.6334269642829895,
+ "num_tokens": 4210688.0,
+ "step": 257
+ },
+ {
+ "entropy": 1.5112392902374268,
+ "epoch": 0.5212121212121212,
+ "grad_norm": 2.0176963806152344,
+ "learning_rate": 1.653872053872054e-05,
+ "loss": 1.5279680490493774,
+ "mean_token_accuracy": 0.640754759311676,
+ "num_tokens": 4227072.0,
+ "step": 258
+ },
+ {
+ "entropy": 1.4898114204406738,
+ "epoch": 0.5232323232323233,
+ "grad_norm": 2.0637850761413574,
+ "learning_rate": 1.6525252525252528e-05,
+ "loss": 1.5081419944763184,
+ "mean_token_accuracy": 0.6375183463096619,
+ "num_tokens": 4243456.0,
+ "step": 259
+ },
+ {
+ "entropy": 1.4768201112747192,
+ "epoch": 0.5252525252525253,
+ "grad_norm": 1.835053563117981,
+ "learning_rate": 1.6511784511784513e-05,
+ "loss": 1.494248390197754,
+ "mean_token_accuracy": 0.6496092081069946,
+ "num_tokens": 4259840.0,
+ "step": 260
+ },
+ {
+ "entropy": 1.655643105506897,
+ "epoch": 0.5272727272727272,
+ "grad_norm": 1.9655805826187134,
+ "learning_rate": 1.64983164983165e-05,
+ "loss": 1.6360158920288086,
+ "mean_token_accuracy": 0.6209086179733276,
+ "num_tokens": 4276224.0,
+ "step": 261
+ },
+ {
+ "entropy": 1.5621604919433594,
+ "epoch": 0.5292929292929293,
+ "grad_norm": 1.9304734468460083,
+ "learning_rate": 1.6484848484848486e-05,
+ "loss": 1.5850739479064941,
+ "mean_token_accuracy": 0.6422203183174133,
+ "num_tokens": 4292608.0,
+ "step": 262
+ },
+ {
+ "entropy": 1.592617392539978,
+ "epoch": 0.5313131313131313,
+ "grad_norm": 1.9590504169464111,
+ "learning_rate": 1.6471380471380474e-05,
+ "loss": 1.583874225616455,
+ "mean_token_accuracy": 0.6230459213256836,
+ "num_tokens": 4308992.0,
+ "step": 263
+ },
+ {
+ "entropy": 1.5268265008926392,
+ "epoch": 0.5333333333333333,
+ "grad_norm": 2.0704433917999268,
+ "learning_rate": 1.645791245791246e-05,
+ "loss": 1.5327314138412476,
+ "mean_token_accuracy": 0.6398388147354126,
+ "num_tokens": 4325376.0,
+ "step": 264
+ },
+ {
+ "entropy": 1.1762311458587646,
+ "epoch": 0.5353535353535354,
+ "grad_norm": 2.0408453941345215,
+ "learning_rate": 1.6444444444444444e-05,
+ "loss": 1.1851191520690918,
+ "mean_token_accuracy": 0.7027967572212219,
+ "num_tokens": 4341760.0,
+ "step": 265
+ },
+ {
+ "entropy": 1.567710518836975,
+ "epoch": 0.5373737373737374,
+ "grad_norm": 1.8334625959396362,
+ "learning_rate": 1.6430976430976432e-05,
+ "loss": 1.564115047454834,
+ "mean_token_accuracy": 0.6340376138687134,
+ "num_tokens": 4358144.0,
+ "step": 266
+ },
+ {
+ "entropy": 1.2294297218322754,
+ "epoch": 0.5393939393939394,
+ "grad_norm": 3.5099127292633057,
+ "learning_rate": 1.641750841750842e-05,
+ "loss": 1.2887773513793945,
+ "mean_token_accuracy": 0.688629686832428,
+ "num_tokens": 4374528.0,
+ "step": 267
+ },
+ {
+ "entropy": 1.5130258798599243,
+ "epoch": 0.5414141414141415,
+ "grad_norm": 1.8295154571533203,
+ "learning_rate": 1.6404040404040405e-05,
+ "loss": 1.581455945968628,
+ "mean_token_accuracy": 0.639655590057373,
+ "num_tokens": 4390912.0,
+ "step": 268
+ },
+ {
+ "entropy": 1.6612389087677002,
+ "epoch": 0.5434343434343434,
+ "grad_norm": 2.1676483154296875,
+ "learning_rate": 1.6390572390572393e-05,
+ "loss": 1.6873081922531128,
+ "mean_token_accuracy": 0.6049095988273621,
+ "num_tokens": 4407296.0,
+ "step": 269
+ },
+ {
+ "entropy": 1.416680932044983,
+ "epoch": 0.5454545454545454,
+ "grad_norm": 2.044541358947754,
+ "learning_rate": 1.6377104377104378e-05,
+ "loss": 1.4432693719863892,
+ "mean_token_accuracy": 0.6580972075462341,
+ "num_tokens": 4423680.0,
+ "step": 270
+ },
+ {
+ "entropy": 1.6253215074539185,
+ "epoch": 0.5474747474747474,
+ "grad_norm": 2.1488161087036133,
+ "learning_rate": 1.6363636363636366e-05,
+ "loss": 1.6667883396148682,
+ "mean_token_accuracy": 0.6122373938560486,
+ "num_tokens": 4440064.0,
+ "step": 271
+ },
+ {
+ "entropy": 1.492803931236267,
+ "epoch": 0.5494949494949495,
+ "grad_norm": 2.0591108798980713,
+ "learning_rate": 1.635016835016835e-05,
+ "loss": 1.5023317337036133,
+ "mean_token_accuracy": 0.6420371532440186,
+ "num_tokens": 4456448.0,
+ "step": 272
+ },
+ {
+ "entropy": 1.5818493366241455,
+ "epoch": 0.5515151515151515,
+ "grad_norm": 1.9004875421524048,
+ "learning_rate": 1.633670033670034e-05,
+ "loss": 1.5762417316436768,
+ "mean_token_accuracy": 0.6375793814659119,
+ "num_tokens": 4472832.0,
+ "step": 273
+ },
+ {
+ "entropy": 1.2263695001602173,
+ "epoch": 0.5535353535353535,
+ "grad_norm": 1.8976677656173706,
+ "learning_rate": 1.6323232323232324e-05,
+ "loss": 1.2397428750991821,
+ "mean_token_accuracy": 0.70658278465271,
+ "num_tokens": 4489216.0,
+ "step": 274
+ },
+ {
+ "entropy": 1.370070219039917,
+ "epoch": 0.5555555555555556,
+ "grad_norm": 1.875389575958252,
+ "learning_rate": 1.6309764309764312e-05,
+ "loss": 1.3603096008300781,
+ "mean_token_accuracy": 0.6843551397323608,
+ "num_tokens": 4505600.0,
+ "step": 275
+ },
+ {
+ "entropy": 1.9219107627868652,
+ "epoch": 0.5575757575757576,
+ "grad_norm": 2.057508945465088,
+ "learning_rate": 1.6296296296296297e-05,
+ "loss": 1.9082988500595093,
+ "mean_token_accuracy": 0.5834758281707764,
+ "num_tokens": 4521984.0,
+ "step": 276
+ },
+ {
+ "entropy": 1.6965564489364624,
+ "epoch": 0.5595959595959596,
+ "grad_norm": 1.9217735528945923,
+ "learning_rate": 1.6282828282828285e-05,
+ "loss": 1.7024450302124023,
+ "mean_token_accuracy": 0.6050928235054016,
+ "num_tokens": 4538368.0,
+ "step": 277
+ },
+ {
+ "entropy": 1.7517896890640259,
+ "epoch": 0.5616161616161616,
+ "grad_norm": 2.029672145843506,
+ "learning_rate": 1.626936026936027e-05,
+ "loss": 1.7533857822418213,
+ "mean_token_accuracy": 0.5986199378967285,
+ "num_tokens": 4554752.0,
+ "step": 278
+ },
+ {
+ "entropy": 1.6086634397506714,
+ "epoch": 0.5636363636363636,
+ "grad_norm": 1.980076551437378,
+ "learning_rate": 1.6255892255892258e-05,
+ "loss": 1.5998293161392212,
+ "mean_token_accuracy": 0.6469223499298096,
+ "num_tokens": 4571136.0,
+ "step": 279
+ },
+ {
+ "entropy": 1.5670934915542603,
+ "epoch": 0.5656565656565656,
+ "grad_norm": 1.9693994522094727,
+ "learning_rate": 1.6242424242424243e-05,
+ "loss": 1.583737850189209,
+ "mean_token_accuracy": 0.6189545392990112,
+ "num_tokens": 4587520.0,
+ "step": 280
+ },
+ {
+ "entropy": 1.5290220975875854,
+ "epoch": 0.5676767676767677,
+ "grad_norm": 1.9464402198791504,
+ "learning_rate": 1.622895622895623e-05,
+ "loss": 1.523187518119812,
+ "mean_token_accuracy": 0.6289692521095276,
+ "num_tokens": 4603904.0,
+ "step": 281
+ },
+ {
+ "entropy": 1.2070591449737549,
+ "epoch": 0.5696969696969697,
+ "grad_norm": 2.1152102947235107,
+ "learning_rate": 1.621548821548822e-05,
+ "loss": 1.2107012271881104,
+ "mean_token_accuracy": 0.695652186870575,
+ "num_tokens": 4620288.0,
+ "step": 282
+ },
+ {
+ "entropy": 1.4039427042007446,
+ "epoch": 0.5717171717171717,
+ "grad_norm": 1.936285376548767,
+ "learning_rate": 1.6202020202020204e-05,
+ "loss": 1.425846815109253,
+ "mean_token_accuracy": 0.663593053817749,
+ "num_tokens": 4636672.0,
+ "step": 283
+ },
+ {
+ "entropy": 1.2910168170928955,
+ "epoch": 0.5737373737373738,
+ "grad_norm": 1.8882776498794556,
+ "learning_rate": 1.618855218855219e-05,
+ "loss": 1.3109780550003052,
+ "mean_token_accuracy": 0.6896067261695862,
+ "num_tokens": 4653056.0,
+ "step": 284
+ },
+ {
+ "entropy": 1.7089474201202393,
+ "epoch": 0.5757575757575758,
+ "grad_norm": 2.0113184452056885,
+ "learning_rate": 1.6175084175084177e-05,
+ "loss": 1.7272329330444336,
+ "mean_token_accuracy": 0.600024402141571,
+ "num_tokens": 4669440.0,
+ "step": 285
+ },
+ {
+ "entropy": 1.12257719039917,
+ "epoch": 0.5777777777777777,
+ "grad_norm": 2.0191409587860107,
+ "learning_rate": 1.616161616161616e-05,
+ "loss": 1.1193959712982178,
+ "mean_token_accuracy": 0.7199560403823853,
+ "num_tokens": 4685824.0,
+ "step": 286
+ },
+ {
+ "entropy": 1.2052935361862183,
+ "epoch": 0.5797979797979798,
+ "grad_norm": 1.9647032022476196,
+ "learning_rate": 1.614814814814815e-05,
+ "loss": 1.2192071676254272,
+ "mean_token_accuracy": 0.6977894306182861,
+ "num_tokens": 4702208.0,
+ "step": 287
+ },
+ {
+ "entropy": 1.5017658472061157,
+ "epoch": 0.5818181818181818,
+ "grad_norm": 2.0078749656677246,
+ "learning_rate": 1.6134680134680138e-05,
+ "loss": 1.5069187879562378,
+ "mean_token_accuracy": 0.6404494643211365,
+ "num_tokens": 4718592.0,
+ "step": 288
+ },
+ {
+ "entropy": 1.4429490566253662,
+ "epoch": 0.5838383838383838,
+ "grad_norm": 1.8666913509368896,
+ "learning_rate": 1.6121212121212123e-05,
+ "loss": 1.4498789310455322,
+ "mean_token_accuracy": 0.6577919125556946,
+ "num_tokens": 4734976.0,
+ "step": 289
+ },
+ {
+ "entropy": 1.216312050819397,
+ "epoch": 0.5858585858585859,
+ "grad_norm": 1.8042014837265015,
+ "learning_rate": 1.610774410774411e-05,
+ "loss": 1.2346259355545044,
+ "mean_token_accuracy": 0.7015144228935242,
+ "num_tokens": 4751360.0,
+ "step": 290
+ },
+ {
+ "entropy": 1.1411411762237549,
+ "epoch": 0.5878787878787879,
+ "grad_norm": 1.7743948698043823,
+ "learning_rate": 1.6094276094276096e-05,
+ "loss": 1.158776044845581,
+ "mean_token_accuracy": 0.7111626863479614,
+ "num_tokens": 4767744.0,
+ "step": 291
+ },
+ {
+ "entropy": 1.5691558122634888,
+ "epoch": 0.5898989898989899,
+ "grad_norm": 2.0790789127349854,
+ "learning_rate": 1.608080808080808e-05,
+ "loss": 1.5900537967681885,
+ "mean_token_accuracy": 0.6259770393371582,
+ "num_tokens": 4784128.0,
+ "step": 292
+ },
+ {
+ "entropy": 1.5600252151489258,
+ "epoch": 0.591919191919192,
+ "grad_norm": 1.8288682699203491,
+ "learning_rate": 1.606734006734007e-05,
+ "loss": 1.5642855167388916,
+ "mean_token_accuracy": 0.6497313380241394,
+ "num_tokens": 4800512.0,
+ "step": 293
+ },
+ {
+ "entropy": 1.316757082939148,
+ "epoch": 0.593939393939394,
+ "grad_norm": 2.051280975341797,
+ "learning_rate": 1.6053872053872053e-05,
+ "loss": 1.3361237049102783,
+ "mean_token_accuracy": 0.6642647981643677,
+ "num_tokens": 4816896.0,
+ "step": 294
+ },
+ {
+ "entropy": 1.4421989917755127,
+ "epoch": 0.5959595959595959,
+ "grad_norm": 2.257932662963867,
+ "learning_rate": 1.604040404040404e-05,
+ "loss": 1.4325928688049316,
+ "mean_token_accuracy": 0.6571201682090759,
+ "num_tokens": 4833280.0,
+ "step": 295
+ },
+ {
+ "entropy": 1.4986436367034912,
+ "epoch": 0.597979797979798,
+ "grad_norm": 2.0747647285461426,
+ "learning_rate": 1.602693602693603e-05,
+ "loss": 1.508068323135376,
+ "mean_token_accuracy": 0.6551050543785095,
+ "num_tokens": 4849664.0,
+ "step": 296
+ },
+ {
+ "entropy": 1.4758206605911255,
+ "epoch": 0.6,
+ "grad_norm": 1.874723196029663,
+ "learning_rate": 1.6013468013468014e-05,
+ "loss": 1.4831781387329102,
+ "mean_token_accuracy": 0.6479604244232178,
+ "num_tokens": 4866048.0,
+ "step": 297
+ },
+ {
+ "entropy": 1.8049858808517456,
+ "epoch": 0.602020202020202,
+ "grad_norm": 2.089683771133423,
+ "learning_rate": 1.6000000000000003e-05,
+ "loss": 1.8325893878936768,
+ "mean_token_accuracy": 0.5932462215423584,
+ "num_tokens": 4882432.0,
+ "step": 298
+ },
+ {
+ "entropy": 1.6346150636672974,
+ "epoch": 0.604040404040404,
+ "grad_norm": 1.9526349306106567,
+ "learning_rate": 1.5986531986531987e-05,
+ "loss": 1.6568968296051025,
+ "mean_token_accuracy": 0.6246336102485657,
+ "num_tokens": 4898816.0,
+ "step": 299
+ },
+ {
+ "entropy": 1.431725025177002,
+ "epoch": 0.6060606060606061,
+ "grad_norm": 2.109987497329712,
+ "learning_rate": 1.5973063973063972e-05,
+ "loss": 1.4448115825653076,
+ "mean_token_accuracy": 0.6481436491012573,
+ "num_tokens": 4915200.0,
+ "step": 300
+ },
+ {
+ "epoch": 0.6060606060606061,
+ "eval_entropy": 1.524387352889584,
+ "eval_loss": 1.5324952602386475,
+ "eval_mean_token_accuracy": 0.6443250953189789,
+ "eval_num_tokens": 4915200.0,
+ "eval_runtime": 43.7714,
+ "eval_samples_per_second": 22.618,
+ "eval_steps_per_second": 2.833,
+ "step": 300
+ },
+ {
+ "entropy": 1.4816409349441528,
+ "epoch": 0.6080808080808081,
+ "grad_norm": 3.3732845783233643,
+ "learning_rate": 1.595959595959596e-05,
+ "loss": 1.4913065433502197,
+ "mean_token_accuracy": 0.658707857131958,
+ "num_tokens": 4931584.0,
+ "step": 301
+ },
+ {
+ "entropy": 1.4020944833755493,
+ "epoch": 0.6101010101010101,
+ "grad_norm": 2.1152141094207764,
+ "learning_rate": 1.594612794612795e-05,
+ "loss": 1.4000526666641235,
+ "mean_token_accuracy": 0.656509518623352,
+ "num_tokens": 4947968.0,
+ "step": 302
+ },
+ {
+ "entropy": 1.4965202808380127,
+ "epoch": 0.6121212121212121,
+ "grad_norm": 2.1385467052459717,
+ "learning_rate": 1.5932659932659933e-05,
+ "loss": 1.5221803188323975,
+ "mean_token_accuracy": 0.6567537784576416,
+ "num_tokens": 4964352.0,
+ "step": 303
+ },
+ {
+ "entropy": 1.4024577140808105,
+ "epoch": 0.6141414141414141,
+ "grad_norm": 1.9162694215774536,
+ "learning_rate": 1.591919191919192e-05,
+ "loss": 1.4050084352493286,
+ "mean_token_accuracy": 0.6670737862586975,
+ "num_tokens": 4980736.0,
+ "step": 304
+ },
+ {
+ "entropy": 1.430273413658142,
+ "epoch": 0.6161616161616161,
+ "grad_norm": 2.0312552452087402,
+ "learning_rate": 1.5905723905723906e-05,
+ "loss": 1.418628454208374,
+ "mean_token_accuracy": 0.6585857272148132,
+ "num_tokens": 4997120.0,
+ "step": 305
+ },
+ {
+ "entropy": 1.700549602508545,
+ "epoch": 0.6181818181818182,
+ "grad_norm": 2.080810546875,
+ "learning_rate": 1.5892255892255895e-05,
+ "loss": 1.740492820739746,
+ "mean_token_accuracy": 0.6042989492416382,
+ "num_tokens": 5013504.0,
+ "step": 306
+ },
+ {
+ "entropy": 1.6465320587158203,
+ "epoch": 0.6202020202020202,
+ "grad_norm": 1.8810964822769165,
+ "learning_rate": 1.587878787878788e-05,
+ "loss": 1.6414886713027954,
+ "mean_token_accuracy": 0.629824161529541,
+ "num_tokens": 5029888.0,
+ "step": 307
+ },
+ {
+ "entropy": 1.3717931509017944,
+ "epoch": 0.6222222222222222,
+ "grad_norm": 1.921536922454834,
+ "learning_rate": 1.5865319865319868e-05,
+ "loss": 1.3700361251831055,
+ "mean_token_accuracy": 0.6630434989929199,
+ "num_tokens": 5046272.0,
+ "step": 308
+ },
+ {
+ "entropy": 1.4906446933746338,
+ "epoch": 0.6242424242424243,
+ "grad_norm": 2.1185555458068848,
+ "learning_rate": 1.5851851851851852e-05,
+ "loss": 1.4850339889526367,
+ "mean_token_accuracy": 0.6402662396430969,
+ "num_tokens": 5062656.0,
+ "step": 309
+ },
+ {
+ "entropy": 1.4417473077774048,
+ "epoch": 0.6262626262626263,
+ "grad_norm": 2.0196895599365234,
+ "learning_rate": 1.583838383838384e-05,
+ "loss": 1.4567816257476807,
+ "mean_token_accuracy": 0.6760503053665161,
+ "num_tokens": 5079040.0,
+ "step": 310
+ },
+ {
+ "entropy": 1.4662094116210938,
+ "epoch": 0.6282828282828283,
+ "grad_norm": 1.8859882354736328,
+ "learning_rate": 1.5824915824915825e-05,
+ "loss": 1.4870874881744385,
+ "mean_token_accuracy": 0.6618221998214722,
+ "num_tokens": 5095424.0,
+ "step": 311
+ },
+ {
+ "entropy": 1.8497077226638794,
+ "epoch": 0.6303030303030303,
+ "grad_norm": 1.974537968635559,
+ "learning_rate": 1.5811447811447813e-05,
+ "loss": 1.840538501739502,
+ "mean_token_accuracy": 0.589093804359436,
+ "num_tokens": 5111808.0,
+ "step": 312
+ },
+ {
+ "entropy": 1.4185130596160889,
+ "epoch": 0.6323232323232323,
+ "grad_norm": 1.8612359762191772,
+ "learning_rate": 1.5797979797979798e-05,
+ "loss": 1.4404246807098389,
+ "mean_token_accuracy": 0.664631187915802,
+ "num_tokens": 5128192.0,
+ "step": 313
+ },
+ {
+ "entropy": 1.8270463943481445,
+ "epoch": 0.6343434343434343,
+ "grad_norm": 2.1050970554351807,
+ "learning_rate": 1.5784511784511786e-05,
+ "loss": 1.8682940006256104,
+ "mean_token_accuracy": 0.5893380641937256,
+ "num_tokens": 5144576.0,
+ "step": 314
+ },
+ {
+ "entropy": 1.5919326543807983,
+ "epoch": 0.6363636363636364,
+ "grad_norm": 1.8464620113372803,
+ "learning_rate": 1.577104377104377e-05,
+ "loss": 1.6087713241577148,
+ "mean_token_accuracy": 0.6350146532058716,
+ "num_tokens": 5160960.0,
+ "step": 315
+ },
+ {
+ "entropy": 1.4776946306228638,
+ "epoch": 0.6383838383838384,
+ "grad_norm": 2.031834602355957,
+ "learning_rate": 1.575757575757576e-05,
+ "loss": 1.5114507675170898,
+ "mean_token_accuracy": 0.6498534679412842,
+ "num_tokens": 5177344.0,
+ "step": 316
+ },
+ {
+ "entropy": 1.3789927959442139,
+ "epoch": 0.6404040404040404,
+ "grad_norm": 1.8604289293289185,
+ "learning_rate": 1.5744107744107748e-05,
+ "loss": 1.3792052268981934,
+ "mean_token_accuracy": 0.6712262034416199,
+ "num_tokens": 5193728.0,
+ "step": 317
+ },
+ {
+ "entropy": 1.5859184265136719,
+ "epoch": 0.6424242424242425,
+ "grad_norm": 2.1134088039398193,
+ "learning_rate": 1.5730639730639732e-05,
+ "loss": 1.595099925994873,
+ "mean_token_accuracy": 0.6351367831230164,
+ "num_tokens": 5210112.0,
+ "step": 318
+ },
+ {
+ "entropy": 1.3681901693344116,
+ "epoch": 0.6444444444444445,
+ "grad_norm": 1.9200235605239868,
+ "learning_rate": 1.5717171717171717e-05,
+ "loss": 1.3700852394104004,
+ "mean_token_accuracy": 0.6671959161758423,
+ "num_tokens": 5226496.0,
+ "step": 319
+ },
+ {
+ "entropy": 1.594885230064392,
+ "epoch": 0.6464646464646465,
+ "grad_norm": 1.7683591842651367,
+ "learning_rate": 1.5703703703703705e-05,
+ "loss": 1.5906047821044922,
+ "mean_token_accuracy": 0.6350757479667664,
+ "num_tokens": 5242880.0,
+ "step": 320
+ },
+ {
+ "entropy": 1.240937352180481,
+ "epoch": 0.6484848484848484,
+ "grad_norm": 2.117145299911499,
+ "learning_rate": 1.569023569023569e-05,
+ "loss": 1.2188963890075684,
+ "mean_token_accuracy": 0.6958353519439697,
+ "num_tokens": 5259264.0,
+ "step": 321
+ },
+ {
+ "entropy": 1.244801640510559,
+ "epoch": 0.6505050505050505,
+ "grad_norm": 1.983111023902893,
+ "learning_rate": 1.5676767676767678e-05,
+ "loss": 1.246530294418335,
+ "mean_token_accuracy": 0.6966902613639832,
+ "num_tokens": 5275648.0,
+ "step": 322
+ },
+ {
+ "entropy": 1.5355960130691528,
+ "epoch": 0.6525252525252525,
+ "grad_norm": 2.126024007797241,
+ "learning_rate": 1.5663299663299666e-05,
+ "loss": 1.576147198677063,
+ "mean_token_accuracy": 0.6414265036582947,
+ "num_tokens": 5292032.0,
+ "step": 323
+ },
+ {
+ "entropy": 1.13125741481781,
+ "epoch": 0.6545454545454545,
+ "grad_norm": 1.8933371305465698,
+ "learning_rate": 1.564983164983165e-05,
+ "loss": 1.1167645454406738,
+ "mean_token_accuracy": 0.7080483436584473,
+ "num_tokens": 5308416.0,
+ "step": 324
+ },
+ {
+ "entropy": 1.6553086042404175,
+ "epoch": 0.6565656565656566,
+ "grad_norm": 1.9253427982330322,
+ "learning_rate": 1.563636363636364e-05,
+ "loss": 1.6746933460235596,
+ "mean_token_accuracy": 0.6138250827789307,
+ "num_tokens": 5324800.0,
+ "step": 325
+ },
+ {
+ "entropy": 1.7187142372131348,
+ "epoch": 0.6585858585858586,
+ "grad_norm": 1.8655303716659546,
+ "learning_rate": 1.5622895622895624e-05,
+ "loss": 1.7368483543395996,
+ "mean_token_accuracy": 0.6073521971702576,
+ "num_tokens": 5341184.0,
+ "step": 326
+ },
+ {
+ "entropy": 1.4150934219360352,
+ "epoch": 0.6606060606060606,
+ "grad_norm": 1.8396921157836914,
+ "learning_rate": 1.560942760942761e-05,
+ "loss": 1.450613260269165,
+ "mean_token_accuracy": 0.6628602743148804,
+ "num_tokens": 5357568.0,
+ "step": 327
+ },
+ {
+ "entropy": 1.6463909149169922,
+ "epoch": 0.6626262626262627,
+ "grad_norm": 2.0179622173309326,
+ "learning_rate": 1.5595959595959597e-05,
+ "loss": 1.6767137050628662,
+ "mean_token_accuracy": 0.6147410869598389,
+ "num_tokens": 5373952.0,
+ "step": 328
+ },
+ {
+ "entropy": 1.2962068319320679,
+ "epoch": 0.6646464646464646,
+ "grad_norm": 1.9322903156280518,
+ "learning_rate": 1.5582491582491582e-05,
+ "loss": 1.288381814956665,
+ "mean_token_accuracy": 0.6877137422561646,
+ "num_tokens": 5390336.0,
+ "step": 329
+ },
+ {
+ "entropy": 1.4675830602645874,
+ "epoch": 0.6666666666666666,
+ "grad_norm": 1.7735666036605835,
+ "learning_rate": 1.556902356902357e-05,
+ "loss": 1.4585455656051636,
+ "mean_token_accuracy": 0.671775758266449,
+ "num_tokens": 5406720.0,
+ "step": 330
+ },
+ {
+ "entropy": 1.55418062210083,
+ "epoch": 0.6686868686868687,
+ "grad_norm": 1.9255602359771729,
+ "learning_rate": 1.555555555555556e-05,
+ "loss": 1.6033880710601807,
+ "mean_token_accuracy": 0.6486321687698364,
+ "num_tokens": 5423104.0,
+ "step": 331
+ },
+ {
+ "entropy": 1.4288218021392822,
+ "epoch": 0.6707070707070707,
+ "grad_norm": 1.8075612783432007,
+ "learning_rate": 1.5542087542087543e-05,
+ "loss": 1.457895278930664,
+ "mean_token_accuracy": 0.6725085377693176,
+ "num_tokens": 5439488.0,
+ "step": 332
+ },
+ {
+ "entropy": 1.3419297933578491,
+ "epoch": 0.6727272727272727,
+ "grad_norm": 2.1730353832244873,
+ "learning_rate": 1.552861952861953e-05,
+ "loss": 1.3681403398513794,
+ "mean_token_accuracy": 0.6664020419120789,
+ "num_tokens": 5455872.0,
+ "step": 333
+ },
+ {
+ "entropy": 1.706923007965088,
+ "epoch": 0.6747474747474748,
+ "grad_norm": 1.9297505617141724,
+ "learning_rate": 1.5515151515151516e-05,
+ "loss": 1.7304211854934692,
+ "mean_token_accuracy": 0.6020395755767822,
+ "num_tokens": 5472256.0,
+ "step": 334
+ },
+ {
+ "entropy": 1.117404818534851,
+ "epoch": 0.6767676767676768,
+ "grad_norm": 1.7363555431365967,
+ "learning_rate": 1.55016835016835e-05,
+ "loss": 1.1002353429794312,
+ "mean_token_accuracy": 0.7285661697387695,
+ "num_tokens": 5488640.0,
+ "step": 335
+ },
+ {
+ "entropy": 1.409253478050232,
+ "epoch": 0.6787878787878788,
+ "grad_norm": 2.029304265975952,
+ "learning_rate": 1.548821548821549e-05,
+ "loss": 1.4001437425613403,
+ "mean_token_accuracy": 0.6604176759719849,
+ "num_tokens": 5505024.0,
+ "step": 336
+ },
+ {
+ "entropy": 1.0427494049072266,
+ "epoch": 0.6808080808080809,
+ "grad_norm": 2.0266847610473633,
+ "learning_rate": 1.5474747474747477e-05,
+ "loss": 1.0636063814163208,
+ "mean_token_accuracy": 0.7298485636711121,
+ "num_tokens": 5521408.0,
+ "step": 337
+ },
+ {
+ "entropy": 1.479506254196167,
+ "epoch": 0.6828282828282828,
+ "grad_norm": 1.937677264213562,
+ "learning_rate": 1.5461279461279462e-05,
+ "loss": 1.4711894989013672,
+ "mean_token_accuracy": 0.6436248421669006,
+ "num_tokens": 5537792.0,
+ "step": 338
+ },
+ {
+ "entropy": 1.636492371559143,
+ "epoch": 0.6848484848484848,
+ "grad_norm": 1.9746829271316528,
+ "learning_rate": 1.544781144781145e-05,
+ "loss": 1.638418436050415,
+ "mean_token_accuracy": 0.6204200983047485,
+ "num_tokens": 5554176.0,
+ "step": 339
+ },
+ {
+ "entropy": 1.6094971895217896,
+ "epoch": 0.6868686868686869,
+ "grad_norm": 1.998382806777954,
+ "learning_rate": 1.5434343434343435e-05,
+ "loss": 1.6025879383087158,
+ "mean_token_accuracy": 0.6193209290504456,
+ "num_tokens": 5570560.0,
+ "step": 340
+ },
+ {
+ "entropy": 1.3113542795181274,
+ "epoch": 0.6888888888888889,
+ "grad_norm": 2.059739589691162,
+ "learning_rate": 1.5420875420875423e-05,
+ "loss": 1.32335364818573,
+ "mean_token_accuracy": 0.6776379942893982,
+ "num_tokens": 5586944.0,
+ "step": 341
+ },
+ {
+ "entropy": 1.3761622905731201,
+ "epoch": 0.6909090909090909,
+ "grad_norm": 1.9774043560028076,
+ "learning_rate": 1.5407407407407408e-05,
+ "loss": 1.3731458187103271,
+ "mean_token_accuracy": 0.681546151638031,
+ "num_tokens": 5603328.0,
+ "step": 342
+ },
+ {
+ "entropy": 1.3822734355926514,
+ "epoch": 0.692929292929293,
+ "grad_norm": 5.41459321975708,
+ "learning_rate": 1.5393939393939393e-05,
+ "loss": 1.4077341556549072,
+ "mean_token_accuracy": 0.6688446402549744,
+ "num_tokens": 5619712.0,
+ "step": 343
+ },
+ {
+ "entropy": 1.4885843992233276,
+ "epoch": 0.694949494949495,
+ "grad_norm": 1.9256173372268677,
+ "learning_rate": 1.538047138047138e-05,
+ "loss": 1.5017898082733154,
+ "mean_token_accuracy": 0.6411211490631104,
+ "num_tokens": 5636096.0,
+ "step": 344
+ },
+ {
+ "entropy": 1.4558689594268799,
+ "epoch": 0.696969696969697,
+ "grad_norm": 1.923056721687317,
+ "learning_rate": 1.536700336700337e-05,
+ "loss": 1.4409637451171875,
+ "mean_token_accuracy": 0.6607840657234192,
+ "num_tokens": 5652480.0,
+ "step": 345
+ },
+ {
+ "entropy": 1.3396257162094116,
+ "epoch": 0.6989898989898989,
+ "grad_norm": 1.9321191310882568,
+ "learning_rate": 1.5353535353535354e-05,
+ "loss": 1.3465441465377808,
+ "mean_token_accuracy": 0.6682339906692505,
+ "num_tokens": 5668864.0,
+ "step": 346
+ },
+ {
+ "entropy": 1.931525468826294,
+ "epoch": 0.701010101010101,
+ "grad_norm": 2.11252498626709,
+ "learning_rate": 1.5340067340067342e-05,
+ "loss": 1.947523832321167,
+ "mean_token_accuracy": 0.5751709938049316,
+ "num_tokens": 5685248.0,
+ "step": 347
+ },
+ {
+ "entropy": 1.4246045351028442,
+ "epoch": 0.703030303030303,
+ "grad_norm": 2.017690658569336,
+ "learning_rate": 1.5326599326599327e-05,
+ "loss": 1.431095004081726,
+ "mean_token_accuracy": 0.6621274948120117,
+ "num_tokens": 5701632.0,
+ "step": 348
+ },
+ {
+ "entropy": 1.2840481996536255,
+ "epoch": 0.705050505050505,
+ "grad_norm": 1.897916555404663,
+ "learning_rate": 1.5313131313131315e-05,
+ "loss": 1.2944300174713135,
+ "mean_token_accuracy": 0.7013311982154846,
+ "num_tokens": 5718016.0,
+ "step": 349
+ },
+ {
+ "entropy": 1.1308226585388184,
+ "epoch": 0.7070707070707071,
+ "grad_norm": 1.8326802253723145,
+ "learning_rate": 1.52996632996633e-05,
+ "loss": 1.1564277410507202,
+ "mean_token_accuracy": 0.714154839515686,
+ "num_tokens": 5734400.0,
+ "step": 350
+ },
+ {
+ "entropy": 1.4157475233078003,
+ "epoch": 0.7090909090909091,
+ "grad_norm": 1.9305709600448608,
+ "learning_rate": 1.5286195286195288e-05,
+ "loss": 1.421119213104248,
+ "mean_token_accuracy": 0.6651197075843811,
+ "num_tokens": 5750784.0,
+ "step": 351
+ },
+ {
+ "entropy": 1.353966474533081,
+ "epoch": 0.7111111111111111,
+ "grad_norm": 2.058542251586914,
+ "learning_rate": 1.5272727272727276e-05,
+ "loss": 1.3575413227081299,
+ "mean_token_accuracy": 0.6724475026130676,
+ "num_tokens": 5767168.0,
+ "step": 352
+ },
+ {
+ "entropy": 1.7946882247924805,
+ "epoch": 0.7131313131313132,
+ "grad_norm": 1.9397060871124268,
+ "learning_rate": 1.525925925925926e-05,
+ "loss": 1.7882269620895386,
+ "mean_token_accuracy": 0.6064972877502441,
+ "num_tokens": 5783552.0,
+ "step": 353
+ },
+ {
+ "entropy": 1.4986019134521484,
+ "epoch": 0.7151515151515152,
+ "grad_norm": 1.9084810018539429,
+ "learning_rate": 1.5245791245791246e-05,
+ "loss": 1.5188498497009277,
+ "mean_token_accuracy": 0.6610283255577087,
+ "num_tokens": 5799936.0,
+ "step": 354
+ },
+ {
+ "entropy": 1.9081071615219116,
+ "epoch": 0.7171717171717171,
+ "grad_norm": 2.1528689861297607,
+ "learning_rate": 1.5232323232323234e-05,
+ "loss": 1.8967041969299316,
+ "mean_token_accuracy": 0.5691255331039429,
+ "num_tokens": 5816320.0,
+ "step": 355
+ },
+ {
+ "entropy": 1.641337513923645,
+ "epoch": 0.7191919191919192,
+ "grad_norm": 1.9197942018508911,
+ "learning_rate": 1.521885521885522e-05,
+ "loss": 1.6193110942840576,
+ "mean_token_accuracy": 0.6286638975143433,
+ "num_tokens": 5832704.0,
+ "step": 356
+ },
+ {
+ "entropy": 1.348304271697998,
+ "epoch": 0.7212121212121212,
+ "grad_norm": 1.9729533195495605,
+ "learning_rate": 1.5205387205387207e-05,
+ "loss": 1.3781111240386963,
+ "mean_token_accuracy": 0.6654250025749207,
+ "num_tokens": 5849088.0,
+ "step": 357
+ },
+ {
+ "entropy": 1.5427544116973877,
+ "epoch": 0.7232323232323232,
+ "grad_norm": 2.0730769634246826,
+ "learning_rate": 1.5191919191919193e-05,
+ "loss": 1.5770654678344727,
+ "mean_token_accuracy": 0.6273815631866455,
+ "num_tokens": 5865472.0,
+ "step": 358
+ },
+ {
+ "entropy": 1.5560660362243652,
+ "epoch": 0.7252525252525253,
+ "grad_norm": 2.0436644554138184,
+ "learning_rate": 1.5178451178451178e-05,
+ "loss": 1.5465143918991089,
+ "mean_token_accuracy": 0.6335490942001343,
+ "num_tokens": 5881856.0,
+ "step": 359
+ },
+ {
+ "entropy": 1.5396223068237305,
+ "epoch": 0.7272727272727273,
+ "grad_norm": 1.9955254793167114,
+ "learning_rate": 1.5164983164983166e-05,
+ "loss": 1.5742223262786865,
+ "mean_token_accuracy": 0.6344650983810425,
+ "num_tokens": 5898240.0,
+ "step": 360
+ },
+ {
+ "entropy": 1.1664644479751587,
+ "epoch": 0.7292929292929293,
+ "grad_norm": 1.6942992210388184,
+ "learning_rate": 1.5151515151515153e-05,
+ "loss": 1.159040927886963,
+ "mean_token_accuracy": 0.7144601941108704,
+ "num_tokens": 5914624.0,
+ "step": 361
+ },
+ {
+ "entropy": 1.356544017791748,
+ "epoch": 0.7313131313131314,
+ "grad_norm": 1.890787124633789,
+ "learning_rate": 1.5138047138047138e-05,
+ "loss": 1.347895622253418,
+ "mean_token_accuracy": 0.6813629865646362,
+ "num_tokens": 5931008.0,
+ "step": 362
+ },
+ {
+ "entropy": 1.6837173700332642,
+ "epoch": 0.7333333333333333,
+ "grad_norm": 2.1780381202697754,
+ "learning_rate": 1.5124579124579126e-05,
+ "loss": 1.6997990608215332,
+ "mean_token_accuracy": 0.6098558902740479,
+ "num_tokens": 5947392.0,
+ "step": 363
+ },
+ {
+ "entropy": 1.1412957906723022,
+ "epoch": 0.7353535353535353,
+ "grad_norm": 1.801152229309082,
+ "learning_rate": 1.5111111111111112e-05,
+ "loss": 1.161649227142334,
+ "mean_token_accuracy": 0.705483615398407,
+ "num_tokens": 5963776.0,
+ "step": 364
+ },
+ {
+ "entropy": 1.2495859861373901,
+ "epoch": 0.7373737373737373,
+ "grad_norm": 1.8535690307617188,
+ "learning_rate": 1.50976430976431e-05,
+ "loss": 1.2595994472503662,
+ "mean_token_accuracy": 0.6926599740982056,
+ "num_tokens": 5980160.0,
+ "step": 365
+ },
+ {
+ "entropy": 1.8037965297698975,
+ "epoch": 0.7393939393939394,
+ "grad_norm": 2.0506303310394287,
+ "learning_rate": 1.5084175084175085e-05,
+ "loss": 1.8499953746795654,
+ "mean_token_accuracy": 0.5861626863479614,
+ "num_tokens": 5996544.0,
+ "step": 366
+ },
+ {
+ "entropy": 1.2877405881881714,
+ "epoch": 0.7414141414141414,
+ "grad_norm": 1.8159914016723633,
+ "learning_rate": 1.5070707070707072e-05,
+ "loss": 1.3165576457977295,
+ "mean_token_accuracy": 0.685271143913269,
+ "num_tokens": 6012928.0,
+ "step": 367
+ },
+ {
+ "entropy": 1.345953106880188,
+ "epoch": 0.7434343434343434,
+ "grad_norm": 2.003962516784668,
+ "learning_rate": 1.505723905723906e-05,
+ "loss": 1.3270621299743652,
+ "mean_token_accuracy": 0.6671348214149475,
+ "num_tokens": 6029312.0,
+ "step": 368
+ },
+ {
+ "entropy": 1.408361792564392,
+ "epoch": 0.7454545454545455,
+ "grad_norm": 1.9815948009490967,
+ "learning_rate": 1.5043771043771045e-05,
+ "loss": 1.4097518920898438,
+ "mean_token_accuracy": 0.652662456035614,
+ "num_tokens": 6045696.0,
+ "step": 369
+ },
+ {
+ "entropy": 1.4007829427719116,
+ "epoch": 0.7474747474747475,
+ "grad_norm": 1.928751826286316,
+ "learning_rate": 1.5030303030303031e-05,
+ "loss": 1.4142816066741943,
+ "mean_token_accuracy": 0.671775758266449,
+ "num_tokens": 6062080.0,
+ "step": 370
+ },
+ {
+ "entropy": 1.7548134326934814,
+ "epoch": 0.7494949494949495,
+ "grad_norm": 2.081939458847046,
+ "learning_rate": 1.5016835016835018e-05,
+ "loss": 1.792219877243042,
+ "mean_token_accuracy": 0.6044821739196777,
+ "num_tokens": 6078464.0,
+ "step": 371
+ },
+ {
+ "entropy": 1.6162793636322021,
+ "epoch": 0.7515151515151515,
+ "grad_norm": 1.8544763326644897,
+ "learning_rate": 1.5003367003367004e-05,
+ "loss": 1.650557041168213,
+ "mean_token_accuracy": 0.6289692521095276,
+ "num_tokens": 6094848.0,
+ "step": 372
+ },
+ {
+ "entropy": 1.6329439878463745,
+ "epoch": 0.7535353535353535,
+ "grad_norm": 1.8025282621383667,
+ "learning_rate": 1.4989898989898992e-05,
+ "loss": 1.6812629699707031,
+ "mean_token_accuracy": 0.6405104994773865,
+ "num_tokens": 6111232.0,
+ "step": 373
+ },
+ {
+ "entropy": 1.2201604843139648,
+ "epoch": 0.7555555555555555,
+ "grad_norm": 1.9814586639404297,
+ "learning_rate": 1.4976430976430977e-05,
+ "loss": 1.2086223363876343,
+ "mean_token_accuracy": 0.7020029425621033,
+ "num_tokens": 6127616.0,
+ "step": 374
+ },
+ {
+ "entropy": 1.8105695247650146,
+ "epoch": 0.7575757575757576,
+ "grad_norm": 2.0217790603637695,
+ "learning_rate": 1.4962962962962964e-05,
+ "loss": 1.8443559408187866,
+ "mean_token_accuracy": 0.5897654891014099,
+ "num_tokens": 6144000.0,
+ "step": 375
+ },
+ {
+ "entropy": 1.197646141052246,
+ "epoch": 0.7595959595959596,
+ "grad_norm": 1.9286304712295532,
+ "learning_rate": 1.4949494949494952e-05,
+ "loss": 1.2183899879455566,
+ "mean_token_accuracy": 0.6925989389419556,
+ "num_tokens": 6160384.0,
+ "step": 376
+ },
+ {
+ "entropy": 1.4825936555862427,
+ "epoch": 0.7616161616161616,
+ "grad_norm": 1.6694327592849731,
+ "learning_rate": 1.4936026936026937e-05,
+ "loss": 1.4815235137939453,
+ "mean_token_accuracy": 0.6572422981262207,
+ "num_tokens": 6176768.0,
+ "step": 377
+ },
+ {
+ "entropy": 1.3497473001480103,
+ "epoch": 0.7636363636363637,
+ "grad_norm": 2.0366406440734863,
+ "learning_rate": 1.4922558922558923e-05,
+ "loss": 1.3637127876281738,
+ "mean_token_accuracy": 0.6690889000892639,
+ "num_tokens": 6193152.0,
+ "step": 378
+ },
+ {
+ "entropy": 1.314116358757019,
+ "epoch": 0.7656565656565657,
+ "grad_norm": 1.742874026298523,
+ "learning_rate": 1.4909090909090911e-05,
+ "loss": 1.300222396850586,
+ "mean_token_accuracy": 0.6977283954620361,
+ "num_tokens": 6209536.0,
+ "step": 379
+ },
+ {
+ "entropy": 1.4347912073135376,
+ "epoch": 0.7676767676767676,
+ "grad_norm": 1.9317418336868286,
+ "learning_rate": 1.4895622895622896e-05,
+ "loss": 1.429826021194458,
+ "mean_token_accuracy": 0.6650586128234863,
+ "num_tokens": 6225920.0,
+ "step": 380
+ },
+ {
+ "entropy": 1.448641061782837,
+ "epoch": 0.7696969696969697,
+ "grad_norm": 1.8417843580245972,
+ "learning_rate": 1.4882154882154884e-05,
+ "loss": 1.4439187049865723,
+ "mean_token_accuracy": 0.6615168452262878,
+ "num_tokens": 6242304.0,
+ "step": 381
+ },
+ {
+ "entropy": 1.2918578386306763,
+ "epoch": 0.7717171717171717,
+ "grad_norm": 1.8720030784606934,
+ "learning_rate": 1.486868686868687e-05,
+ "loss": 1.3158597946166992,
+ "mean_token_accuracy": 0.6865534782409668,
+ "num_tokens": 6258688.0,
+ "step": 382
+ },
+ {
+ "entropy": 1.5822116136550903,
+ "epoch": 0.7737373737373737,
+ "grad_norm": 1.9301713705062866,
+ "learning_rate": 1.4855218855218856e-05,
+ "loss": 1.6046046018600464,
+ "mean_token_accuracy": 0.6361138224601746,
+ "num_tokens": 6275072.0,
+ "step": 383
+ },
+ {
+ "entropy": 1.6837408542633057,
+ "epoch": 0.7757575757575758,
+ "grad_norm": 2.270636796951294,
+ "learning_rate": 1.4841750841750844e-05,
+ "loss": 1.7323150634765625,
+ "mean_token_accuracy": 0.6006350517272949,
+ "num_tokens": 6291456.0,
+ "step": 384
+ },
+ {
+ "entropy": 1.4837889671325684,
+ "epoch": 0.7777777777777778,
+ "grad_norm": 1.908146619796753,
+ "learning_rate": 1.482828282828283e-05,
+ "loss": 1.4938652515411377,
+ "mean_token_accuracy": 0.64667809009552,
+ "num_tokens": 6307840.0,
+ "step": 385
+ },
+ {
+ "entropy": 1.680192232131958,
+ "epoch": 0.7797979797979798,
+ "grad_norm": 2.2369024753570557,
+ "learning_rate": 1.4814814814814815e-05,
+ "loss": 1.692443609237671,
+ "mean_token_accuracy": 0.6279922127723694,
+ "num_tokens": 6324224.0,
+ "step": 386
+ },
+ {
+ "entropy": 1.686415195465088,
+ "epoch": 0.7818181818181819,
+ "grad_norm": 1.8252373933792114,
+ "learning_rate": 1.4801346801346803e-05,
+ "loss": 1.7022712230682373,
+ "mean_token_accuracy": 0.6302515864372253,
+ "num_tokens": 6340608.0,
+ "step": 387
+ },
+ {
+ "entropy": 1.6967185735702515,
+ "epoch": 0.7838383838383839,
+ "grad_norm": 2.102458953857422,
+ "learning_rate": 1.478787878787879e-05,
+ "loss": 1.6931922435760498,
+ "mean_token_accuracy": 0.6149853467941284,
+ "num_tokens": 6356992.0,
+ "step": 388
+ },
+ {
+ "entropy": 1.201438307762146,
+ "epoch": 0.7858585858585858,
+ "grad_norm": 1.8396239280700684,
+ "learning_rate": 1.4774410774410774e-05,
+ "loss": 1.2081820964813232,
+ "mean_token_accuracy": 0.6987664699554443,
+ "num_tokens": 6373376.0,
+ "step": 389
+ },
+ {
+ "entropy": 1.5559548139572144,
+ "epoch": 0.7878787878787878,
+ "grad_norm": 2.063498020172119,
+ "learning_rate": 1.4760942760942763e-05,
+ "loss": 1.545433759689331,
+ "mean_token_accuracy": 0.6267709136009216,
+ "num_tokens": 6389760.0,
+ "step": 390
+ },
+ {
+ "entropy": 1.7186503410339355,
+ "epoch": 0.7898989898989899,
+ "grad_norm": 2.0129787921905518,
+ "learning_rate": 1.4747474747474747e-05,
+ "loss": 1.717472791671753,
+ "mean_token_accuracy": 0.6540058851242065,
+ "num_tokens": 6406144.0,
+ "step": 391
+ },
+ {
+ "entropy": 1.643048882484436,
+ "epoch": 0.7919191919191919,
+ "grad_norm": 1.9443073272705078,
+ "learning_rate": 1.4734006734006736e-05,
+ "loss": 1.6441841125488281,
+ "mean_token_accuracy": 0.6458231806755066,
+ "num_tokens": 6422528.0,
+ "step": 392
+ },
+ {
+ "entropy": 1.3627581596374512,
+ "epoch": 0.793939393939394,
+ "grad_norm": 1.9852598905563354,
+ "learning_rate": 1.4720538720538722e-05,
+ "loss": 1.3558071851730347,
+ "mean_token_accuracy": 0.6720200181007385,
+ "num_tokens": 6438912.0,
+ "step": 393
+ },
+ {
+ "entropy": 0.8946540951728821,
+ "epoch": 0.795959595959596,
+ "grad_norm": 1.54441237449646,
+ "learning_rate": 1.4707070707070707e-05,
+ "loss": 0.8874151706695557,
+ "mean_token_accuracy": 0.7685027122497559,
+ "num_tokens": 6455296.0,
+ "step": 394
+ },
+ {
+ "entropy": 1.35861074924469,
+ "epoch": 0.797979797979798,
+ "grad_norm": 2.158553123474121,
+ "learning_rate": 1.4693602693602695e-05,
+ "loss": 1.4291828870773315,
+ "mean_token_accuracy": 0.6647533178329468,
+ "num_tokens": 6471680.0,
+ "step": 395
+ },
+ {
+ "entropy": 1.7047442197799683,
+ "epoch": 0.8,
+ "grad_norm": 2.1242175102233887,
+ "learning_rate": 1.4680134680134681e-05,
+ "loss": 1.7065966129302979,
+ "mean_token_accuracy": 0.6058256030082703,
+ "num_tokens": 6488064.0,
+ "step": 396
+ },
+ {
+ "entropy": 1.7153913974761963,
+ "epoch": 0.802020202020202,
+ "grad_norm": 1.9085419178009033,
+ "learning_rate": 1.4666666666666666e-05,
+ "loss": 1.7373592853546143,
+ "mean_token_accuracy": 0.6020395755767822,
+ "num_tokens": 6504448.0,
+ "step": 397
+ },
+ {
+ "entropy": 1.1062475442886353,
+ "epoch": 0.804040404040404,
+ "grad_norm": 1.9173986911773682,
+ "learning_rate": 1.4653198653198654e-05,
+ "loss": 1.0991405248641968,
+ "mean_token_accuracy": 0.7195896506309509,
+ "num_tokens": 6520832.0,
+ "step": 398
+ },
+ {
+ "entropy": 1.5717406272888184,
+ "epoch": 0.806060606060606,
+ "grad_norm": 2.0077219009399414,
+ "learning_rate": 1.4639730639730641e-05,
+ "loss": 1.595954418182373,
+ "mean_token_accuracy": 0.6270151734352112,
+ "num_tokens": 6537216.0,
+ "step": 399
+ },
+ {
+ "entropy": 1.3960795402526855,
+ "epoch": 0.8080808080808081,
+ "grad_norm": 2.2864415645599365,
+ "learning_rate": 1.4626262626262629e-05,
+ "loss": 1.416130781173706,
+ "mean_token_accuracy": 0.6631656289100647,
+ "num_tokens": 6553600.0,
+ "step": 400
+ },
+ {
+ "epoch": 0.8080808080808081,
+ "eval_entropy": 1.510355769626556,
+ "eval_loss": 1.5199862718582153,
+ "eval_mean_token_accuracy": 0.6461201098657423,
+ "eval_num_tokens": 6553600.0,
+ "eval_runtime": 43.8634,
+ "eval_samples_per_second": 22.57,
+ "eval_steps_per_second": 2.827,
+ "step": 400
+ },
+ {
+ "entropy": 1.5662645101547241,
+ "epoch": 0.8101010101010101,
+ "grad_norm": 2.1806085109710693,
+ "learning_rate": 1.4612794612794614e-05,
+ "loss": 1.5763804912567139,
+ "mean_token_accuracy": 0.6330605745315552,
+ "num_tokens": 6569984.0,
+ "step": 401
+ },
+ {
+ "entropy": 1.4046087265014648,
+ "epoch": 0.8121212121212121,
+ "grad_norm": 1.9650650024414062,
+ "learning_rate": 1.45993265993266e-05,
+ "loss": 1.396510124206543,
+ "mean_token_accuracy": 0.6568148732185364,
+ "num_tokens": 6586368.0,
+ "step": 402
+ },
+ {
+ "entropy": 1.084455966949463,
+ "epoch": 0.8141414141414142,
+ "grad_norm": 1.926537275314331,
+ "learning_rate": 1.4585858585858587e-05,
+ "loss": 1.096333622932434,
+ "mean_token_accuracy": 0.7233145833015442,
+ "num_tokens": 6602752.0,
+ "step": 403
+ },
+ {
+ "entropy": 1.379700779914856,
+ "epoch": 0.8161616161616162,
+ "grad_norm": 1.957202672958374,
+ "learning_rate": 1.4572390572390573e-05,
+ "loss": 1.4035027027130127,
+ "mean_token_accuracy": 0.674462616443634,
+ "num_tokens": 6619136.0,
+ "step": 404
+ },
+ {
+ "entropy": 1.3775221109390259,
+ "epoch": 0.8181818181818182,
+ "grad_norm": 1.9095491170883179,
+ "learning_rate": 1.455892255892256e-05,
+ "loss": 1.3915553092956543,
+ "mean_token_accuracy": 0.6811187267303467,
+ "num_tokens": 6635520.0,
+ "step": 405
+ },
+ {
+ "entropy": 1.4745222330093384,
+ "epoch": 0.8202020202020202,
+ "grad_norm": 2.0017123222351074,
+ "learning_rate": 1.4545454545454546e-05,
+ "loss": 1.4845868349075317,
+ "mean_token_accuracy": 0.6557767391204834,
+ "num_tokens": 6651904.0,
+ "step": 406
+ },
+ {
+ "entropy": 1.5129855871200562,
+ "epoch": 0.8222222222222222,
+ "grad_norm": 2.127979040145874,
+ "learning_rate": 1.4531986531986533e-05,
+ "loss": 1.5081605911254883,
+ "mean_token_accuracy": 0.6425256729125977,
+ "num_tokens": 6668288.0,
+ "step": 407
+ },
+ {
+ "entropy": 1.1438099145889282,
+ "epoch": 0.8242424242424242,
+ "grad_norm": 1.7389986515045166,
+ "learning_rate": 1.4518518518518521e-05,
+ "loss": 1.1446681022644043,
+ "mean_token_accuracy": 0.7180629968643188,
+ "num_tokens": 6684672.0,
+ "step": 408
+ },
+ {
+ "entropy": 1.448157787322998,
+ "epoch": 0.8262626262626263,
+ "grad_norm": 1.8483424186706543,
+ "learning_rate": 1.4505050505050506e-05,
+ "loss": 1.4317030906677246,
+ "mean_token_accuracy": 0.6659746170043945,
+ "num_tokens": 6701056.0,
+ "step": 409
+ },
+ {
+ "entropy": 1.419940710067749,
+ "epoch": 0.8282828282828283,
+ "grad_norm": 1.961125135421753,
+ "learning_rate": 1.4491582491582492e-05,
+ "loss": 1.42207670211792,
+ "mean_token_accuracy": 0.672874927520752,
+ "num_tokens": 6717440.0,
+ "step": 410
+ },
+ {
+ "entropy": 1.5707522630691528,
+ "epoch": 0.8303030303030303,
+ "grad_norm": 2.0862255096435547,
+ "learning_rate": 1.447811447811448e-05,
+ "loss": 1.5721523761749268,
+ "mean_token_accuracy": 0.6305569410324097,
+ "num_tokens": 6733824.0,
+ "step": 411
+ },
+ {
+ "entropy": 1.4308481216430664,
+ "epoch": 0.8323232323232324,
+ "grad_norm": 2.1505820751190186,
+ "learning_rate": 1.4464646464646465e-05,
+ "loss": 1.4522666931152344,
+ "mean_token_accuracy": 0.6502198576927185,
+ "num_tokens": 6750208.0,
+ "step": 412
+ },
+ {
+ "entropy": 1.7424806356430054,
+ "epoch": 0.8343434343434344,
+ "grad_norm": 2.1787710189819336,
+ "learning_rate": 1.4451178451178452e-05,
+ "loss": 1.7719662189483643,
+ "mean_token_accuracy": 0.5897654891014099,
+ "num_tokens": 6766592.0,
+ "step": 413
+ },
+ {
+ "entropy": 1.9777010679244995,
+ "epoch": 0.8363636363636363,
+ "grad_norm": 2.1178369522094727,
+ "learning_rate": 1.443771043771044e-05,
+ "loss": 1.9885730743408203,
+ "mean_token_accuracy": 0.5628358721733093,
+ "num_tokens": 6782976.0,
+ "step": 414
+ },
+ {
+ "entropy": 1.701235055923462,
+ "epoch": 0.8383838383838383,
+ "grad_norm": 1.9038937091827393,
+ "learning_rate": 1.4424242424242425e-05,
+ "loss": 1.6970818042755127,
+ "mean_token_accuracy": 0.6209086179733276,
+ "num_tokens": 6799360.0,
+ "step": 415
+ },
+ {
+ "entropy": 1.5118876695632935,
+ "epoch": 0.8404040404040404,
+ "grad_norm": 1.8924365043640137,
+ "learning_rate": 1.4410774410774413e-05,
+ "loss": 1.5288946628570557,
+ "mean_token_accuracy": 0.6480215191841125,
+ "num_tokens": 6815744.0,
+ "step": 416
+ },
+ {
+ "entropy": 1.5526877641677856,
+ "epoch": 0.8424242424242424,
+ "grad_norm": 2.109499216079712,
+ "learning_rate": 1.43973063973064e-05,
+ "loss": 1.5581820011138916,
+ "mean_token_accuracy": 0.6351978778839111,
+ "num_tokens": 6832128.0,
+ "step": 417
+ },
+ {
+ "entropy": 1.8132506608963013,
+ "epoch": 0.8444444444444444,
+ "grad_norm": 2.0389890670776367,
+ "learning_rate": 1.4383838383838384e-05,
+ "loss": 1.8378231525421143,
+ "mean_token_accuracy": 0.592391312122345,
+ "num_tokens": 6848512.0,
+ "step": 418
+ },
+ {
+ "entropy": 1.3766752481460571,
+ "epoch": 0.8464646464646465,
+ "grad_norm": 1.8586320877075195,
+ "learning_rate": 1.4370370370370372e-05,
+ "loss": 1.3741214275360107,
+ "mean_token_accuracy": 0.6796531677246094,
+ "num_tokens": 6864896.0,
+ "step": 419
+ },
+ {
+ "entropy": 1.1854407787322998,
+ "epoch": 0.8484848484848485,
+ "grad_norm": 1.875899314880371,
+ "learning_rate": 1.4356902356902359e-05,
+ "loss": 1.2393877506256104,
+ "mean_token_accuracy": 0.704323410987854,
+ "num_tokens": 6881280.0,
+ "step": 420
+ },
+ {
+ "entropy": 1.498558759689331,
+ "epoch": 0.8505050505050505,
+ "grad_norm": 1.898848295211792,
+ "learning_rate": 1.4343434343434344e-05,
+ "loss": 1.4963322877883911,
+ "mean_token_accuracy": 0.6542501449584961,
+ "num_tokens": 6897664.0,
+ "step": 421
+ },
+ {
+ "entropy": 1.5070384740829468,
+ "epoch": 0.8525252525252526,
+ "grad_norm": 2.095853090286255,
+ "learning_rate": 1.4329966329966332e-05,
+ "loss": 1.5026702880859375,
+ "mean_token_accuracy": 0.6561431288719177,
+ "num_tokens": 6914048.0,
+ "step": 422
+ },
+ {
+ "entropy": 1.7848025560379028,
+ "epoch": 0.8545454545454545,
+ "grad_norm": 1.9265769720077515,
+ "learning_rate": 1.4316498316498317e-05,
+ "loss": 1.8066134452819824,
+ "mean_token_accuracy": 0.5936736464500427,
+ "num_tokens": 6930432.0,
+ "step": 423
+ },
+ {
+ "entropy": 1.4278018474578857,
+ "epoch": 0.8565656565656565,
+ "grad_norm": 1.900985836982727,
+ "learning_rate": 1.4303030303030305e-05,
+ "loss": 1.4045631885528564,
+ "mean_token_accuracy": 0.6630434989929199,
+ "num_tokens": 6946816.0,
+ "step": 424
+ },
+ {
+ "entropy": 1.728216290473938,
+ "epoch": 0.8585858585858586,
+ "grad_norm": 1.952841877937317,
+ "learning_rate": 1.4289562289562291e-05,
+ "loss": 1.728804349899292,
+ "mean_token_accuracy": 0.613031268119812,
+ "num_tokens": 6963200.0,
+ "step": 425
+ },
+ {
+ "entropy": 1.6632241010665894,
+ "epoch": 0.8606060606060606,
+ "grad_norm": 2.187973737716675,
+ "learning_rate": 1.4276094276094276e-05,
+ "loss": 1.6703932285308838,
+ "mean_token_accuracy": 0.6150463819503784,
+ "num_tokens": 6979584.0,
+ "step": 426
+ },
+ {
+ "entropy": 1.761826992034912,
+ "epoch": 0.8626262626262626,
+ "grad_norm": 2.203636646270752,
+ "learning_rate": 1.4262626262626264e-05,
+ "loss": 1.727257490158081,
+ "mean_token_accuracy": 0.5861626863479614,
+ "num_tokens": 6995968.0,
+ "step": 427
+ },
+ {
+ "entropy": 1.4528838396072388,
+ "epoch": 0.8646464646464647,
+ "grad_norm": 1.7967053651809692,
+ "learning_rate": 1.424915824915825e-05,
+ "loss": 1.4786958694458008,
+ "mean_token_accuracy": 0.6589521169662476,
+ "num_tokens": 7012352.0,
+ "step": 428
+ },
+ {
+ "entropy": 1.2221835851669312,
+ "epoch": 0.8666666666666667,
+ "grad_norm": 1.8118345737457275,
+ "learning_rate": 1.4235690235690235e-05,
+ "loss": 1.247727632522583,
+ "mean_token_accuracy": 0.7024303674697876,
+ "num_tokens": 7028736.0,
+ "step": 429
+ },
+ {
+ "entropy": 1.5066863298416138,
+ "epoch": 0.8686868686868687,
+ "grad_norm": 1.8390358686447144,
+ "learning_rate": 1.4222222222222224e-05,
+ "loss": 1.5283207893371582,
+ "mean_token_accuracy": 0.6553493142127991,
+ "num_tokens": 7045120.0,
+ "step": 430
+ },
+ {
+ "entropy": 1.346379041671753,
+ "epoch": 0.8707070707070707,
+ "grad_norm": 2.070962905883789,
+ "learning_rate": 1.420875420875421e-05,
+ "loss": 1.386117935180664,
+ "mean_token_accuracy": 0.6690889000892639,
+ "num_tokens": 7061504.0,
+ "step": 431
+ },
+ {
+ "entropy": 1.8691747188568115,
+ "epoch": 0.8727272727272727,
+ "grad_norm": 1.7868962287902832,
+ "learning_rate": 1.4195286195286198e-05,
+ "loss": 1.9285637140274048,
+ "mean_token_accuracy": 0.5680874586105347,
+ "num_tokens": 7077888.0,
+ "step": 432
+ },
+ {
+ "entropy": 1.8500115871429443,
+ "epoch": 0.8747474747474747,
+ "grad_norm": 1.8853468894958496,
+ "learning_rate": 1.4181818181818183e-05,
+ "loss": 1.8733386993408203,
+ "mean_token_accuracy": 0.5956888198852539,
+ "num_tokens": 7094272.0,
+ "step": 433
+ },
+ {
+ "entropy": 1.3737305402755737,
+ "epoch": 0.8767676767676768,
+ "grad_norm": 1.8363661766052246,
+ "learning_rate": 1.416835016835017e-05,
+ "loss": 1.4020421504974365,
+ "mean_token_accuracy": 0.6723253726959229,
+ "num_tokens": 7110656.0,
+ "step": 434
+ },
+ {
+ "entropy": 1.5069265365600586,
+ "epoch": 0.8787878787878788,
+ "grad_norm": 2.1154301166534424,
+ "learning_rate": 1.4154882154882156e-05,
+ "loss": 1.5137630701065063,
+ "mean_token_accuracy": 0.628724992275238,
+ "num_tokens": 7127040.0,
+ "step": 435
+ },
+ {
+ "entropy": 1.6280715465545654,
+ "epoch": 0.8808080808080808,
+ "grad_norm": 1.9827117919921875,
+ "learning_rate": 1.4141414141414143e-05,
+ "loss": 1.6629055738449097,
+ "mean_token_accuracy": 0.6300684213638306,
+ "num_tokens": 7143424.0,
+ "step": 436
+ },
+ {
+ "entropy": 1.0767987966537476,
+ "epoch": 0.8828282828282829,
+ "grad_norm": 1.8503392934799194,
+ "learning_rate": 1.4127946127946129e-05,
+ "loss": 1.0964105129241943,
+ "mean_token_accuracy": 0.7118954658508301,
+ "num_tokens": 7159808.0,
+ "step": 437
+ },
+ {
+ "entropy": 1.6979929208755493,
+ "epoch": 0.8848484848484849,
+ "grad_norm": 2.114170551300049,
+ "learning_rate": 1.4114478114478116e-05,
+ "loss": 1.7339307069778442,
+ "mean_token_accuracy": 0.6071690320968628,
+ "num_tokens": 7176192.0,
+ "step": 438
+ },
+ {
+ "entropy": 1.840633511543274,
+ "epoch": 0.8868686868686869,
+ "grad_norm": 2.1661946773529053,
+ "learning_rate": 1.4101010101010102e-05,
+ "loss": 1.866344690322876,
+ "mean_token_accuracy": 0.5988031029701233,
+ "num_tokens": 7192576.0,
+ "step": 439
+ },
+ {
+ "entropy": 1.8278590440750122,
+ "epoch": 0.8888888888888888,
+ "grad_norm": 1.9278390407562256,
+ "learning_rate": 1.4087542087542087e-05,
+ "loss": 1.81584632396698,
+ "mean_token_accuracy": 0.5878114104270935,
+ "num_tokens": 7208960.0,
+ "step": 440
+ },
+ {
+ "entropy": 1.405017375946045,
+ "epoch": 0.8909090909090909,
+ "grad_norm": 1.9446384906768799,
+ "learning_rate": 1.4074074074074075e-05,
+ "loss": 1.4040675163269043,
+ "mean_token_accuracy": 0.6647533178329468,
+ "num_tokens": 7225344.0,
+ "step": 441
+ },
+ {
+ "entropy": 1.0967023372650146,
+ "epoch": 0.8929292929292929,
+ "grad_norm": 1.8880785703659058,
+ "learning_rate": 1.4060606060606061e-05,
+ "loss": 1.0798790454864502,
+ "mean_token_accuracy": 0.7167806625366211,
+ "num_tokens": 7241728.0,
+ "step": 442
+ },
+ {
+ "entropy": 1.6126327514648438,
+ "epoch": 0.8949494949494949,
+ "grad_norm": 2.2159016132354736,
+ "learning_rate": 1.404713804713805e-05,
+ "loss": 1.620417833328247,
+ "mean_token_accuracy": 0.6215192675590515,
+ "num_tokens": 7258112.0,
+ "step": 443
+ },
+ {
+ "entropy": 1.4352757930755615,
+ "epoch": 0.896969696969697,
+ "grad_norm": 2.070910692214966,
+ "learning_rate": 1.4033670033670034e-05,
+ "loss": 1.409663200378418,
+ "mean_token_accuracy": 0.658646821975708,
+ "num_tokens": 7274496.0,
+ "step": 444
+ },
+ {
+ "entropy": 1.350129246711731,
+ "epoch": 0.898989898989899,
+ "grad_norm": 1.8672778606414795,
+ "learning_rate": 1.4020202020202021e-05,
+ "loss": 1.3485581874847412,
+ "mean_token_accuracy": 0.6743404865264893,
+ "num_tokens": 7290880.0,
+ "step": 445
+ },
+ {
+ "entropy": 1.0900049209594727,
+ "epoch": 0.901010101010101,
+ "grad_norm": 1.8460185527801514,
+ "learning_rate": 1.4006734006734009e-05,
+ "loss": 1.074802279472351,
+ "mean_token_accuracy": 0.727161705493927,
+ "num_tokens": 7307264.0,
+ "step": 446
+ },
+ {
+ "entropy": 1.6430293321609497,
+ "epoch": 0.9030303030303031,
+ "grad_norm": 2.0190517902374268,
+ "learning_rate": 1.3993265993265994e-05,
+ "loss": 1.6505115032196045,
+ "mean_token_accuracy": 0.6073521971702576,
+ "num_tokens": 7323648.0,
+ "step": 447
+ },
+ {
+ "entropy": 1.465686559677124,
+ "epoch": 0.9050505050505051,
+ "grad_norm": 2.0139927864074707,
+ "learning_rate": 1.397979797979798e-05,
+ "loss": 1.480743408203125,
+ "mean_token_accuracy": 0.6593796014785767,
+ "num_tokens": 7340032.0,
+ "step": 448
+ },
+ {
+ "entropy": 2.152183771133423,
+ "epoch": 0.907070707070707,
+ "grad_norm": 2.2376039028167725,
+ "learning_rate": 1.3966329966329969e-05,
+ "loss": 2.117891788482666,
+ "mean_token_accuracy": 0.5404250025749207,
+ "num_tokens": 7356416.0,
+ "step": 449
+ },
+ {
+ "entropy": 1.6066724061965942,
+ "epoch": 0.9090909090909091,
+ "grad_norm": 1.895409107208252,
+ "learning_rate": 1.3952861952861953e-05,
+ "loss": 1.6247167587280273,
+ "mean_token_accuracy": 0.6281143426895142,
+ "num_tokens": 7372800.0,
+ "step": 450
+ },
+ {
+ "entropy": 1.7240369319915771,
+ "epoch": 0.9111111111111111,
+ "grad_norm": 1.9780583381652832,
+ "learning_rate": 1.3939393939393942e-05,
+ "loss": 1.7521916627883911,
+ "mean_token_accuracy": 0.6022838354110718,
+ "num_tokens": 7389184.0,
+ "step": 451
+ },
+ {
+ "entropy": 1.168492317199707,
+ "epoch": 0.9131313131313131,
+ "grad_norm": 1.676292896270752,
+ "learning_rate": 1.3925925925925928e-05,
+ "loss": 1.1678345203399658,
+ "mean_token_accuracy": 0.7136663198471069,
+ "num_tokens": 7405568.0,
+ "step": 452
+ },
+ {
+ "entropy": 1.4993922710418701,
+ "epoch": 0.9151515151515152,
+ "grad_norm": 2.051370620727539,
+ "learning_rate": 1.3912457912457913e-05,
+ "loss": 1.5273159742355347,
+ "mean_token_accuracy": 0.6378847360610962,
+ "num_tokens": 7421952.0,
+ "step": 453
+ },
+ {
+ "entropy": 1.3478108644485474,
+ "epoch": 0.9171717171717172,
+ "grad_norm": 1.9196240901947021,
+ "learning_rate": 1.3898989898989901e-05,
+ "loss": 1.3547455072402954,
+ "mean_token_accuracy": 0.666829526424408,
+ "num_tokens": 7438336.0,
+ "step": 454
+ },
+ {
+ "entropy": 1.1411141157150269,
+ "epoch": 0.9191919191919192,
+ "grad_norm": 1.9581170082092285,
+ "learning_rate": 1.3885521885521886e-05,
+ "loss": 1.1604664325714111,
+ "mean_token_accuracy": 0.7018197178840637,
+ "num_tokens": 7454720.0,
+ "step": 455
+ },
+ {
+ "entropy": 1.1814258098602295,
+ "epoch": 0.9212121212121213,
+ "grad_norm": 2.005451202392578,
+ "learning_rate": 1.3872053872053872e-05,
+ "loss": 1.1768745183944702,
+ "mean_token_accuracy": 0.7044455409049988,
+ "num_tokens": 7471104.0,
+ "step": 456
+ },
+ {
+ "entropy": 1.335093379020691,
+ "epoch": 0.9232323232323232,
+ "grad_norm": 1.928747534751892,
+ "learning_rate": 1.385858585858586e-05,
+ "loss": 1.3504598140716553,
+ "mean_token_accuracy": 0.6774548292160034,
+ "num_tokens": 7487488.0,
+ "step": 457
+ },
+ {
+ "entropy": 1.6866769790649414,
+ "epoch": 0.9252525252525252,
+ "grad_norm": 1.9078223705291748,
+ "learning_rate": 1.3845117845117845e-05,
+ "loss": 1.688689947128296,
+ "mean_token_accuracy": 0.6077796816825867,
+ "num_tokens": 7503872.0,
+ "step": 458
+ },
+ {
+ "entropy": 1.2842025756835938,
+ "epoch": 0.9272727272727272,
+ "grad_norm": 1.909472942352295,
+ "learning_rate": 1.3831649831649833e-05,
+ "loss": 1.3128941059112549,
+ "mean_token_accuracy": 0.6872252225875854,
+ "num_tokens": 7520256.0,
+ "step": 459
+ },
+ {
+ "entropy": 1.7797539234161377,
+ "epoch": 0.9292929292929293,
+ "grad_norm": 2.246411085128784,
+ "learning_rate": 1.381818181818182e-05,
+ "loss": 1.8016175031661987,
+ "mean_token_accuracy": 0.6049095988273621,
+ "num_tokens": 7536640.0,
+ "step": 460
+ },
+ {
+ "entropy": 1.3464831113815308,
+ "epoch": 0.9313131313131313,
+ "grad_norm": 1.9999858140945435,
+ "learning_rate": 1.3804713804713805e-05,
+ "loss": 1.3523657321929932,
+ "mean_token_accuracy": 0.6787371635437012,
+ "num_tokens": 7553024.0,
+ "step": 461
+ },
+ {
+ "entropy": 1.5901066064834595,
+ "epoch": 0.9333333333333333,
+ "grad_norm": 1.9632524251937866,
+ "learning_rate": 1.3791245791245793e-05,
+ "loss": 1.596254587173462,
+ "mean_token_accuracy": 0.6428309679031372,
+ "num_tokens": 7569408.0,
+ "step": 462
+ },
+ {
+ "entropy": 1.2511993646621704,
+ "epoch": 0.9353535353535354,
+ "grad_norm": 1.7678464651107788,
+ "learning_rate": 1.377777777777778e-05,
+ "loss": 1.254664659500122,
+ "mean_token_accuracy": 0.6981558203697205,
+ "num_tokens": 7585792.0,
+ "step": 463
+ },
+ {
+ "entropy": 1.546912431716919,
+ "epoch": 0.9373737373737374,
+ "grad_norm": 2.0353212356567383,
+ "learning_rate": 1.3764309764309764e-05,
+ "loss": 1.54923677444458,
+ "mean_token_accuracy": 0.6403273344039917,
+ "num_tokens": 7602176.0,
+ "step": 464
+ },
+ {
+ "entropy": 1.6535273790359497,
+ "epoch": 0.9393939393939394,
+ "grad_norm": 2.1584854125976562,
+ "learning_rate": 1.3750841750841752e-05,
+ "loss": 1.6464964151382446,
+ "mean_token_accuracy": 0.6218246221542358,
+ "num_tokens": 7618560.0,
+ "step": 465
+ },
+ {
+ "entropy": 1.1885185241699219,
+ "epoch": 0.9414141414141414,
+ "grad_norm": 1.8068963289260864,
+ "learning_rate": 1.3737373737373739e-05,
+ "loss": 1.1980109214782715,
+ "mean_token_accuracy": 0.6990718245506287,
+ "num_tokens": 7634944.0,
+ "step": 466
+ },
+ {
+ "entropy": 1.2306157350540161,
+ "epoch": 0.9434343434343434,
+ "grad_norm": 1.9044586420059204,
+ "learning_rate": 1.3723905723905725e-05,
+ "loss": 1.2291686534881592,
+ "mean_token_accuracy": 0.6996824741363525,
+ "num_tokens": 7651328.0,
+ "step": 467
+ },
+ {
+ "entropy": 1.2496092319488525,
+ "epoch": 0.9454545454545454,
+ "grad_norm": 1.7264868021011353,
+ "learning_rate": 1.3710437710437712e-05,
+ "loss": 1.2440049648284912,
+ "mean_token_accuracy": 0.6877137422561646,
+ "num_tokens": 7667712.0,
+ "step": 468
+ },
+ {
+ "entropy": 1.8231102228164673,
+ "epoch": 0.9474747474747475,
+ "grad_norm": 2.099104166030884,
+ "learning_rate": 1.3696969696969698e-05,
+ "loss": 1.84971284866333,
+ "mean_token_accuracy": 0.5869565010070801,
+ "num_tokens": 7684096.0,
+ "step": 469
+ },
+ {
+ "entropy": 1.537786841392517,
+ "epoch": 0.9494949494949495,
+ "grad_norm": 1.872962474822998,
+ "learning_rate": 1.3683501683501685e-05,
+ "loss": 1.5448193550109863,
+ "mean_token_accuracy": 0.6465559601783752,
+ "num_tokens": 7700480.0,
+ "step": 470
+ },
+ {
+ "entropy": 1.8270875215530396,
+ "epoch": 0.9515151515151515,
+ "grad_norm": 2.2747695446014404,
+ "learning_rate": 1.3670033670033671e-05,
+ "loss": 1.854095697402954,
+ "mean_token_accuracy": 0.5834758281707764,
+ "num_tokens": 7716864.0,
+ "step": 471
+ },
+ {
+ "entropy": 1.505604863166809,
+ "epoch": 0.9535353535353536,
+ "grad_norm": 1.9903687238693237,
+ "learning_rate": 1.3656565656565656e-05,
+ "loss": 1.5269451141357422,
+ "mean_token_accuracy": 0.6475940346717834,
+ "num_tokens": 7733248.0,
+ "step": 472
+ },
+ {
+ "entropy": 1.8228932619094849,
+ "epoch": 0.9555555555555556,
+ "grad_norm": 2.4003078937530518,
+ "learning_rate": 1.3643097643097644e-05,
+ "loss": 1.846016764640808,
+ "mean_token_accuracy": 0.5900097489356995,
+ "num_tokens": 7749632.0,
+ "step": 473
+ },
+ {
+ "entropy": 1.2786308526992798,
+ "epoch": 0.9575757575757575,
+ "grad_norm": 1.9355131387710571,
+ "learning_rate": 1.362962962962963e-05,
+ "loss": 1.3152062892913818,
+ "mean_token_accuracy": 0.6703712940216064,
+ "num_tokens": 7766016.0,
+ "step": 474
+ },
+ {
+ "entropy": 1.1032856702804565,
+ "epoch": 0.9595959595959596,
+ "grad_norm": 1.7720186710357666,
+ "learning_rate": 1.3616161616161619e-05,
+ "loss": 1.1242918968200684,
+ "mean_token_accuracy": 0.7250854969024658,
+ "num_tokens": 7782400.0,
+ "step": 475
+ },
+ {
+ "entropy": 1.5261021852493286,
+ "epoch": 0.9616161616161616,
+ "grad_norm": 1.8472843170166016,
+ "learning_rate": 1.3602693602693604e-05,
+ "loss": 1.54013991355896,
+ "mean_token_accuracy": 0.6471055150032043,
+ "num_tokens": 7798784.0,
+ "step": 476
+ },
+ {
+ "entropy": 0.9697252511978149,
+ "epoch": 0.9636363636363636,
+ "grad_norm": 1.8871296644210815,
+ "learning_rate": 1.358922558922559e-05,
+ "loss": 0.9874091148376465,
+ "mean_token_accuracy": 0.7430996298789978,
+ "num_tokens": 7815168.0,
+ "step": 477
+ },
+ {
+ "entropy": 1.6003227233886719,
+ "epoch": 0.9656565656565657,
+ "grad_norm": 2.371385097503662,
+ "learning_rate": 1.3575757575757578e-05,
+ "loss": 1.580587387084961,
+ "mean_token_accuracy": 0.6273815631866455,
+ "num_tokens": 7831552.0,
+ "step": 478
+ },
+ {
+ "entropy": 1.556900978088379,
+ "epoch": 0.9676767676767677,
+ "grad_norm": 1.839432954788208,
+ "learning_rate": 1.3562289562289563e-05,
+ "loss": 1.5154083967208862,
+ "mean_token_accuracy": 0.628724992275238,
+ "num_tokens": 7847936.0,
+ "step": 479
+ },
+ {
+ "entropy": 1.2394448518753052,
+ "epoch": 0.9696969696969697,
+ "grad_norm": 1.6638867855072021,
+ "learning_rate": 1.354882154882155e-05,
+ "loss": 1.2016921043395996,
+ "mean_token_accuracy": 0.6991939544677734,
+ "num_tokens": 7864320.0,
+ "step": 480
+ },
+ {
+ "entropy": 1.4200247526168823,
+ "epoch": 0.9717171717171718,
+ "grad_norm": 1.8742305040359497,
+ "learning_rate": 1.3535353535353538e-05,
+ "loss": 1.4267700910568237,
+ "mean_token_accuracy": 0.6604176759719849,
+ "num_tokens": 7880704.0,
+ "step": 481
+ },
+ {
+ "entropy": 2.0156707763671875,
+ "epoch": 0.9737373737373738,
+ "grad_norm": 2.1772451400756836,
+ "learning_rate": 1.3521885521885523e-05,
+ "loss": 2.0306758880615234,
+ "mean_token_accuracy": 0.5579506754875183,
+ "num_tokens": 7897088.0,
+ "step": 482
+ },
+ {
+ "entropy": 1.2802858352661133,
+ "epoch": 0.9757575757575757,
+ "grad_norm": 1.8951549530029297,
+ "learning_rate": 1.3508417508417509e-05,
+ "loss": 1.2761621475219727,
+ "mean_token_accuracy": 0.6825231909751892,
+ "num_tokens": 7913472.0,
+ "step": 483
+ },
+ {
+ "entropy": 1.3758989572525024,
+ "epoch": 0.9777777777777777,
+ "grad_norm": 1.8072963953018188,
+ "learning_rate": 1.3494949494949497e-05,
+ "loss": 1.3532235622406006,
+ "mean_token_accuracy": 0.6749511361122131,
+ "num_tokens": 7929856.0,
+ "step": 484
+ },
+ {
+ "entropy": 1.4488792419433594,
+ "epoch": 0.9797979797979798,
+ "grad_norm": 1.7842215299606323,
+ "learning_rate": 1.3481481481481482e-05,
+ "loss": 1.4513285160064697,
+ "mean_token_accuracy": 0.6539447903633118,
+ "num_tokens": 7946240.0,
+ "step": 485
+ },
+ {
+ "entropy": 1.2957613468170166,
+ "epoch": 0.9818181818181818,
+ "grad_norm": 1.8612068891525269,
+ "learning_rate": 1.346801346801347e-05,
+ "loss": 1.3016514778137207,
+ "mean_token_accuracy": 0.6848436594009399,
+ "num_tokens": 7962624.0,
+ "step": 486
+ },
+ {
+ "entropy": 1.4321255683898926,
+ "epoch": 0.9838383838383838,
+ "grad_norm": 1.7746661901474,
+ "learning_rate": 1.3454545454545455e-05,
+ "loss": 1.4230387210845947,
+ "mean_token_accuracy": 0.664692223072052,
+ "num_tokens": 7979008.0,
+ "step": 487
+ },
+ {
+ "entropy": 1.7615760564804077,
+ "epoch": 0.9858585858585859,
+ "grad_norm": 1.9986366033554077,
+ "learning_rate": 1.3441077441077441e-05,
+ "loss": 1.7871575355529785,
+ "mean_token_accuracy": 0.5994748473167419,
+ "num_tokens": 7995392.0,
+ "step": 488
+ },
+ {
+ "entropy": 1.349948763847351,
+ "epoch": 0.9878787878787879,
+ "grad_norm": 2.0217247009277344,
+ "learning_rate": 1.342760942760943e-05,
+ "loss": 1.352069616317749,
+ "mean_token_accuracy": 0.6762335300445557,
+ "num_tokens": 8011776.0,
+ "step": 489
+ },
+ {
+ "entropy": 1.6317073106765747,
+ "epoch": 0.98989898989899,
+ "grad_norm": 1.9479029178619385,
+ "learning_rate": 1.3414141414141414e-05,
+ "loss": 1.6507500410079956,
+ "mean_token_accuracy": 0.6186492443084717,
+ "num_tokens": 8028160.0,
+ "step": 490
+ },
+ {
+ "entropy": 2.0288875102996826,
+ "epoch": 0.9919191919191919,
+ "grad_norm": 2.1520888805389404,
+ "learning_rate": 1.3400673400673401e-05,
+ "loss": 2.06215500831604,
+ "mean_token_accuracy": 0.5530043840408325,
+ "num_tokens": 8044544.0,
+ "step": 491
+ },
+ {
+ "entropy": 1.8297137022018433,
+ "epoch": 0.9939393939393939,
+ "grad_norm": 1.949404001235962,
+ "learning_rate": 1.3387205387205389e-05,
+ "loss": 1.8859628438949585,
+ "mean_token_accuracy": 0.5876282453536987,
+ "num_tokens": 8060928.0,
+ "step": 492
+ },
+ {
+ "entropy": 1.587094783782959,
+ "epoch": 0.9959595959595959,
+ "grad_norm": 1.9205344915390015,
+ "learning_rate": 1.3373737373737374e-05,
+ "loss": 1.6053454875946045,
+ "mean_token_accuracy": 0.6257938742637634,
+ "num_tokens": 8077312.0,
+ "step": 493
+ },
+ {
+ "entropy": 1.6583648920059204,
+ "epoch": 0.997979797979798,
+ "grad_norm": 1.976946234703064,
+ "learning_rate": 1.3360269360269362e-05,
+ "loss": 1.6656805276870728,
+ "mean_token_accuracy": 0.6186492443084717,
+ "num_tokens": 8093696.0,
+ "step": 494
+ },
+ {
+ "entropy": 1.1685364246368408,
+ "epoch": 1.0,
+ "grad_norm": 1.8258967399597168,
+ "learning_rate": 1.3346801346801349e-05,
+ "loss": 1.165276288986206,
+ "mean_token_accuracy": 0.7082315683364868,
+ "num_tokens": 8110080.0,
+ "step": 495
+ },
+ {
+ "entropy": 1.1211013793945312,
+ "epoch": 1.002020202020202,
+ "grad_norm": 2.27962327003479,
+ "learning_rate": 1.3333333333333333e-05,
+ "loss": 0.9540231227874756,
+ "mean_token_accuracy": 0.747801661491394,
+ "num_tokens": 8126464.0,
+ "step": 496
+ },
+ {
+ "entropy": 1.370996356010437,
+ "epoch": 1.004040404040404,
+ "grad_norm": 2.3193519115448,
+ "learning_rate": 1.3319865319865321e-05,
+ "loss": 1.240065097808838,
+ "mean_token_accuracy": 0.6976062655448914,
+ "num_tokens": 8142848.0,
+ "step": 497
+ },
+ {
+ "entropy": 1.4673359394073486,
+ "epoch": 1.006060606060606,
+ "grad_norm": 2.071213483810425,
+ "learning_rate": 1.3306397306397308e-05,
+ "loss": 1.3351866006851196,
+ "mean_token_accuracy": 0.6921104192733765,
+ "num_tokens": 8159232.0,
+ "step": 498
+ },
+ {
+ "entropy": 1.1151381731033325,
+ "epoch": 1.0080808080808081,
+ "grad_norm": 2.1310431957244873,
+ "learning_rate": 1.3292929292929293e-05,
+ "loss": 1.0092531442642212,
+ "mean_token_accuracy": 0.7261846661567688,
+ "num_tokens": 8175616.0,
+ "step": 499
+ },
+ {
+ "entropy": 1.4734766483306885,
+ "epoch": 1.0101010101010102,
+ "grad_norm": 2.015787124633789,
+ "learning_rate": 1.3279461279461281e-05,
+ "loss": 1.3748562335968018,
+ "mean_token_accuracy": 0.6682339906692505,
+ "num_tokens": 8192000.0,
+ "step": 500
+ },
+ {
+ "epoch": 1.0101010101010102,
+ "eval_entropy": 1.3614274196086391,
+ "eval_loss": 1.5201176404953003,
+ "eval_mean_token_accuracy": 0.647282806134993,
+ "eval_num_tokens": 8192000.0,
+ "eval_runtime": 43.8232,
+ "eval_samples_per_second": 22.591,
+ "eval_steps_per_second": 2.83,
+ "step": 500
+ },
+ {
+ "entropy": 1.3656656742095947,
+ "epoch": 1.0121212121212122,
+ "grad_norm": 2.102975845336914,
+ "learning_rate": 1.3265993265993267e-05,
+ "loss": 1.2834689617156982,
+ "mean_token_accuracy": 0.6831338405609131,
+ "num_tokens": 8208384.0,
+ "step": 501
+ },
+ {
+ "entropy": 1.0829076766967773,
+ "epoch": 1.0141414141414142,
+ "grad_norm": 1.8827977180480957,
+ "learning_rate": 1.3252525252525254e-05,
+ "loss": 1.070042610168457,
+ "mean_token_accuracy": 0.72007817029953,
+ "num_tokens": 8224768.0,
+ "step": 502
+ },
+ {
+ "entropy": 1.2298822402954102,
+ "epoch": 1.0161616161616163,
+ "grad_norm": 1.9913817644119263,
+ "learning_rate": 1.323905723905724e-05,
+ "loss": 1.2619516849517822,
+ "mean_token_accuracy": 0.7024303674697876,
+ "num_tokens": 8241152.0,
+ "step": 503
+ },
+ {
+ "entropy": 0.9216552972793579,
+ "epoch": 1.018181818181818,
+ "grad_norm": 1.8755308389663696,
+ "learning_rate": 1.3225589225589225e-05,
+ "loss": 0.9175798892974854,
+ "mean_token_accuracy": 0.7650830745697021,
+ "num_tokens": 8257536.0,
+ "step": 504
+ },
+ {
+ "entropy": 0.7169107794761658,
+ "epoch": 1.02020202020202,
+ "grad_norm": 2.100609064102173,
+ "learning_rate": 1.3212121212121213e-05,
+ "loss": 0.7609860897064209,
+ "mean_token_accuracy": 0.7913410067558289,
+ "num_tokens": 8273920.0,
+ "step": 505
+ },
+ {
+ "entropy": 1.5552464723587036,
+ "epoch": 1.0222222222222221,
+ "grad_norm": 2.5406126976013184,
+ "learning_rate": 1.31986531986532e-05,
+ "loss": 1.6440303325653076,
+ "mean_token_accuracy": 0.6193820238113403,
+ "num_tokens": 8290304.0,
+ "step": 506
+ },
+ {
+ "entropy": 0.9679869413375854,
+ "epoch": 1.0242424242424242,
+ "grad_norm": 2.2049074172973633,
+ "learning_rate": 1.3185185185185185e-05,
+ "loss": 1.0273747444152832,
+ "mean_token_accuracy": 0.729421079158783,
+ "num_tokens": 8306688.0,
+ "step": 507
+ },
+ {
+ "entropy": 1.5491999387741089,
+ "epoch": 1.0262626262626262,
+ "grad_norm": 2.2690296173095703,
+ "learning_rate": 1.3171717171717173e-05,
+ "loss": 1.6104233264923096,
+ "mean_token_accuracy": 0.6261602640151978,
+ "num_tokens": 8323072.0,
+ "step": 508
+ },
+ {
+ "entropy": 1.157227635383606,
+ "epoch": 1.0282828282828282,
+ "grad_norm": 2.5081140995025635,
+ "learning_rate": 1.315824915824916e-05,
+ "loss": 1.2283058166503906,
+ "mean_token_accuracy": 0.7013311982154846,
+ "num_tokens": 8339456.0,
+ "step": 509
+ },
+ {
+ "entropy": 1.2934587001800537,
+ "epoch": 1.0303030303030303,
+ "grad_norm": 2.0315630435943604,
+ "learning_rate": 1.3144781144781147e-05,
+ "loss": 1.313960075378418,
+ "mean_token_accuracy": 0.6683561205863953,
+ "num_tokens": 8355840.0,
+ "step": 510
+ },
+ {
+ "entropy": 1.058363437652588,
+ "epoch": 1.0323232323232323,
+ "grad_norm": 2.0326106548309326,
+ "learning_rate": 1.3131313131313132e-05,
+ "loss": 1.060279130935669,
+ "mean_token_accuracy": 0.7263678312301636,
+ "num_tokens": 8372224.0,
+ "step": 511
+ },
+ {
+ "entropy": 1.166068196296692,
+ "epoch": 1.0343434343434343,
+ "grad_norm": 1.9384857416152954,
+ "learning_rate": 1.3117845117845119e-05,
+ "loss": 1.1466734409332275,
+ "mean_token_accuracy": 0.7182462215423584,
+ "num_tokens": 8388608.0,
+ "step": 512
+ },
+ {
+ "entropy": 1.1295984983444214,
+ "epoch": 1.0363636363636364,
+ "grad_norm": 2.0500752925872803,
+ "learning_rate": 1.3104377104377107e-05,
+ "loss": 1.1119377613067627,
+ "mean_token_accuracy": 0.7249022722244263,
+ "num_tokens": 8404992.0,
+ "step": 513
+ },
+ {
+ "entropy": 1.2471498250961304,
+ "epoch": 1.0383838383838384,
+ "grad_norm": 2.057642698287964,
+ "learning_rate": 1.3090909090909092e-05,
+ "loss": 1.2348787784576416,
+ "mean_token_accuracy": 0.69840008020401,
+ "num_tokens": 8421376.0,
+ "step": 514
+ },
+ {
+ "entropy": 1.2146048545837402,
+ "epoch": 1.0404040404040404,
+ "grad_norm": 1.8724642992019653,
+ "learning_rate": 1.3077441077441078e-05,
+ "loss": 1.1992437839508057,
+ "mean_token_accuracy": 0.7179408669471741,
+ "num_tokens": 8437760.0,
+ "step": 515
+ },
+ {
+ "entropy": 1.119163990020752,
+ "epoch": 1.0424242424242425,
+ "grad_norm": 2.013307571411133,
+ "learning_rate": 1.3063973063973066e-05,
+ "loss": 1.0880366563796997,
+ "mean_token_accuracy": 0.7274059653282166,
+ "num_tokens": 8454144.0,
+ "step": 516
+ },
+ {
+ "entropy": 1.055690884590149,
+ "epoch": 1.0444444444444445,
+ "grad_norm": 2.9534032344818115,
+ "learning_rate": 1.3050505050505051e-05,
+ "loss": 1.0625638961791992,
+ "mean_token_accuracy": 0.7302759885787964,
+ "num_tokens": 8470528.0,
+ "step": 517
+ },
+ {
+ "entropy": 1.629905343055725,
+ "epoch": 1.0464646464646465,
+ "grad_norm": 1.9111285209655762,
+ "learning_rate": 1.303703703703704e-05,
+ "loss": 1.642975926399231,
+ "mean_token_accuracy": 0.638067901134491,
+ "num_tokens": 8486912.0,
+ "step": 518
+ },
+ {
+ "entropy": 1.2197786569595337,
+ "epoch": 1.0484848484848486,
+ "grad_norm": 2.0819573402404785,
+ "learning_rate": 1.3023569023569024e-05,
+ "loss": 1.178088903427124,
+ "mean_token_accuracy": 0.7040180563926697,
+ "num_tokens": 8503296.0,
+ "step": 519
+ },
+ {
+ "entropy": 1.4044541120529175,
+ "epoch": 1.0505050505050506,
+ "grad_norm": 9.860201835632324,
+ "learning_rate": 1.301010101010101e-05,
+ "loss": 1.6388373374938965,
+ "mean_token_accuracy": 0.6572422981262207,
+ "num_tokens": 8519680.0,
+ "step": 520
+ },
+ {
+ "entropy": 0.9115698337554932,
+ "epoch": 1.0525252525252524,
+ "grad_norm": 2.1228489875793457,
+ "learning_rate": 1.2996632996632999e-05,
+ "loss": 0.8868257999420166,
+ "mean_token_accuracy": 0.7558622360229492,
+ "num_tokens": 8536064.0,
+ "step": 521
+ },
+ {
+ "entropy": 1.0348355770111084,
+ "epoch": 1.0545454545454545,
+ "grad_norm": 1.8459810018539429,
+ "learning_rate": 1.2983164983164984e-05,
+ "loss": 1.0149322748184204,
+ "mean_token_accuracy": 0.7421836853027344,
+ "num_tokens": 8552448.0,
+ "step": 522
+ },
+ {
+ "entropy": 1.0639865398406982,
+ "epoch": 1.0565656565656565,
+ "grad_norm": 2.0999417304992676,
+ "learning_rate": 1.296969696969697e-05,
+ "loss": 1.0582343339920044,
+ "mean_token_accuracy": 0.7242916226387024,
+ "num_tokens": 8568832.0,
+ "step": 523
+ },
+ {
+ "entropy": 1.2032114267349243,
+ "epoch": 1.0585858585858585,
+ "grad_norm": 2.0500640869140625,
+ "learning_rate": 1.2956228956228958e-05,
+ "loss": 1.2051773071289062,
+ "mean_token_accuracy": 0.7076819539070129,
+ "num_tokens": 8585216.0,
+ "step": 524
+ },
+ {
+ "entropy": 1.2425649166107178,
+ "epoch": 1.0606060606060606,
+ "grad_norm": 2.1104960441589355,
+ "learning_rate": 1.2942760942760943e-05,
+ "loss": 1.2576614618301392,
+ "mean_token_accuracy": 0.6907669901847839,
+ "num_tokens": 8601600.0,
+ "step": 525
+ },
+ {
+ "entropy": 1.2280266284942627,
+ "epoch": 1.0626262626262626,
+ "grad_norm": 2.0641212463378906,
+ "learning_rate": 1.2929292929292931e-05,
+ "loss": 1.2378020286560059,
+ "mean_token_accuracy": 0.6943697929382324,
+ "num_tokens": 8617984.0,
+ "step": 526
+ },
+ {
+ "entropy": 0.9312539100646973,
+ "epoch": 1.0646464646464646,
+ "grad_norm": 2.0006070137023926,
+ "learning_rate": 1.2915824915824918e-05,
+ "loss": 0.952136754989624,
+ "mean_token_accuracy": 0.7538471221923828,
+ "num_tokens": 8634368.0,
+ "step": 527
+ },
+ {
+ "entropy": 1.5708260536193848,
+ "epoch": 1.0666666666666667,
+ "grad_norm": 1.873006820678711,
+ "learning_rate": 1.2902356902356902e-05,
+ "loss": 1.5705666542053223,
+ "mean_token_accuracy": 0.6449071764945984,
+ "num_tokens": 8650752.0,
+ "step": 528
+ },
+ {
+ "entropy": 0.9064179062843323,
+ "epoch": 1.0686868686868687,
+ "grad_norm": 2.109816551208496,
+ "learning_rate": 1.288888888888889e-05,
+ "loss": 0.9165636301040649,
+ "mean_token_accuracy": 0.754824161529541,
+ "num_tokens": 8667136.0,
+ "step": 529
+ },
+ {
+ "entropy": 1.4197264909744263,
+ "epoch": 1.0707070707070707,
+ "grad_norm": 2.2213833332061768,
+ "learning_rate": 1.2875420875420877e-05,
+ "loss": 1.4330552816390991,
+ "mean_token_accuracy": 0.6589521169662476,
+ "num_tokens": 8683520.0,
+ "step": 530
+ },
+ {
+ "entropy": 1.3674145936965942,
+ "epoch": 1.0727272727272728,
+ "grad_norm": 2.2046334743499756,
+ "learning_rate": 1.2861952861952862e-05,
+ "loss": 1.3838258981704712,
+ "mean_token_accuracy": 0.6660356521606445,
+ "num_tokens": 8699904.0,
+ "step": 531
+ },
+ {
+ "entropy": 1.0396678447723389,
+ "epoch": 1.0747474747474748,
+ "grad_norm": 1.9246002435684204,
+ "learning_rate": 1.284848484848485e-05,
+ "loss": 1.0304534435272217,
+ "mean_token_accuracy": 0.7421836853027344,
+ "num_tokens": 8716288.0,
+ "step": 532
+ },
+ {
+ "entropy": 0.8541752696037292,
+ "epoch": 1.0767676767676768,
+ "grad_norm": 1.9644383192062378,
+ "learning_rate": 1.2835016835016837e-05,
+ "loss": 0.8437563180923462,
+ "mean_token_accuracy": 0.7683805823326111,
+ "num_tokens": 8732672.0,
+ "step": 533
+ },
+ {
+ "entropy": 0.962359607219696,
+ "epoch": 1.0787878787878789,
+ "grad_norm": 1.8767060041427612,
+ "learning_rate": 1.2821548821548821e-05,
+ "loss": 0.9354047775268555,
+ "mean_token_accuracy": 0.7493893504142761,
+ "num_tokens": 8749056.0,
+ "step": 534
+ },
+ {
+ "entropy": 1.1609500646591187,
+ "epoch": 1.0808080808080809,
+ "grad_norm": 2.316633939743042,
+ "learning_rate": 1.280808080808081e-05,
+ "loss": 1.1338696479797363,
+ "mean_token_accuracy": 0.7132388949394226,
+ "num_tokens": 8765440.0,
+ "step": 535
+ },
+ {
+ "entropy": 0.9635956883430481,
+ "epoch": 1.082828282828283,
+ "grad_norm": 1.8842763900756836,
+ "learning_rate": 1.2794612794612794e-05,
+ "loss": 0.9779557585716248,
+ "mean_token_accuracy": 0.7515876889228821,
+ "num_tokens": 8781824.0,
+ "step": 536
+ },
+ {
+ "entropy": 1.5283102989196777,
+ "epoch": 1.084848484848485,
+ "grad_norm": 2.2277283668518066,
+ "learning_rate": 1.2781144781144783e-05,
+ "loss": 1.5118470191955566,
+ "mean_token_accuracy": 0.6464338302612305,
+ "num_tokens": 8798208.0,
+ "step": 537
+ },
+ {
+ "entropy": 1.1301655769348145,
+ "epoch": 1.0868686868686868,
+ "grad_norm": 2.0282225608825684,
+ "learning_rate": 1.2767676767676769e-05,
+ "loss": 1.0959392786026,
+ "mean_token_accuracy": 0.7204445600509644,
+ "num_tokens": 8814592.0,
+ "step": 538
+ },
+ {
+ "entropy": 1.3107094764709473,
+ "epoch": 1.0888888888888888,
+ "grad_norm": 1.9527665376663208,
+ "learning_rate": 1.2754208754208754e-05,
+ "loss": 1.2717593908309937,
+ "mean_token_accuracy": 0.6916218996047974,
+ "num_tokens": 8830976.0,
+ "step": 539
+ },
+ {
+ "entropy": 1.3232251405715942,
+ "epoch": 1.0909090909090908,
+ "grad_norm": 2.134962320327759,
+ "learning_rate": 1.2740740740740742e-05,
+ "loss": 1.3589959144592285,
+ "mean_token_accuracy": 0.6701880693435669,
+ "num_tokens": 8847360.0,
+ "step": 540
+ },
+ {
+ "entropy": 1.5372718572616577,
+ "epoch": 1.0929292929292929,
+ "grad_norm": 1.8864489793777466,
+ "learning_rate": 1.2727272727272728e-05,
+ "loss": 1.5291733741760254,
+ "mean_token_accuracy": 0.6487542986869812,
+ "num_tokens": 8863744.0,
+ "step": 541
+ },
+ {
+ "entropy": 1.2037636041641235,
+ "epoch": 1.094949494949495,
+ "grad_norm": 1.8530166149139404,
+ "learning_rate": 1.2713804713804713e-05,
+ "loss": 1.1905983686447144,
+ "mean_token_accuracy": 0.6986443400382996,
+ "num_tokens": 8880128.0,
+ "step": 542
+ },
+ {
+ "entropy": 1.256689429283142,
+ "epoch": 1.096969696969697,
+ "grad_norm": 1.9364465475082397,
+ "learning_rate": 1.2700336700336701e-05,
+ "loss": 1.275427222251892,
+ "mean_token_accuracy": 0.689728856086731,
+ "num_tokens": 8896512.0,
+ "step": 543
+ },
+ {
+ "entropy": 1.154582142829895,
+ "epoch": 1.098989898989899,
+ "grad_norm": 2.097226619720459,
+ "learning_rate": 1.2686868686868688e-05,
+ "loss": 1.1638195514678955,
+ "mean_token_accuracy": 0.7220322489738464,
+ "num_tokens": 8912896.0,
+ "step": 544
+ },
+ {
+ "entropy": 1.3060903549194336,
+ "epoch": 1.101010101010101,
+ "grad_norm": 2.1401753425598145,
+ "learning_rate": 1.2673400673400676e-05,
+ "loss": 1.337110996246338,
+ "mean_token_accuracy": 0.6781876087188721,
+ "num_tokens": 8929280.0,
+ "step": 545
+ },
+ {
+ "entropy": 1.2550262212753296,
+ "epoch": 1.103030303030303,
+ "grad_norm": 1.8216264247894287,
+ "learning_rate": 1.2659932659932661e-05,
+ "loss": 1.2674927711486816,
+ "mean_token_accuracy": 0.6860649585723877,
+ "num_tokens": 8945664.0,
+ "step": 546
+ },
+ {
+ "entropy": 1.1487771272659302,
+ "epoch": 1.105050505050505,
+ "grad_norm": 1.8050950765609741,
+ "learning_rate": 1.2646464646464647e-05,
+ "loss": 1.1384658813476562,
+ "mean_token_accuracy": 0.734245240688324,
+ "num_tokens": 8962048.0,
+ "step": 547
+ },
+ {
+ "entropy": 1.2411508560180664,
+ "epoch": 1.107070707070707,
+ "grad_norm": 2.0532772541046143,
+ "learning_rate": 1.2632996632996634e-05,
+ "loss": 1.2501752376556396,
+ "mean_token_accuracy": 0.7154982686042786,
+ "num_tokens": 8978432.0,
+ "step": 548
+ },
+ {
+ "entropy": 1.383638620376587,
+ "epoch": 1.1090909090909091,
+ "grad_norm": 1.9422906637191772,
+ "learning_rate": 1.261952861952862e-05,
+ "loss": 1.4019615650177002,
+ "mean_token_accuracy": 0.677760124206543,
+ "num_tokens": 8994816.0,
+ "step": 549
+ },
+ {
+ "entropy": 1.302299976348877,
+ "epoch": 1.1111111111111112,
+ "grad_norm": 2.008962869644165,
+ "learning_rate": 1.2606060606060607e-05,
+ "loss": 1.2934541702270508,
+ "mean_token_accuracy": 0.7078040838241577,
+ "num_tokens": 9011200.0,
+ "step": 550
+ },
+ {
+ "entropy": 1.2423378229141235,
+ "epoch": 1.1131313131313132,
+ "grad_norm": 1.9461158514022827,
+ "learning_rate": 1.2592592592592593e-05,
+ "loss": 1.241358995437622,
+ "mean_token_accuracy": 0.7046898007392883,
+ "num_tokens": 9027584.0,
+ "step": 551
+ },
+ {
+ "entropy": 1.3034340143203735,
+ "epoch": 1.1151515151515152,
+ "grad_norm": 1.8627954721450806,
+ "learning_rate": 1.257912457912458e-05,
+ "loss": 1.3082997798919678,
+ "mean_token_accuracy": 0.6942476630210876,
+ "num_tokens": 9043968.0,
+ "step": 552
+ },
+ {
+ "entropy": 1.2468732595443726,
+ "epoch": 1.1171717171717173,
+ "grad_norm": 1.8033503293991089,
+ "learning_rate": 1.2565656565656568e-05,
+ "loss": 1.2398372888565063,
+ "mean_token_accuracy": 0.7059721350669861,
+ "num_tokens": 9060352.0,
+ "step": 553
+ },
+ {
+ "entropy": 1.3763928413391113,
+ "epoch": 1.1191919191919193,
+ "grad_norm": 2.109393358230591,
+ "learning_rate": 1.2552188552188553e-05,
+ "loss": 1.3908042907714844,
+ "mean_token_accuracy": 0.6643258333206177,
+ "num_tokens": 9076736.0,
+ "step": 554
+ },
+ {
+ "entropy": 1.0741946697235107,
+ "epoch": 1.121212121212121,
+ "grad_norm": 1.9498950242996216,
+ "learning_rate": 1.253872053872054e-05,
+ "loss": 1.0534818172454834,
+ "mean_token_accuracy": 0.7283830046653748,
+ "num_tokens": 9093120.0,
+ "step": 555
+ },
+ {
+ "entropy": 1.2464348077774048,
+ "epoch": 1.1232323232323231,
+ "grad_norm": 1.7962212562561035,
+ "learning_rate": 1.2525252525252527e-05,
+ "loss": 1.225571632385254,
+ "mean_token_accuracy": 0.695652186870575,
+ "num_tokens": 9109504.0,
+ "step": 556
+ },
+ {
+ "entropy": 1.1979366540908813,
+ "epoch": 1.1252525252525252,
+ "grad_norm": 2.0588653087615967,
+ "learning_rate": 1.2511784511784512e-05,
+ "loss": 1.1866135597229004,
+ "mean_token_accuracy": 0.705422580242157,
+ "num_tokens": 9125888.0,
+ "step": 557
+ },
+ {
+ "entropy": 0.9303950071334839,
+ "epoch": 1.1272727272727272,
+ "grad_norm": 1.8577561378479004,
+ "learning_rate": 1.2498316498316499e-05,
+ "loss": 0.9317413568496704,
+ "mean_token_accuracy": 0.7565950155258179,
+ "num_tokens": 9142272.0,
+ "step": 558
+ },
+ {
+ "entropy": 0.857255756855011,
+ "epoch": 1.1292929292929292,
+ "grad_norm": 1.8342596292495728,
+ "learning_rate": 1.2484848484848487e-05,
+ "loss": 0.8612891435623169,
+ "mean_token_accuracy": 0.7796775698661804,
+ "num_tokens": 9158656.0,
+ "step": 559
+ },
+ {
+ "entropy": 0.928502082824707,
+ "epoch": 1.1313131313131313,
+ "grad_norm": 1.9290180206298828,
+ "learning_rate": 1.2471380471380472e-05,
+ "loss": 0.9316297769546509,
+ "mean_token_accuracy": 0.7509160041809082,
+ "num_tokens": 9175040.0,
+ "step": 560
+ },
+ {
+ "entropy": 1.4019330739974976,
+ "epoch": 1.1333333333333333,
+ "grad_norm": 2.1189591884613037,
+ "learning_rate": 1.245791245791246e-05,
+ "loss": 1.3917508125305176,
+ "mean_token_accuracy": 0.6691499948501587,
+ "num_tokens": 9191424.0,
+ "step": 561
+ },
+ {
+ "entropy": 1.4660407304763794,
+ "epoch": 1.1353535353535353,
+ "grad_norm": 2.417506217956543,
+ "learning_rate": 1.2444444444444446e-05,
+ "loss": 1.4893054962158203,
+ "mean_token_accuracy": 0.6543111801147461,
+ "num_tokens": 9207808.0,
+ "step": 562
+ },
+ {
+ "entropy": 1.4580096006393433,
+ "epoch": 1.1373737373737374,
+ "grad_norm": 2.1689300537109375,
+ "learning_rate": 1.2430976430976431e-05,
+ "loss": 1.498686671257019,
+ "mean_token_accuracy": 0.6479604244232178,
+ "num_tokens": 9224192.0,
+ "step": 563
+ },
+ {
+ "entropy": 1.5781100988388062,
+ "epoch": 1.1393939393939394,
+ "grad_norm": 1.9947389364242554,
+ "learning_rate": 1.241750841750842e-05,
+ "loss": 1.604151725769043,
+ "mean_token_accuracy": 0.6561431288719177,
+ "num_tokens": 9240576.0,
+ "step": 564
+ },
+ {
+ "entropy": 0.898691713809967,
+ "epoch": 1.1414141414141414,
+ "grad_norm": 2.0886003971099854,
+ "learning_rate": 1.2404040404040406e-05,
+ "loss": 0.9051169157028198,
+ "mean_token_accuracy": 0.7479848265647888,
+ "num_tokens": 9256960.0,
+ "step": 565
+ },
+ {
+ "entropy": 1.2837234735488892,
+ "epoch": 1.1434343434343435,
+ "grad_norm": 2.1896820068359375,
+ "learning_rate": 1.239057239057239e-05,
+ "loss": 1.2775177955627441,
+ "mean_token_accuracy": 0.6811187267303467,
+ "num_tokens": 9273344.0,
+ "step": 566
+ },
+ {
+ "entropy": 1.1134051084518433,
+ "epoch": 1.1454545454545455,
+ "grad_norm": 1.883119821548462,
+ "learning_rate": 1.2377104377104379e-05,
+ "loss": 1.1263504028320312,
+ "mean_token_accuracy": 0.7154982686042786,
+ "num_tokens": 9289728.0,
+ "step": 567
+ },
+ {
+ "entropy": 1.1212451457977295,
+ "epoch": 1.1474747474747475,
+ "grad_norm": 2.0665414333343506,
+ "learning_rate": 1.2363636363636364e-05,
+ "loss": 1.1123074293136597,
+ "mean_token_accuracy": 0.7117122411727905,
+ "num_tokens": 9306112.0,
+ "step": 568
+ },
+ {
+ "entropy": 1.1619912385940552,
+ "epoch": 1.1494949494949496,
+ "grad_norm": 1.986931562423706,
+ "learning_rate": 1.2350168350168352e-05,
+ "loss": 1.133022665977478,
+ "mean_token_accuracy": 0.7123839855194092,
+ "num_tokens": 9322496.0,
+ "step": 569
+ },
+ {
+ "entropy": 1.433761477470398,
+ "epoch": 1.1515151515151516,
+ "grad_norm": 1.8984476327896118,
+ "learning_rate": 1.2336700336700338e-05,
+ "loss": 1.434887170791626,
+ "mean_token_accuracy": 0.6634098887443542,
+ "num_tokens": 9338880.0,
+ "step": 570
+ },
+ {
+ "entropy": 1.2688939571380615,
+ "epoch": 1.1535353535353536,
+ "grad_norm": 1.9783724546432495,
+ "learning_rate": 1.2323232323232323e-05,
+ "loss": 1.2475486993789673,
+ "mean_token_accuracy": 0.7003541588783264,
+ "num_tokens": 9355264.0,
+ "step": 571
+ },
+ {
+ "entropy": 1.3158199787139893,
+ "epoch": 1.1555555555555554,
+ "grad_norm": 2.1993207931518555,
+ "learning_rate": 1.2309764309764311e-05,
+ "loss": 1.2641775608062744,
+ "mean_token_accuracy": 0.6900342106819153,
+ "num_tokens": 9371648.0,
+ "step": 572
+ },
+ {
+ "entropy": 1.5717110633850098,
+ "epoch": 1.1575757575757575,
+ "grad_norm": 2.0973942279815674,
+ "learning_rate": 1.2296296296296298e-05,
+ "loss": 1.5607821941375732,
+ "mean_token_accuracy": 0.6434416174888611,
+ "num_tokens": 9388032.0,
+ "step": 573
+ },
+ {
+ "entropy": 1.1339603662490845,
+ "epoch": 1.1595959595959595,
+ "grad_norm": 2.1611239910125732,
+ "learning_rate": 1.2282828282828282e-05,
+ "loss": 1.1485066413879395,
+ "mean_token_accuracy": 0.7079262137413025,
+ "num_tokens": 9404416.0,
+ "step": 574
+ },
+ {
+ "entropy": 0.7215157151222229,
+ "epoch": 1.1616161616161615,
+ "grad_norm": 1.7199064493179321,
+ "learning_rate": 1.226936026936027e-05,
+ "loss": 0.6996407508850098,
+ "mean_token_accuracy": 0.8110039234161377,
+ "num_tokens": 9420800.0,
+ "step": 575
+ },
+ {
+ "entropy": 1.207773208618164,
+ "epoch": 1.1636363636363636,
+ "grad_norm": 1.9363049268722534,
+ "learning_rate": 1.2255892255892257e-05,
+ "loss": 1.2100518941879272,
+ "mean_token_accuracy": 0.6953468322753906,
+ "num_tokens": 9437184.0,
+ "step": 576
+ },
+ {
+ "entropy": 1.1832739114761353,
+ "epoch": 1.1656565656565656,
+ "grad_norm": 1.9463727474212646,
+ "learning_rate": 1.2242424242424242e-05,
+ "loss": 1.1800014972686768,
+ "mean_token_accuracy": 0.7148265838623047,
+ "num_tokens": 9453568.0,
+ "step": 577
+ },
+ {
+ "entropy": 0.969874918460846,
+ "epoch": 1.1676767676767676,
+ "grad_norm": 1.9776639938354492,
+ "learning_rate": 1.222895622895623e-05,
+ "loss": 0.9835254549980164,
+ "mean_token_accuracy": 0.7401074767112732,
+ "num_tokens": 9469952.0,
+ "step": 578
+ },
+ {
+ "entropy": 1.2477227449417114,
+ "epoch": 1.1696969696969697,
+ "grad_norm": 2.054093599319458,
+ "learning_rate": 1.2215488215488217e-05,
+ "loss": 1.2632465362548828,
+ "mean_token_accuracy": 0.6811797618865967,
+ "num_tokens": 9486336.0,
+ "step": 579
+ },
+ {
+ "entropy": 1.3652271032333374,
+ "epoch": 1.1717171717171717,
+ "grad_norm": 2.033511161804199,
+ "learning_rate": 1.2202020202020203e-05,
+ "loss": 1.3719289302825928,
+ "mean_token_accuracy": 0.6808744668960571,
+ "num_tokens": 9502720.0,
+ "step": 580
+ },
+ {
+ "entropy": 1.0066897869110107,
+ "epoch": 1.1737373737373737,
+ "grad_norm": 1.906139612197876,
+ "learning_rate": 1.218855218855219e-05,
+ "loss": 1.0317155122756958,
+ "mean_token_accuracy": 0.7470688819885254,
+ "num_tokens": 9519104.0,
+ "step": 581
+ },
+ {
+ "entropy": 1.1423532962799072,
+ "epoch": 1.1757575757575758,
+ "grad_norm": 2.0951437950134277,
+ "learning_rate": 1.2175084175084176e-05,
+ "loss": 1.1328619718551636,
+ "mean_token_accuracy": 0.7098802924156189,
+ "num_tokens": 9535488.0,
+ "step": 582
+ },
+ {
+ "entropy": 0.9396458864212036,
+ "epoch": 1.1777777777777778,
+ "grad_norm": 2.001469373703003,
+ "learning_rate": 1.2161616161616162e-05,
+ "loss": 0.9508752822875977,
+ "mean_token_accuracy": 0.7453590631484985,
+ "num_tokens": 9551872.0,
+ "step": 583
+ },
+ {
+ "entropy": 1.2478508949279785,
+ "epoch": 1.1797979797979798,
+ "grad_norm": 2.1304337978363037,
+ "learning_rate": 1.2148148148148149e-05,
+ "loss": 1.2533063888549805,
+ "mean_token_accuracy": 0.6894235610961914,
+ "num_tokens": 9568256.0,
+ "step": 584
+ },
+ {
+ "entropy": 0.8429924249649048,
+ "epoch": 1.1818181818181819,
+ "grad_norm": 1.891184687614441,
+ "learning_rate": 1.2134680134680134e-05,
+ "loss": 0.8334753513336182,
+ "mean_token_accuracy": 0.7737542986869812,
+ "num_tokens": 9584640.0,
+ "step": 585
+ },
+ {
+ "entropy": 1.1487921476364136,
+ "epoch": 1.183838383838384,
+ "grad_norm": 2.100118637084961,
+ "learning_rate": 1.2121212121212122e-05,
+ "loss": 1.1393694877624512,
+ "mean_token_accuracy": 0.7046287059783936,
+ "num_tokens": 9601024.0,
+ "step": 586
+ },
+ {
+ "entropy": 1.406325101852417,
+ "epoch": 1.185858585858586,
+ "grad_norm": 2.0391998291015625,
+ "learning_rate": 1.2107744107744108e-05,
+ "loss": 1.395864725112915,
+ "mean_token_accuracy": 0.6802638173103333,
+ "num_tokens": 9617408.0,
+ "step": 587
+ },
+ {
+ "entropy": 1.192980170249939,
+ "epoch": 1.187878787878788,
+ "grad_norm": 2.105341911315918,
+ "learning_rate": 1.2094276094276097e-05,
+ "loss": 1.2203993797302246,
+ "mean_token_accuracy": 0.6992549896240234,
+ "num_tokens": 9633792.0,
+ "step": 588
+ },
+ {
+ "entropy": 1.4770102500915527,
+ "epoch": 1.1898989898989898,
+ "grad_norm": 1.959194302558899,
+ "learning_rate": 1.2080808080808081e-05,
+ "loss": 1.4826602935791016,
+ "mean_token_accuracy": 0.6475329995155334,
+ "num_tokens": 9650176.0,
+ "step": 589
+ },
+ {
+ "entropy": 1.440915584564209,
+ "epoch": 1.1919191919191918,
+ "grad_norm": 2.2827632427215576,
+ "learning_rate": 1.2067340067340068e-05,
+ "loss": 1.4349164962768555,
+ "mean_token_accuracy": 0.6570591330528259,
+ "num_tokens": 9666560.0,
+ "step": 590
+ },
+ {
+ "entropy": 0.9503017067909241,
+ "epoch": 1.1939393939393939,
+ "grad_norm": 1.8947645425796509,
+ "learning_rate": 1.2053872053872056e-05,
+ "loss": 0.9562087655067444,
+ "mean_token_accuracy": 0.7511602640151978,
+ "num_tokens": 9682944.0,
+ "step": 591
+ },
+ {
+ "entropy": 0.935599148273468,
+ "epoch": 1.195959595959596,
+ "grad_norm": 1.8788244724273682,
+ "learning_rate": 1.2040404040404041e-05,
+ "loss": 0.9235643148422241,
+ "mean_token_accuracy": 0.7525647282600403,
+ "num_tokens": 9699328.0,
+ "step": 592
+ },
+ {
+ "entropy": 1.3421721458435059,
+ "epoch": 1.197979797979798,
+ "grad_norm": 2.038698673248291,
+ "learning_rate": 1.2026936026936027e-05,
+ "loss": 1.3459725379943848,
+ "mean_token_accuracy": 0.6803248524665833,
+ "num_tokens": 9715712.0,
+ "step": 593
+ },
+ {
+ "entropy": 1.4244537353515625,
+ "epoch": 1.2,
+ "grad_norm": 2.0669443607330322,
+ "learning_rate": 1.2013468013468016e-05,
+ "loss": 1.462540626525879,
+ "mean_token_accuracy": 0.6539447903633118,
+ "num_tokens": 9732096.0,
+ "step": 594
+ },
+ {
+ "entropy": 1.3264589309692383,
+ "epoch": 1.202020202020202,
+ "grad_norm": 2.2563624382019043,
+ "learning_rate": 1.2e-05,
+ "loss": 1.3387104272842407,
+ "mean_token_accuracy": 0.6822789311408997,
+ "num_tokens": 9748480.0,
+ "step": 595
+ },
+ {
+ "entropy": 1.250544548034668,
+ "epoch": 1.204040404040404,
+ "grad_norm": 1.9611469507217407,
+ "learning_rate": 1.1986531986531988e-05,
+ "loss": 1.2569901943206787,
+ "mean_token_accuracy": 0.6947361826896667,
+ "num_tokens": 9764864.0,
+ "step": 596
+ },
+ {
+ "entropy": 1.3189572095870972,
+ "epoch": 1.206060606060606,
+ "grad_norm": 1.9136087894439697,
+ "learning_rate": 1.1973063973063975e-05,
+ "loss": 1.3251599073410034,
+ "mean_token_accuracy": 0.6856985688209534,
+ "num_tokens": 9781248.0,
+ "step": 597
+ },
+ {
+ "entropy": 1.1400903463363647,
+ "epoch": 1.208080808080808,
+ "grad_norm": 1.9812469482421875,
+ "learning_rate": 1.195959595959596e-05,
+ "loss": 1.1514993906021118,
+ "mean_token_accuracy": 0.7175744771957397,
+ "num_tokens": 9797632.0,
+ "step": 598
+ },
+ {
+ "entropy": 1.263106346130371,
+ "epoch": 1.2101010101010101,
+ "grad_norm": 2.1461827754974365,
+ "learning_rate": 1.1946127946127948e-05,
+ "loss": 1.2711405754089355,
+ "mean_token_accuracy": 0.6880801320075989,
+ "num_tokens": 9814016.0,
+ "step": 599
+ },
+ {
+ "entropy": 1.1541659832000732,
+ "epoch": 1.2121212121212122,
+ "grad_norm": 1.8964859247207642,
+ "learning_rate": 1.1932659932659933e-05,
+ "loss": 1.1726963520050049,
+ "mean_token_accuracy": 0.7143380641937256,
+ "num_tokens": 9830400.0,
+ "step": 600
+ },
+ {
+ "epoch": 1.2121212121212122,
+ "eval_entropy": 1.3044754386909547,
+ "eval_loss": 1.5363746881484985,
+ "eval_mean_token_accuracy": 0.6458704312962871,
+ "eval_num_tokens": 9830400.0,
+ "eval_runtime": 43.9189,
+ "eval_samples_per_second": 22.542,
+ "eval_steps_per_second": 2.823,
+ "step": 600
+ },
+ {
+ "entropy": 1.4692459106445312,
+ "epoch": 1.2141414141414142,
+ "grad_norm": 1.9165432453155518,
+ "learning_rate": 1.191919191919192e-05,
+ "loss": 1.444916009902954,
+ "mean_token_accuracy": 0.6709208488464355,
+ "num_tokens": 9846784.0,
+ "step": 601
+ },
+ {
+ "entropy": 1.2140837907791138,
+ "epoch": 1.2161616161616162,
+ "grad_norm": 2.184677839279175,
+ "learning_rate": 1.1905723905723907e-05,
+ "loss": 1.1810932159423828,
+ "mean_token_accuracy": 0.7119565010070801,
+ "num_tokens": 9863168.0,
+ "step": 602
+ },
+ {
+ "entropy": 1.2266523838043213,
+ "epoch": 1.2181818181818183,
+ "grad_norm": 2.0835864543914795,
+ "learning_rate": 1.1892255892255892e-05,
+ "loss": 1.216700792312622,
+ "mean_token_accuracy": 0.702125072479248,
+ "num_tokens": 9879552.0,
+ "step": 603
+ },
+ {
+ "entropy": 1.153336524963379,
+ "epoch": 1.2202020202020203,
+ "grad_norm": 1.959340214729309,
+ "learning_rate": 1.187878787878788e-05,
+ "loss": 1.1497607231140137,
+ "mean_token_accuracy": 0.7254518866539001,
+ "num_tokens": 9895936.0,
+ "step": 604
+ },
+ {
+ "entropy": 1.1328232288360596,
+ "epoch": 1.2222222222222223,
+ "grad_norm": 2.165703773498535,
+ "learning_rate": 1.1865319865319867e-05,
+ "loss": 1.1248265504837036,
+ "mean_token_accuracy": 0.7068881392478943,
+ "num_tokens": 9912320.0,
+ "step": 605
+ },
+ {
+ "entropy": 1.0859829187393188,
+ "epoch": 1.2242424242424241,
+ "grad_norm": 2.0244224071502686,
+ "learning_rate": 1.1851851851851852e-05,
+ "loss": 1.0981967449188232,
+ "mean_token_accuracy": 0.717391312122345,
+ "num_tokens": 9928704.0,
+ "step": 606
+ },
+ {
+ "entropy": 0.9204480648040771,
+ "epoch": 1.2262626262626264,
+ "grad_norm": 1.7957704067230225,
+ "learning_rate": 1.183838383838384e-05,
+ "loss": 0.9045253992080688,
+ "mean_token_accuracy": 0.7639839053153992,
+ "num_tokens": 9945088.0,
+ "step": 607
+ },
+ {
+ "entropy": 1.1566129922866821,
+ "epoch": 1.2282828282828282,
+ "grad_norm": 2.0805907249450684,
+ "learning_rate": 1.1824915824915826e-05,
+ "loss": 1.1485098600387573,
+ "mean_token_accuracy": 0.7118343710899353,
+ "num_tokens": 9961472.0,
+ "step": 608
+ },
+ {
+ "entropy": 1.108127236366272,
+ "epoch": 1.2303030303030302,
+ "grad_norm": 1.977317214012146,
+ "learning_rate": 1.1811447811447811e-05,
+ "loss": 1.1131000518798828,
+ "mean_token_accuracy": 0.7167806625366211,
+ "num_tokens": 9977856.0,
+ "step": 609
+ },
+ {
+ "entropy": 0.8749650716781616,
+ "epoch": 1.2323232323232323,
+ "grad_norm": 2.009488344192505,
+ "learning_rate": 1.17979797979798e-05,
+ "loss": 0.9132063388824463,
+ "mean_token_accuracy": 0.7530532479286194,
+ "num_tokens": 9994240.0,
+ "step": 610
+ },
+ {
+ "entropy": 1.2267354726791382,
+ "epoch": 1.2343434343434343,
+ "grad_norm": 2.075202465057373,
+ "learning_rate": 1.1784511784511786e-05,
+ "loss": 1.2091751098632812,
+ "mean_token_accuracy": 0.70652174949646,
+ "num_tokens": 10010624.0,
+ "step": 611
+ },
+ {
+ "entropy": 1.3990167379379272,
+ "epoch": 1.2363636363636363,
+ "grad_norm": 2.2843780517578125,
+ "learning_rate": 1.1771043771043772e-05,
+ "loss": 1.3952152729034424,
+ "mean_token_accuracy": 0.6685393452644348,
+ "num_tokens": 10027008.0,
+ "step": 612
+ },
+ {
+ "entropy": 1.3222941160202026,
+ "epoch": 1.2383838383838384,
+ "grad_norm": 2.1916584968566895,
+ "learning_rate": 1.1757575757575759e-05,
+ "loss": 1.322625994682312,
+ "mean_token_accuracy": 0.6761114001274109,
+ "num_tokens": 10043392.0,
+ "step": 613
+ },
+ {
+ "entropy": 1.3163132667541504,
+ "epoch": 1.2404040404040404,
+ "grad_norm": 2.1066598892211914,
+ "learning_rate": 1.1744107744107745e-05,
+ "loss": 1.3350703716278076,
+ "mean_token_accuracy": 0.6811797618865967,
+ "num_tokens": 10059776.0,
+ "step": 614
+ },
+ {
+ "entropy": 1.3258253335952759,
+ "epoch": 1.2424242424242424,
+ "grad_norm": 2.16098952293396,
+ "learning_rate": 1.1730639730639732e-05,
+ "loss": 1.3305881023406982,
+ "mean_token_accuracy": 0.6830117106437683,
+ "num_tokens": 10076160.0,
+ "step": 615
+ },
+ {
+ "entropy": 0.9164132475852966,
+ "epoch": 1.2444444444444445,
+ "grad_norm": 1.950881838798523,
+ "learning_rate": 1.1717171717171718e-05,
+ "loss": 0.9203709959983826,
+ "mean_token_accuracy": 0.7569614052772522,
+ "num_tokens": 10092544.0,
+ "step": 616
+ },
+ {
+ "entropy": 1.1378085613250732,
+ "epoch": 1.2464646464646465,
+ "grad_norm": 1.99736487865448,
+ "learning_rate": 1.1703703703703703e-05,
+ "loss": 1.1409945487976074,
+ "mean_token_accuracy": 0.7130556702613831,
+ "num_tokens": 10108928.0,
+ "step": 617
+ },
+ {
+ "entropy": 1.010982871055603,
+ "epoch": 1.2484848484848485,
+ "grad_norm": 1.8678277730941772,
+ "learning_rate": 1.1690235690235691e-05,
+ "loss": 1.0040092468261719,
+ "mean_token_accuracy": 0.7382755279541016,
+ "num_tokens": 10125312.0,
+ "step": 618
+ },
+ {
+ "entropy": 1.3451189994812012,
+ "epoch": 1.2505050505050506,
+ "grad_norm": 2.1611146926879883,
+ "learning_rate": 1.1676767676767678e-05,
+ "loss": 1.3347704410552979,
+ "mean_token_accuracy": 0.6760503053665161,
+ "num_tokens": 10141696.0,
+ "step": 619
+ },
+ {
+ "entropy": 1.5893058776855469,
+ "epoch": 1.2525252525252526,
+ "grad_norm": 2.5862345695495605,
+ "learning_rate": 1.1663299663299666e-05,
+ "loss": 1.6102404594421387,
+ "mean_token_accuracy": 0.6231069564819336,
+ "num_tokens": 10158080.0,
+ "step": 620
+ },
+ {
+ "entropy": 1.4471327066421509,
+ "epoch": 1.2545454545454544,
+ "grad_norm": 1.8857513666152954,
+ "learning_rate": 1.164983164983165e-05,
+ "loss": 1.4646904468536377,
+ "mean_token_accuracy": 0.6591353416442871,
+ "num_tokens": 10174464.0,
+ "step": 621
+ },
+ {
+ "entropy": 1.1457897424697876,
+ "epoch": 1.2565656565656567,
+ "grad_norm": 1.9036853313446045,
+ "learning_rate": 1.1636363636363637e-05,
+ "loss": 1.136209487915039,
+ "mean_token_accuracy": 0.7179408669471741,
+ "num_tokens": 10190848.0,
+ "step": 622
+ },
+ {
+ "entropy": 1.3230067491531372,
+ "epoch": 1.2585858585858585,
+ "grad_norm": 1.969241738319397,
+ "learning_rate": 1.1622895622895625e-05,
+ "loss": 1.3203729391098022,
+ "mean_token_accuracy": 0.6909501552581787,
+ "num_tokens": 10207232.0,
+ "step": 623
+ },
+ {
+ "entropy": 0.9331954717636108,
+ "epoch": 1.2606060606060607,
+ "grad_norm": 1.8972336053848267,
+ "learning_rate": 1.160942760942761e-05,
+ "loss": 0.9346438050270081,
+ "mean_token_accuracy": 0.7507327795028687,
+ "num_tokens": 10223616.0,
+ "step": 624
+ },
+ {
+ "entropy": 1.0203893184661865,
+ "epoch": 1.2626262626262625,
+ "grad_norm": 1.9246903657913208,
+ "learning_rate": 1.1595959595959597e-05,
+ "loss": 1.0318937301635742,
+ "mean_token_accuracy": 0.7396799921989441,
+ "num_tokens": 10240000.0,
+ "step": 625
+ },
+ {
+ "entropy": 1.0469106435775757,
+ "epoch": 1.2646464646464646,
+ "grad_norm": 2.0860235691070557,
+ "learning_rate": 1.1582491582491585e-05,
+ "loss": 1.0661770105361938,
+ "mean_token_accuracy": 0.7215437293052673,
+ "num_tokens": 10256384.0,
+ "step": 626
+ },
+ {
+ "entropy": 0.9044442772865295,
+ "epoch": 1.2666666666666666,
+ "grad_norm": 1.922624111175537,
+ "learning_rate": 1.156902356902357e-05,
+ "loss": 0.8972492218017578,
+ "mean_token_accuracy": 0.7624572515487671,
+ "num_tokens": 10272768.0,
+ "step": 627
+ },
+ {
+ "entropy": 1.7082630395889282,
+ "epoch": 1.2686868686868686,
+ "grad_norm": 2.0335583686828613,
+ "learning_rate": 1.1555555555555556e-05,
+ "loss": 1.7194339036941528,
+ "mean_token_accuracy": 0.6278089880943298,
+ "num_tokens": 10289152.0,
+ "step": 628
+ },
+ {
+ "entropy": 1.1749809980392456,
+ "epoch": 1.2707070707070707,
+ "grad_norm": 2.3157317638397217,
+ "learning_rate": 1.1542087542087544e-05,
+ "loss": 1.1912015676498413,
+ "mean_token_accuracy": 0.6991939544677734,
+ "num_tokens": 10305536.0,
+ "step": 629
+ },
+ {
+ "entropy": 1.0230133533477783,
+ "epoch": 1.2727272727272727,
+ "grad_norm": 1.9861854314804077,
+ "learning_rate": 1.1528619528619529e-05,
+ "loss": 1.030585765838623,
+ "mean_token_accuracy": 0.7328407168388367,
+ "num_tokens": 10321920.0,
+ "step": 630
+ },
+ {
+ "entropy": 0.9128814935684204,
+ "epoch": 1.2747474747474747,
+ "grad_norm": 5.766819000244141,
+ "learning_rate": 1.1515151515151517e-05,
+ "loss": 0.9959099292755127,
+ "mean_token_accuracy": 0.7520151734352112,
+ "num_tokens": 10338304.0,
+ "step": 631
+ },
+ {
+ "entropy": 1.3987287282943726,
+ "epoch": 1.2767676767676768,
+ "grad_norm": 1.934733510017395,
+ "learning_rate": 1.1501683501683502e-05,
+ "loss": 1.3910452127456665,
+ "mean_token_accuracy": 0.675561785697937,
+ "num_tokens": 10354688.0,
+ "step": 632
+ },
+ {
+ "entropy": 1.7590066194534302,
+ "epoch": 1.2787878787878788,
+ "grad_norm": 2.003899574279785,
+ "learning_rate": 1.1488215488215488e-05,
+ "loss": 1.771543264389038,
+ "mean_token_accuracy": 0.6064972877502441,
+ "num_tokens": 10371072.0,
+ "step": 633
+ },
+ {
+ "entropy": 1.1135497093200684,
+ "epoch": 1.2808080808080808,
+ "grad_norm": 1.9084051847457886,
+ "learning_rate": 1.1474747474747477e-05,
+ "loss": 1.1299227476119995,
+ "mean_token_accuracy": 0.7198339104652405,
+ "num_tokens": 10387456.0,
+ "step": 634
+ },
+ {
+ "entropy": 1.4671945571899414,
+ "epoch": 1.2828282828282829,
+ "grad_norm": 1.9825108051300049,
+ "learning_rate": 1.1461279461279461e-05,
+ "loss": 1.486473560333252,
+ "mean_token_accuracy": 0.6673180460929871,
+ "num_tokens": 10403840.0,
+ "step": 635
+ },
+ {
+ "entropy": 1.2157530784606934,
+ "epoch": 1.284848484848485,
+ "grad_norm": 2.048701286315918,
+ "learning_rate": 1.1447811447811448e-05,
+ "loss": 1.196000099182129,
+ "mean_token_accuracy": 0.7078040838241577,
+ "num_tokens": 10420224.0,
+ "step": 636
+ },
+ {
+ "entropy": 0.8687402009963989,
+ "epoch": 1.286868686868687,
+ "grad_norm": 1.8956520557403564,
+ "learning_rate": 1.1434343434343436e-05,
+ "loss": 0.86002117395401,
+ "mean_token_accuracy": 0.7663654088973999,
+ "num_tokens": 10436608.0,
+ "step": 637
+ },
+ {
+ "entropy": 1.1118111610412598,
+ "epoch": 1.2888888888888888,
+ "grad_norm": 1.9835058450698853,
+ "learning_rate": 1.142087542087542e-05,
+ "loss": 1.1115164756774902,
+ "mean_token_accuracy": 0.7170859575271606,
+ "num_tokens": 10452992.0,
+ "step": 638
+ },
+ {
+ "entropy": 1.1080268621444702,
+ "epoch": 1.290909090909091,
+ "grad_norm": 1.9650659561157227,
+ "learning_rate": 1.1407407407407409e-05,
+ "loss": 1.0900099277496338,
+ "mean_token_accuracy": 0.7143990993499756,
+ "num_tokens": 10469376.0,
+ "step": 639
+ },
+ {
+ "entropy": 0.9909329414367676,
+ "epoch": 1.2929292929292928,
+ "grad_norm": 2.1029200553894043,
+ "learning_rate": 1.1393939393939395e-05,
+ "loss": 0.9947470426559448,
+ "mean_token_accuracy": 0.7403517365455627,
+ "num_tokens": 10485760.0,
+ "step": 640
+ },
+ {
+ "entropy": 1.1272226572036743,
+ "epoch": 1.294949494949495,
+ "grad_norm": 1.9888865947723389,
+ "learning_rate": 1.138047138047138e-05,
+ "loss": 1.1474848985671997,
+ "mean_token_accuracy": 0.7138495445251465,
+ "num_tokens": 10502144.0,
+ "step": 641
+ },
+ {
+ "entropy": 1.5329686403274536,
+ "epoch": 1.2969696969696969,
+ "grad_norm": 2.046448230743408,
+ "learning_rate": 1.1367003367003368e-05,
+ "loss": 1.556037425994873,
+ "mean_token_accuracy": 0.642953097820282,
+ "num_tokens": 10518528.0,
+ "step": 642
+ },
+ {
+ "entropy": 1.177565336227417,
+ "epoch": 1.298989898989899,
+ "grad_norm": 2.1602399349212646,
+ "learning_rate": 1.1353535353535355e-05,
+ "loss": 1.1836936473846436,
+ "mean_token_accuracy": 0.6971787810325623,
+ "num_tokens": 10534912.0,
+ "step": 643
+ },
+ {
+ "entropy": 0.8429449796676636,
+ "epoch": 1.301010101010101,
+ "grad_norm": 1.7227115631103516,
+ "learning_rate": 1.134006734006734e-05,
+ "loss": 0.8446363806724548,
+ "mean_token_accuracy": 0.7757083773612976,
+ "num_tokens": 10551296.0,
+ "step": 644
+ },
+ {
+ "entropy": 1.397851824760437,
+ "epoch": 1.303030303030303,
+ "grad_norm": 2.137099504470825,
+ "learning_rate": 1.1326599326599328e-05,
+ "loss": 1.3887293338775635,
+ "mean_token_accuracy": 0.6610283255577087,
+ "num_tokens": 10567680.0,
+ "step": 645
+ },
+ {
+ "entropy": 1.2101221084594727,
+ "epoch": 1.305050505050505,
+ "grad_norm": 2.073007583618164,
+ "learning_rate": 1.1313131313131314e-05,
+ "loss": 1.1941102743148804,
+ "mean_token_accuracy": 0.7100635170936584,
+ "num_tokens": 10584064.0,
+ "step": 646
+ },
+ {
+ "entropy": 1.626240849494934,
+ "epoch": 1.307070707070707,
+ "grad_norm": 2.073517084121704,
+ "learning_rate": 1.1299663299663301e-05,
+ "loss": 1.6281862258911133,
+ "mean_token_accuracy": 0.6289692521095276,
+ "num_tokens": 10600448.0,
+ "step": 647
+ },
+ {
+ "entropy": 1.0951712131500244,
+ "epoch": 1.309090909090909,
+ "grad_norm": 2.126551866531372,
+ "learning_rate": 1.1286195286195287e-05,
+ "loss": 1.105857014656067,
+ "mean_token_accuracy": 0.7147044539451599,
+ "num_tokens": 10616832.0,
+ "step": 648
+ },
+ {
+ "entropy": 1.2107304334640503,
+ "epoch": 1.3111111111111111,
+ "grad_norm": 2.067368507385254,
+ "learning_rate": 1.1272727272727272e-05,
+ "loss": 1.2249386310577393,
+ "mean_token_accuracy": 0.7076819539070129,
+ "num_tokens": 10633216.0,
+ "step": 649
+ },
+ {
+ "entropy": 1.4364724159240723,
+ "epoch": 1.3131313131313131,
+ "grad_norm": 1.9724938869476318,
+ "learning_rate": 1.125925925925926e-05,
+ "loss": 1.4194120168685913,
+ "mean_token_accuracy": 0.6655471324920654,
+ "num_tokens": 10649600.0,
+ "step": 650
+ },
+ {
+ "entropy": 0.9386561512947083,
+ "epoch": 1.3151515151515152,
+ "grad_norm": 1.9386829137802124,
+ "learning_rate": 1.1245791245791247e-05,
+ "loss": 0.9313067197799683,
+ "mean_token_accuracy": 0.7560454607009888,
+ "num_tokens": 10665984.0,
+ "step": 651
+ },
+ {
+ "entropy": 1.3809468746185303,
+ "epoch": 1.3171717171717172,
+ "grad_norm": 2.1187713146209717,
+ "learning_rate": 1.1232323232323232e-05,
+ "loss": 1.3900315761566162,
+ "mean_token_accuracy": 0.6675012111663818,
+ "num_tokens": 10682368.0,
+ "step": 652
+ },
+ {
+ "entropy": 1.1068683862686157,
+ "epoch": 1.3191919191919192,
+ "grad_norm": 1.9972929954528809,
+ "learning_rate": 1.121885521885522e-05,
+ "loss": 1.1015276908874512,
+ "mean_token_accuracy": 0.7225818037986755,
+ "num_tokens": 10698752.0,
+ "step": 653
+ },
+ {
+ "entropy": 0.8208921551704407,
+ "epoch": 1.3212121212121213,
+ "grad_norm": 1.9667426347732544,
+ "learning_rate": 1.1205387205387206e-05,
+ "loss": 0.8475431799888611,
+ "mean_token_accuracy": 0.773937463760376,
+ "num_tokens": 10715136.0,
+ "step": 654
+ },
+ {
+ "entropy": 1.2287651300430298,
+ "epoch": 1.3232323232323233,
+ "grad_norm": 2.038112163543701,
+ "learning_rate": 1.1191919191919194e-05,
+ "loss": 1.2165236473083496,
+ "mean_token_accuracy": 0.7084147334098816,
+ "num_tokens": 10731520.0,
+ "step": 655
+ },
+ {
+ "entropy": 1.3068023920059204,
+ "epoch": 1.3252525252525253,
+ "grad_norm": 2.001093626022339,
+ "learning_rate": 1.117845117845118e-05,
+ "loss": 1.3044464588165283,
+ "mean_token_accuracy": 0.6809965968132019,
+ "num_tokens": 10747904.0,
+ "step": 656
+ },
+ {
+ "entropy": 1.0627591609954834,
+ "epoch": 1.3272727272727272,
+ "grad_norm": 2.204498529434204,
+ "learning_rate": 1.1164983164983166e-05,
+ "loss": 1.07448410987854,
+ "mean_token_accuracy": 0.7211162447929382,
+ "num_tokens": 10764288.0,
+ "step": 657
+ },
+ {
+ "entropy": 1.1071605682373047,
+ "epoch": 1.3292929292929294,
+ "grad_norm": 2.2397496700286865,
+ "learning_rate": 1.1151515151515154e-05,
+ "loss": 1.1061029434204102,
+ "mean_token_accuracy": 0.7100635170936584,
+ "num_tokens": 10780672.0,
+ "step": 658
+ },
+ {
+ "entropy": 1.2299960851669312,
+ "epoch": 1.3313131313131312,
+ "grad_norm": 2.0838606357574463,
+ "learning_rate": 1.1138047138047139e-05,
+ "loss": 1.2077662944793701,
+ "mean_token_accuracy": 0.6886907815933228,
+ "num_tokens": 10797056.0,
+ "step": 659
+ },
+ {
+ "entropy": 1.2082552909851074,
+ "epoch": 1.3333333333333333,
+ "grad_norm": 2.0905637741088867,
+ "learning_rate": 1.1124579124579125e-05,
+ "loss": 1.2045608758926392,
+ "mean_token_accuracy": 0.6978505253791809,
+ "num_tokens": 10813440.0,
+ "step": 660
+ },
+ {
+ "entropy": 1.4011980295181274,
+ "epoch": 1.3353535353535353,
+ "grad_norm": 2.227360725402832,
+ "learning_rate": 1.1111111111111113e-05,
+ "loss": 1.4221925735473633,
+ "mean_token_accuracy": 0.6541890501976013,
+ "num_tokens": 10829824.0,
+ "step": 661
+ },
+ {
+ "entropy": 0.7984166145324707,
+ "epoch": 1.3373737373737373,
+ "grad_norm": 1.7253413200378418,
+ "learning_rate": 1.1097643097643098e-05,
+ "loss": 0.7818700671195984,
+ "mean_token_accuracy": 0.7938446402549744,
+ "num_tokens": 10846208.0,
+ "step": 662
+ },
+ {
+ "entropy": 1.0456970930099487,
+ "epoch": 1.3393939393939394,
+ "grad_norm": 2.098661422729492,
+ "learning_rate": 1.1084175084175086e-05,
+ "loss": 1.084232211112976,
+ "mean_token_accuracy": 0.735405445098877,
+ "num_tokens": 10862592.0,
+ "step": 663
+ },
+ {
+ "entropy": 1.1313530206680298,
+ "epoch": 1.3414141414141414,
+ "grad_norm": 1.8916199207305908,
+ "learning_rate": 1.1070707070707071e-05,
+ "loss": 1.1359466314315796,
+ "mean_token_accuracy": 0.7338177561759949,
+ "num_tokens": 10878976.0,
+ "step": 664
+ },
+ {
+ "entropy": 0.9603059887886047,
+ "epoch": 1.3434343434343434,
+ "grad_norm": 2.061540365219116,
+ "learning_rate": 1.1057239057239058e-05,
+ "loss": 0.9516211748123169,
+ "mean_token_accuracy": 0.7465803623199463,
+ "num_tokens": 10895360.0,
+ "step": 665
+ },
+ {
+ "entropy": 0.9865843057632446,
+ "epoch": 1.3454545454545455,
+ "grad_norm": 1.9756824970245361,
+ "learning_rate": 1.1043771043771046e-05,
+ "loss": 0.995586633682251,
+ "mean_token_accuracy": 0.7349780201911926,
+ "num_tokens": 10911744.0,
+ "step": 666
+ },
+ {
+ "entropy": 1.098454475402832,
+ "epoch": 1.3474747474747475,
+ "grad_norm": 1.936257243156433,
+ "learning_rate": 1.103030303030303e-05,
+ "loss": 1.0999541282653809,
+ "mean_token_accuracy": 0.7220322489738464,
+ "num_tokens": 10928128.0,
+ "step": 667
+ },
+ {
+ "entropy": 1.1791200637817383,
+ "epoch": 1.3494949494949495,
+ "grad_norm": 2.2319350242614746,
+ "learning_rate": 1.1016835016835017e-05,
+ "loss": 1.1725637912750244,
+ "mean_token_accuracy": 0.6991328597068787,
+ "num_tokens": 10944512.0,
+ "step": 668
+ },
+ {
+ "entropy": 1.2307621240615845,
+ "epoch": 1.3515151515151516,
+ "grad_norm": 1.9910095930099487,
+ "learning_rate": 1.1003367003367005e-05,
+ "loss": 1.2371525764465332,
+ "mean_token_accuracy": 0.6916218996047974,
+ "num_tokens": 10960896.0,
+ "step": 669
+ },
+ {
+ "entropy": 1.2578529119491577,
+ "epoch": 1.3535353535353536,
+ "grad_norm": 2.1415979862213135,
+ "learning_rate": 1.098989898989899e-05,
+ "loss": 1.2753688097000122,
+ "mean_token_accuracy": 0.6794699430465698,
+ "num_tokens": 10977280.0,
+ "step": 670
+ },
+ {
+ "entropy": 1.0493848323822021,
+ "epoch": 1.3555555555555556,
+ "grad_norm": 1.8233927488327026,
+ "learning_rate": 1.0976430976430978e-05,
+ "loss": 1.0482254028320312,
+ "mean_token_accuracy": 0.744076669216156,
+ "num_tokens": 10993664.0,
+ "step": 671
+ },
+ {
+ "entropy": 0.8162504434585571,
+ "epoch": 1.3575757575757577,
+ "grad_norm": 1.7477539777755737,
+ "learning_rate": 1.0962962962962965e-05,
+ "loss": 0.8021665811538696,
+ "mean_token_accuracy": 0.7877381443977356,
+ "num_tokens": 11010048.0,
+ "step": 672
+ },
+ {
+ "entropy": 1.5074793100357056,
+ "epoch": 1.3595959595959597,
+ "grad_norm": 2.096250057220459,
+ "learning_rate": 1.094949494949495e-05,
+ "loss": 1.4992775917053223,
+ "mean_token_accuracy": 0.6537005305290222,
+ "num_tokens": 11026432.0,
+ "step": 673
+ },
+ {
+ "entropy": 0.866222620010376,
+ "epoch": 1.3616161616161615,
+ "grad_norm": 1.9504317045211792,
+ "learning_rate": 1.0936026936026938e-05,
+ "loss": 0.8860467672348022,
+ "mean_token_accuracy": 0.7655715942382812,
+ "num_tokens": 11042816.0,
+ "step": 674
+ },
+ {
+ "entropy": 1.0089516639709473,
+ "epoch": 1.3636363636363638,
+ "grad_norm": 2.218254566192627,
+ "learning_rate": 1.0922558922558924e-05,
+ "loss": 1.005429983139038,
+ "mean_token_accuracy": 0.7360160946846008,
+ "num_tokens": 11059200.0,
+ "step": 675
+ },
+ {
+ "entropy": 1.2176142930984497,
+ "epoch": 1.3656565656565656,
+ "grad_norm": 2.1257565021514893,
+ "learning_rate": 1.0909090909090909e-05,
+ "loss": 1.286620020866394,
+ "mean_token_accuracy": 0.7042012810707092,
+ "num_tokens": 11075584.0,
+ "step": 676
+ },
+ {
+ "entropy": 1.1085315942764282,
+ "epoch": 1.3676767676767676,
+ "grad_norm": 2.065206527709961,
+ "learning_rate": 1.0895622895622897e-05,
+ "loss": 1.1294854879379272,
+ "mean_token_accuracy": 0.7205055952072144,
+ "num_tokens": 11091968.0,
+ "step": 677
+ },
+ {
+ "entropy": 1.2821027040481567,
+ "epoch": 1.3696969696969696,
+ "grad_norm": 2.126866340637207,
+ "learning_rate": 1.0882154882154884e-05,
+ "loss": 1.2661428451538086,
+ "mean_token_accuracy": 0.6976673007011414,
+ "num_tokens": 11108352.0,
+ "step": 678
+ },
+ {
+ "entropy": 0.9285033941268921,
+ "epoch": 1.3717171717171717,
+ "grad_norm": 1.9017925262451172,
+ "learning_rate": 1.0868686868686868e-05,
+ "loss": 0.9141640663146973,
+ "mean_token_accuracy": 0.7670371532440186,
+ "num_tokens": 11124736.0,
+ "step": 679
+ },
+ {
+ "entropy": 0.7374900579452515,
+ "epoch": 1.3737373737373737,
+ "grad_norm": 1.645703911781311,
+ "learning_rate": 1.0855218855218857e-05,
+ "loss": 0.7550391554832458,
+ "mean_token_accuracy": 0.7958598136901855,
+ "num_tokens": 11141120.0,
+ "step": 680
+ },
+ {
+ "entropy": 1.2956714630126953,
+ "epoch": 1.3757575757575757,
+ "grad_norm": 1.9934611320495605,
+ "learning_rate": 1.0841750841750841e-05,
+ "loss": 1.318368673324585,
+ "mean_token_accuracy": 0.686370313167572,
+ "num_tokens": 11157504.0,
+ "step": 681
+ },
+ {
+ "entropy": 1.0948729515075684,
+ "epoch": 1.3777777777777778,
+ "grad_norm": 1.8784940242767334,
+ "learning_rate": 1.082828282828283e-05,
+ "loss": 1.0897748470306396,
+ "mean_token_accuracy": 0.7231314182281494,
+ "num_tokens": 11173888.0,
+ "step": 682
+ },
+ {
+ "entropy": 1.5263433456420898,
+ "epoch": 1.3797979797979798,
+ "grad_norm": 1.9812215566635132,
+ "learning_rate": 1.0814814814814816e-05,
+ "loss": 1.535595178604126,
+ "mean_token_accuracy": 0.6683561205863953,
+ "num_tokens": 11190272.0,
+ "step": 683
+ },
+ {
+ "entropy": 1.3359065055847168,
+ "epoch": 1.3818181818181818,
+ "grad_norm": 2.1247689723968506,
+ "learning_rate": 1.08013468013468e-05,
+ "loss": 1.3298604488372803,
+ "mean_token_accuracy": 0.6813629865646362,
+ "num_tokens": 11206656.0,
+ "step": 684
+ },
+ {
+ "entropy": 1.2030925750732422,
+ "epoch": 1.3838383838383839,
+ "grad_norm": 2.178311586380005,
+ "learning_rate": 1.0787878787878789e-05,
+ "loss": 1.1808526515960693,
+ "mean_token_accuracy": 0.7067660093307495,
+ "num_tokens": 11223040.0,
+ "step": 685
+ },
+ {
+ "entropy": 0.954181969165802,
+ "epoch": 1.385858585858586,
+ "grad_norm": 1.9475486278533936,
+ "learning_rate": 1.0774410774410775e-05,
+ "loss": 0.9589706063270569,
+ "mean_token_accuracy": 0.7527479529380798,
+ "num_tokens": 11239424.0,
+ "step": 686
+ },
+ {
+ "entropy": 1.310365080833435,
+ "epoch": 1.387878787878788,
+ "grad_norm": 2.1845083236694336,
+ "learning_rate": 1.076094276094276e-05,
+ "loss": 1.2851444482803345,
+ "mean_token_accuracy": 0.6789203882217407,
+ "num_tokens": 11255808.0,
+ "step": 687
+ },
+ {
+ "entropy": 1.1187219619750977,
+ "epoch": 1.38989898989899,
+ "grad_norm": 1.9923648834228516,
+ "learning_rate": 1.0747474747474748e-05,
+ "loss": 1.1112818717956543,
+ "mean_token_accuracy": 0.7198339104652405,
+ "num_tokens": 11272192.0,
+ "step": 688
+ },
+ {
+ "entropy": 1.1317063570022583,
+ "epoch": 1.391919191919192,
+ "grad_norm": 2.003026008605957,
+ "learning_rate": 1.0734006734006735e-05,
+ "loss": 1.1108404397964478,
+ "mean_token_accuracy": 0.7223986387252808,
+ "num_tokens": 11288576.0,
+ "step": 689
+ },
+ {
+ "entropy": 1.1476070880889893,
+ "epoch": 1.393939393939394,
+ "grad_norm": 2.190676689147949,
+ "learning_rate": 1.0720538720538723e-05,
+ "loss": 1.1344139575958252,
+ "mean_token_accuracy": 0.7170249223709106,
+ "num_tokens": 11304960.0,
+ "step": 690
+ },
+ {
+ "entropy": 0.8959853053092957,
+ "epoch": 1.3959595959595958,
+ "grad_norm": 2.078281879425049,
+ "learning_rate": 1.0707070707070708e-05,
+ "loss": 0.9233289361000061,
+ "mean_token_accuracy": 0.7490229606628418,
+ "num_tokens": 11321344.0,
+ "step": 691
+ },
+ {
+ "entropy": 0.8892031311988831,
+ "epoch": 1.397979797979798,
+ "grad_norm": 1.8343734741210938,
+ "learning_rate": 1.0693602693602694e-05,
+ "loss": 0.8952107429504395,
+ "mean_token_accuracy": 0.7662432789802551,
+ "num_tokens": 11337728.0,
+ "step": 692
+ },
+ {
+ "entropy": 1.0275429487228394,
+ "epoch": 1.4,
+ "grad_norm": 1.8822777271270752,
+ "learning_rate": 1.0680134680134683e-05,
+ "loss": 1.0251365900039673,
+ "mean_token_accuracy": 0.7409623861312866,
+ "num_tokens": 11354112.0,
+ "step": 693
+ },
+ {
+ "entropy": 1.2872905731201172,
+ "epoch": 1.402020202020202,
+ "grad_norm": 1.9260203838348389,
+ "learning_rate": 1.0666666666666667e-05,
+ "loss": 1.3024024963378906,
+ "mean_token_accuracy": 0.6938202381134033,
+ "num_tokens": 11370496.0,
+ "step": 694
+ },
+ {
+ "entropy": 1.0969563722610474,
+ "epoch": 1.404040404040404,
+ "grad_norm": 2.07882022857666,
+ "learning_rate": 1.0653198653198654e-05,
+ "loss": 1.086198091506958,
+ "mean_token_accuracy": 0.7195285558700562,
+ "num_tokens": 11386880.0,
+ "step": 695
+ },
+ {
+ "entropy": 1.4206339120864868,
+ "epoch": 1.406060606060606,
+ "grad_norm": 2.110828399658203,
+ "learning_rate": 1.063973063973064e-05,
+ "loss": 1.4270105361938477,
+ "mean_token_accuracy": 0.6843551397323608,
+ "num_tokens": 11403264.0,
+ "step": 696
+ },
+ {
+ "entropy": 1.3813765048980713,
+ "epoch": 1.408080808080808,
+ "grad_norm": 1.8829882144927979,
+ "learning_rate": 1.0626262626262627e-05,
+ "loss": 1.4026310443878174,
+ "mean_token_accuracy": 0.663593053817749,
+ "num_tokens": 11419648.0,
+ "step": 697
+ },
+ {
+ "entropy": 0.8580052256584167,
+ "epoch": 1.41010101010101,
+ "grad_norm": 2.0236165523529053,
+ "learning_rate": 1.0612794612794615e-05,
+ "loss": 0.8655292391777039,
+ "mean_token_accuracy": 0.7634953856468201,
+ "num_tokens": 11436032.0,
+ "step": 698
+ },
+ {
+ "entropy": 1.0694427490234375,
+ "epoch": 1.412121212121212,
+ "grad_norm": 1.8688758611679077,
+ "learning_rate": 1.05993265993266e-05,
+ "loss": 1.089691162109375,
+ "mean_token_accuracy": 0.7183072566986084,
+ "num_tokens": 11452416.0,
+ "step": 699
+ },
+ {
+ "entropy": 1.2054415941238403,
+ "epoch": 1.4141414141414141,
+ "grad_norm": 2.0692546367645264,
+ "learning_rate": 1.0585858585858586e-05,
+ "loss": 1.2053213119506836,
+ "mean_token_accuracy": 0.7134220600128174,
+ "num_tokens": 11468800.0,
+ "step": 700
+ },
+ {
+ "epoch": 1.4141414141414141,
+ "eval_entropy": 1.2882747703021573,
+ "eval_loss": 1.535590410232544,
+ "eval_mean_token_accuracy": 0.6466120765093835,
+ "eval_num_tokens": 11468800.0,
+ "eval_runtime": 43.9264,
+ "eval_samples_per_second": 22.538,
+ "eval_steps_per_second": 2.823,
+ "step": 700
+ },
+ {
+ "entropy": 1.3370676040649414,
+ "epoch": 1.4161616161616162,
+ "grad_norm": 2.075853109359741,
+ "learning_rate": 1.0572390572390574e-05,
+ "loss": 1.3726792335510254,
+ "mean_token_accuracy": 0.6731802821159363,
+ "num_tokens": 11485184.0,
+ "step": 701
+ },
+ {
+ "entropy": 1.1095269918441772,
+ "epoch": 1.4181818181818182,
+ "grad_norm": 2.0868141651153564,
+ "learning_rate": 1.055892255892256e-05,
+ "loss": 1.124086856842041,
+ "mean_token_accuracy": 0.7162310481071472,
+ "num_tokens": 11501568.0,
+ "step": 702
+ },
+ {
+ "entropy": 1.1948339939117432,
+ "epoch": 1.4202020202020202,
+ "grad_norm": 2.091768741607666,
+ "learning_rate": 1.0545454545454546e-05,
+ "loss": 1.208143949508667,
+ "mean_token_accuracy": 0.7008426785469055,
+ "num_tokens": 11517952.0,
+ "step": 703
+ },
+ {
+ "entropy": 0.8022822141647339,
+ "epoch": 1.4222222222222223,
+ "grad_norm": 1.799911379814148,
+ "learning_rate": 1.0531986531986534e-05,
+ "loss": 0.7870017290115356,
+ "mean_token_accuracy": 0.789631187915802,
+ "num_tokens": 11534336.0,
+ "step": 704
+ },
+ {
+ "entropy": 1.0242470502853394,
+ "epoch": 1.4242424242424243,
+ "grad_norm": 1.8621245622634888,
+ "learning_rate": 1.0518518518518519e-05,
+ "loss": 1.031789779663086,
+ "mean_token_accuracy": 0.7375427484512329,
+ "num_tokens": 11550720.0,
+ "step": 705
+ },
+ {
+ "entropy": 0.9582471251487732,
+ "epoch": 1.4262626262626263,
+ "grad_norm": 2.0054924488067627,
+ "learning_rate": 1.0505050505050507e-05,
+ "loss": 0.9522819519042969,
+ "mean_token_accuracy": 0.739130437374115,
+ "num_tokens": 11567104.0,
+ "step": 706
+ },
+ {
+ "entropy": 1.172863483428955,
+ "epoch": 1.4282828282828284,
+ "grad_norm": 1.8495111465454102,
+ "learning_rate": 1.0491582491582493e-05,
+ "loss": 1.1783076524734497,
+ "mean_token_accuracy": 0.7132999300956726,
+ "num_tokens": 11583488.0,
+ "step": 707
+ },
+ {
+ "entropy": 1.2278543710708618,
+ "epoch": 1.4303030303030302,
+ "grad_norm": 2.095710277557373,
+ "learning_rate": 1.0478114478114478e-05,
+ "loss": 1.2355749607086182,
+ "mean_token_accuracy": 0.6905227303504944,
+ "num_tokens": 11599872.0,
+ "step": 708
+ },
+ {
+ "entropy": 0.9415866732597351,
+ "epoch": 1.4323232323232324,
+ "grad_norm": 1.9720549583435059,
+ "learning_rate": 1.0464646464646466e-05,
+ "loss": 0.9524072408676147,
+ "mean_token_accuracy": 0.7501221299171448,
+ "num_tokens": 11616256.0,
+ "step": 709
+ },
+ {
+ "entropy": 1.371761441230774,
+ "epoch": 1.4343434343434343,
+ "grad_norm": 2.2934458255767822,
+ "learning_rate": 1.0451178451178453e-05,
+ "loss": 1.3502682447433472,
+ "mean_token_accuracy": 0.665730357170105,
+ "num_tokens": 11632640.0,
+ "step": 710
+ },
+ {
+ "entropy": 1.1605411767959595,
+ "epoch": 1.4363636363636363,
+ "grad_norm": 1.978593349456787,
+ "learning_rate": 1.0437710437710438e-05,
+ "loss": 1.166311264038086,
+ "mean_token_accuracy": 0.6971177458763123,
+ "num_tokens": 11649024.0,
+ "step": 711
+ },
+ {
+ "entropy": 1.2717126607894897,
+ "epoch": 1.4383838383838383,
+ "grad_norm": 2.077765703201294,
+ "learning_rate": 1.0424242424242426e-05,
+ "loss": 1.2650127410888672,
+ "mean_token_accuracy": 0.6827064156532288,
+ "num_tokens": 11665408.0,
+ "step": 712
+ },
+ {
+ "entropy": 1.42205810546875,
+ "epoch": 1.4404040404040404,
+ "grad_norm": 2.152578592300415,
+ "learning_rate": 1.041077441077441e-05,
+ "loss": 1.4107255935668945,
+ "mean_token_accuracy": 0.6676233410835266,
+ "num_tokens": 11681792.0,
+ "step": 713
+ },
+ {
+ "entropy": 1.8077080249786377,
+ "epoch": 1.4424242424242424,
+ "grad_norm": 2.0739355087280273,
+ "learning_rate": 1.0397306397306399e-05,
+ "loss": 1.83225417137146,
+ "mean_token_accuracy": 0.6088177561759949,
+ "num_tokens": 11698176.0,
+ "step": 714
+ },
+ {
+ "entropy": 1.3185032606124878,
+ "epoch": 1.4444444444444444,
+ "grad_norm": 2.1207802295684814,
+ "learning_rate": 1.0383838383838385e-05,
+ "loss": 1.3216420412063599,
+ "mean_token_accuracy": 0.6762335300445557,
+ "num_tokens": 11714560.0,
+ "step": 715
+ },
+ {
+ "entropy": 1.1990821361541748,
+ "epoch": 1.4464646464646465,
+ "grad_norm": 1.9028881788253784,
+ "learning_rate": 1.037037037037037e-05,
+ "loss": 1.2123594284057617,
+ "mean_token_accuracy": 0.7231314182281494,
+ "num_tokens": 11730944.0,
+ "step": 716
+ },
+ {
+ "entropy": 1.1971170902252197,
+ "epoch": 1.4484848484848485,
+ "grad_norm": 2.071993827819824,
+ "learning_rate": 1.0356902356902358e-05,
+ "loss": 1.1753935813903809,
+ "mean_token_accuracy": 0.7173302173614502,
+ "num_tokens": 11747328.0,
+ "step": 717
+ },
+ {
+ "entropy": 1.16183602809906,
+ "epoch": 1.4505050505050505,
+ "grad_norm": 2.0574443340301514,
+ "learning_rate": 1.0343434343434345e-05,
+ "loss": 1.171074390411377,
+ "mean_token_accuracy": 0.7076819539070129,
+ "num_tokens": 11763712.0,
+ "step": 718
+ },
+ {
+ "entropy": 1.1637240648269653,
+ "epoch": 1.4525252525252526,
+ "grad_norm": 1.9361884593963623,
+ "learning_rate": 1.032996632996633e-05,
+ "loss": 1.1374025344848633,
+ "mean_token_accuracy": 0.7241694927215576,
+ "num_tokens": 11780096.0,
+ "step": 719
+ },
+ {
+ "entropy": 1.0883492231369019,
+ "epoch": 1.4545454545454546,
+ "grad_norm": 2.0204899311065674,
+ "learning_rate": 1.0316498316498318e-05,
+ "loss": 1.0666342973709106,
+ "mean_token_accuracy": 0.7216047644615173,
+ "num_tokens": 11796480.0,
+ "step": 720
+ },
+ {
+ "entropy": 1.3728728294372559,
+ "epoch": 1.4565656565656566,
+ "grad_norm": 1.8690729141235352,
+ "learning_rate": 1.0303030303030304e-05,
+ "loss": 1.3879103660583496,
+ "mean_token_accuracy": 0.6769052147865295,
+ "num_tokens": 11812864.0,
+ "step": 721
+ },
+ {
+ "entropy": 1.1204687356948853,
+ "epoch": 1.4585858585858587,
+ "grad_norm": 2.091113567352295,
+ "learning_rate": 1.0289562289562289e-05,
+ "loss": 1.1190675497055054,
+ "mean_token_accuracy": 0.7191621661186218,
+ "num_tokens": 11829248.0,
+ "step": 722
+ },
+ {
+ "entropy": 1.479430913925171,
+ "epoch": 1.4606060606060607,
+ "grad_norm": 2.194871187210083,
+ "learning_rate": 1.0276094276094277e-05,
+ "loss": 1.4967867136001587,
+ "mean_token_accuracy": 0.6475329995155334,
+ "num_tokens": 11845632.0,
+ "step": 723
+ },
+ {
+ "entropy": 1.083875060081482,
+ "epoch": 1.4626262626262627,
+ "grad_norm": 1.9257639646530151,
+ "learning_rate": 1.0262626262626264e-05,
+ "loss": 1.1053729057312012,
+ "mean_token_accuracy": 0.7168416976928711,
+ "num_tokens": 11862016.0,
+ "step": 724
+ },
+ {
+ "entropy": 1.074636697769165,
+ "epoch": 1.4646464646464645,
+ "grad_norm": 1.956766963005066,
+ "learning_rate": 1.024915824915825e-05,
+ "loss": 1.0775985717773438,
+ "mean_token_accuracy": 0.7348558902740479,
+ "num_tokens": 11878400.0,
+ "step": 725
+ },
+ {
+ "entropy": 1.1595367193222046,
+ "epoch": 1.4666666666666668,
+ "grad_norm": 2.0679080486297607,
+ "learning_rate": 1.0235690235690236e-05,
+ "loss": 1.1976954936981201,
+ "mean_token_accuracy": 0.7014533281326294,
+ "num_tokens": 11894784.0,
+ "step": 726
+ },
+ {
+ "entropy": 1.0546326637268066,
+ "epoch": 1.4686868686868686,
+ "grad_norm": 1.8939472436904907,
+ "learning_rate": 1.0222222222222223e-05,
+ "loss": 1.0483310222625732,
+ "mean_token_accuracy": 0.7339398860931396,
+ "num_tokens": 11911168.0,
+ "step": 727
+ },
+ {
+ "entropy": 1.3418383598327637,
+ "epoch": 1.4707070707070706,
+ "grad_norm": 2.239595413208008,
+ "learning_rate": 1.020875420875421e-05,
+ "loss": 1.35018789768219,
+ "mean_token_accuracy": 0.6685393452644348,
+ "num_tokens": 11927552.0,
+ "step": 728
+ },
+ {
+ "entropy": 0.9122293591499329,
+ "epoch": 1.4727272727272727,
+ "grad_norm": 2.1075761318206787,
+ "learning_rate": 1.0195286195286196e-05,
+ "loss": 0.9050356149673462,
+ "mean_token_accuracy": 0.7633121609687805,
+ "num_tokens": 11943936.0,
+ "step": 729
+ },
+ {
+ "entropy": 0.9527847766876221,
+ "epoch": 1.4747474747474747,
+ "grad_norm": 2.017185688018799,
+ "learning_rate": 1.0181818181818182e-05,
+ "loss": 0.9490136504173279,
+ "mean_token_accuracy": 0.7539081573486328,
+ "num_tokens": 11960320.0,
+ "step": 730
+ },
+ {
+ "entropy": 1.2855827808380127,
+ "epoch": 1.4767676767676767,
+ "grad_norm": 1.9821733236312866,
+ "learning_rate": 1.0168350168350169e-05,
+ "loss": 1.2902567386627197,
+ "mean_token_accuracy": 0.6854543089866638,
+ "num_tokens": 11976704.0,
+ "step": 731
+ },
+ {
+ "entropy": 1.143114686012268,
+ "epoch": 1.4787878787878788,
+ "grad_norm": 2.0281424522399902,
+ "learning_rate": 1.0154882154882155e-05,
+ "loss": 1.1373090744018555,
+ "mean_token_accuracy": 0.7071934342384338,
+ "num_tokens": 11993088.0,
+ "step": 732
+ },
+ {
+ "entropy": 1.1341931819915771,
+ "epoch": 1.4808080808080808,
+ "grad_norm": 2.075814723968506,
+ "learning_rate": 1.0141414141414144e-05,
+ "loss": 1.1449689865112305,
+ "mean_token_accuracy": 0.7180019617080688,
+ "num_tokens": 12009472.0,
+ "step": 733
+ },
+ {
+ "entropy": 0.9200477004051208,
+ "epoch": 1.4828282828282828,
+ "grad_norm": 1.7193031311035156,
+ "learning_rate": 1.0127946127946128e-05,
+ "loss": 0.9225804209709167,
+ "mean_token_accuracy": 0.7621519565582275,
+ "num_tokens": 12025856.0,
+ "step": 734
+ },
+ {
+ "entropy": 1.3266046047210693,
+ "epoch": 1.4848484848484849,
+ "grad_norm": 2.0543408393859863,
+ "learning_rate": 1.0114478114478115e-05,
+ "loss": 1.3158385753631592,
+ "mean_token_accuracy": 0.6758060455322266,
+ "num_tokens": 12042240.0,
+ "step": 735
+ },
+ {
+ "entropy": 1.2785359621047974,
+ "epoch": 1.486868686868687,
+ "grad_norm": 2.0174546241760254,
+ "learning_rate": 1.0101010101010103e-05,
+ "loss": 1.269796371459961,
+ "mean_token_accuracy": 0.7070102691650391,
+ "num_tokens": 12058624.0,
+ "step": 736
+ },
+ {
+ "entropy": 1.3418751955032349,
+ "epoch": 1.488888888888889,
+ "grad_norm": 2.0906383991241455,
+ "learning_rate": 1.0087542087542088e-05,
+ "loss": 1.35174560546875,
+ "mean_token_accuracy": 0.6758060455322266,
+ "num_tokens": 12075008.0,
+ "step": 737
+ },
+ {
+ "entropy": 1.266552448272705,
+ "epoch": 1.490909090909091,
+ "grad_norm": 1.7485158443450928,
+ "learning_rate": 1.0074074074074074e-05,
+ "loss": 1.2538254261016846,
+ "mean_token_accuracy": 0.7070713043212891,
+ "num_tokens": 12091392.0,
+ "step": 738
+ },
+ {
+ "entropy": 0.9562437534332275,
+ "epoch": 1.492929292929293,
+ "grad_norm": 1.9106290340423584,
+ "learning_rate": 1.0060606060606062e-05,
+ "loss": 0.938219428062439,
+ "mean_token_accuracy": 0.7383365631103516,
+ "num_tokens": 12107776.0,
+ "step": 739
+ },
+ {
+ "entropy": 1.3171569108963013,
+ "epoch": 1.494949494949495,
+ "grad_norm": 2.127641439437866,
+ "learning_rate": 1.0047138047138047e-05,
+ "loss": 1.3213756084442139,
+ "mean_token_accuracy": 0.6819125413894653,
+ "num_tokens": 12124160.0,
+ "step": 740
+ },
+ {
+ "entropy": 1.0634039640426636,
+ "epoch": 1.496969696969697,
+ "grad_norm": 2.0710532665252686,
+ "learning_rate": 1.0033670033670035e-05,
+ "loss": 1.0797393321990967,
+ "mean_token_accuracy": 0.7150708436965942,
+ "num_tokens": 12140544.0,
+ "step": 741
+ },
+ {
+ "entropy": 1.3235414028167725,
+ "epoch": 1.4989898989898989,
+ "grad_norm": 2.040738105773926,
+ "learning_rate": 1.0020202020202022e-05,
+ "loss": 1.3383063077926636,
+ "mean_token_accuracy": 0.6737298369407654,
+ "num_tokens": 12156928.0,
+ "step": 742
+ },
+ {
+ "entropy": 1.2156025171279907,
+ "epoch": 1.5010101010101011,
+ "grad_norm": 2.196033239364624,
+ "learning_rate": 1.0006734006734007e-05,
+ "loss": 1.2233551740646362,
+ "mean_token_accuracy": 0.7015144228935242,
+ "num_tokens": 12173312.0,
+ "step": 743
+ },
+ {
+ "entropy": 0.9380976557731628,
+ "epoch": 1.503030303030303,
+ "grad_norm": 1.951636791229248,
+ "learning_rate": 9.993265993265993e-06,
+ "loss": 0.9319983124732971,
+ "mean_token_accuracy": 0.7494503855705261,
+ "num_tokens": 12189696.0,
+ "step": 744
+ },
+ {
+ "entropy": 1.3272504806518555,
+ "epoch": 1.5050505050505052,
+ "grad_norm": 2.045417308807373,
+ "learning_rate": 9.97979797979798e-06,
+ "loss": 1.3266823291778564,
+ "mean_token_accuracy": 0.6848436594009399,
+ "num_tokens": 12206080.0,
+ "step": 745
+ },
+ {
+ "entropy": 0.955014705657959,
+ "epoch": 1.507070707070707,
+ "grad_norm": 1.7301888465881348,
+ "learning_rate": 9.966329966329968e-06,
+ "loss": 0.9721895456314087,
+ "mean_token_accuracy": 0.7561064958572388,
+ "num_tokens": 12222464.0,
+ "step": 746
+ },
+ {
+ "entropy": 1.0614056587219238,
+ "epoch": 1.509090909090909,
+ "grad_norm": 2.235422372817993,
+ "learning_rate": 9.952861952861954e-06,
+ "loss": 1.081586241722107,
+ "mean_token_accuracy": 0.7147044539451599,
+ "num_tokens": 12238848.0,
+ "step": 747
+ },
+ {
+ "entropy": 1.075492024421692,
+ "epoch": 1.511111111111111,
+ "grad_norm": 2.0352933406829834,
+ "learning_rate": 9.939393939393939e-06,
+ "loss": 1.083856463432312,
+ "mean_token_accuracy": 0.7148265838623047,
+ "num_tokens": 12255232.0,
+ "step": 748
+ },
+ {
+ "entropy": 1.1964384317398071,
+ "epoch": 1.513131313131313,
+ "grad_norm": 1.9096226692199707,
+ "learning_rate": 9.925925925925927e-06,
+ "loss": 1.1777125597000122,
+ "mean_token_accuracy": 0.7061553597450256,
+ "num_tokens": 12271616.0,
+ "step": 749
+ },
+ {
+ "entropy": 1.2923556566238403,
+ "epoch": 1.5151515151515151,
+ "grad_norm": 2.1002533435821533,
+ "learning_rate": 9.912457912457914e-06,
+ "loss": 1.2916597127914429,
+ "mean_token_accuracy": 0.6882632970809937,
+ "num_tokens": 12288000.0,
+ "step": 750
+ },
+ {
+ "entropy": 1.008608341217041,
+ "epoch": 1.5171717171717172,
+ "grad_norm": 2.1212828159332275,
+ "learning_rate": 9.8989898989899e-06,
+ "loss": 1.0104637145996094,
+ "mean_token_accuracy": 0.7343673706054688,
+ "num_tokens": 12304384.0,
+ "step": 751
+ },
+ {
+ "entropy": 1.550642967224121,
+ "epoch": 1.5191919191919192,
+ "grad_norm": 1.9693539142608643,
+ "learning_rate": 9.885521885521887e-06,
+ "loss": 1.5664982795715332,
+ "mean_token_accuracy": 0.6830728054046631,
+ "num_tokens": 12320768.0,
+ "step": 752
+ },
+ {
+ "entropy": 1.3540711402893066,
+ "epoch": 1.5212121212121212,
+ "grad_norm": 2.181079149246216,
+ "learning_rate": 9.872053872053873e-06,
+ "loss": 1.3578659296035767,
+ "mean_token_accuracy": 0.666829526424408,
+ "num_tokens": 12337152.0,
+ "step": 753
+ },
+ {
+ "entropy": 1.002195119857788,
+ "epoch": 1.5232323232323233,
+ "grad_norm": 2.051185369491577,
+ "learning_rate": 9.85858585858586e-06,
+ "loss": 0.9798580408096313,
+ "mean_token_accuracy": 0.7434660196304321,
+ "num_tokens": 12353536.0,
+ "step": 754
+ },
+ {
+ "entropy": 1.2805551290512085,
+ "epoch": 1.5252525252525253,
+ "grad_norm": 2.0427937507629395,
+ "learning_rate": 9.845117845117846e-06,
+ "loss": 1.2829053401947021,
+ "mean_token_accuracy": 0.6933317184448242,
+ "num_tokens": 12369920.0,
+ "step": 755
+ },
+ {
+ "entropy": 1.3517152070999146,
+ "epoch": 1.5272727272727273,
+ "grad_norm": 2.0065114498138428,
+ "learning_rate": 9.831649831649833e-06,
+ "loss": 1.3636512756347656,
+ "mean_token_accuracy": 0.6757450103759766,
+ "num_tokens": 12386304.0,
+ "step": 756
+ },
+ {
+ "entropy": 0.7547817826271057,
+ "epoch": 1.5292929292929291,
+ "grad_norm": 3.2289278507232666,
+ "learning_rate": 9.81818181818182e-06,
+ "loss": 0.7594918012619019,
+ "mean_token_accuracy": 0.7917684316635132,
+ "num_tokens": 12402688.0,
+ "step": 757
+ },
+ {
+ "entropy": 1.4882941246032715,
+ "epoch": 1.5313131313131314,
+ "grad_norm": 1.9724371433258057,
+ "learning_rate": 9.804713804713806e-06,
+ "loss": 1.4832470417022705,
+ "mean_token_accuracy": 0.6436858773231506,
+ "num_tokens": 12419072.0,
+ "step": 758
+ },
+ {
+ "entropy": 0.9879036545753479,
+ "epoch": 1.5333333333333332,
+ "grad_norm": 1.9205714464187622,
+ "learning_rate": 9.791245791245792e-06,
+ "loss": 0.9901232123374939,
+ "mean_token_accuracy": 0.7465192675590515,
+ "num_tokens": 12435456.0,
+ "step": 759
+ },
+ {
+ "entropy": 0.9692004919052124,
+ "epoch": 1.5353535353535355,
+ "grad_norm": 1.8762351274490356,
+ "learning_rate": 9.777777777777779e-06,
+ "loss": 0.9544116258621216,
+ "mean_token_accuracy": 0.7518319487571716,
+ "num_tokens": 12451840.0,
+ "step": 760
+ },
+ {
+ "entropy": 0.8084391951560974,
+ "epoch": 1.5373737373737373,
+ "grad_norm": 1.9590117931365967,
+ "learning_rate": 9.764309764309765e-06,
+ "loss": 0.8268290162086487,
+ "mean_token_accuracy": 0.7758305072784424,
+ "num_tokens": 12468224.0,
+ "step": 761
+ },
+ {
+ "entropy": 1.2048877477645874,
+ "epoch": 1.5393939393939395,
+ "grad_norm": 2.1473560333251953,
+ "learning_rate": 9.750841750841752e-06,
+ "loss": 1.1967238187789917,
+ "mean_token_accuracy": 0.7075598239898682,
+ "num_tokens": 12484608.0,
+ "step": 762
+ },
+ {
+ "entropy": 1.330485224723816,
+ "epoch": 1.5414141414141413,
+ "grad_norm": 2.0944108963012695,
+ "learning_rate": 9.737373737373738e-06,
+ "loss": 1.3377408981323242,
+ "mean_token_accuracy": 0.6920493245124817,
+ "num_tokens": 12500992.0,
+ "step": 763
+ },
+ {
+ "entropy": 0.994591474533081,
+ "epoch": 1.5434343434343434,
+ "grad_norm": 2.224015712738037,
+ "learning_rate": 9.723905723905725e-06,
+ "loss": 0.9741864204406738,
+ "mean_token_accuracy": 0.734306275844574,
+ "num_tokens": 12517376.0,
+ "step": 764
+ },
+ {
+ "entropy": 1.3140320777893066,
+ "epoch": 1.5454545454545454,
+ "grad_norm": 2.0910675525665283,
+ "learning_rate": 9.710437710437711e-06,
+ "loss": 1.3168857097625732,
+ "mean_token_accuracy": 0.6908280253410339,
+ "num_tokens": 12533760.0,
+ "step": 765
+ },
+ {
+ "entropy": 0.5885581374168396,
+ "epoch": 1.5474747474747474,
+ "grad_norm": 1.6729995012283325,
+ "learning_rate": 9.696969696969698e-06,
+ "loss": 0.587052583694458,
+ "mean_token_accuracy": 0.8340253829956055,
+ "num_tokens": 12550144.0,
+ "step": 766
+ },
+ {
+ "entropy": 1.1873608827590942,
+ "epoch": 1.5494949494949495,
+ "grad_norm": 2.0278220176696777,
+ "learning_rate": 9.683501683501684e-06,
+ "loss": 1.1988582611083984,
+ "mean_token_accuracy": 0.708109438419342,
+ "num_tokens": 12566528.0,
+ "step": 767
+ },
+ {
+ "entropy": 1.600446343421936,
+ "epoch": 1.5515151515151515,
+ "grad_norm": 2.1766979694366455,
+ "learning_rate": 9.67003367003367e-06,
+ "loss": 1.6514582633972168,
+ "mean_token_accuracy": 0.6134586930274963,
+ "num_tokens": 12582912.0,
+ "step": 768
+ },
+ {
+ "entropy": 1.3398208618164062,
+ "epoch": 1.5535353535353535,
+ "grad_norm": 2.3216426372528076,
+ "learning_rate": 9.656565656565657e-06,
+ "loss": 1.3702855110168457,
+ "mean_token_accuracy": 0.6650586128234863,
+ "num_tokens": 12599296.0,
+ "step": 769
+ },
+ {
+ "entropy": 1.061170220375061,
+ "epoch": 1.5555555555555556,
+ "grad_norm": 1.9703660011291504,
+ "learning_rate": 9.643097643097643e-06,
+ "loss": 1.0738681554794312,
+ "mean_token_accuracy": 0.7376648783683777,
+ "num_tokens": 12615680.0,
+ "step": 770
+ },
+ {
+ "entropy": 1.1564687490463257,
+ "epoch": 1.5575757575757576,
+ "grad_norm": 2.0951433181762695,
+ "learning_rate": 9.62962962962963e-06,
+ "loss": 1.1778810024261475,
+ "mean_token_accuracy": 0.7112237215042114,
+ "num_tokens": 12632064.0,
+ "step": 771
+ },
+ {
+ "entropy": 0.8667913675308228,
+ "epoch": 1.5595959595959596,
+ "grad_norm": 1.9066787958145142,
+ "learning_rate": 9.616161616161616e-06,
+ "loss": 0.8634734153747559,
+ "mean_token_accuracy": 0.769052267074585,
+ "num_tokens": 12648448.0,
+ "step": 772
+ },
+ {
+ "entropy": 1.20845627784729,
+ "epoch": 1.5616161616161617,
+ "grad_norm": 2.0367836952209473,
+ "learning_rate": 9.602693602693603e-06,
+ "loss": 1.2191519737243652,
+ "mean_token_accuracy": 0.69840008020401,
+ "num_tokens": 12664832.0,
+ "step": 773
+ },
+ {
+ "entropy": 1.3325128555297852,
+ "epoch": 1.5636363636363635,
+ "grad_norm": 2.000307083129883,
+ "learning_rate": 9.589225589225591e-06,
+ "loss": 1.3183135986328125,
+ "mean_token_accuracy": 0.6904006004333496,
+ "num_tokens": 12681216.0,
+ "step": 774
+ },
+ {
+ "entropy": 1.3372687101364136,
+ "epoch": 1.5656565656565657,
+ "grad_norm": 2.2447569370269775,
+ "learning_rate": 9.575757575757576e-06,
+ "loss": 1.3026032447814941,
+ "mean_token_accuracy": 0.6769663095474243,
+ "num_tokens": 12697600.0,
+ "step": 775
+ },
+ {
+ "entropy": 1.1477705240249634,
+ "epoch": 1.5676767676767676,
+ "grad_norm": 1.9308643341064453,
+ "learning_rate": 9.562289562289562e-06,
+ "loss": 1.1485618352890015,
+ "mean_token_accuracy": 0.7186126112937927,
+ "num_tokens": 12713984.0,
+ "step": 776
+ },
+ {
+ "entropy": 1.021490454673767,
+ "epoch": 1.5696969696969698,
+ "grad_norm": 1.9314731359481812,
+ "learning_rate": 9.548821548821549e-06,
+ "loss": 1.002140998840332,
+ "mean_token_accuracy": 0.735344409942627,
+ "num_tokens": 12730368.0,
+ "step": 777
+ },
+ {
+ "entropy": 1.1351706981658936,
+ "epoch": 1.5717171717171716,
+ "grad_norm": 1.9402823448181152,
+ "learning_rate": 9.535353535353537e-06,
+ "loss": 1.1265199184417725,
+ "mean_token_accuracy": 0.716292142868042,
+ "num_tokens": 12746752.0,
+ "step": 778
+ },
+ {
+ "entropy": 0.9608347415924072,
+ "epoch": 1.5737373737373739,
+ "grad_norm": 1.9762189388275146,
+ "learning_rate": 9.521885521885522e-06,
+ "loss": 0.9408327341079712,
+ "mean_token_accuracy": 0.7556179761886597,
+ "num_tokens": 12763136.0,
+ "step": 779
+ },
+ {
+ "entropy": 1.2921335697174072,
+ "epoch": 1.5757575757575757,
+ "grad_norm": 1.9008197784423828,
+ "learning_rate": 9.508417508417508e-06,
+ "loss": 1.298734426498413,
+ "mean_token_accuracy": 0.6901563405990601,
+ "num_tokens": 12779520.0,
+ "step": 780
+ },
+ {
+ "entropy": 1.1303656101226807,
+ "epoch": 1.5777777777777777,
+ "grad_norm": 2.368401527404785,
+ "learning_rate": 9.494949494949497e-06,
+ "loss": 1.1172974109649658,
+ "mean_token_accuracy": 0.7112848162651062,
+ "num_tokens": 12795904.0,
+ "step": 781
+ },
+ {
+ "entropy": 1.5801783800125122,
+ "epoch": 1.5797979797979798,
+ "grad_norm": 2.1036503314971924,
+ "learning_rate": 9.481481481481483e-06,
+ "loss": 1.5825071334838867,
+ "mean_token_accuracy": 0.6334269642829895,
+ "num_tokens": 12812288.0,
+ "step": 782
+ },
+ {
+ "entropy": 1.1247191429138184,
+ "epoch": 1.5818181818181818,
+ "grad_norm": 2.0482163429260254,
+ "learning_rate": 9.468013468013468e-06,
+ "loss": 1.1644407510757446,
+ "mean_token_accuracy": 0.7239863276481628,
+ "num_tokens": 12828672.0,
+ "step": 783
+ },
+ {
+ "entropy": 1.6274546384811401,
+ "epoch": 1.5838383838383838,
+ "grad_norm": 2.075209379196167,
+ "learning_rate": 9.454545454545456e-06,
+ "loss": 1.6390056610107422,
+ "mean_token_accuracy": 0.6265266537666321,
+ "num_tokens": 12845056.0,
+ "step": 784
+ },
+ {
+ "entropy": 1.1876834630966187,
+ "epoch": 1.5858585858585859,
+ "grad_norm": 2.3457517623901367,
+ "learning_rate": 9.441077441077442e-06,
+ "loss": 1.229914903640747,
+ "mean_token_accuracy": 0.686431348323822,
+ "num_tokens": 12861440.0,
+ "step": 785
+ },
+ {
+ "entropy": 1.0943474769592285,
+ "epoch": 1.587878787878788,
+ "grad_norm": 2.2637226581573486,
+ "learning_rate": 9.427609427609429e-06,
+ "loss": 1.1246657371520996,
+ "mean_token_accuracy": 0.7178187370300293,
+ "num_tokens": 12877824.0,
+ "step": 786
+ },
+ {
+ "entropy": 1.1606156826019287,
+ "epoch": 1.58989898989899,
+ "grad_norm": 2.1095638275146484,
+ "learning_rate": 9.414141414141414e-06,
+ "loss": 1.146558403968811,
+ "mean_token_accuracy": 0.7142769694328308,
+ "num_tokens": 12894208.0,
+ "step": 787
+ },
+ {
+ "entropy": 0.7914054989814758,
+ "epoch": 1.591919191919192,
+ "grad_norm": 2.0504069328308105,
+ "learning_rate": 9.400673400673402e-06,
+ "loss": 0.7990944385528564,
+ "mean_token_accuracy": 0.7863947153091431,
+ "num_tokens": 12910592.0,
+ "step": 788
+ },
+ {
+ "entropy": 1.2664366960525513,
+ "epoch": 1.593939393939394,
+ "grad_norm": 2.174043893814087,
+ "learning_rate": 9.387205387205388e-06,
+ "loss": 1.263155221939087,
+ "mean_token_accuracy": 0.6905227303504944,
+ "num_tokens": 12926976.0,
+ "step": 789
+ },
+ {
+ "entropy": 1.6059703826904297,
+ "epoch": 1.595959595959596,
+ "grad_norm": 2.0089824199676514,
+ "learning_rate": 9.373737373737375e-06,
+ "loss": 1.628819465637207,
+ "mean_token_accuracy": 0.6278700828552246,
+ "num_tokens": 12943360.0,
+ "step": 790
+ },
+ {
+ "entropy": 1.4796923398971558,
+ "epoch": 1.5979797979797978,
+ "grad_norm": 2.4513726234436035,
+ "learning_rate": 9.360269360269361e-06,
+ "loss": 1.5144509077072144,
+ "mean_token_accuracy": 0.6381289958953857,
+ "num_tokens": 12959744.0,
+ "step": 791
+ },
+ {
+ "entropy": 1.1704087257385254,
+ "epoch": 1.6,
+ "grad_norm": 2.079355478286743,
+ "learning_rate": 9.346801346801348e-06,
+ "loss": 1.1726913452148438,
+ "mean_token_accuracy": 0.709208607673645,
+ "num_tokens": 12976128.0,
+ "step": 792
+ },
+ {
+ "entropy": 1.2573802471160889,
+ "epoch": 1.602020202020202,
+ "grad_norm": 2.087904453277588,
+ "learning_rate": 9.333333333333334e-06,
+ "loss": 1.2652850151062012,
+ "mean_token_accuracy": 0.6868588328361511,
+ "num_tokens": 12992512.0,
+ "step": 793
+ },
+ {
+ "entropy": 1.5265566110610962,
+ "epoch": 1.6040404040404042,
+ "grad_norm": 2.1586968898773193,
+ "learning_rate": 9.31986531986532e-06,
+ "loss": 1.5234975814819336,
+ "mean_token_accuracy": 0.6436248421669006,
+ "num_tokens": 13008896.0,
+ "step": 794
+ },
+ {
+ "entropy": 1.2587038278579712,
+ "epoch": 1.606060606060606,
+ "grad_norm": 2.0685694217681885,
+ "learning_rate": 9.306397306397307e-06,
+ "loss": 1.2585840225219727,
+ "mean_token_accuracy": 0.6930874586105347,
+ "num_tokens": 13025280.0,
+ "step": 795
+ },
+ {
+ "entropy": 1.1857022047042847,
+ "epoch": 1.6080808080808082,
+ "grad_norm": 2.0476908683776855,
+ "learning_rate": 9.292929292929294e-06,
+ "loss": 1.2005798816680908,
+ "mean_token_accuracy": 0.693453848361969,
+ "num_tokens": 13041664.0,
+ "step": 796
+ },
+ {
+ "entropy": 1.5042216777801514,
+ "epoch": 1.61010101010101,
+ "grad_norm": 1.973929762840271,
+ "learning_rate": 9.27946127946128e-06,
+ "loss": 1.5095748901367188,
+ "mean_token_accuracy": 0.6579140424728394,
+ "num_tokens": 13058048.0,
+ "step": 797
+ },
+ {
+ "entropy": 1.5203564167022705,
+ "epoch": 1.612121212121212,
+ "grad_norm": 1.957383155822754,
+ "learning_rate": 9.265993265993267e-06,
+ "loss": 1.4991892576217651,
+ "mean_token_accuracy": 0.6539447903633118,
+ "num_tokens": 13074432.0,
+ "step": 798
+ },
+ {
+ "entropy": 1.3984291553497314,
+ "epoch": 1.614141414141414,
+ "grad_norm": 2.143216848373413,
+ "learning_rate": 9.252525252525253e-06,
+ "loss": 1.3863308429718018,
+ "mean_token_accuracy": 0.6649364829063416,
+ "num_tokens": 13090816.0,
+ "step": 799
+ },
+ {
+ "entropy": 1.662933111190796,
+ "epoch": 1.6161616161616161,
+ "grad_norm": 2.111069440841675,
+ "learning_rate": 9.23905723905724e-06,
+ "loss": 1.6813087463378906,
+ "mean_token_accuracy": 0.6262823939323425,
+ "num_tokens": 13107200.0,
+ "step": 800
+ },
+ {
+ "epoch": 1.6161616161616161,
+ "eval_entropy": 1.3006088190501737,
+ "eval_loss": 1.5314096212387085,
+ "eval_mean_token_accuracy": 0.6469700956536878,
+ "eval_num_tokens": 13107200.0,
+ "eval_runtime": 43.8956,
+ "eval_samples_per_second": 22.554,
+ "eval_steps_per_second": 2.825,
+ "step": 800
+ },
+ {
+ "entropy": 1.709677815437317,
+ "epoch": 1.6181818181818182,
+ "grad_norm": 2.790998935699463,
+ "learning_rate": 9.225589225589226e-06,
+ "loss": 1.740628719329834,
+ "mean_token_accuracy": 0.6136419177055359,
+ "num_tokens": 13123584.0,
+ "step": 801
+ },
+ {
+ "entropy": 1.2373168468475342,
+ "epoch": 1.6202020202020202,
+ "grad_norm": 2.49934458732605,
+ "learning_rate": 9.212121212121213e-06,
+ "loss": 1.2430124282836914,
+ "mean_token_accuracy": 0.6856985688209534,
+ "num_tokens": 13139968.0,
+ "step": 802
+ },
+ {
+ "entropy": 1.4974594116210938,
+ "epoch": 1.6222222222222222,
+ "grad_norm": 2.015254259109497,
+ "learning_rate": 9.1986531986532e-06,
+ "loss": 1.5037147998809814,
+ "mean_token_accuracy": 0.6581583023071289,
+ "num_tokens": 13156352.0,
+ "step": 803
+ },
+ {
+ "entropy": 1.0019752979278564,
+ "epoch": 1.6242424242424243,
+ "grad_norm": 1.9795469045639038,
+ "learning_rate": 9.185185185185186e-06,
+ "loss": 1.0053883790969849,
+ "mean_token_accuracy": 0.7297264337539673,
+ "num_tokens": 13172736.0,
+ "step": 804
+ },
+ {
+ "entropy": 1.2690199613571167,
+ "epoch": 1.6262626262626263,
+ "grad_norm": 2.1013784408569336,
+ "learning_rate": 9.171717171717172e-06,
+ "loss": 1.2500054836273193,
+ "mean_token_accuracy": 0.6856985688209534,
+ "num_tokens": 13189120.0,
+ "step": 805
+ },
+ {
+ "entropy": 1.1282989978790283,
+ "epoch": 1.6282828282828283,
+ "grad_norm": 2.2034029960632324,
+ "learning_rate": 9.15824915824916e-06,
+ "loss": 1.1393691301345825,
+ "mean_token_accuracy": 0.7051783204078674,
+ "num_tokens": 13205504.0,
+ "step": 806
+ },
+ {
+ "entropy": 1.1744720935821533,
+ "epoch": 1.6303030303030304,
+ "grad_norm": 1.9687124490737915,
+ "learning_rate": 9.144781144781145e-06,
+ "loss": 1.1625237464904785,
+ "mean_token_accuracy": 0.7002931237220764,
+ "num_tokens": 13221888.0,
+ "step": 807
+ },
+ {
+ "entropy": 0.9849017858505249,
+ "epoch": 1.6323232323232322,
+ "grad_norm": 1.9646586179733276,
+ "learning_rate": 9.131313131313132e-06,
+ "loss": 0.9992689490318298,
+ "mean_token_accuracy": 0.7460307478904724,
+ "num_tokens": 13238272.0,
+ "step": 808
+ },
+ {
+ "entropy": 1.142819881439209,
+ "epoch": 1.6343434343434344,
+ "grad_norm": 1.975189208984375,
+ "learning_rate": 9.117845117845118e-06,
+ "loss": 1.1389580965042114,
+ "mean_token_accuracy": 0.710368812084198,
+ "num_tokens": 13254656.0,
+ "step": 809
+ },
+ {
+ "entropy": 1.1362377405166626,
+ "epoch": 1.6363636363636362,
+ "grad_norm": 2.0360374450683594,
+ "learning_rate": 9.104377104377106e-06,
+ "loss": 1.15363609790802,
+ "mean_token_accuracy": 0.7113458514213562,
+ "num_tokens": 13271040.0,
+ "step": 810
+ },
+ {
+ "entropy": 1.6209924221038818,
+ "epoch": 1.6383838383838385,
+ "grad_norm": 2.2165496349334717,
+ "learning_rate": 9.090909090909091e-06,
+ "loss": 1.637102484703064,
+ "mean_token_accuracy": 0.6318392753601074,
+ "num_tokens": 13287424.0,
+ "step": 811
+ },
+ {
+ "entropy": 1.050283432006836,
+ "epoch": 1.6404040404040403,
+ "grad_norm": 2.0473668575286865,
+ "learning_rate": 9.077441077441078e-06,
+ "loss": 1.0430940389633179,
+ "mean_token_accuracy": 0.7264899611473083,
+ "num_tokens": 13303808.0,
+ "step": 812
+ },
+ {
+ "entropy": 1.0707964897155762,
+ "epoch": 1.6424242424242426,
+ "grad_norm": 2.013641357421875,
+ "learning_rate": 9.063973063973066e-06,
+ "loss": 1.0929813385009766,
+ "mean_token_accuracy": 0.7231924533843994,
+ "num_tokens": 13320192.0,
+ "step": 813
+ },
+ {
+ "entropy": 0.8777739405632019,
+ "epoch": 1.6444444444444444,
+ "grad_norm": 1.815849781036377,
+ "learning_rate": 9.050505050505052e-06,
+ "loss": 0.868852972984314,
+ "mean_token_accuracy": 0.7692965269088745,
+ "num_tokens": 13336576.0,
+ "step": 814
+ },
+ {
+ "entropy": 1.1157307624816895,
+ "epoch": 1.6464646464646466,
+ "grad_norm": 2.1264355182647705,
+ "learning_rate": 9.037037037037037e-06,
+ "loss": 1.1249959468841553,
+ "mean_token_accuracy": 0.7153151035308838,
+ "num_tokens": 13352960.0,
+ "step": 815
+ },
+ {
+ "entropy": 1.348052740097046,
+ "epoch": 1.6484848484848484,
+ "grad_norm": 1.951642394065857,
+ "learning_rate": 9.023569023569025e-06,
+ "loss": 1.3196947574615479,
+ "mean_token_accuracy": 0.6911333799362183,
+ "num_tokens": 13369344.0,
+ "step": 816
+ },
+ {
+ "entropy": 1.2172725200653076,
+ "epoch": 1.6505050505050505,
+ "grad_norm": 2.2433671951293945,
+ "learning_rate": 9.010101010101012e-06,
+ "loss": 1.1833471059799194,
+ "mean_token_accuracy": 0.6938812732696533,
+ "num_tokens": 13385728.0,
+ "step": 817
+ },
+ {
+ "entropy": 1.5047324895858765,
+ "epoch": 1.6525252525252525,
+ "grad_norm": 2.015489101409912,
+ "learning_rate": 8.996632996632996e-06,
+ "loss": 1.48842453956604,
+ "mean_token_accuracy": 0.6552271842956543,
+ "num_tokens": 13402112.0,
+ "step": 818
+ },
+ {
+ "entropy": 1.3752695322036743,
+ "epoch": 1.6545454545454545,
+ "grad_norm": 2.024895191192627,
+ "learning_rate": 8.983164983164983e-06,
+ "loss": 1.3467098474502563,
+ "mean_token_accuracy": 0.681546151638031,
+ "num_tokens": 13418496.0,
+ "step": 819
+ },
+ {
+ "entropy": 1.1657299995422363,
+ "epoch": 1.6565656565656566,
+ "grad_norm": 2.4099085330963135,
+ "learning_rate": 8.969696969696971e-06,
+ "loss": 1.1612054109573364,
+ "mean_token_accuracy": 0.696140706539154,
+ "num_tokens": 13434880.0,
+ "step": 820
+ },
+ {
+ "entropy": 1.3764209747314453,
+ "epoch": 1.6585858585858586,
+ "grad_norm": 1.9971355199813843,
+ "learning_rate": 8.956228956228958e-06,
+ "loss": 1.3754298686981201,
+ "mean_token_accuracy": 0.6687836050987244,
+ "num_tokens": 13451264.0,
+ "step": 821
+ },
+ {
+ "entropy": 1.2135001420974731,
+ "epoch": 1.6606060606060606,
+ "grad_norm": 1.9414972066879272,
+ "learning_rate": 8.942760942760942e-06,
+ "loss": 1.2148737907409668,
+ "mean_token_accuracy": 0.7018808126449585,
+ "num_tokens": 13467648.0,
+ "step": 822
+ },
+ {
+ "entropy": 1.2993255853652954,
+ "epoch": 1.6626262626262627,
+ "grad_norm": 2.149869203567505,
+ "learning_rate": 8.92929292929293e-06,
+ "loss": 1.2947715520858765,
+ "mean_token_accuracy": 0.6836223602294922,
+ "num_tokens": 13484032.0,
+ "step": 823
+ },
+ {
+ "entropy": 1.2745556831359863,
+ "epoch": 1.6646464646464647,
+ "grad_norm": 2.1189515590667725,
+ "learning_rate": 8.915824915824917e-06,
+ "loss": 1.2992188930511475,
+ "mean_token_accuracy": 0.6871030926704407,
+ "num_tokens": 13500416.0,
+ "step": 824
+ },
+ {
+ "entropy": 1.1959195137023926,
+ "epoch": 1.6666666666666665,
+ "grad_norm": 2.2528860569000244,
+ "learning_rate": 8.902356902356904e-06,
+ "loss": 1.200485110282898,
+ "mean_token_accuracy": 0.6969956159591675,
+ "num_tokens": 13516800.0,
+ "step": 825
+ },
+ {
+ "entropy": 1.358959674835205,
+ "epoch": 1.6686868686868688,
+ "grad_norm": 2.0133697986602783,
+ "learning_rate": 8.888888888888888e-06,
+ "loss": 1.3683624267578125,
+ "mean_token_accuracy": 0.6767830848693848,
+ "num_tokens": 13533184.0,
+ "step": 826
+ },
+ {
+ "entropy": 1.4699946641921997,
+ "epoch": 1.6707070707070706,
+ "grad_norm": 1.9599978923797607,
+ "learning_rate": 8.875420875420876e-06,
+ "loss": 1.4912409782409668,
+ "mean_token_accuracy": 0.663593053817749,
+ "num_tokens": 13549568.0,
+ "step": 827
+ },
+ {
+ "entropy": 1.1315391063690186,
+ "epoch": 1.6727272727272728,
+ "grad_norm": 2.053920269012451,
+ "learning_rate": 8.861952861952863e-06,
+ "loss": 1.140032172203064,
+ "mean_token_accuracy": 0.7090864777565002,
+ "num_tokens": 13565952.0,
+ "step": 828
+ },
+ {
+ "entropy": 1.5379900932312012,
+ "epoch": 1.6747474747474747,
+ "grad_norm": 2.007784128189087,
+ "learning_rate": 8.84848484848485e-06,
+ "loss": 1.5630019903182983,
+ "mean_token_accuracy": 0.6397777199745178,
+ "num_tokens": 13582336.0,
+ "step": 829
+ },
+ {
+ "entropy": 1.241722822189331,
+ "epoch": 1.676767676767677,
+ "grad_norm": 2.060314416885376,
+ "learning_rate": 8.835016835016836e-06,
+ "loss": 1.24226975440979,
+ "mean_token_accuracy": 0.691316545009613,
+ "num_tokens": 13598720.0,
+ "step": 830
+ },
+ {
+ "entropy": 1.0310786962509155,
+ "epoch": 1.6787878787878787,
+ "grad_norm": 1.9201405048370361,
+ "learning_rate": 8.821548821548822e-06,
+ "loss": 1.0244829654693604,
+ "mean_token_accuracy": 0.7352222800254822,
+ "num_tokens": 13615104.0,
+ "step": 831
+ },
+ {
+ "entropy": 0.8319131731987,
+ "epoch": 1.680808080808081,
+ "grad_norm": 1.8874962329864502,
+ "learning_rate": 8.808080808080809e-06,
+ "loss": 0.8486998081207275,
+ "mean_token_accuracy": 0.7804714441299438,
+ "num_tokens": 13631488.0,
+ "step": 832
+ },
+ {
+ "entropy": 1.3878363370895386,
+ "epoch": 1.6828282828282828,
+ "grad_norm": 2.034135580062866,
+ "learning_rate": 8.794612794612795e-06,
+ "loss": 1.389417052268982,
+ "mean_token_accuracy": 0.6738519668579102,
+ "num_tokens": 13647872.0,
+ "step": 833
+ },
+ {
+ "entropy": 1.2692970037460327,
+ "epoch": 1.6848484848484848,
+ "grad_norm": 2.251734972000122,
+ "learning_rate": 8.781144781144782e-06,
+ "loss": 1.3026533126831055,
+ "mean_token_accuracy": 0.6725085377693176,
+ "num_tokens": 13664256.0,
+ "step": 834
+ },
+ {
+ "entropy": 1.030187726020813,
+ "epoch": 1.6868686868686869,
+ "grad_norm": 2.072669506072998,
+ "learning_rate": 8.767676767676768e-06,
+ "loss": 1.0316928625106812,
+ "mean_token_accuracy": 0.7219711542129517,
+ "num_tokens": 13680640.0,
+ "step": 835
+ },
+ {
+ "entropy": 0.9987007975578308,
+ "epoch": 1.6888888888888889,
+ "grad_norm": 1.9975026845932007,
+ "learning_rate": 8.754208754208755e-06,
+ "loss": 0.9982929229736328,
+ "mean_token_accuracy": 0.7407181262969971,
+ "num_tokens": 13697024.0,
+ "step": 836
+ },
+ {
+ "entropy": 1.335033655166626,
+ "epoch": 1.690909090909091,
+ "grad_norm": 2.9397995471954346,
+ "learning_rate": 8.740740740740741e-06,
+ "loss": 1.3791459798812866,
+ "mean_token_accuracy": 0.6587689518928528,
+ "num_tokens": 13713408.0,
+ "step": 837
+ },
+ {
+ "entropy": 1.064363718032837,
+ "epoch": 1.692929292929293,
+ "grad_norm": 1.859521508216858,
+ "learning_rate": 8.727272727272728e-06,
+ "loss": 1.0654652118682861,
+ "mean_token_accuracy": 0.7314362525939941,
+ "num_tokens": 13729792.0,
+ "step": 838
+ },
+ {
+ "entropy": 1.2016503810882568,
+ "epoch": 1.694949494949495,
+ "grad_norm": 1.9492405652999878,
+ "learning_rate": 8.713804713804714e-06,
+ "loss": 1.2206454277038574,
+ "mean_token_accuracy": 0.6963849663734436,
+ "num_tokens": 13746176.0,
+ "step": 839
+ },
+ {
+ "entropy": 1.0941510200500488,
+ "epoch": 1.696969696969697,
+ "grad_norm": 1.9697571992874146,
+ "learning_rate": 8.7003367003367e-06,
+ "loss": 1.0997231006622314,
+ "mean_token_accuracy": 0.7194675207138062,
+ "num_tokens": 13762560.0,
+ "step": 840
+ },
+ {
+ "entropy": 1.2805196046829224,
+ "epoch": 1.698989898989899,
+ "grad_norm": 2.183673620223999,
+ "learning_rate": 8.686868686868687e-06,
+ "loss": 1.2889196872711182,
+ "mean_token_accuracy": 0.6751953959465027,
+ "num_tokens": 13778944.0,
+ "step": 841
+ },
+ {
+ "entropy": 1.3549253940582275,
+ "epoch": 1.7010101010101009,
+ "grad_norm": 2.1223721504211426,
+ "learning_rate": 8.673400673400674e-06,
+ "loss": 1.351933240890503,
+ "mean_token_accuracy": 0.6756228804588318,
+ "num_tokens": 13795328.0,
+ "step": 842
+ },
+ {
+ "entropy": 1.333280324935913,
+ "epoch": 1.7030303030303031,
+ "grad_norm": 2.0345280170440674,
+ "learning_rate": 8.65993265993266e-06,
+ "loss": 1.3359044790267944,
+ "mean_token_accuracy": 0.6783707737922668,
+ "num_tokens": 13811712.0,
+ "step": 843
+ },
+ {
+ "entropy": 1.0658303499221802,
+ "epoch": 1.705050505050505,
+ "grad_norm": 1.926890254020691,
+ "learning_rate": 8.646464646464647e-06,
+ "loss": 1.0638974905014038,
+ "mean_token_accuracy": 0.7286272644996643,
+ "num_tokens": 13828096.0,
+ "step": 844
+ },
+ {
+ "entropy": 0.9549879431724548,
+ "epoch": 1.7070707070707072,
+ "grad_norm": 2.124664306640625,
+ "learning_rate": 8.632996632996635e-06,
+ "loss": 0.9564298987388611,
+ "mean_token_accuracy": 0.7528700828552246,
+ "num_tokens": 13844480.0,
+ "step": 845
+ },
+ {
+ "entropy": 1.6893866062164307,
+ "epoch": 1.709090909090909,
+ "grad_norm": 2.070680856704712,
+ "learning_rate": 8.61952861952862e-06,
+ "loss": 1.6929473876953125,
+ "mean_token_accuracy": 0.6165730357170105,
+ "num_tokens": 13860864.0,
+ "step": 846
+ },
+ {
+ "entropy": 0.8587391972541809,
+ "epoch": 1.7111111111111112,
+ "grad_norm": 1.9683139324188232,
+ "learning_rate": 8.606060606060606e-06,
+ "loss": 0.8467673659324646,
+ "mean_token_accuracy": 0.7707620859146118,
+ "num_tokens": 13877248.0,
+ "step": 847
+ },
+ {
+ "entropy": 1.6370538473129272,
+ "epoch": 1.713131313131313,
+ "grad_norm": 2.1608047485351562,
+ "learning_rate": 8.592592592592593e-06,
+ "loss": 1.6496617794036865,
+ "mean_token_accuracy": 0.6143136024475098,
+ "num_tokens": 13893632.0,
+ "step": 848
+ },
+ {
+ "entropy": 1.23484206199646,
+ "epoch": 1.7151515151515153,
+ "grad_norm": 2.0296223163604736,
+ "learning_rate": 8.57912457912458e-06,
+ "loss": 1.2227973937988281,
+ "mean_token_accuracy": 0.689667820930481,
+ "num_tokens": 13910016.0,
+ "step": 849
+ },
+ {
+ "entropy": 1.2687101364135742,
+ "epoch": 1.7171717171717171,
+ "grad_norm": 2.204806089401245,
+ "learning_rate": 8.565656565656566e-06,
+ "loss": 1.2694804668426514,
+ "mean_token_accuracy": 0.6865534782409668,
+ "num_tokens": 13926400.0,
+ "step": 850
+ },
+ {
+ "entropy": 1.2225878238677979,
+ "epoch": 1.7191919191919192,
+ "grad_norm": 2.1541168689727783,
+ "learning_rate": 8.552188552188552e-06,
+ "loss": 1.2527751922607422,
+ "mean_token_accuracy": 0.6800195574760437,
+ "num_tokens": 13942784.0,
+ "step": 851
+ },
+ {
+ "entropy": 1.2106293439865112,
+ "epoch": 1.7212121212121212,
+ "grad_norm": 2.0455424785614014,
+ "learning_rate": 8.53872053872054e-06,
+ "loss": 1.2005270719528198,
+ "mean_token_accuracy": 0.6949804425239563,
+ "num_tokens": 13959168.0,
+ "step": 852
+ },
+ {
+ "entropy": 1.169908046722412,
+ "epoch": 1.7232323232323232,
+ "grad_norm": 2.0055043697357178,
+ "learning_rate": 8.525252525252527e-06,
+ "loss": 1.1640398502349854,
+ "mean_token_accuracy": 0.7149487137794495,
+ "num_tokens": 13975552.0,
+ "step": 853
+ },
+ {
+ "entropy": 1.1458942890167236,
+ "epoch": 1.7252525252525253,
+ "grad_norm": 2.0365331172943115,
+ "learning_rate": 8.511784511784512e-06,
+ "loss": 1.1413549184799194,
+ "mean_token_accuracy": 0.7126282453536987,
+ "num_tokens": 13991936.0,
+ "step": 854
+ },
+ {
+ "entropy": 0.8760596513748169,
+ "epoch": 1.7272727272727273,
+ "grad_norm": 1.9858993291854858,
+ "learning_rate": 8.4983164983165e-06,
+ "loss": 0.8733371496200562,
+ "mean_token_accuracy": 0.760869562625885,
+ "num_tokens": 14008320.0,
+ "step": 855
+ },
+ {
+ "entropy": 1.3949522972106934,
+ "epoch": 1.7292929292929293,
+ "grad_norm": 2.6730244159698486,
+ "learning_rate": 8.484848484848486e-06,
+ "loss": 1.4128756523132324,
+ "mean_token_accuracy": 0.6792256832122803,
+ "num_tokens": 14024704.0,
+ "step": 856
+ },
+ {
+ "entropy": 1.4345626831054688,
+ "epoch": 1.7313131313131314,
+ "grad_norm": 1.9446519613265991,
+ "learning_rate": 8.471380471380473e-06,
+ "loss": 1.450282096862793,
+ "mean_token_accuracy": 0.664631187915802,
+ "num_tokens": 14041088.0,
+ "step": 857
+ },
+ {
+ "entropy": 1.1180377006530762,
+ "epoch": 1.7333333333333334,
+ "grad_norm": 2.051783323287964,
+ "learning_rate": 8.457912457912457e-06,
+ "loss": 1.1588772535324097,
+ "mean_token_accuracy": 0.7128114104270935,
+ "num_tokens": 14057472.0,
+ "step": 858
+ },
+ {
+ "entropy": 1.2605851888656616,
+ "epoch": 1.7353535353535352,
+ "grad_norm": 2.0577232837677,
+ "learning_rate": 8.444444444444446e-06,
+ "loss": 1.2173504829406738,
+ "mean_token_accuracy": 0.6938812732696533,
+ "num_tokens": 14073856.0,
+ "step": 859
+ },
+ {
+ "entropy": 1.2957148551940918,
+ "epoch": 1.7373737373737375,
+ "grad_norm": 2.14068603515625,
+ "learning_rate": 8.430976430976432e-06,
+ "loss": 1.301836609840393,
+ "mean_token_accuracy": 0.6832559704780579,
+ "num_tokens": 14090240.0,
+ "step": 860
+ },
+ {
+ "entropy": 0.9871225953102112,
+ "epoch": 1.7393939393939393,
+ "grad_norm": 1.9262301921844482,
+ "learning_rate": 8.417508417508419e-06,
+ "loss": 1.0081474781036377,
+ "mean_token_accuracy": 0.7432828545570374,
+ "num_tokens": 14106624.0,
+ "step": 861
+ },
+ {
+ "entropy": 1.2774608135223389,
+ "epoch": 1.7414141414141415,
+ "grad_norm": 1.8633641004562378,
+ "learning_rate": 8.404040404040405e-06,
+ "loss": 1.2942487001419067,
+ "mean_token_accuracy": 0.6910722851753235,
+ "num_tokens": 14123008.0,
+ "step": 862
+ },
+ {
+ "entropy": 1.3229460716247559,
+ "epoch": 1.7434343434343433,
+ "grad_norm": 2.1861679553985596,
+ "learning_rate": 8.390572390572392e-06,
+ "loss": 1.3244147300720215,
+ "mean_token_accuracy": 0.6719589829444885,
+ "num_tokens": 14139392.0,
+ "step": 863
+ },
+ {
+ "entropy": 1.306138277053833,
+ "epoch": 1.7454545454545456,
+ "grad_norm": 2.1364035606384277,
+ "learning_rate": 8.377104377104378e-06,
+ "loss": 1.3266233205795288,
+ "mean_token_accuracy": 0.6864924430847168,
+ "num_tokens": 14155776.0,
+ "step": 864
+ },
+ {
+ "entropy": 0.9926152229309082,
+ "epoch": 1.7474747474747474,
+ "grad_norm": 1.784975528717041,
+ "learning_rate": 8.363636363636365e-06,
+ "loss": 1.0243797302246094,
+ "mean_token_accuracy": 0.7546409368515015,
+ "num_tokens": 14172160.0,
+ "step": 865
+ },
+ {
+ "entropy": 1.7003151178359985,
+ "epoch": 1.7494949494949497,
+ "grad_norm": 2.081186056137085,
+ "learning_rate": 8.350168350168351e-06,
+ "loss": 1.699352502822876,
+ "mean_token_accuracy": 0.6183438897132874,
+ "num_tokens": 14188544.0,
+ "step": 866
+ },
+ {
+ "entropy": 1.7050426006317139,
+ "epoch": 1.7515151515151515,
+ "grad_norm": 1.8780465126037598,
+ "learning_rate": 8.336700336700338e-06,
+ "loss": 1.7067642211914062,
+ "mean_token_accuracy": 0.6159623861312866,
+ "num_tokens": 14204928.0,
+ "step": 867
+ },
+ {
+ "entropy": 1.0717660188674927,
+ "epoch": 1.7535353535353535,
+ "grad_norm": 2.0461418628692627,
+ "learning_rate": 8.323232323232324e-06,
+ "loss": 1.0874615907669067,
+ "mean_token_accuracy": 0.7315583825111389,
+ "num_tokens": 14221312.0,
+ "step": 868
+ },
+ {
+ "entropy": 1.3650007247924805,
+ "epoch": 1.7555555555555555,
+ "grad_norm": 1.9097316265106201,
+ "learning_rate": 8.30976430976431e-06,
+ "loss": 1.3483806848526,
+ "mean_token_accuracy": 0.6889350414276123,
+ "num_tokens": 14237696.0,
+ "step": 869
+ },
+ {
+ "entropy": 0.9856852293014526,
+ "epoch": 1.7575757575757576,
+ "grad_norm": 1.9712227582931519,
+ "learning_rate": 8.296296296296297e-06,
+ "loss": 0.975986897945404,
+ "mean_token_accuracy": 0.7417562007904053,
+ "num_tokens": 14254080.0,
+ "step": 870
+ },
+ {
+ "entropy": 0.8876434564590454,
+ "epoch": 1.7595959595959596,
+ "grad_norm": 1.8536908626556396,
+ "learning_rate": 8.282828282828283e-06,
+ "loss": 0.8868429660797119,
+ "mean_token_accuracy": 0.7714338302612305,
+ "num_tokens": 14270464.0,
+ "step": 871
+ },
+ {
+ "entropy": 0.7506260871887207,
+ "epoch": 1.7616161616161616,
+ "grad_norm": 1.7068337202072144,
+ "learning_rate": 8.26936026936027e-06,
+ "loss": 0.7422510385513306,
+ "mean_token_accuracy": 0.8030654788017273,
+ "num_tokens": 14286848.0,
+ "step": 872
+ },
+ {
+ "entropy": 1.726669192314148,
+ "epoch": 1.7636363636363637,
+ "grad_norm": 2.157275676727295,
+ "learning_rate": 8.255892255892256e-06,
+ "loss": 1.7225112915039062,
+ "mean_token_accuracy": 0.6207864880561829,
+ "num_tokens": 14303232.0,
+ "step": 873
+ },
+ {
+ "entropy": 0.8383756279945374,
+ "epoch": 1.7656565656565657,
+ "grad_norm": 1.8378020524978638,
+ "learning_rate": 8.242424242424243e-06,
+ "loss": 0.8363397717475891,
+ "mean_token_accuracy": 0.772838294506073,
+ "num_tokens": 14319616.0,
+ "step": 874
+ },
+ {
+ "entropy": 1.1146882772445679,
+ "epoch": 1.7676767676767677,
+ "grad_norm": 2.144796371459961,
+ "learning_rate": 8.22895622895623e-06,
+ "loss": 1.124634027481079,
+ "mean_token_accuracy": 0.711467981338501,
+ "num_tokens": 14336000.0,
+ "step": 875
+ },
+ {
+ "entropy": 1.4330589771270752,
+ "epoch": 1.7696969696969695,
+ "grad_norm": 2.268897533416748,
+ "learning_rate": 8.215488215488216e-06,
+ "loss": 1.4430513381958008,
+ "mean_token_accuracy": 0.6437469720840454,
+ "num_tokens": 14352384.0,
+ "step": 876
+ },
+ {
+ "entropy": 1.1955921649932861,
+ "epoch": 1.7717171717171718,
+ "grad_norm": 2.109701156616211,
+ "learning_rate": 8.202020202020202e-06,
+ "loss": 1.1778371334075928,
+ "mean_token_accuracy": 0.6951025724411011,
+ "num_tokens": 14368768.0,
+ "step": 877
+ },
+ {
+ "entropy": 1.0419657230377197,
+ "epoch": 1.7737373737373736,
+ "grad_norm": 2.002164840698242,
+ "learning_rate": 8.188552188552189e-06,
+ "loss": 1.0472073554992676,
+ "mean_token_accuracy": 0.7313751578330994,
+ "num_tokens": 14385152.0,
+ "step": 878
+ },
+ {
+ "entropy": 1.324011206626892,
+ "epoch": 1.7757575757575759,
+ "grad_norm": 2.233525514602661,
+ "learning_rate": 8.175084175084175e-06,
+ "loss": 1.3433442115783691,
+ "mean_token_accuracy": 0.666890561580658,
+ "num_tokens": 14401536.0,
+ "step": 879
+ },
+ {
+ "entropy": 1.0424224138259888,
+ "epoch": 1.7777777777777777,
+ "grad_norm": 1.8213649988174438,
+ "learning_rate": 8.161616161616162e-06,
+ "loss": 1.0525646209716797,
+ "mean_token_accuracy": 0.7383976578712463,
+ "num_tokens": 14417920.0,
+ "step": 880
+ },
+ {
+ "entropy": 1.3322721719741821,
+ "epoch": 1.77979797979798,
+ "grad_norm": 2.0028865337371826,
+ "learning_rate": 8.148148148148148e-06,
+ "loss": 1.356465458869934,
+ "mean_token_accuracy": 0.6836223602294922,
+ "num_tokens": 14434304.0,
+ "step": 881
+ },
+ {
+ "entropy": 1.2495578527450562,
+ "epoch": 1.7818181818181817,
+ "grad_norm": 2.021883249282837,
+ "learning_rate": 8.134680134680135e-06,
+ "loss": 1.2500100135803223,
+ "mean_token_accuracy": 0.7239863276481628,
+ "num_tokens": 14450688.0,
+ "step": 882
+ },
+ {
+ "entropy": 1.3212776184082031,
+ "epoch": 1.783838383838384,
+ "grad_norm": 1.932453989982605,
+ "learning_rate": 8.121212121212121e-06,
+ "loss": 1.3274552822113037,
+ "mean_token_accuracy": 0.6889350414276123,
+ "num_tokens": 14467072.0,
+ "step": 883
+ },
+ {
+ "entropy": 1.1536792516708374,
+ "epoch": 1.7858585858585858,
+ "grad_norm": 1.944832682609558,
+ "learning_rate": 8.10774410774411e-06,
+ "loss": 1.1310882568359375,
+ "mean_token_accuracy": 0.7239863276481628,
+ "num_tokens": 14483456.0,
+ "step": 884
+ },
+ {
+ "entropy": 1.335244059562683,
+ "epoch": 1.7878787878787878,
+ "grad_norm": 1.9081436395645142,
+ "learning_rate": 8.094276094276094e-06,
+ "loss": 1.3430235385894775,
+ "mean_token_accuracy": 0.6779433488845825,
+ "num_tokens": 14499840.0,
+ "step": 885
+ },
+ {
+ "entropy": 0.9776304364204407,
+ "epoch": 1.7898989898989899,
+ "grad_norm": 2.077418088912964,
+ "learning_rate": 8.08080808080808e-06,
+ "loss": 0.9834706783294678,
+ "mean_token_accuracy": 0.7418172955513,
+ "num_tokens": 14516224.0,
+ "step": 886
+ },
+ {
+ "entropy": 0.9367865920066833,
+ "epoch": 1.791919191919192,
+ "grad_norm": 1.9657868146896362,
+ "learning_rate": 8.067340067340069e-06,
+ "loss": 0.9443397521972656,
+ "mean_token_accuracy": 0.7449926733970642,
+ "num_tokens": 14532608.0,
+ "step": 887
+ },
+ {
+ "entropy": 1.352593183517456,
+ "epoch": 1.793939393939394,
+ "grad_norm": 2.039900779724121,
+ "learning_rate": 8.053872053872055e-06,
+ "loss": 1.3500186204910278,
+ "mean_token_accuracy": 0.6840498447418213,
+ "num_tokens": 14548992.0,
+ "step": 888
+ },
+ {
+ "entropy": 1.21983802318573,
+ "epoch": 1.795959595959596,
+ "grad_norm": 2.04494309425354,
+ "learning_rate": 8.04040404040404e-06,
+ "loss": 1.2253628969192505,
+ "mean_token_accuracy": 0.6975451707839966,
+ "num_tokens": 14565376.0,
+ "step": 889
+ },
+ {
+ "entropy": 1.1911543607711792,
+ "epoch": 1.797979797979798,
+ "grad_norm": 2.09802508354187,
+ "learning_rate": 8.026936026936027e-06,
+ "loss": 1.2186518907546997,
+ "mean_token_accuracy": 0.6941866278648376,
+ "num_tokens": 14581760.0,
+ "step": 890
+ },
+ {
+ "entropy": 1.2681410312652588,
+ "epoch": 1.8,
+ "grad_norm": 2.242945432662964,
+ "learning_rate": 8.013468013468015e-06,
+ "loss": 1.2717292308807373,
+ "mean_token_accuracy": 0.6806302070617676,
+ "num_tokens": 14598144.0,
+ "step": 891
+ },
+ {
+ "entropy": 1.0779279470443726,
+ "epoch": 1.802020202020202,
+ "grad_norm": 2.0271153450012207,
+ "learning_rate": 8.000000000000001e-06,
+ "loss": 1.070134162902832,
+ "mean_token_accuracy": 0.7261846661567688,
+ "num_tokens": 14614528.0,
+ "step": 892
+ },
+ {
+ "entropy": 1.9025505781173706,
+ "epoch": 1.8040404040404039,
+ "grad_norm": 2.221864938735962,
+ "learning_rate": 7.986531986531986e-06,
+ "loss": 1.9073364734649658,
+ "mean_token_accuracy": 0.5933683514595032,
+ "num_tokens": 14630912.0,
+ "step": 893
+ },
+ {
+ "entropy": 1.0104752779006958,
+ "epoch": 1.8060606060606061,
+ "grad_norm": 2.165552854537964,
+ "learning_rate": 7.973063973063974e-06,
+ "loss": 1.0037627220153809,
+ "mean_token_accuracy": 0.7404738664627075,
+ "num_tokens": 14647296.0,
+ "step": 894
+ },
+ {
+ "entropy": 1.5633832216262817,
+ "epoch": 1.808080808080808,
+ "grad_norm": 2.1008827686309814,
+ "learning_rate": 7.95959595959596e-06,
+ "loss": 1.5352282524108887,
+ "mean_token_accuracy": 0.6398388147354126,
+ "num_tokens": 14663680.0,
+ "step": 895
+ },
+ {
+ "entropy": 1.080839991569519,
+ "epoch": 1.8101010101010102,
+ "grad_norm": 1.9256923198699951,
+ "learning_rate": 7.946127946127947e-06,
+ "loss": 1.0921295881271362,
+ "mean_token_accuracy": 0.7247801423072815,
+ "num_tokens": 14680064.0,
+ "step": 896
+ },
+ {
+ "entropy": 0.9788808226585388,
+ "epoch": 1.812121212121212,
+ "grad_norm": 1.9602711200714111,
+ "learning_rate": 7.932659932659934e-06,
+ "loss": 0.9705313444137573,
+ "mean_token_accuracy": 0.7442598938941956,
+ "num_tokens": 14696448.0,
+ "step": 897
+ },
+ {
+ "entropy": 1.1658309698104858,
+ "epoch": 1.8141414141414143,
+ "grad_norm": 1.8840819597244263,
+ "learning_rate": 7.91919191919192e-06,
+ "loss": 1.1346904039382935,
+ "mean_token_accuracy": 0.7229481935501099,
+ "num_tokens": 14712832.0,
+ "step": 898
+ },
+ {
+ "entropy": 0.8824312686920166,
+ "epoch": 1.816161616161616,
+ "grad_norm": 2.1427361965179443,
+ "learning_rate": 7.905723905723907e-06,
+ "loss": 0.9055237770080566,
+ "mean_token_accuracy": 0.759770393371582,
+ "num_tokens": 14729216.0,
+ "step": 899
+ },
+ {
+ "entropy": 1.0914777517318726,
+ "epoch": 1.8181818181818183,
+ "grad_norm": 2.282011032104492,
+ "learning_rate": 7.892255892255893e-06,
+ "loss": 1.101895809173584,
+ "mean_token_accuracy": 0.7096360325813293,
+ "num_tokens": 14745600.0,
+ "step": 900
+ },
+ {
+ "epoch": 1.8181818181818183,
+ "eval_entropy": 1.303592009890464,
+ "eval_loss": 1.5282857418060303,
+ "eval_mean_token_accuracy": 0.6474241422068688,
+ "eval_num_tokens": 14745600.0,
+ "eval_runtime": 43.8637,
+ "eval_samples_per_second": 22.57,
+ "eval_steps_per_second": 2.827,
+ "step": 900
+ },
+ {
+ "entropy": 1.4693570137023926,
+ "epoch": 1.8202020202020202,
+ "grad_norm": 2.0938942432403564,
+ "learning_rate": 7.87878787878788e-06,
+ "loss": 1.4499380588531494,
+ "mean_token_accuracy": 0.6446018815040588,
+ "num_tokens": 14761984.0,
+ "step": 901
+ },
+ {
+ "entropy": 1.0810022354125977,
+ "epoch": 1.8222222222222222,
+ "grad_norm": 2.0279886722564697,
+ "learning_rate": 7.865319865319866e-06,
+ "loss": 1.0581843852996826,
+ "mean_token_accuracy": 0.723375678062439,
+ "num_tokens": 14778368.0,
+ "step": 902
+ },
+ {
+ "entropy": 1.0680485963821411,
+ "epoch": 1.8242424242424242,
+ "grad_norm": 2.015047788619995,
+ "learning_rate": 7.851851851851853e-06,
+ "loss": 1.0612249374389648,
+ "mean_token_accuracy": 0.7264289259910583,
+ "num_tokens": 14794752.0,
+ "step": 903
+ },
+ {
+ "entropy": 0.981645941734314,
+ "epoch": 1.8262626262626263,
+ "grad_norm": 2.0122671127319336,
+ "learning_rate": 7.838383838383839e-06,
+ "loss": 0.9650060534477234,
+ "mean_token_accuracy": 0.7337567210197449,
+ "num_tokens": 14811136.0,
+ "step": 904
+ },
+ {
+ "entropy": 1.0827676057815552,
+ "epoch": 1.8282828282828283,
+ "grad_norm": 2.1086010932922363,
+ "learning_rate": 7.824915824915826e-06,
+ "loss": 1.097700595855713,
+ "mean_token_accuracy": 0.7087811231613159,
+ "num_tokens": 14827520.0,
+ "step": 905
+ },
+ {
+ "entropy": 0.9498234391212463,
+ "epoch": 1.8303030303030303,
+ "grad_norm": 1.9012964963912964,
+ "learning_rate": 7.811447811447812e-06,
+ "loss": 0.9704345464706421,
+ "mean_token_accuracy": 0.7472520470619202,
+ "num_tokens": 14843904.0,
+ "step": 906
+ },
+ {
+ "entropy": 0.8871595859527588,
+ "epoch": 1.8323232323232324,
+ "grad_norm": 1.9970269203186035,
+ "learning_rate": 7.797979797979799e-06,
+ "loss": 0.8876000642776489,
+ "mean_token_accuracy": 0.7578163146972656,
+ "num_tokens": 14860288.0,
+ "step": 907
+ },
+ {
+ "entropy": 1.1324913501739502,
+ "epoch": 1.8343434343434344,
+ "grad_norm": 2.1133675575256348,
+ "learning_rate": 7.784511784511785e-06,
+ "loss": 1.142140507698059,
+ "mean_token_accuracy": 0.7180629968643188,
+ "num_tokens": 14876672.0,
+ "step": 908
+ },
+ {
+ "entropy": 1.1514050960540771,
+ "epoch": 1.8363636363636364,
+ "grad_norm": 2.073002338409424,
+ "learning_rate": 7.771043771043772e-06,
+ "loss": 1.1670506000518799,
+ "mean_token_accuracy": 0.7009037733078003,
+ "num_tokens": 14893056.0,
+ "step": 909
+ },
+ {
+ "entropy": 1.17017662525177,
+ "epoch": 1.8383838383838382,
+ "grad_norm": 2.17962384223938,
+ "learning_rate": 7.757575757575758e-06,
+ "loss": 1.1800150871276855,
+ "mean_token_accuracy": 0.7028578519821167,
+ "num_tokens": 14909440.0,
+ "step": 910
+ },
+ {
+ "entropy": 0.9377864599227905,
+ "epoch": 1.8404040404040405,
+ "grad_norm": 2.2141757011413574,
+ "learning_rate": 7.744107744107745e-06,
+ "loss": 0.9712649583816528,
+ "mean_token_accuracy": 0.7405349016189575,
+ "num_tokens": 14925824.0,
+ "step": 911
+ },
+ {
+ "entropy": 1.2093497514724731,
+ "epoch": 1.8424242424242423,
+ "grad_norm": 2.0677497386932373,
+ "learning_rate": 7.730639730639731e-06,
+ "loss": 1.210886001586914,
+ "mean_token_accuracy": 0.7037737965583801,
+ "num_tokens": 14942208.0,
+ "step": 912
+ },
+ {
+ "entropy": 0.8966845273971558,
+ "epoch": 1.8444444444444446,
+ "grad_norm": 2.0689940452575684,
+ "learning_rate": 7.717171717171717e-06,
+ "loss": 0.9142885208129883,
+ "mean_token_accuracy": 0.7523815631866455,
+ "num_tokens": 14958592.0,
+ "step": 913
+ },
+ {
+ "entropy": 1.4647231101989746,
+ "epoch": 1.8464646464646464,
+ "grad_norm": 2.1815598011016846,
+ "learning_rate": 7.703703703703704e-06,
+ "loss": 1.4676027297973633,
+ "mean_token_accuracy": 0.6506472826004028,
+ "num_tokens": 14974976.0,
+ "step": 914
+ },
+ {
+ "entropy": 1.5831035375595093,
+ "epoch": 1.8484848484848486,
+ "grad_norm": 2.028407096862793,
+ "learning_rate": 7.69023569023569e-06,
+ "loss": 1.5933328866958618,
+ "mean_token_accuracy": 0.6368466019630432,
+ "num_tokens": 14991360.0,
+ "step": 915
+ },
+ {
+ "entropy": 1.3269871473312378,
+ "epoch": 1.8505050505050504,
+ "grad_norm": 2.173727035522461,
+ "learning_rate": 7.676767676767677e-06,
+ "loss": 1.3404830694198608,
+ "mean_token_accuracy": 0.6750122308731079,
+ "num_tokens": 15007744.0,
+ "step": 916
+ },
+ {
+ "entropy": 1.378677487373352,
+ "epoch": 1.8525252525252527,
+ "grad_norm": 2.0357983112335205,
+ "learning_rate": 7.663299663299663e-06,
+ "loss": 1.3752634525299072,
+ "mean_token_accuracy": 0.6711651086807251,
+ "num_tokens": 15024128.0,
+ "step": 917
+ },
+ {
+ "entropy": 1.5462778806686401,
+ "epoch": 1.8545454545454545,
+ "grad_norm": 1.9574400186538696,
+ "learning_rate": 7.64983164983165e-06,
+ "loss": 1.5374811887741089,
+ "mean_token_accuracy": 0.6375183463096619,
+ "num_tokens": 15040512.0,
+ "step": 918
+ },
+ {
+ "entropy": 1.2211424112319946,
+ "epoch": 1.8565656565656565,
+ "grad_norm": 2.0203700065612793,
+ "learning_rate": 7.636363636363638e-06,
+ "loss": 1.2375779151916504,
+ "mean_token_accuracy": 0.6878358721733093,
+ "num_tokens": 15056896.0,
+ "step": 919
+ },
+ {
+ "entropy": 1.493086338043213,
+ "epoch": 1.8585858585858586,
+ "grad_norm": 2.1047544479370117,
+ "learning_rate": 7.622895622895623e-06,
+ "loss": 1.471447229385376,
+ "mean_token_accuracy": 0.6543111801147461,
+ "num_tokens": 15073280.0,
+ "step": 920
+ },
+ {
+ "entropy": 1.351884126663208,
+ "epoch": 1.8606060606060606,
+ "grad_norm": 2.2803215980529785,
+ "learning_rate": 7.60942760942761e-06,
+ "loss": 1.3451658487319946,
+ "mean_token_accuracy": 0.6725696325302124,
+ "num_tokens": 15089664.0,
+ "step": 921
+ },
+ {
+ "entropy": 1.4343167543411255,
+ "epoch": 1.8626262626262626,
+ "grad_norm": 1.9821034669876099,
+ "learning_rate": 7.595959595959597e-06,
+ "loss": 1.4694747924804688,
+ "mean_token_accuracy": 0.6622496247291565,
+ "num_tokens": 15106048.0,
+ "step": 922
+ },
+ {
+ "entropy": 1.3977091312408447,
+ "epoch": 1.8646464646464647,
+ "grad_norm": 2.0600125789642334,
+ "learning_rate": 7.582491582491583e-06,
+ "loss": 1.4120681285858154,
+ "mean_token_accuracy": 0.6642037034034729,
+ "num_tokens": 15122432.0,
+ "step": 923
+ },
+ {
+ "entropy": 1.4182765483856201,
+ "epoch": 1.8666666666666667,
+ "grad_norm": 1.9010684490203857,
+ "learning_rate": 7.569023569023569e-06,
+ "loss": 1.4160431623458862,
+ "mean_token_accuracy": 0.6696995496749878,
+ "num_tokens": 15138816.0,
+ "step": 924
+ },
+ {
+ "entropy": 1.1520206928253174,
+ "epoch": 1.8686868686868687,
+ "grad_norm": 1.8414452075958252,
+ "learning_rate": 7.555555555555556e-06,
+ "loss": 1.154735803604126,
+ "mean_token_accuracy": 0.7159867882728577,
+ "num_tokens": 15155200.0,
+ "step": 925
+ },
+ {
+ "entropy": 0.8958664536476135,
+ "epoch": 1.8707070707070708,
+ "grad_norm": 1.82782781124115,
+ "learning_rate": 7.542087542087543e-06,
+ "loss": 0.8718663454055786,
+ "mean_token_accuracy": 0.7771739363670349,
+ "num_tokens": 15171584.0,
+ "step": 926
+ },
+ {
+ "entropy": 1.4068299531936646,
+ "epoch": 1.8727272727272726,
+ "grad_norm": 1.9482192993164062,
+ "learning_rate": 7.52861952861953e-06,
+ "loss": 1.3947927951812744,
+ "mean_token_accuracy": 0.6699438095092773,
+ "num_tokens": 15187968.0,
+ "step": 927
+ },
+ {
+ "entropy": 0.8980593085289001,
+ "epoch": 1.8747474747474748,
+ "grad_norm": 2.040997266769409,
+ "learning_rate": 7.515151515151516e-06,
+ "loss": 0.9092239141464233,
+ "mean_token_accuracy": 0.7550684213638306,
+ "num_tokens": 15204352.0,
+ "step": 928
+ },
+ {
+ "entropy": 1.2259284257888794,
+ "epoch": 1.8767676767676766,
+ "grad_norm": 2.1932485103607178,
+ "learning_rate": 7.501683501683502e-06,
+ "loss": 1.2324566841125488,
+ "mean_token_accuracy": 0.6918050646781921,
+ "num_tokens": 15220736.0,
+ "step": 929
+ },
+ {
+ "entropy": 1.3737105131149292,
+ "epoch": 1.878787878787879,
+ "grad_norm": 1.9829115867614746,
+ "learning_rate": 7.4882154882154886e-06,
+ "loss": 1.4132641553878784,
+ "mean_token_accuracy": 0.6854543089866638,
+ "num_tokens": 15237120.0,
+ "step": 930
+ },
+ {
+ "entropy": 1.1767487525939941,
+ "epoch": 1.8808080808080807,
+ "grad_norm": 2.279439926147461,
+ "learning_rate": 7.474747474747476e-06,
+ "loss": 1.1658971309661865,
+ "mean_token_accuracy": 0.711467981338501,
+ "num_tokens": 15253504.0,
+ "step": 931
+ },
+ {
+ "entropy": 1.0110876560211182,
+ "epoch": 1.882828282828283,
+ "grad_norm": 2.144711494445801,
+ "learning_rate": 7.4612794612794615e-06,
+ "loss": 1.0287975072860718,
+ "mean_token_accuracy": 0.7346726655960083,
+ "num_tokens": 15269888.0,
+ "step": 932
+ },
+ {
+ "entropy": 1.0826390981674194,
+ "epoch": 1.8848484848484848,
+ "grad_norm": 2.1858105659484863,
+ "learning_rate": 7.447811447811448e-06,
+ "loss": 1.1194939613342285,
+ "mean_token_accuracy": 0.7099413871765137,
+ "num_tokens": 15286272.0,
+ "step": 933
+ },
+ {
+ "entropy": 1.430822491645813,
+ "epoch": 1.886868686868687,
+ "grad_norm": 1.9496047496795654,
+ "learning_rate": 7.434343434343435e-06,
+ "loss": 1.452394723892212,
+ "mean_token_accuracy": 0.6591353416442871,
+ "num_tokens": 15302656.0,
+ "step": 934
+ },
+ {
+ "entropy": 0.9242729544639587,
+ "epoch": 1.8888888888888888,
+ "grad_norm": 2.1464316844940186,
+ "learning_rate": 7.420875420875422e-06,
+ "loss": 0.9562792778015137,
+ "mean_token_accuracy": 0.7407791614532471,
+ "num_tokens": 15319040.0,
+ "step": 935
+ },
+ {
+ "entropy": 1.5215003490447998,
+ "epoch": 1.8909090909090909,
+ "grad_norm": 2.105729818344116,
+ "learning_rate": 7.4074074074074075e-06,
+ "loss": 1.5162043571472168,
+ "mean_token_accuracy": 0.6430141925811768,
+ "num_tokens": 15335424.0,
+ "step": 936
+ },
+ {
+ "entropy": 1.1809922456741333,
+ "epoch": 1.892929292929293,
+ "grad_norm": 2.1825973987579346,
+ "learning_rate": 7.393939393939395e-06,
+ "loss": 1.192020058631897,
+ "mean_token_accuracy": 0.693453848361969,
+ "num_tokens": 15351808.0,
+ "step": 937
+ },
+ {
+ "entropy": 1.0707145929336548,
+ "epoch": 1.894949494949495,
+ "grad_norm": 2.033658027648926,
+ "learning_rate": 7.380471380471381e-06,
+ "loss": 1.0794811248779297,
+ "mean_token_accuracy": 0.7291157841682434,
+ "num_tokens": 15368192.0,
+ "step": 938
+ },
+ {
+ "entropy": 0.8133494257926941,
+ "epoch": 1.896969696969697,
+ "grad_norm": 1.9038894176483154,
+ "learning_rate": 7.367003367003368e-06,
+ "loss": 0.817105233669281,
+ "mean_token_accuracy": 0.785906195640564,
+ "num_tokens": 15384576.0,
+ "step": 939
+ },
+ {
+ "entropy": 1.1366889476776123,
+ "epoch": 1.898989898989899,
+ "grad_norm": 2.3373923301696777,
+ "learning_rate": 7.353535353535353e-06,
+ "loss": 1.1590774059295654,
+ "mean_token_accuracy": 0.7004152536392212,
+ "num_tokens": 15400960.0,
+ "step": 940
+ },
+ {
+ "entropy": 1.0165143013000488,
+ "epoch": 1.901010101010101,
+ "grad_norm": 1.8290495872497559,
+ "learning_rate": 7.340067340067341e-06,
+ "loss": 0.9955418109893799,
+ "mean_token_accuracy": 0.7454811930656433,
+ "num_tokens": 15417344.0,
+ "step": 941
+ },
+ {
+ "entropy": 1.536665916442871,
+ "epoch": 1.903030303030303,
+ "grad_norm": 2.2636616230010986,
+ "learning_rate": 7.326599326599327e-06,
+ "loss": 1.5516541004180908,
+ "mean_token_accuracy": 0.6290913820266724,
+ "num_tokens": 15433728.0,
+ "step": 942
+ },
+ {
+ "entropy": 1.0885748863220215,
+ "epoch": 1.905050505050505,
+ "grad_norm": 2.104462146759033,
+ "learning_rate": 7.3131313131313146e-06,
+ "loss": 1.096163272857666,
+ "mean_token_accuracy": 0.7129946351051331,
+ "num_tokens": 15450112.0,
+ "step": 943
+ },
+ {
+ "entropy": 1.0774754285812378,
+ "epoch": 1.907070707070707,
+ "grad_norm": 2.006735324859619,
+ "learning_rate": 7.2996632996633e-06,
+ "loss": 1.0466490983963013,
+ "mean_token_accuracy": 0.7385197877883911,
+ "num_tokens": 15466496.0,
+ "step": 944
+ },
+ {
+ "entropy": 1.3489689826965332,
+ "epoch": 1.9090909090909092,
+ "grad_norm": 2.148700714111328,
+ "learning_rate": 7.286195286195287e-06,
+ "loss": 1.317432165145874,
+ "mean_token_accuracy": 0.6951025724411011,
+ "num_tokens": 15482880.0,
+ "step": 945
+ },
+ {
+ "entropy": 0.957065999507904,
+ "epoch": 1.911111111111111,
+ "grad_norm": 2.118453025817871,
+ "learning_rate": 7.272727272727273e-06,
+ "loss": 0.9499804377555847,
+ "mean_token_accuracy": 0.7590376138687134,
+ "num_tokens": 15499264.0,
+ "step": 946
+ },
+ {
+ "entropy": 1.1132235527038574,
+ "epoch": 1.9131313131313132,
+ "grad_norm": 1.8254663944244385,
+ "learning_rate": 7.2592592592592605e-06,
+ "loss": 1.0975847244262695,
+ "mean_token_accuracy": 0.7224596738815308,
+ "num_tokens": 15515648.0,
+ "step": 947
+ },
+ {
+ "entropy": 1.1259424686431885,
+ "epoch": 1.915151515151515,
+ "grad_norm": 2.105797529220581,
+ "learning_rate": 7.245791245791246e-06,
+ "loss": 1.1171300411224365,
+ "mean_token_accuracy": 0.7150097489356995,
+ "num_tokens": 15532032.0,
+ "step": 948
+ },
+ {
+ "entropy": 1.3454829454421997,
+ "epoch": 1.9171717171717173,
+ "grad_norm": 2.2715842723846436,
+ "learning_rate": 7.232323232323233e-06,
+ "loss": 1.359604835510254,
+ "mean_token_accuracy": 0.6568759083747864,
+ "num_tokens": 15548416.0,
+ "step": 949
+ },
+ {
+ "entropy": 1.2873634099960327,
+ "epoch": 1.9191919191919191,
+ "grad_norm": 1.8735847473144531,
+ "learning_rate": 7.21885521885522e-06,
+ "loss": 1.2842814922332764,
+ "mean_token_accuracy": 0.6951025724411011,
+ "num_tokens": 15564800.0,
+ "step": 950
+ },
+ {
+ "entropy": 1.154240369796753,
+ "epoch": 1.9212121212121214,
+ "grad_norm": 1.9821181297302246,
+ "learning_rate": 7.2053872053872064e-06,
+ "loss": 1.1434863805770874,
+ "mean_token_accuracy": 0.7203834652900696,
+ "num_tokens": 15581184.0,
+ "step": 951
+ },
+ {
+ "entropy": 1.05129075050354,
+ "epoch": 1.9232323232323232,
+ "grad_norm": 1.990182638168335,
+ "learning_rate": 7.191919191919192e-06,
+ "loss": 1.0694432258605957,
+ "mean_token_accuracy": 0.734245240688324,
+ "num_tokens": 15597568.0,
+ "step": 952
+ },
+ {
+ "entropy": 1.0232638120651245,
+ "epoch": 1.9252525252525252,
+ "grad_norm": 2.017350673675537,
+ "learning_rate": 7.178451178451179e-06,
+ "loss": 1.0090606212615967,
+ "mean_token_accuracy": 0.7305202484130859,
+ "num_tokens": 15613952.0,
+ "step": 953
+ },
+ {
+ "entropy": 0.8146723508834839,
+ "epoch": 1.9272727272727272,
+ "grad_norm": 1.8920353651046753,
+ "learning_rate": 7.164983164983166e-06,
+ "loss": 0.8227875232696533,
+ "mean_token_accuracy": 0.778761625289917,
+ "num_tokens": 15630336.0,
+ "step": 954
+ },
+ {
+ "entropy": 0.6415520906448364,
+ "epoch": 1.9292929292929293,
+ "grad_norm": 1.7636535167694092,
+ "learning_rate": 7.151515151515152e-06,
+ "loss": 0.6381626725196838,
+ "mean_token_accuracy": 0.8171104192733765,
+ "num_tokens": 15646720.0,
+ "step": 955
+ },
+ {
+ "entropy": 1.4309028387069702,
+ "epoch": 1.9313131313131313,
+ "grad_norm": 1.8857070207595825,
+ "learning_rate": 7.138047138047138e-06,
+ "loss": 1.4779207706451416,
+ "mean_token_accuracy": 0.6556546092033386,
+ "num_tokens": 15663104.0,
+ "step": 956
+ },
+ {
+ "entropy": 1.1201391220092773,
+ "epoch": 1.9333333333333333,
+ "grad_norm": 1.833791732788086,
+ "learning_rate": 7.124579124579125e-06,
+ "loss": 1.0972872972488403,
+ "mean_token_accuracy": 0.7187347412109375,
+ "num_tokens": 15679488.0,
+ "step": 957
+ },
+ {
+ "entropy": 1.1922115087509155,
+ "epoch": 1.9353535353535354,
+ "grad_norm": 1.8989986181259155,
+ "learning_rate": 7.111111111111112e-06,
+ "loss": 1.2037092447280884,
+ "mean_token_accuracy": 0.7104299068450928,
+ "num_tokens": 15695872.0,
+ "step": 958
+ },
+ {
+ "entropy": 0.9312219023704529,
+ "epoch": 1.9373737373737374,
+ "grad_norm": 2.0068769454956055,
+ "learning_rate": 7.097643097643099e-06,
+ "loss": 0.9126096963882446,
+ "mean_token_accuracy": 0.7556790709495544,
+ "num_tokens": 15712256.0,
+ "step": 959
+ },
+ {
+ "entropy": 1.1126971244812012,
+ "epoch": 1.9393939393939394,
+ "grad_norm": 1.9915603399276733,
+ "learning_rate": 7.084175084175085e-06,
+ "loss": 1.1312336921691895,
+ "mean_token_accuracy": 0.7178187370300293,
+ "num_tokens": 15728640.0,
+ "step": 960
+ },
+ {
+ "entropy": 1.487212896347046,
+ "epoch": 1.9414141414141413,
+ "grad_norm": 2.068739414215088,
+ "learning_rate": 7.070707070707071e-06,
+ "loss": 1.4876431226730347,
+ "mean_token_accuracy": 0.6568759083747864,
+ "num_tokens": 15745024.0,
+ "step": 961
+ },
+ {
+ "entropy": 1.207465648651123,
+ "epoch": 1.9434343434343435,
+ "grad_norm": 1.9962066411972046,
+ "learning_rate": 7.057239057239058e-06,
+ "loss": 1.2003669738769531,
+ "mean_token_accuracy": 0.7024914622306824,
+ "num_tokens": 15761408.0,
+ "step": 962
+ },
+ {
+ "entropy": 1.7148067951202393,
+ "epoch": 1.9454545454545453,
+ "grad_norm": 2.3441736698150635,
+ "learning_rate": 7.043771043771043e-06,
+ "loss": 1.7310147285461426,
+ "mean_token_accuracy": 0.6288471221923828,
+ "num_tokens": 15777792.0,
+ "step": 963
+ },
+ {
+ "entropy": 1.3103407621383667,
+ "epoch": 1.9474747474747476,
+ "grad_norm": 1.915368676185608,
+ "learning_rate": 7.030303030303031e-06,
+ "loss": 1.3412322998046875,
+ "mean_token_accuracy": 0.6845383644104004,
+ "num_tokens": 15794176.0,
+ "step": 964
+ },
+ {
+ "entropy": 0.9835378527641296,
+ "epoch": 1.9494949494949494,
+ "grad_norm": 1.9611473083496094,
+ "learning_rate": 7.016835016835017e-06,
+ "loss": 0.9903290271759033,
+ "mean_token_accuracy": 0.7569003701210022,
+ "num_tokens": 15810560.0,
+ "step": 965
+ },
+ {
+ "entropy": 1.4645164012908936,
+ "epoch": 1.9515151515151516,
+ "grad_norm": 2.017500162124634,
+ "learning_rate": 7.0033670033670045e-06,
+ "loss": 1.4612789154052734,
+ "mean_token_accuracy": 0.6558378338813782,
+ "num_tokens": 15826944.0,
+ "step": 966
+ },
+ {
+ "entropy": 1.2586326599121094,
+ "epoch": 1.9535353535353535,
+ "grad_norm": 2.0969791412353516,
+ "learning_rate": 6.98989898989899e-06,
+ "loss": 1.224534273147583,
+ "mean_token_accuracy": 0.6958353519439697,
+ "num_tokens": 15843328.0,
+ "step": 967
+ },
+ {
+ "entropy": 0.8170226216316223,
+ "epoch": 1.9555555555555557,
+ "grad_norm": 1.9889802932739258,
+ "learning_rate": 6.976430976430977e-06,
+ "loss": 0.8214236497879028,
+ "mean_token_accuracy": 0.7777845859527588,
+ "num_tokens": 15859712.0,
+ "step": 968
+ },
+ {
+ "entropy": 0.8966978788375854,
+ "epoch": 1.9575757575757575,
+ "grad_norm": 1.8261324167251587,
+ "learning_rate": 6.962962962962964e-06,
+ "loss": 0.8880574107170105,
+ "mean_token_accuracy": 0.7681363224983215,
+ "num_tokens": 15876096.0,
+ "step": 969
+ },
+ {
+ "entropy": 1.4782947301864624,
+ "epoch": 1.9595959595959596,
+ "grad_norm": 2.2666432857513428,
+ "learning_rate": 6.9494949494949505e-06,
+ "loss": 1.5416191816329956,
+ "mean_token_accuracy": 0.6491206884384155,
+ "num_tokens": 15892480.0,
+ "step": 970
+ },
+ {
+ "entropy": 1.6055350303649902,
+ "epoch": 1.9616161616161616,
+ "grad_norm": 2.0585474967956543,
+ "learning_rate": 6.936026936026936e-06,
+ "loss": 1.627528190612793,
+ "mean_token_accuracy": 0.6376404762268066,
+ "num_tokens": 15908864.0,
+ "step": 971
+ },
+ {
+ "entropy": 1.092020869255066,
+ "epoch": 1.9636363636363636,
+ "grad_norm": 2.002554416656494,
+ "learning_rate": 6.922558922558923e-06,
+ "loss": 1.0792224407196045,
+ "mean_token_accuracy": 0.7230092883110046,
+ "num_tokens": 15925248.0,
+ "step": 972
+ },
+ {
+ "entropy": 0.7485284805297852,
+ "epoch": 1.9656565656565657,
+ "grad_norm": 1.9477611780166626,
+ "learning_rate": 6.90909090909091e-06,
+ "loss": 0.7471826076507568,
+ "mean_token_accuracy": 0.7961040735244751,
+ "num_tokens": 15941632.0,
+ "step": 973
+ },
+ {
+ "entropy": 1.3997598886489868,
+ "epoch": 1.9676767676767677,
+ "grad_norm": 2.1589887142181396,
+ "learning_rate": 6.895622895622896e-06,
+ "loss": 1.3947563171386719,
+ "mean_token_accuracy": 0.6626160144805908,
+ "num_tokens": 15958016.0,
+ "step": 974
+ },
+ {
+ "entropy": 1.2053040266036987,
+ "epoch": 1.9696969696969697,
+ "grad_norm": 2.1021888256073,
+ "learning_rate": 6.882154882154882e-06,
+ "loss": 1.2196800708770752,
+ "mean_token_accuracy": 0.6948583126068115,
+ "num_tokens": 15974400.0,
+ "step": 975
+ },
+ {
+ "entropy": 1.1462217569351196,
+ "epoch": 1.9717171717171718,
+ "grad_norm": 2.1295032501220703,
+ "learning_rate": 6.868686868686869e-06,
+ "loss": 1.1645277738571167,
+ "mean_token_accuracy": 0.717391312122345,
+ "num_tokens": 15990784.0,
+ "step": 976
+ },
+ {
+ "entropy": 0.9189468026161194,
+ "epoch": 1.9737373737373738,
+ "grad_norm": 1.9384510517120361,
+ "learning_rate": 6.855218855218856e-06,
+ "loss": 0.8912248015403748,
+ "mean_token_accuracy": 0.7561064958572388,
+ "num_tokens": 16007168.0,
+ "step": 977
+ },
+ {
+ "entropy": 1.2767856121063232,
+ "epoch": 1.9757575757575756,
+ "grad_norm": 2.086789608001709,
+ "learning_rate": 6.841750841750842e-06,
+ "loss": 1.2951477766036987,
+ "mean_token_accuracy": 0.6822178959846497,
+ "num_tokens": 16023552.0,
+ "step": 978
+ },
+ {
+ "entropy": 1.4691493511199951,
+ "epoch": 1.9777777777777779,
+ "grad_norm": 2.081007719039917,
+ "learning_rate": 6.828282828282828e-06,
+ "loss": 1.4899260997772217,
+ "mean_token_accuracy": 0.6618832349777222,
+ "num_tokens": 16039936.0,
+ "step": 979
+ },
+ {
+ "entropy": 1.5689647197723389,
+ "epoch": 1.9797979797979797,
+ "grad_norm": 2.168604850769043,
+ "learning_rate": 6.814814814814815e-06,
+ "loss": 1.57144296169281,
+ "mean_token_accuracy": 0.6405715942382812,
+ "num_tokens": 16056320.0,
+ "step": 980
+ },
+ {
+ "entropy": 1.4429019689559937,
+ "epoch": 1.981818181818182,
+ "grad_norm": 1.9658228158950806,
+ "learning_rate": 6.801346801346802e-06,
+ "loss": 1.473623275756836,
+ "mean_token_accuracy": 0.6576697826385498,
+ "num_tokens": 16072704.0,
+ "step": 981
+ },
+ {
+ "entropy": 1.0032570362091064,
+ "epoch": 1.9838383838383837,
+ "grad_norm": 1.9377433061599731,
+ "learning_rate": 6.787878787878789e-06,
+ "loss": 0.9838194847106934,
+ "mean_token_accuracy": 0.7490839958190918,
+ "num_tokens": 16089088.0,
+ "step": 982
+ },
+ {
+ "entropy": 0.9308913946151733,
+ "epoch": 1.985858585858586,
+ "grad_norm": 1.9042673110961914,
+ "learning_rate": 6.774410774410775e-06,
+ "loss": 0.9097878932952881,
+ "mean_token_accuracy": 0.7631289958953857,
+ "num_tokens": 16105472.0,
+ "step": 983
+ },
+ {
+ "entropy": 1.0642609596252441,
+ "epoch": 1.9878787878787878,
+ "grad_norm": 2.1934077739715576,
+ "learning_rate": 6.760942760942761e-06,
+ "loss": 1.0470092296600342,
+ "mean_token_accuracy": 0.7263067960739136,
+ "num_tokens": 16121856.0,
+ "step": 984
+ },
+ {
+ "entropy": 1.2610602378845215,
+ "epoch": 1.98989898989899,
+ "grad_norm": 2.2924704551696777,
+ "learning_rate": 6.747474747474749e-06,
+ "loss": 1.246216058731079,
+ "mean_token_accuracy": 0.6873473525047302,
+ "num_tokens": 16138240.0,
+ "step": 985
+ },
+ {
+ "entropy": 1.4189343452453613,
+ "epoch": 1.9919191919191919,
+ "grad_norm": 1.906319260597229,
+ "learning_rate": 6.734006734006735e-06,
+ "loss": 1.4098241329193115,
+ "mean_token_accuracy": 0.6734855771064758,
+ "num_tokens": 16154624.0,
+ "step": 986
+ },
+ {
+ "entropy": 1.339056134223938,
+ "epoch": 1.993939393939394,
+ "grad_norm": 2.034353733062744,
+ "learning_rate": 6.720538720538721e-06,
+ "loss": 1.3571393489837646,
+ "mean_token_accuracy": 0.678859293460846,
+ "num_tokens": 16171008.0,
+ "step": 987
+ },
+ {
+ "entropy": 1.057302474975586,
+ "epoch": 1.995959595959596,
+ "grad_norm": 2.1716647148132324,
+ "learning_rate": 6.707070707070707e-06,
+ "loss": 1.042845606803894,
+ "mean_token_accuracy": 0.7322300672531128,
+ "num_tokens": 16187392.0,
+ "step": 988
+ },
+ {
+ "entropy": 1.2057303190231323,
+ "epoch": 1.997979797979798,
+ "grad_norm": 1.9809064865112305,
+ "learning_rate": 6.6936026936026945e-06,
+ "loss": 1.2091717720031738,
+ "mean_token_accuracy": 0.7068270444869995,
+ "num_tokens": 16203776.0,
+ "step": 989
+ },
+ {
+ "entropy": 1.3678697347640991,
+ "epoch": 2.0,
+ "grad_norm": 2.1474292278289795,
+ "learning_rate": 6.680134680134681e-06,
+ "loss": 1.359403133392334,
+ "mean_token_accuracy": 0.6702491641044617,
+ "num_tokens": 16220160.0,
+ "step": 990
+ },
+ {
+ "entropy": 1.6815240383148193,
+ "epoch": 2.002020202020202,
+ "grad_norm": 2.209095001220703,
+ "learning_rate": 6.666666666666667e-06,
+ "loss": 1.5768096446990967,
+ "mean_token_accuracy": 0.6408768892288208,
+ "num_tokens": 16236544.0,
+ "step": 991
+ },
+ {
+ "entropy": 1.1142786741256714,
+ "epoch": 2.004040404040404,
+ "grad_norm": 2.8133344650268555,
+ "learning_rate": 6.653198653198654e-06,
+ "loss": 0.937615692615509,
+ "mean_token_accuracy": 0.7556179761886597,
+ "num_tokens": 16252928.0,
+ "step": 992
+ },
+ {
+ "entropy": 1.2004083395004272,
+ "epoch": 2.006060606060606,
+ "grad_norm": 2.753380060195923,
+ "learning_rate": 6.6397306397306405e-06,
+ "loss": 1.0454245805740356,
+ "mean_token_accuracy": 0.7423058152198792,
+ "num_tokens": 16269312.0,
+ "step": 993
+ },
+ {
+ "entropy": 0.9945010542869568,
+ "epoch": 2.008080808080808,
+ "grad_norm": 2.3262150287628174,
+ "learning_rate": 6.626262626262627e-06,
+ "loss": 0.8623301982879639,
+ "mean_token_accuracy": 0.7791890501976013,
+ "num_tokens": 16285696.0,
+ "step": 994
+ },
+ {
+ "entropy": 1.0330396890640259,
+ "epoch": 2.01010101010101,
+ "grad_norm": 2.3145525455474854,
+ "learning_rate": 6.612794612794613e-06,
+ "loss": 0.9156472682952881,
+ "mean_token_accuracy": 0.7564118504524231,
+ "num_tokens": 16302080.0,
+ "step": 995
+ },
+ {
+ "entropy": 1.109296202659607,
+ "epoch": 2.012121212121212,
+ "grad_norm": 1.9910767078399658,
+ "learning_rate": 6.5993265993266e-06,
+ "loss": 1.0271074771881104,
+ "mean_token_accuracy": 0.7449315786361694,
+ "num_tokens": 16318464.0,
+ "step": 996
+ },
+ {
+ "entropy": 1.3861018419265747,
+ "epoch": 2.014141414141414,
+ "grad_norm": 2.2054574489593506,
+ "learning_rate": 6.585858585858586e-06,
+ "loss": 1.3088997602462769,
+ "mean_token_accuracy": 0.6818515062332153,
+ "num_tokens": 16334848.0,
+ "step": 997
+ },
+ {
+ "entropy": 1.1170406341552734,
+ "epoch": 2.0161616161616163,
+ "grad_norm": 2.3239619731903076,
+ "learning_rate": 6.572390572390574e-06,
+ "loss": 1.036240577697754,
+ "mean_token_accuracy": 0.735405445098877,
+ "num_tokens": 16351232.0,
+ "step": 998
+ },
+ {
+ "entropy": 0.8306671977043152,
+ "epoch": 2.018181818181818,
+ "grad_norm": 1.844927430152893,
+ "learning_rate": 6.558922558922559e-06,
+ "loss": 0.7926205396652222,
+ "mean_token_accuracy": 0.790730357170105,
+ "num_tokens": 16367616.0,
+ "step": 999
+ },
+ {
+ "entropy": 1.0606129169464111,
+ "epoch": 2.0202020202020203,
+ "grad_norm": 1.884960412979126,
+ "learning_rate": 6.545454545454546e-06,
+ "loss": 1.0186505317687988,
+ "mean_token_accuracy": 0.7497557401657104,
+ "num_tokens": 16384000.0,
+ "step": 1000
+ },
+ {
+ "epoch": 2.0202020202020203,
+ "eval_entropy": 1.1442755407864047,
+ "eval_loss": 1.5800968408584595,
+ "eval_mean_token_accuracy": 0.6456370079709638,
+ "eval_num_tokens": 16384000.0,
+ "eval_runtime": 43.8308,
+ "eval_samples_per_second": 22.587,
+ "eval_steps_per_second": 2.829,
+ "step": 1000
+ }
+ ],
+ "logging_steps": 1,
+ "max_steps": 1485,
+ "num_input_tokens_seen": 0,
+ "num_train_epochs": 3,
+ "save_steps": 500,
+ "stateful_callbacks": {
+ "TrainerControl": {
+ "args": {
+ "should_epoch_stop": false,
+ "should_evaluate": false,
+ "should_log": false,
+ "should_save": true,
+ "should_training_stop": false
+ },
+ "attributes": {}
+ }
+ },
+ "total_flos": 1.38550645358592e+17,
+ "train_batch_size": 4,
+ "trial_name": null,
+ "trial_params": null
+}
diff --git a/qwen3-1.7b-teutonic-2e5/checkpoint-1000/training_args.bin b/qwen3-1.7b-teutonic-2e5/checkpoint-1000/training_args.bin
new file mode 100644
index 0000000000000000000000000000000000000000..98c558883023185245bfac817f2ca585caa9d94f
--- /dev/null
+++ b/qwen3-1.7b-teutonic-2e5/checkpoint-1000/training_args.bin
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:2c573bdac95c796f8f7368ab9af6d35c5687f53373737ea965b8102d518df7cb
+size 7121
diff --git a/qwen3-1.7b-teutonic-2e5/checkpoint-1000/zero_to_fp32.py b/qwen3-1.7b-teutonic-2e5/checkpoint-1000/zero_to_fp32.py
new file mode 100644
index 0000000000000000000000000000000000000000..3970548c400fd4361bd923b763db90e963ddaf8c
--- /dev/null
+++ b/qwen3-1.7b-teutonic-2e5/checkpoint-1000/zero_to_fp32.py
@@ -0,0 +1,854 @@
+#!/usr/bin/env python
+
+# Copyright (c) Microsoft Corporation.
+# SPDX-License-Identifier: Apache-2.0
+
+# DeepSpeed Team
+
+# This script extracts fp32 consolidated weights from a zero 1, 2 and 3 DeepSpeed checkpoints. It gets
+# copied into the top level checkpoint dir, so the user can easily do the conversion at any point in
+# the future. Once extracted, the weights don't require DeepSpeed and can be used in any
+# application.
+#
+# example:
+# python zero_to_fp32.py . output_dir/
+# or
+# python zero_to_fp32.py . output_dir/ --safe_serialization
+
+import argparse
+import torch
+import glob
+import math
+import os
+import re
+import gc
+import json
+import numpy as np
+from tqdm import tqdm
+from collections import OrderedDict
+from dataclasses import dataclass
+
+# while this script doesn't use deepspeed to recover data, since the checkpoints are pickled with
+# DeepSpeed data structures it has to be available in the current python environment.
+from deepspeed.utils import logger
+from deepspeed.checkpoint.constants import (DS_VERSION, OPTIMIZER_STATE_DICT, SINGLE_PARTITION_OF_FP32_GROUPS,
+ FP32_FLAT_GROUPS, ZERO_STAGE, PARTITION_COUNT, PARAM_SHAPES, BUFFER_NAMES,
+ FROZEN_PARAM_SHAPES, FROZEN_PARAM_FRAGMENTS, AUTOEP_LAYERS_KEY,
+ AUTOEP_LAYERS_KEY_LEGACY, AUTOEP_ZERO3_EXPERT_STATE_FORMAT_KEY,
+ AUTOEP_ZERO3_PARTITIONED_EXPERT_STATE_FORMAT, PARAM_ALIGNMENT_PADDINGS)
+
+
+@dataclass
+class zero_model_state:
+ buffers: dict()
+ param_shapes: dict()
+ shared_params: list
+ ds_version: int
+ frozen_param_shapes: dict()
+ frozen_param_fragments: dict()
+
+
+debug = 0
+
+# load to cpu
+device = torch.device('cpu')
+
+OUTPUT_DTYPE_NAMES = {
+ 'float32': torch.float32,
+ 'fp32': torch.float32,
+ 'float16': torch.float16,
+ 'fp16': torch.float16,
+ 'bfloat16': torch.bfloat16,
+ 'bf16': torch.bfloat16,
+}
+
+
+def _resolve_output_dtype(dtype):
+ requested_dtype = dtype
+ if isinstance(dtype, str):
+ dtype_name = dtype[6:] if dtype.startswith('torch.') else dtype
+ dtype = OUTPUT_DTYPE_NAMES.get(dtype_name.lower())
+ if dtype not in set(OUTPUT_DTYPE_NAMES.values()):
+ supported = ', '.join(sorted(OUTPUT_DTYPE_NAMES))
+ raise ValueError(f"Unsupported output dtype {requested_dtype!r}. Choose one of: {supported}")
+ return dtype
+
+
+def atoi(text):
+ return int(text) if text.isdigit() else text
+
+
+def natural_keys(text):
+ '''
+ alist.sort(key=natural_keys) sorts in human order
+ http://nedbatchelder.com/blog/200712/human_sorting.html
+ (See Toothy's implementation in the comments)
+ '''
+ return [atoi(c) for c in re.split(r'(\d+)', text)]
+
+
+def get_model_state_file(checkpoint_dir, zero_stage):
+ if not os.path.isdir(checkpoint_dir):
+ raise FileNotFoundError(f"Directory '{checkpoint_dir}' doesn't exist")
+
+ # there should be only one file
+ if zero_stage <= 2:
+ file = os.path.join(checkpoint_dir, "mp_rank_00_model_states.pt")
+ elif zero_stage == 3:
+ file = os.path.join(checkpoint_dir, "zero_pp_rank_0_mp_rank_00_model_states.pt")
+
+ if not os.path.exists(file):
+ raise FileNotFoundError(f"can't find model states file at '{file}'")
+
+ return file
+
+
+def get_checkpoint_files(checkpoint_dir, glob_pattern):
+ # XXX: need to test that this simple glob rule works for multi-node setup too
+ ckpt_files = sorted(glob.glob(os.path.join(checkpoint_dir, glob_pattern)), key=natural_keys)
+
+ if len(ckpt_files) == 0:
+ raise FileNotFoundError(f"can't find {glob_pattern} files in directory '{checkpoint_dir}'")
+
+ return ckpt_files
+
+
+def get_optim_files(checkpoint_dir):
+ return get_checkpoint_files(checkpoint_dir, "*_optim_states.pt")
+
+
+def get_model_state_files(checkpoint_dir):
+ return get_checkpoint_files(checkpoint_dir, "*_model_states.pt")
+
+
+def _has_autoep_zero3_partitioned_metadata(state_dict):
+ autoep_layers = state_dict.get(AUTOEP_LAYERS_KEY)
+ if autoep_layers is None:
+ autoep_layers = state_dict.get(AUTOEP_LAYERS_KEY_LEGACY)
+ if not isinstance(autoep_layers, list):
+ return False
+ return any(
+ isinstance(entry, dict)
+ and entry.get(AUTOEP_ZERO3_EXPERT_STATE_FORMAT_KEY) == AUTOEP_ZERO3_PARTITIONED_EXPERT_STATE_FORMAT
+ for entry in autoep_layers)
+
+
+def _raise_if_autoep_zero3_partitioned_state(state_dict):
+ if _has_autoep_zero3_partitioned_metadata(state_dict):
+ raise NotImplementedError("zero_to_fp32 does not support AutoEP ZeRO-3 partition-native checkpoints. "
+ "AutoEP expert parameters are partitioned over expert replica groups, so "
+ "global data-parallel consolidation would produce incomplete expert tensors. "
+ "Use ds_to_universal.py for expert-aware conversion.")
+
+
+def _raise_if_autoep_zero3_partitioned_checkpoint(model_files):
+ for file in model_files:
+ state_dict = torch.load(file, map_location=device, weights_only=False)
+ _raise_if_autoep_zero3_partitioned_state(state_dict)
+
+
+def parse_model_states(files):
+ zero_model_states = []
+ for file in files:
+ state_dict = torch.load(file, map_location=device, weights_only=False)
+ _raise_if_autoep_zero3_partitioned_state(state_dict)
+
+ if BUFFER_NAMES not in state_dict:
+ raise ValueError(f"{file} is not a model state checkpoint")
+ buffer_names = state_dict[BUFFER_NAMES]
+ if debug:
+ print("Found buffers:", buffer_names)
+
+ # recover just the buffers while restoring them to fp32 if they were saved in fp16
+ buffers = {k: v.float() for k, v in state_dict["module"].items() if k in buffer_names}
+ param_shapes = state_dict[PARAM_SHAPES]
+
+ # collect parameters that are included in param_shapes
+ param_names = []
+ for s in param_shapes:
+ for name in s.keys():
+ param_names.append(name)
+
+ # update with frozen parameters
+ frozen_param_shapes = state_dict.get(FROZEN_PARAM_SHAPES, None)
+ if frozen_param_shapes is not None:
+ if debug:
+ print(f"Found frozen_param_shapes: {frozen_param_shapes}")
+ param_names += list(frozen_param_shapes.keys())
+
+ # handle shared params
+ shared_params = [[k, v] for k, v in state_dict["shared_params"].items()]
+
+ ds_version = state_dict.get(DS_VERSION, None)
+
+ frozen_param_fragments = state_dict.get(FROZEN_PARAM_FRAGMENTS, None)
+
+ z_model_state = zero_model_state(buffers=buffers,
+ param_shapes=param_shapes,
+ shared_params=shared_params,
+ ds_version=ds_version,
+ frozen_param_shapes=frozen_param_shapes,
+ frozen_param_fragments=frozen_param_fragments)
+ zero_model_states.append(z_model_state)
+
+ return zero_model_states
+
+
+def parse_optim_states(files, ds_checkpoint_dir):
+ total_files = len(files)
+ state_dicts = []
+ for f in tqdm(files, desc='Loading checkpoint shards'):
+ state_dict = torch.load(f, map_location=device, mmap=True, weights_only=False)
+ # immediately discard the potentially huge 2 optimizer states as we only care for fp32 master weights
+ # and also handle the case where it was already removed by another helper script
+ state_dict["optimizer_state_dict"].pop("optimizer_state_dict", None)
+ state_dicts.append(state_dict)
+
+ if ZERO_STAGE not in state_dicts[0][OPTIMIZER_STATE_DICT]:
+ raise ValueError(f"{files[0]} is not a zero checkpoint")
+ zero_stage = state_dicts[0][OPTIMIZER_STATE_DICT][ZERO_STAGE]
+ world_size = state_dicts[0][OPTIMIZER_STATE_DICT][PARTITION_COUNT]
+
+ # For ZeRO-2 each param group can have different partition_count as data parallelism for expert
+ # parameters can be different from data parallelism for non-expert parameters. So we can just
+ # use the max of the partition_count to get the dp world_size.
+
+ if type(world_size) is list:
+ world_size = max(world_size)
+
+ if world_size != total_files:
+ raise ValueError(
+ f"Expected {world_size} of '*_optim_states.pt' under '{ds_checkpoint_dir}' but found {total_files} files. "
+ "Possibly due to an overwrite of an old checkpoint, or a checkpoint didn't get saved by one or more processes."
+ )
+
+ # the groups are named differently in each stage
+ if zero_stage <= 2:
+ fp32_groups_key = SINGLE_PARTITION_OF_FP32_GROUPS
+ elif zero_stage == 3:
+ fp32_groups_key = FP32_FLAT_GROUPS
+ else:
+ raise ValueError(f"unknown zero stage {zero_stage}")
+
+ fp32_flat_groups = [state_dicts[i][OPTIMIZER_STATE_DICT][fp32_groups_key] for i in range(len(state_dicts))]
+ param_alignment_paddings = state_dicts[0][OPTIMIZER_STATE_DICT].get(PARAM_ALIGNMENT_PADDINGS)
+ return zero_stage, world_size, fp32_flat_groups, param_alignment_paddings
+
+
+def _get_fp32_state_dict_from_zero_checkpoint(ds_checkpoint_dir, exclude_frozen_parameters):
+ """
+ Returns fp32 state_dict reconstructed from ds checkpoint
+
+ Args:
+ - ``ds_checkpoint_dir``: path to the deepspeed checkpoint folder (where the optimizer files are)
+
+ """
+ print(f"Processing zero checkpoint '{ds_checkpoint_dir}'")
+
+ # parse_model_states rejects AutoEP ZeRO-3 partition-native checkpoints
+ # before the expensive optimizer-shard load below.
+ model_files = get_model_state_files(ds_checkpoint_dir)
+ zero_model_states = parse_model_states(model_files)
+ print(f'Parsing checkpoint created by deepspeed=={zero_model_states[0].ds_version}')
+
+ optim_files = get_optim_files(ds_checkpoint_dir)
+ zero_stage, world_size, fp32_flat_groups, param_alignment_paddings = parse_optim_states(
+ optim_files, ds_checkpoint_dir)
+ print(f"Detected checkpoint of type zero stage {zero_stage}, world_size: {world_size}")
+
+ if zero_stage <= 2:
+ return _get_fp32_state_dict_from_zero2_checkpoint(world_size, fp32_flat_groups, zero_model_states,
+ exclude_frozen_parameters, param_alignment_paddings)
+ elif zero_stage == 3:
+ return _get_fp32_state_dict_from_zero3_checkpoint(world_size, fp32_flat_groups, zero_model_states,
+ exclude_frozen_parameters)
+
+
+def _zero2_merge_frozen_params(state_dict, zero_model_states):
+ if zero_model_states[0].frozen_param_shapes is None or len(zero_model_states[0].frozen_param_shapes) == 0:
+ return
+
+ frozen_param_shapes = zero_model_states[0].frozen_param_shapes
+ frozen_param_fragments = zero_model_states[0].frozen_param_fragments
+
+ if debug:
+ num_elem = sum(s.numel() for s in frozen_param_shapes.values())
+ print(f'rank 0: {FROZEN_PARAM_SHAPES}.numel = {num_elem}')
+
+ wanted_params = len(frozen_param_shapes)
+ wanted_numel = sum(s.numel() for s in frozen_param_shapes.values())
+ avail_numel = sum([p.numel() for p in frozen_param_fragments.values()])
+ print(f'Frozen params: Have {avail_numel} numels to process.')
+ print(f'Frozen params: Need {wanted_numel} numels in {wanted_params} params')
+
+ total_params = 0
+ total_numel = 0
+ for name, shape in frozen_param_shapes.items():
+ total_params += 1
+ unpartitioned_numel = shape.numel()
+ total_numel += unpartitioned_numel
+
+ state_dict[name] = frozen_param_fragments[name]
+
+ if debug:
+ print(f"{name} full shape: {shape} unpartitioned numel {unpartitioned_numel} ")
+
+ print(f"Reconstructed Frozen fp32 state dict with {total_params} params {total_numel} elements")
+
+
+def _has_callable(obj, fn):
+ attr = getattr(obj, fn, None)
+ return callable(attr)
+
+
+def _zero2_merge_trainable_params(state_dict,
+ world_size,
+ fp32_flat_groups,
+ zero_model_states,
+ param_alignment_paddings=None):
+ param_shapes = zero_model_states[0].param_shapes
+
+ # Reconstruction protocol:
+ #
+ # XXX: document this
+
+ if debug:
+ for i in range(world_size):
+ for j in range(len(fp32_flat_groups[0])):
+ print(f"{FP32_FLAT_GROUPS}[{i}][{j}].shape={fp32_flat_groups[i][j].shape}")
+
+ # XXX: memory usage doubles here (zero2)
+ num_param_groups = len(fp32_flat_groups[0])
+ merged_single_partition_of_fp32_groups = []
+ for i in range(num_param_groups):
+ merged_partitions = [sd[i] for sd in fp32_flat_groups]
+ full_single_fp32_vector = torch.cat(merged_partitions, 0)
+ merged_single_partition_of_fp32_groups.append(full_single_fp32_vector)
+ avail_numel = sum(
+ [full_single_fp32_vector.numel() for full_single_fp32_vector in merged_single_partition_of_fp32_groups])
+
+ if debug:
+ wanted_params = sum([len(shapes) for shapes in param_shapes])
+ wanted_numel = sum([sum(shape.numel() for shape in shapes.values()) for shapes in param_shapes])
+ # not asserting if there is a mismatch due to possible padding
+ print(f"Have {avail_numel} numels to process.")
+ print(f"Need {wanted_numel} numels in {wanted_params} params.")
+
+ # params
+ # XXX: for huge models that can't fit into the host's RAM we will have to recode this to support
+ # out-of-core computing solution
+ total_numel = 0
+ total_params = 0
+ for group_idx, (shapes,
+ full_single_fp32_vector) in enumerate(zip(param_shapes, merged_single_partition_of_fp32_groups)):
+ offset = 0
+ avail_numel = full_single_fp32_vector.numel()
+ group_alignment_paddings = None
+ if param_alignment_paddings is not None:
+ group_alignment_paddings = param_alignment_paddings[group_idx]
+ if len(group_alignment_paddings) != len(shapes):
+ raise ValueError(f"Expected {len(shapes)} parameter alignment paddings for group {group_idx}, "
+ f"but found {len(group_alignment_paddings)}")
+
+ for param_idx, (name, shape) in enumerate(shapes.items()):
+
+ unpartitioned_numel = shape.numel() if _has_callable(shape, 'numel') else math.prod(shape)
+ total_numel += unpartitioned_numel
+ total_params += 1
+
+ if debug:
+ print(f"{name} full shape: {shape} unpartitioned numel {unpartitioned_numel} ")
+ state_dict[name] = full_single_fp32_vector.narrow(0, offset, unpartitioned_numel).view(shape)
+ offset += unpartitioned_numel
+ if group_alignment_paddings is not None:
+ offset += group_alignment_paddings[param_idx]
+
+ # Z2 started to align to 2*world_size to improve nccl performance. Therefore both offset and
+ # avail_numel can differ by anywhere between 0..2*world_size. Due to two unrelated complex
+ # paddings performed in the code it's almost impossible to predict the exact numbers w/o the
+ # live optimizer object, so we are checking that the numbers are within the right range
+ align_to = 2 * world_size
+
+ def zero2_align(x):
+ return align_to * math.ceil(x / align_to)
+
+ if debug:
+ print(f"original offset={offset}, avail_numel={avail_numel}")
+
+ offset = zero2_align(offset)
+ avail_numel = zero2_align(avail_numel)
+
+ if debug:
+ print(f"aligned offset={offset}, avail_numel={avail_numel}")
+
+ # Sanity check
+ if offset != avail_numel:
+ raise ValueError(f"consumed {offset} numels out of {avail_numel} - something is wrong")
+
+ print(f"Reconstructed fp32 state dict with {total_params} params {total_numel} elements")
+
+
+def _get_fp32_state_dict_from_zero2_checkpoint(world_size,
+ fp32_flat_groups,
+ zero_model_states,
+ exclude_frozen_parameters,
+ param_alignment_paddings=None):
+ state_dict = OrderedDict()
+
+ # buffers
+ buffers = zero_model_states[0].buffers
+ state_dict.update(buffers)
+ if debug:
+ print(f"added {len(buffers)} buffers")
+
+ if not exclude_frozen_parameters:
+ _zero2_merge_frozen_params(state_dict, zero_model_states)
+
+ _zero2_merge_trainable_params(state_dict, world_size, fp32_flat_groups, zero_model_states,
+ param_alignment_paddings)
+
+ # recover shared parameters
+ for pair in zero_model_states[0].shared_params:
+ if pair[1] in state_dict:
+ state_dict[pair[0]] = state_dict[pair[1]]
+
+ return state_dict
+
+
+def zero3_partitioned_param_info(unpartitioned_numel, world_size):
+ remainder = unpartitioned_numel % world_size
+ padding_numel = (world_size - remainder) if remainder else 0
+ partitioned_numel = math.ceil(unpartitioned_numel / world_size)
+ return partitioned_numel, padding_numel
+
+
+def _zero3_merge_frozen_params(state_dict, world_size, zero_model_states):
+ if zero_model_states[0].frozen_param_shapes is None or len(zero_model_states[0].frozen_param_shapes) == 0:
+ return
+
+ if debug:
+ for i in range(world_size):
+ num_elem = sum(s.numel() for s in zero_model_states[i].frozen_param_fragments.values())
+ print(f'rank {i}: {FROZEN_PARAM_SHAPES}.numel = {num_elem}')
+
+ frozen_param_shapes = zero_model_states[0].frozen_param_shapes
+ wanted_params = len(frozen_param_shapes)
+ wanted_numel = sum(s.numel() for s in frozen_param_shapes.values())
+ avail_numel = sum([p.numel() for p in zero_model_states[0].frozen_param_fragments.values()]) * world_size
+ print(f'Frozen params: Have {avail_numel} numels to process.')
+ print(f'Frozen params: Need {wanted_numel} numels in {wanted_params} params')
+
+ total_params = 0
+ total_numel = 0
+ for name, shape in zero_model_states[0].frozen_param_shapes.items():
+ total_params += 1
+ unpartitioned_numel = shape.numel()
+ total_numel += unpartitioned_numel
+
+ param_frags = tuple(model_state.frozen_param_fragments[name] for model_state in zero_model_states)
+ state_dict[name] = torch.cat(param_frags, 0).narrow(0, 0, unpartitioned_numel).view(shape)
+
+ partitioned_numel, partitioned_padding_numel = zero3_partitioned_param_info(unpartitioned_numel, world_size)
+
+ if debug:
+ print(
+ f"Frozen params: {total_params} {name} full shape: {shape} partition0 numel={partitioned_numel} partitioned_padding_numel={partitioned_padding_numel}"
+ )
+
+ print(f"Reconstructed Frozen fp32 state dict with {total_params} params {total_numel} elements")
+
+
+class GatheredTensor:
+ """
+ A pseudo tensor that collects partitioned weights.
+ It is more memory efficient when there are multiple groups.
+ """
+
+ def __init__(self, flat_groups, flat_groups_offset, offset, partitioned_numel, shape):
+ self.flat_groups = flat_groups
+ self.flat_groups_offset = flat_groups_offset
+ self.offset = offset
+ self.partitioned_numel = partitioned_numel
+ self.shape = shape
+ self.dtype = self.flat_groups[0][0].dtype
+
+ def contiguous(self):
+ """
+ Merge partitioned weights from flat_groups into a single tensor.
+ """
+ end_idx = self.offset + self.partitioned_numel
+ world_size = len(self.flat_groups)
+ pad_flat_param_chunks = []
+
+ for rank_i in range(world_size):
+ # for each rank, we need to collect weights from related group/groups
+ flat_groups_at_rank_i = self.flat_groups[rank_i]
+ start_group_id = None
+ end_group_id = None
+ for group_id in range(len(self.flat_groups_offset)):
+ if self.flat_groups_offset[group_id] <= self.offset < self.flat_groups_offset[group_id + 1]:
+ start_group_id = group_id
+ if self.flat_groups_offset[group_id] < end_idx <= self.flat_groups_offset[group_id + 1]:
+ end_group_id = group_id
+ break
+ # collect weights from related group/groups
+ for group_id in range(start_group_id, end_group_id + 1):
+ flat_tensor = flat_groups_at_rank_i[group_id]
+ start_offset = self.offset - self.flat_groups_offset[group_id]
+ end_offset = min(end_idx, self.flat_groups_offset[group_id + 1]) - self.flat_groups_offset[group_id]
+ pad_flat_param_chunks.append(flat_tensor[start_offset:end_offset])
+
+ # collect weights from all ranks
+ pad_flat_param = torch.cat(pad_flat_param_chunks, dim=0)
+ param = pad_flat_param[:self.shape.numel()].view(self.shape).contiguous()
+ return param
+
+
+def _zero3_merge_trainable_params(state_dict, world_size, fp32_flat_groups, zero_model_states):
+ param_shapes = zero_model_states[0].param_shapes
+ avail_numel = sum([flat_group.numel() for flat_group in fp32_flat_groups[0]]) * world_size
+
+ # Reconstruction protocol: For zero3 we need to zip the partitions together at boundary of each
+ # param, re-consolidating each param, while dealing with padding if any
+
+ # merge list of dicts, preserving order
+ param_shapes = {k: v for d in param_shapes for k, v in d.items()}
+
+ if debug:
+ for i in range(world_size):
+ print(f"{FP32_FLAT_GROUPS}[{i}].shape={fp32_flat_groups[i].shape}")
+
+ wanted_params = len(param_shapes)
+ wanted_numel = sum(shape.numel() for shape in param_shapes.values())
+ # not asserting if there is a mismatch due to possible padding
+ avail_numel = fp32_flat_groups[0].numel() * world_size
+ print(f"Trainable params: Have {avail_numel} numels to process.")
+ print(f"Trainable params: Need {wanted_numel} numels in {wanted_params} params.")
+
+ # params
+ # XXX: for huge models that can't fit into the host's RAM we will have to recode this to support
+ # out-of-core computing solution
+ offset = 0
+ total_numel = 0
+ total_params = 0
+ flat_groups_offset = [0] + list(np.cumsum([flat_tensor.numel() for flat_tensor in fp32_flat_groups[0]]))
+ for name, shape in tqdm(param_shapes.items(), desc='Gathering sharded weights'):
+ unpartitioned_numel = shape.numel()
+ total_numel += unpartitioned_numel
+ total_params += 1
+ partitioned_numel, partitioned_padding_numel = zero3_partitioned_param_info(unpartitioned_numel, world_size)
+
+ if debug:
+ print(
+ f"Trainable params: {total_params} {name} full shape: {shape} partition0 numel={partitioned_numel} partitioned_padding_numel={partitioned_padding_numel}"
+ )
+
+ # memory efficient tensor
+ tensor = GatheredTensor(fp32_flat_groups, flat_groups_offset, offset, partitioned_numel, shape)
+ state_dict[name] = tensor
+ offset += partitioned_numel
+
+ offset *= world_size
+
+ # Sanity check
+ if offset != avail_numel:
+ raise ValueError(f"consumed {offset} numels out of {avail_numel} - something is wrong")
+
+ print(f"Reconstructed Trainable fp32 state dict with {total_params} params {total_numel} elements")
+
+
+def _get_fp32_state_dict_from_zero3_checkpoint(world_size, fp32_flat_groups, zero_model_states,
+ exclude_frozen_parameters):
+ state_dict = OrderedDict()
+
+ # buffers
+ buffers = zero_model_states[0].buffers
+ state_dict.update(buffers)
+ if debug:
+ print(f"added {len(buffers)} buffers")
+
+ if not exclude_frozen_parameters:
+ _zero3_merge_frozen_params(state_dict, world_size, zero_model_states)
+
+ _zero3_merge_trainable_params(state_dict, world_size, fp32_flat_groups, zero_model_states)
+
+ # recover shared parameters
+ for pair in zero_model_states[0].shared_params:
+ if pair[1] in state_dict:
+ state_dict[pair[0]] = state_dict[pair[1]]
+
+ return state_dict
+
+
+def to_torch_tensor(state_dict, return_empty_tensor=False, dtype=None):
+ """
+ Convert state_dict of GatheredTensor to torch tensor
+ """
+ if dtype is not None:
+ dtype = _resolve_output_dtype(dtype)
+
+ torch_state_dict = {}
+ converted_tensors = {}
+ for name, tensor in state_dict.items():
+ tensor_id = id(tensor)
+ if tensor_id in converted_tensors: # shared tensors
+ shared_tensor = torch_state_dict[converted_tensors[tensor_id]]
+ torch_state_dict[name] = shared_tensor
+ else:
+ converted_tensors[tensor_id] = name
+ if return_empty_tensor:
+ torch_state_dict[name] = torch.empty(tensor.shape, dtype=dtype or tensor.dtype)
+ else:
+ contiguous_tensor = tensor.contiguous()
+ torch_state_dict[name] = contiguous_tensor.to(dtype=dtype) if dtype else contiguous_tensor
+ return torch_state_dict
+
+
+def get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir,
+ tag=None,
+ exclude_frozen_parameters=False,
+ lazy_mode=False):
+ """
+ Convert ZeRO 2 or 3 checkpoint into a single fp32 consolidated state_dict that can be loaded with
+ ``load_state_dict()`` and used for training without DeepSpeed or shared with others, for example
+ via a model hub.
+
+ Args:
+ - ``checkpoint_dir``: path to the desired checkpoint folder
+ - ``tag``: checkpoint tag used as a unique identifier for checkpoint. If not provided will attempt to load tag in 'latest' file. e.g., ``global_step14``
+ - ``exclude_frozen_parameters``: exclude frozen parameters
+ - ``lazy_mode``: get state_dict in lazy mode. It returns a dict of pesduo tensor instead of torch tensor, which is more memory efficient.
+ Convert the pesduo tensor to torch tensor by ``.contiguous()``
+
+ Returns:
+ - pytorch ``state_dict``
+
+ A typical usage might be ::
+
+ from deepspeed.utils.zero_to_fp32 import get_fp32_state_dict_from_zero_checkpoint
+ # do the training and checkpoint saving
+ state_dict = get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir) # already on cpu
+ model = model.cpu() # move to cpu
+ model.load_state_dict(state_dict)
+ # submit to model hub or save the model to share with others
+
+ In this example the ``model`` will no longer be usable in the deepspeed context of the same
+ application. i.e. you will need to re-initialize the deepspeed engine, since
+ ``model.load_state_dict(state_dict)`` will remove all the deepspeed magic from it.
+
+ If you want it all done for you, use ``load_state_dict_from_zero_checkpoint`` instead.
+
+ Note: the above usage may not work if your application doesn't have sufficient free CPU memory.
+ You may need to use the offline approach using the ``zero_to_fp32.py`` script that is saved with
+ the checkpoint. Or you can load state_dict in lazy mode ::
+
+ from deepspeed.utils.zero_to_fp32 import get_fp32_state_dict_from_zero_checkpoint
+ state_dict = get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir, lazy_mode=True) # not on cpu
+ for name, lazy_tensor in state_dict.item():
+ tensor = lazy_tensor.contiguous() # to cpu
+ print(name, tensor)
+ # del tensor to release memory if it no longer in use
+ """
+ if tag is None:
+ latest_path = os.path.join(checkpoint_dir, 'latest')
+ if os.path.isfile(latest_path):
+ with open(latest_path, 'r') as fd:
+ tag = fd.read().strip()
+ else:
+ raise ValueError(f"Unable to find 'latest' file at {latest_path}")
+
+ ds_checkpoint_dir = os.path.join(checkpoint_dir, tag)
+
+ if not os.path.isdir(ds_checkpoint_dir):
+ raise FileNotFoundError(f"Directory '{ds_checkpoint_dir}' doesn't exist")
+
+ state_dict = _get_fp32_state_dict_from_zero_checkpoint(ds_checkpoint_dir, exclude_frozen_parameters)
+ if lazy_mode:
+ return state_dict
+ else:
+ return to_torch_tensor(state_dict)
+
+
+def convert_zero_checkpoint_to_state_dict(checkpoint_dir,
+ output_dir,
+ dtype=torch.float32,
+ max_shard_size="5GB",
+ safe_serialization=False,
+ tag=None,
+ exclude_frozen_parameters=False):
+ """
+ Convert ZeRO 2 or 3 checkpoint into a consolidated ``state_dict`` file that can be
+ loaded with ``torch.load(file)`` + ``load_state_dict()`` and used for training without DeepSpeed.
+
+ Args:
+ - ``checkpoint_dir``: path to the desired checkpoint folder. (one that contains the tag-folder, like ``global_step14``)
+ - ``output_dir``: directory for the PyTorch state_dict output files
+ - ``dtype``: output tensor dtype. Supports float32, float16, and bfloat16 as strings or torch dtypes.
+ - ``max_shard_size``: the maximum size for a checkpoint before being sharded, default value is 5GB
+ - ``safe_serialization``: whether to save the model using `safetensors` or the traditional PyTorch way (that uses `pickle`).
+ - ``tag``: checkpoint tag used as a unique identifier for checkpoint. If not provided will attempt to load tag in the file named ``latest`` in the checkpoint folder, e.g., ``global_step14``
+ - ``exclude_frozen_parameters``: exclude frozen parameters
+ """
+
+ dtype = _resolve_output_dtype(dtype)
+
+ # Dependency pre-check
+ if safe_serialization:
+ try:
+ from safetensors.torch import save_file
+ except ImportError:
+ print('If you want to use `safe_serialization`, please `pip install safetensors`')
+ raise
+ if max_shard_size is not None:
+ try:
+ from huggingface_hub import split_torch_state_dict_into_shards
+ except ImportError:
+ print('If you want to use `max_shard_size`, please `pip install huggingface_hub`')
+ raise
+
+ # Convert zero checkpoint to state_dict
+ state_dict = get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir,
+ tag,
+ exclude_frozen_parameters,
+ lazy_mode=True)
+
+ # Shard the model if it is too big.
+ weights_name = "model.safetensors" if safe_serialization else "pytorch_model.bin"
+ if max_shard_size is not None:
+ filename_pattern = weights_name.replace(".bin", "{suffix}.bin").replace(".safetensors", "{suffix}.safetensors")
+ # an memory-efficient approach for sharding
+ empty_state_dict = to_torch_tensor(state_dict, return_empty_tensor=True, dtype=dtype)
+ state_dict_split = split_torch_state_dict_into_shards(empty_state_dict,
+ filename_pattern=filename_pattern,
+ max_shard_size=max_shard_size)
+ else:
+ from collections import namedtuple
+ StateDictSplit = namedtuple("StateDictSplit", ["is_sharded", "filename_to_tensors"])
+ state_dict_split = StateDictSplit(is_sharded=False,
+ filename_to_tensors={weights_name: list(state_dict.keys())})
+
+ # Save the model by shard
+ os.makedirs(output_dir, exist_ok=True)
+ filename_to_tensors = state_dict_split.filename_to_tensors.items()
+ for shard_file, tensors in tqdm(filename_to_tensors, desc="Saving checkpoint shards"):
+ shard_state_dict = {tensor_name: state_dict[tensor_name] for tensor_name in tensors}
+ shard_state_dict = to_torch_tensor(shard_state_dict, dtype=dtype)
+ output_path = os.path.join(output_dir, shard_file)
+ if safe_serialization:
+ save_file(shard_state_dict, output_path, metadata={"format": "pt"})
+ else:
+ torch.save(shard_state_dict, output_path)
+ # release the memory of current shard
+ for tensor_name in list(shard_state_dict.keys()):
+ del state_dict[tensor_name]
+ del shard_state_dict[tensor_name]
+ del shard_state_dict
+ gc.collect()
+
+ # Save index if sharded
+ if state_dict_split.is_sharded:
+ index = {
+ "metadata": state_dict_split.metadata,
+ "weight_map": state_dict_split.tensor_to_filename,
+ }
+ save_index_file = "model.safetensors.index.json" if safe_serialization else "pytorch_model.bin.index.json"
+ save_index_file = os.path.join(output_dir, save_index_file)
+ with open(save_index_file, "w", encoding="utf-8") as f:
+ content = json.dumps(index, indent=2, sort_keys=True) + "\n"
+ f.write(content)
+
+
+def convert_zero_checkpoint_to_fp32_state_dict(checkpoint_dir,
+ output_dir,
+ max_shard_size="5GB",
+ safe_serialization=False,
+ tag=None,
+ exclude_frozen_parameters=False):
+ """Backward-compatible fp32 checkpoint conversion."""
+ return convert_zero_checkpoint_to_state_dict(checkpoint_dir,
+ output_dir,
+ dtype=torch.float32,
+ max_shard_size=max_shard_size,
+ safe_serialization=safe_serialization,
+ tag=tag,
+ exclude_frozen_parameters=exclude_frozen_parameters)
+
+
+def load_state_dict_from_zero_checkpoint(model, checkpoint_dir, tag=None):
+ """
+ 1. Put the provided model to cpu
+ 2. Convert ZeRO 2 or 3 checkpoint into a single fp32 consolidated ``state_dict``
+ 3. Load it into the provided model
+
+ Args:
+ - ``model``: the model object to update
+ - ``checkpoint_dir``: path to the desired checkpoint folder. (one that contains the tag-folder, like ``global_step14``)
+ - ``tag``: checkpoint tag used as a unique identifier for checkpoint. If not provided will attempt to load tag in the file named ``latest`` in the checkpoint folder, e.g., ``global_step14``
+
+ Returns:
+ - ``model`: modified model
+
+ Make sure you have plenty of CPU memory available before you call this function. If you don't
+ have enough use the ``zero_to_fp32.py`` utility to do the conversion. You will find it
+ conveniently placed for you in the checkpoint folder.
+
+ A typical usage might be ::
+
+ from deepspeed.utils.zero_to_fp32 import load_state_dict_from_zero_checkpoint
+ model = load_state_dict_from_zero_checkpoint(trainer.model, checkpoint_dir)
+ # submit to model hub or save the model to share with others
+
+ Note, that once this was run, the ``model`` will no longer be usable in the deepspeed context
+ of the same application. i.e. you will need to re-initialize the deepspeed engine, since
+ ``model.load_state_dict(state_dict)`` will remove all the deepspeed magic from it.
+
+ """
+ logger.info("Extracting fp32 weights")
+ state_dict = get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir, tag)
+
+ logger.info("Overwriting model with fp32 weights")
+ model = model.cpu()
+ model.load_state_dict(state_dict, strict=False)
+
+ return model
+
+
+if __name__ == "__main__":
+ parser = argparse.ArgumentParser()
+ parser.add_argument("checkpoint_dir",
+ type=str,
+ help="path to the desired checkpoint folder, e.g., path/checkpoint-12")
+ parser.add_argument("output_dir",
+ type=str,
+ help="directory to the pytorch fp32 state_dict output files"
+ "(e.g. path/checkpoint-12-output/)")
+ parser.add_argument(
+ "--max_shard_size",
+ type=str,
+ default="5GB",
+ help="The maximum size for a checkpoint before being sharded. Checkpoints shard will then be each of size"
+ "lower than this size. If expressed as a string, needs to be digits followed by a unit (like `5MB`"
+ "We default it to 5GB in order for models to be able to run easily on free-tier google colab instances"
+ "without CPU OOM issues.")
+ parser.add_argument(
+ "--safe_serialization",
+ default=False,
+ action='store_true',
+ help="Whether to save the model using `safetensors` or the traditional PyTorch way (that uses `pickle`).")
+ parser.add_argument("-t",
+ "--tag",
+ type=str,
+ default=None,
+ help="checkpoint tag used as a unique identifier for checkpoint. e.g., global_step1")
+ parser.add_argument("--exclude_frozen_parameters", action='store_true', help="exclude frozen parameters")
+ parser.add_argument("-d", "--debug", action='store_true', help="enable debug")
+ args = parser.parse_args()
+
+ debug = args.debug
+
+ convert_zero_checkpoint_to_fp32_state_dict(args.checkpoint_dir,
+ args.output_dir,
+ max_shard_size=args.max_shard_size,
+ safe_serialization=args.safe_serialization,
+ tag=args.tag,
+ exclude_frozen_parameters=args.exclude_frozen_parameters)
diff --git a/qwen3-1.7b-teutonic-2e5/checkpoint-1000/zero_to_torch.py b/qwen3-1.7b-teutonic-2e5/checkpoint-1000/zero_to_torch.py
new file mode 100644
index 0000000000000000000000000000000000000000..81312e252400d77f03cc1a88522f8f18ebe70ee7
--- /dev/null
+++ b/qwen3-1.7b-teutonic-2e5/checkpoint-1000/zero_to_torch.py
@@ -0,0 +1,54 @@
+#!/usr/bin/env python
+
+# SPDX-License-Identifier: Apache-2.0
+# DeepSpeed Team
+
+import argparse
+
+if __package__:
+ from . import zero_to_fp32
+else:
+ import zero_to_fp32
+
+
+def main(args=None):
+ parser = argparse.ArgumentParser(
+ description="Convert a DeepSpeed ZeRO checkpoint to float32, float16, or bfloat16 PyTorch weights.")
+ parser.add_argument("checkpoint_dir",
+ type=str,
+ help="path to the desired checkpoint folder, e.g., path/checkpoint-12")
+ parser.add_argument("output_dir", type=str, help="directory for the converted PyTorch state_dict files")
+ parser.add_argument("--dtype",
+ type=str,
+ choices=sorted(zero_to_fp32.OUTPUT_DTYPE_NAMES),
+ required=True,
+ help="output tensor dtype")
+ parser.add_argument("--max_shard_size",
+ type=str,
+ default="5GB",
+ help="maximum size of each checkpoint shard, such as 5GB or 500MB")
+ parser.add_argument("--safe_serialization",
+ default=False,
+ action='store_true',
+ help="save with safetensors instead of PyTorch pickle serialization")
+ parser.add_argument("-t",
+ "--tag",
+ type=str,
+ default=None,
+ help="checkpoint tag used as a unique identifier, e.g., global_step1")
+ parser.add_argument("--exclude_frozen_parameters", action='store_true', help="exclude frozen parameters")
+ parser.add_argument("-d", "--debug", action='store_true', help="enable debug output")
+ parsed_args = parser.parse_args(args)
+
+ zero_to_fp32.debug = parsed_args.debug
+ zero_to_fp32.convert_zero_checkpoint_to_state_dict(parsed_args.checkpoint_dir,
+ parsed_args.output_dir,
+ dtype=parsed_args.dtype,
+ max_shard_size=parsed_args.max_shard_size,
+ safe_serialization=parsed_args.safe_serialization,
+ tag=parsed_args.tag,
+ exclude_frozen_parameters=parsed_args.exclude_frozen_parameters)
+
+
+if __name__ == "__main__":
+ main()
diff --git a/qwen3-1.7b-teutonic-2e5/checkpoint-1485/chat_template.jinja b/qwen3-1.7b-teutonic-2e5/checkpoint-1485/chat_template.jinja
new file mode 100644
index 0000000000000000000000000000000000000000..01be9b307daa2d425f7c168c9fb145a286e0afb4
--- /dev/null
+++ b/qwen3-1.7b-teutonic-2e5/checkpoint-1485/chat_template.jinja
@@ -0,0 +1,89 @@
+{%- if tools %}
+ {{- '<|im_start|>system\n' }}
+ {%- if messages[0].role == 'system' %}
+ {{- messages[0].content + '\n\n' }}
+ {%- endif %}
+ {{- "# Tools\n\nYou may call one or more functions to assist with the user query.\n\nYou are provided with function signatures within XML tags:\n" }}
+ {%- for tool in tools %}
+ {{- "\n" }}
+ {{- tool | tojson }}
+ {%- endfor %}
+ {{- "\n\n\nFor each function call, return a json object with function name and arguments within XML tags:\n\n{\"name\": , \"arguments\": }\n<|im_end|>\n" }}
+{%- else %}
+ {%- if messages[0].role == 'system' %}
+ {{- '<|im_start|>system\n' + messages[0].content + '<|im_end|>\n' }}
+ {%- endif %}
+{%- endif %}
+{%- set ns = namespace(multi_step_tool=true, last_query_index=messages|length - 1) %}
+{%- for message in messages[::-1] %}
+ {%- set index = (messages|length - 1) - loop.index0 %}
+ {%- if ns.multi_step_tool and message.role == "user" and message.content is string and not(message.content.startswith('') and message.content.endswith('')) %}
+ {%- set ns.multi_step_tool = false %}
+ {%- set ns.last_query_index = index %}
+ {%- endif %}
+{%- endfor %}
+{%- for message in messages %}
+ {%- if message.content is string %}
+ {%- set content = message.content %}
+ {%- else %}
+ {%- set content = '' %}
+ {%- endif %}
+ {%- if (message.role == "user") or (message.role == "system" and not loop.first) %}
+ {{- '<|im_start|>' + message.role + '\n' + content + '<|im_end|>' + '\n' }}
+ {%- elif message.role == "assistant" %}
+ {%- set reasoning_content = '' %}
+ {%- if message.reasoning_content is string %}
+ {%- set reasoning_content = message.reasoning_content %}
+ {%- else %}
+ {%- if '' in content %}
+ {%- set reasoning_content = content.split('')[0].rstrip('\n').split('')[-1].lstrip('\n') %}
+ {%- set content = content.split('')[-1].lstrip('\n') %}
+ {%- endif %}
+ {%- endif %}
+ {%- if loop.index0 > ns.last_query_index %}
+ {%- if loop.last or (not loop.last and reasoning_content) %}
+ {{- '<|im_start|>' + message.role + '\n\n' + reasoning_content.strip('\n') + '\n\n\n' + content.lstrip('\n') }}
+ {%- else %}
+ {{- '<|im_start|>' + message.role + '\n' + content }}
+ {%- endif %}
+ {%- else %}
+ {{- '<|im_start|>' + message.role + '\n' + content }}
+ {%- endif %}
+ {%- if message.tool_calls %}
+ {%- for tool_call in message.tool_calls %}
+ {%- if (loop.first and content) or (not loop.first) %}
+ {{- '\n' }}
+ {%- endif %}
+ {%- if tool_call.function %}
+ {%- set tool_call = tool_call.function %}
+ {%- endif %}
+ {{- '\n{"name": "' }}
+ {{- tool_call.name }}
+ {{- '", "arguments": ' }}
+ {%- if tool_call.arguments is string %}
+ {{- tool_call.arguments }}
+ {%- else %}
+ {{- tool_call.arguments | tojson }}
+ {%- endif %}
+ {{- '}\n' }}
+ {%- endfor %}
+ {%- endif %}
+ {{- '<|im_end|>\n' }}
+ {%- elif message.role == "tool" %}
+ {%- if loop.first or (messages[loop.index0 - 1].role != "tool") %}
+ {{- '<|im_start|>user' }}
+ {%- endif %}
+ {{- '\n\n' }}
+ {{- content }}
+ {{- '\n' }}
+ {%- if loop.last or (messages[loop.index0 + 1].role != "tool") %}
+ {{- '<|im_end|>\n' }}
+ {%- endif %}
+ {%- endif %}
+{%- endfor %}
+{%- if add_generation_prompt %}
+ {{- '<|im_start|>assistant\n' }}
+ {%- if enable_thinking is defined and enable_thinking is false %}
+ {{- '\n\n\n\n' }}
+ {%- endif %}
+{%- endif %}
\ No newline at end of file
diff --git a/qwen3-1.7b-teutonic-2e5/checkpoint-1485/config.json b/qwen3-1.7b-teutonic-2e5/checkpoint-1485/config.json
new file mode 100644
index 0000000000000000000000000000000000000000..5d9cef07396baadff5390e4508eb33025967a029
--- /dev/null
+++ b/qwen3-1.7b-teutonic-2e5/checkpoint-1485/config.json
@@ -0,0 +1,63 @@
+{
+ "architectures": [
+ "Qwen3ForCausalLM"
+ ],
+ "attention_bias": false,
+ "attention_dropout": 0.0,
+ "bos_token_id": null,
+ "dtype": "bfloat16",
+ "eos_token_id": 151645,
+ "head_dim": 128,
+ "hidden_act": "silu",
+ "hidden_size": 2048,
+ "initializer_range": 0.02,
+ "intermediate_size": 6144,
+ "layer_types": [
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention"
+ ],
+ "max_position_embeddings": 40960,
+ "max_window_layers": 28,
+ "model_type": "qwen3",
+ "num_attention_heads": 16,
+ "num_hidden_layers": 28,
+ "num_key_value_heads": 8,
+ "pad_token_id": 151643,
+ "rms_norm_eps": 1e-06,
+ "rope_parameters": {
+ "rope_theta": 1000000,
+ "rope_type": "default"
+ },
+ "sliding_window": null,
+ "tie_word_embeddings": true,
+ "transformers_version": "5.17.0",
+ "use_cache": false,
+ "use_sliding_window": false,
+ "vocab_size": 151936
+}
diff --git a/qwen3-1.7b-teutonic-2e5/checkpoint-1485/generation_config.json b/qwen3-1.7b-teutonic-2e5/checkpoint-1485/generation_config.json
new file mode 100644
index 0000000000000000000000000000000000000000..e6941fe4b04f26113d6eef6255d005c4d7090bda
--- /dev/null
+++ b/qwen3-1.7b-teutonic-2e5/checkpoint-1485/generation_config.json
@@ -0,0 +1,12 @@
+{
+ "do_sample": true,
+ "eos_token_id": [
+ 151645,
+ 151643
+ ],
+ "pad_token_id": 151643,
+ "temperature": 0.6,
+ "top_k": 20,
+ "top_p": 0.95,
+ "transformers_version": "5.17.0"
+}
diff --git a/qwen3-1.7b-teutonic-2e5/checkpoint-1485/global_step1485/bf16_zero_pp_rank_0_mp_rank_00_optim_states.pt b/qwen3-1.7b-teutonic-2e5/checkpoint-1485/global_step1485/bf16_zero_pp_rank_0_mp_rank_00_optim_states.pt
new file mode 100644
index 0000000000000000000000000000000000000000..809da4f60abb1d9acb3929306b031e2480982777
--- /dev/null
+++ b/qwen3-1.7b-teutonic-2e5/checkpoint-1485/global_step1485/bf16_zero_pp_rank_0_mp_rank_00_optim_states.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:8d310f5466bf901784c39740d2ba6dcabb6904e9280df6b5865f7360cc07f04e
+size 10323466465
diff --git a/qwen3-1.7b-teutonic-2e5/checkpoint-1485/global_step1485/bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt b/qwen3-1.7b-teutonic-2e5/checkpoint-1485/global_step1485/bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt
new file mode 100644
index 0000000000000000000000000000000000000000..74eb12e9db93a5cc269643adadf4e2aa5fd15267
--- /dev/null
+++ b/qwen3-1.7b-teutonic-2e5/checkpoint-1485/global_step1485/bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:eda7d5c3c134c7f2ad759957a202b8b021838ec2f20d5185dc1a52989a7d6ccd
+size 10323469601
diff --git a/qwen3-1.7b-teutonic-2e5/checkpoint-1485/global_step1485/mp_rank_00_model_states.pt b/qwen3-1.7b-teutonic-2e5/checkpoint-1485/global_step1485/mp_rank_00_model_states.pt
new file mode 100644
index 0000000000000000000000000000000000000000..b98d09a6533fbc751cd09741b8c11f5b0e501b8e
--- /dev/null
+++ b/qwen3-1.7b-teutonic-2e5/checkpoint-1485/global_step1485/mp_rank_00_model_states.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a8fc309912ee1059d08a4b85263fc00dc1c46571b061b11e2eb9439979613e40
+size 3441239653
diff --git a/qwen3-1.7b-teutonic-2e5/checkpoint-1485/latest b/qwen3-1.7b-teutonic-2e5/checkpoint-1485/latest
new file mode 100644
index 0000000000000000000000000000000000000000..fd99e82f9d7bc8b55f38e85ef5fb215628a602ae
--- /dev/null
+++ b/qwen3-1.7b-teutonic-2e5/checkpoint-1485/latest
@@ -0,0 +1 @@
+global_step1485
\ No newline at end of file
diff --git a/qwen3-1.7b-teutonic-2e5/checkpoint-1485/model.safetensors b/qwen3-1.7b-teutonic-2e5/checkpoint-1485/model.safetensors
new file mode 100644
index 0000000000000000000000000000000000000000..9b2666228db53911589c4cd2096c38c2c35ae728
--- /dev/null
+++ b/qwen3-1.7b-teutonic-2e5/checkpoint-1485/model.safetensors
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:5cdc018ad7b517ba5e4c7c4f00035390b4e85dec1fc68e477966614a72cda70c
+size 4063515640
diff --git a/qwen3-1.7b-teutonic-2e5/checkpoint-1485/rng_state_0.pth b/qwen3-1.7b-teutonic-2e5/checkpoint-1485/rng_state_0.pth
new file mode 100644
index 0000000000000000000000000000000000000000..52579bb9bd83b7479b7cdb80fe3042c4577cd189
--- /dev/null
+++ b/qwen3-1.7b-teutonic-2e5/checkpoint-1485/rng_state_0.pth
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b6d0c8203f86526d5059f583df0d5b1c246458bd95b5ae63a840d384d95e1811
+size 14917
diff --git a/qwen3-1.7b-teutonic-2e5/checkpoint-1485/rng_state_1.pth b/qwen3-1.7b-teutonic-2e5/checkpoint-1485/rng_state_1.pth
new file mode 100644
index 0000000000000000000000000000000000000000..6f434945a684215579d53189338adf884fdb5335
--- /dev/null
+++ b/qwen3-1.7b-teutonic-2e5/checkpoint-1485/rng_state_1.pth
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f13fc77f33fd9dd376569b0baf2c19ef17d863b26d8028118541e9f7a36ca898
+size 14917
diff --git a/qwen3-1.7b-teutonic-2e5/checkpoint-1485/scheduler.pt b/qwen3-1.7b-teutonic-2e5/checkpoint-1485/scheduler.pt
new file mode 100644
index 0000000000000000000000000000000000000000..73fffb33c0d4f174edd508c067265127c6db6acc
--- /dev/null
+++ b/qwen3-1.7b-teutonic-2e5/checkpoint-1485/scheduler.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d0945608dfeb2871410bf8eb511453bb37137eb1373134300fb59491a6a0b399
+size 1465
diff --git a/qwen3-1.7b-teutonic-2e5/checkpoint-1485/tokenizer.json b/qwen3-1.7b-teutonic-2e5/checkpoint-1485/tokenizer.json
new file mode 100644
index 0000000000000000000000000000000000000000..c7afbed2efcdf019f88ab0572ec29d3bf595dfe2
--- /dev/null
+++ b/qwen3-1.7b-teutonic-2e5/checkpoint-1485/tokenizer.json
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:be75606093db2094d7cd20f3c2f385c212750648bd6ea4fb2bf507a6a4c55506
+size 11422650
diff --git a/qwen3-1.7b-teutonic-2e5/checkpoint-1485/tokenizer_config.json b/qwen3-1.7b-teutonic-2e5/checkpoint-1485/tokenizer_config.json
new file mode 100644
index 0000000000000000000000000000000000000000..770e41d6c92519d525eede4cbcf3ba27f6425311
--- /dev/null
+++ b/qwen3-1.7b-teutonic-2e5/checkpoint-1485/tokenizer_config.json
@@ -0,0 +1,30 @@
+{
+ "add_prefix_space": false,
+ "backend": "tokenizers",
+ "bos_token": null,
+ "clean_up_tokenization_spaces": false,
+ "eos_token": "<|im_end|>",
+ "errors": "replace",
+ "extra_special_tokens": [
+ "<|im_start|>",
+ "<|im_end|>",
+ "<|object_ref_start|>",
+ "<|object_ref_end|>",
+ "<|box_start|>",
+ "<|box_end|>",
+ "<|quad_start|>",
+ "<|quad_end|>",
+ "<|vision_start|>",
+ "<|vision_end|>",
+ "<|vision_pad|>",
+ "<|image_pad|>",
+ "<|video_pad|>"
+ ],
+ "is_local": false,
+ "local_files_only": false,
+ "model_max_length": 131072,
+ "pad_token": "<|endoftext|>",
+ "split_special_tokens": false,
+ "tokenizer_class": "Qwen2Tokenizer",
+ "unk_token": null
+}
diff --git a/qwen3-1.7b-teutonic-2e5/checkpoint-1485/trainer_state.json b/qwen3-1.7b-teutonic-2e5/checkpoint-1485/trainer_state.json
new file mode 100644
index 0000000000000000000000000000000000000000..ea7c9cbeb8183667542e19f03ace901c6ae558ea
--- /dev/null
+++ b/qwen3-1.7b-teutonic-2e5/checkpoint-1485/trainer_state.json
@@ -0,0 +1,15049 @@
+{
+ "best_global_step": 400,
+ "best_metric": 1.5199862718582153,
+ "best_model_checkpoint": null,
+ "epoch": 3.0,
+ "eval_steps": 100,
+ "global_step": 1485,
+ "is_hyper_param_search": false,
+ "is_local_process_zero": true,
+ "is_world_process_zero": true,
+ "log_history": [
+ {
+ "entropy": 0.8295683860778809,
+ "epoch": 0.00202020202020202,
+ "grad_norm": 10.885663032531738,
+ "learning_rate": 2e-05,
+ "loss": 1.5185801982879639,
+ "mean_token_accuracy": 0.672874927520752,
+ "num_tokens": 16384.0,
+ "step": 1
+ },
+ {
+ "entropy": 1.3341505527496338,
+ "epoch": 0.00404040404040404,
+ "grad_norm": 9.276060104370117,
+ "learning_rate": 1.9986531986531986e-05,
+ "loss": 2.0802578926086426,
+ "mean_token_accuracy": 0.5727283954620361,
+ "num_tokens": 32768.0,
+ "step": 2
+ },
+ {
+ "entropy": 1.159377932548523,
+ "epoch": 0.006060606060606061,
+ "grad_norm": 3.999565839767456,
+ "learning_rate": 1.9973063973063975e-05,
+ "loss": 1.4128143787384033,
+ "mean_token_accuracy": 0.6849657893180847,
+ "num_tokens": 49152.0,
+ "step": 3
+ },
+ {
+ "entropy": 1.504081130027771,
+ "epoch": 0.00808080808080808,
+ "grad_norm": 3.018310546875,
+ "learning_rate": 1.995959595959596e-05,
+ "loss": 1.5479036569595337,
+ "mean_token_accuracy": 0.647777259349823,
+ "num_tokens": 65536.0,
+ "step": 4
+ },
+ {
+ "entropy": 2.1211423873901367,
+ "epoch": 0.010101010101010102,
+ "grad_norm": 4.352237701416016,
+ "learning_rate": 1.9946127946127948e-05,
+ "loss": 1.7965757846832275,
+ "mean_token_accuracy": 0.5992916226387024,
+ "num_tokens": 81920.0,
+ "step": 5
+ },
+ {
+ "entropy": 1.6345494985580444,
+ "epoch": 0.012121212121212121,
+ "grad_norm": 3.2360711097717285,
+ "learning_rate": 1.9932659932659936e-05,
+ "loss": 1.4024577140808105,
+ "mean_token_accuracy": 0.6747679710388184,
+ "num_tokens": 98304.0,
+ "step": 6
+ },
+ {
+ "entropy": 2.105585813522339,
+ "epoch": 0.014141414141414142,
+ "grad_norm": 2.84971284866333,
+ "learning_rate": 1.991919191919192e-05,
+ "loss": 1.9915739297866821,
+ "mean_token_accuracy": 0.5882999300956726,
+ "num_tokens": 114688.0,
+ "step": 7
+ },
+ {
+ "entropy": 1.7230671644210815,
+ "epoch": 0.01616161616161616,
+ "grad_norm": 2.686546802520752,
+ "learning_rate": 1.990572390572391e-05,
+ "loss": 1.630468726158142,
+ "mean_token_accuracy": 0.6269540786743164,
+ "num_tokens": 131072.0,
+ "step": 8
+ },
+ {
+ "entropy": 2.048288106918335,
+ "epoch": 0.01818181818181818,
+ "grad_norm": 2.6931567192077637,
+ "learning_rate": 1.9892255892255894e-05,
+ "loss": 2.212648391723633,
+ "mean_token_accuracy": 0.5294333100318909,
+ "num_tokens": 147456.0,
+ "step": 9
+ },
+ {
+ "entropy": 1.3021576404571533,
+ "epoch": 0.020202020202020204,
+ "grad_norm": 2.5286052227020264,
+ "learning_rate": 1.9878787878787878e-05,
+ "loss": 1.4357099533081055,
+ "mean_token_accuracy": 0.6633487939834595,
+ "num_tokens": 163840.0,
+ "step": 10
+ },
+ {
+ "entropy": 1.6998594999313354,
+ "epoch": 0.022222222222222223,
+ "grad_norm": 2.673692226409912,
+ "learning_rate": 1.9865319865319866e-05,
+ "loss": 1.8616552352905273,
+ "mean_token_accuracy": 0.6129091382026672,
+ "num_tokens": 180224.0,
+ "step": 11
+ },
+ {
+ "entropy": 1.5349891185760498,
+ "epoch": 0.024242424242424242,
+ "grad_norm": 2.882964611053467,
+ "learning_rate": 1.9851851851851855e-05,
+ "loss": 1.7540476322174072,
+ "mean_token_accuracy": 0.6111993193626404,
+ "num_tokens": 196608.0,
+ "step": 12
+ },
+ {
+ "entropy": 1.4202098846435547,
+ "epoch": 0.026262626262626262,
+ "grad_norm": 2.3892722129821777,
+ "learning_rate": 1.983838383838384e-05,
+ "loss": 1.5638885498046875,
+ "mean_token_accuracy": 0.642892062664032,
+ "num_tokens": 212992.0,
+ "step": 13
+ },
+ {
+ "entropy": 1.462074875831604,
+ "epoch": 0.028282828282828285,
+ "grad_norm": 2.357353925704956,
+ "learning_rate": 1.9824915824915828e-05,
+ "loss": 1.5339899063110352,
+ "mean_token_accuracy": 0.643991231918335,
+ "num_tokens": 229376.0,
+ "step": 14
+ },
+ {
+ "entropy": 1.6386674642562866,
+ "epoch": 0.030303030303030304,
+ "grad_norm": 2.1600515842437744,
+ "learning_rate": 1.9811447811447812e-05,
+ "loss": 1.6242434978485107,
+ "mean_token_accuracy": 0.6472887396812439,
+ "num_tokens": 245760.0,
+ "step": 15
+ },
+ {
+ "entropy": 1.882306456565857,
+ "epoch": 0.03232323232323232,
+ "grad_norm": 2.478746175765991,
+ "learning_rate": 1.97979797979798e-05,
+ "loss": 1.83274245262146,
+ "mean_token_accuracy": 0.5993527173995972,
+ "num_tokens": 262144.0,
+ "step": 16
+ },
+ {
+ "entropy": 1.5836528539657593,
+ "epoch": 0.03434343434343434,
+ "grad_norm": 2.5016613006591797,
+ "learning_rate": 1.9784511784511785e-05,
+ "loss": 1.5364878177642822,
+ "mean_token_accuracy": 0.6441743969917297,
+ "num_tokens": 278528.0,
+ "step": 17
+ },
+ {
+ "entropy": 1.5695534944534302,
+ "epoch": 0.03636363636363636,
+ "grad_norm": 2.2971248626708984,
+ "learning_rate": 1.9771043771043774e-05,
+ "loss": 1.5179738998413086,
+ "mean_token_accuracy": 0.6437469720840454,
+ "num_tokens": 294912.0,
+ "step": 18
+ },
+ {
+ "entropy": 1.3835400342941284,
+ "epoch": 0.03838383838383838,
+ "grad_norm": 2.1497576236724854,
+ "learning_rate": 1.975757575757576e-05,
+ "loss": 1.3505406379699707,
+ "mean_token_accuracy": 0.6729970574378967,
+ "num_tokens": 311296.0,
+ "step": 19
+ },
+ {
+ "entropy": 1.4660464525222778,
+ "epoch": 0.04040404040404041,
+ "grad_norm": 2.048318862915039,
+ "learning_rate": 1.9744107744107747e-05,
+ "loss": 1.3898100852966309,
+ "mean_token_accuracy": 0.6654250025749207,
+ "num_tokens": 327680.0,
+ "step": 20
+ },
+ {
+ "entropy": 1.959633469581604,
+ "epoch": 0.04242424242424243,
+ "grad_norm": 2.2084245681762695,
+ "learning_rate": 1.973063973063973e-05,
+ "loss": 1.9230711460113525,
+ "mean_token_accuracy": 0.5726062655448914,
+ "num_tokens": 344064.0,
+ "step": 21
+ },
+ {
+ "entropy": 1.4872093200683594,
+ "epoch": 0.044444444444444446,
+ "grad_norm": 2.116982936859131,
+ "learning_rate": 1.971717171717172e-05,
+ "loss": 1.514385461807251,
+ "mean_token_accuracy": 0.6615168452262878,
+ "num_tokens": 360448.0,
+ "step": 22
+ },
+ {
+ "entropy": 1.845888614654541,
+ "epoch": 0.046464646464646465,
+ "grad_norm": 2.4377331733703613,
+ "learning_rate": 1.9703703703703704e-05,
+ "loss": 1.8750195503234863,
+ "mean_token_accuracy": 0.5962994694709778,
+ "num_tokens": 376832.0,
+ "step": 23
+ },
+ {
+ "entropy": 1.5549575090408325,
+ "epoch": 0.048484848484848485,
+ "grad_norm": 2.44534969329834,
+ "learning_rate": 1.9690235690235692e-05,
+ "loss": 1.627150058746338,
+ "mean_token_accuracy": 0.6215803623199463,
+ "num_tokens": 393216.0,
+ "step": 24
+ },
+ {
+ "entropy": 1.4516701698303223,
+ "epoch": 0.050505050505050504,
+ "grad_norm": 2.2948875427246094,
+ "learning_rate": 1.9676767676767677e-05,
+ "loss": 1.4912647008895874,
+ "mean_token_accuracy": 0.6431363224983215,
+ "num_tokens": 409600.0,
+ "step": 25
+ },
+ {
+ "entropy": 1.3493973016738892,
+ "epoch": 0.052525252525252523,
+ "grad_norm": 2.1061222553253174,
+ "learning_rate": 1.9663299663299665e-05,
+ "loss": 1.44138765335083,
+ "mean_token_accuracy": 0.6621885895729065,
+ "num_tokens": 425984.0,
+ "step": 26
+ },
+ {
+ "entropy": 1.7119438648223877,
+ "epoch": 0.05454545454545454,
+ "grad_norm": 2.5149970054626465,
+ "learning_rate": 1.9649831649831654e-05,
+ "loss": 1.776426076889038,
+ "mean_token_accuracy": 0.5944675207138062,
+ "num_tokens": 442368.0,
+ "step": 27
+ },
+ {
+ "entropy": 1.361685872077942,
+ "epoch": 0.05656565656565657,
+ "grad_norm": 2.343843460083008,
+ "learning_rate": 1.963636363636364e-05,
+ "loss": 1.404232144355774,
+ "mean_token_accuracy": 0.6639594435691833,
+ "num_tokens": 458752.0,
+ "step": 28
+ },
+ {
+ "entropy": 1.2213996648788452,
+ "epoch": 0.05858585858585859,
+ "grad_norm": 1.9002994298934937,
+ "learning_rate": 1.9622895622895623e-05,
+ "loss": 1.249969244003296,
+ "mean_token_accuracy": 0.6933927536010742,
+ "num_tokens": 475136.0,
+ "step": 29
+ },
+ {
+ "entropy": 1.7157055139541626,
+ "epoch": 0.06060606060606061,
+ "grad_norm": 2.21567702293396,
+ "learning_rate": 1.960942760942761e-05,
+ "loss": 1.7103240489959717,
+ "mean_token_accuracy": 0.6218856573104858,
+ "num_tokens": 491520.0,
+ "step": 30
+ },
+ {
+ "entropy": 1.8233915567398071,
+ "epoch": 0.06262626262626263,
+ "grad_norm": 2.0189297199249268,
+ "learning_rate": 1.9595959595959596e-05,
+ "loss": 1.820570707321167,
+ "mean_token_accuracy": 0.6126648783683777,
+ "num_tokens": 507904.0,
+ "step": 31
+ },
+ {
+ "entropy": 1.82878839969635,
+ "epoch": 0.06464646464646465,
+ "grad_norm": 2.556058883666992,
+ "learning_rate": 1.9582491582491584e-05,
+ "loss": 1.773336410522461,
+ "mean_token_accuracy": 0.5889716744422913,
+ "num_tokens": 524288.0,
+ "step": 32
+ },
+ {
+ "entropy": 1.4608416557312012,
+ "epoch": 0.06666666666666667,
+ "grad_norm": 2.0254712104797363,
+ "learning_rate": 1.956902356902357e-05,
+ "loss": 1.4261068105697632,
+ "mean_token_accuracy": 0.6561431288719177,
+ "num_tokens": 540672.0,
+ "step": 33
+ },
+ {
+ "entropy": 1.4706993103027344,
+ "epoch": 0.06868686868686869,
+ "grad_norm": 2.1344153881073,
+ "learning_rate": 1.9555555555555557e-05,
+ "loss": 1.4610451459884644,
+ "mean_token_accuracy": 0.6472276449203491,
+ "num_tokens": 557056.0,
+ "step": 34
+ },
+ {
+ "entropy": 1.9629759788513184,
+ "epoch": 0.0707070707070707,
+ "grad_norm": 2.209073066711426,
+ "learning_rate": 1.9542087542087545e-05,
+ "loss": 1.9688678979873657,
+ "mean_token_accuracy": 0.5702857971191406,
+ "num_tokens": 573440.0,
+ "step": 35
+ },
+ {
+ "entropy": 1.5821412801742554,
+ "epoch": 0.07272727272727272,
+ "grad_norm": 1.9711852073669434,
+ "learning_rate": 1.952861952861953e-05,
+ "loss": 1.5614874362945557,
+ "mean_token_accuracy": 0.639167070388794,
+ "num_tokens": 589824.0,
+ "step": 36
+ },
+ {
+ "entropy": 1.4696686267852783,
+ "epoch": 0.07474747474747474,
+ "grad_norm": 2.07849383354187,
+ "learning_rate": 1.9515151515151515e-05,
+ "loss": 1.497229814529419,
+ "mean_token_accuracy": 0.6555324792861938,
+ "num_tokens": 606208.0,
+ "step": 37
+ },
+ {
+ "entropy": 1.481307864189148,
+ "epoch": 0.07676767676767676,
+ "grad_norm": 2.028839588165283,
+ "learning_rate": 1.9501683501683503e-05,
+ "loss": 1.495100975036621,
+ "mean_token_accuracy": 0.6472276449203491,
+ "num_tokens": 622592.0,
+ "step": 38
+ },
+ {
+ "entropy": 1.515073537826538,
+ "epoch": 0.07878787878787878,
+ "grad_norm": 2.2417402267456055,
+ "learning_rate": 1.9488215488215488e-05,
+ "loss": 1.6139262914657593,
+ "mean_token_accuracy": 0.6312286257743835,
+ "num_tokens": 638976.0,
+ "step": 39
+ },
+ {
+ "entropy": 1.9048941135406494,
+ "epoch": 0.08080808080808081,
+ "grad_norm": 2.232630968093872,
+ "learning_rate": 1.9474747474747476e-05,
+ "loss": 1.9573429822921753,
+ "mean_token_accuracy": 0.5796287059783936,
+ "num_tokens": 655360.0,
+ "step": 40
+ },
+ {
+ "entropy": 1.7393985986709595,
+ "epoch": 0.08282828282828283,
+ "grad_norm": 2.1077799797058105,
+ "learning_rate": 1.9461279461279464e-05,
+ "loss": 1.825897216796875,
+ "mean_token_accuracy": 0.5946506857872009,
+ "num_tokens": 671744.0,
+ "step": 41
+ },
+ {
+ "entropy": 1.6294668912887573,
+ "epoch": 0.08484848484848485,
+ "grad_norm": 2.299520492553711,
+ "learning_rate": 1.944781144781145e-05,
+ "loss": 1.6667912006378174,
+ "mean_token_accuracy": 0.6225574016571045,
+ "num_tokens": 688128.0,
+ "step": 42
+ },
+ {
+ "entropy": 1.4176290035247803,
+ "epoch": 0.08686868686868687,
+ "grad_norm": 1.9385138750076294,
+ "learning_rate": 1.9434343434343437e-05,
+ "loss": 1.422924280166626,
+ "mean_token_accuracy": 0.6719589829444885,
+ "num_tokens": 704512.0,
+ "step": 43
+ },
+ {
+ "entropy": 1.6570991277694702,
+ "epoch": 0.08888888888888889,
+ "grad_norm": 2.1650922298431396,
+ "learning_rate": 1.9420875420875422e-05,
+ "loss": 1.6351218223571777,
+ "mean_token_accuracy": 0.6342818737030029,
+ "num_tokens": 720896.0,
+ "step": 44
+ },
+ {
+ "entropy": 1.520472764968872,
+ "epoch": 0.09090909090909091,
+ "grad_norm": 2.116955280303955,
+ "learning_rate": 1.9407407407407407e-05,
+ "loss": 1.4788711071014404,
+ "mean_token_accuracy": 0.6557767391204834,
+ "num_tokens": 737280.0,
+ "step": 45
+ },
+ {
+ "entropy": 1.3090498447418213,
+ "epoch": 0.09292929292929293,
+ "grad_norm": 2.0493314266204834,
+ "learning_rate": 1.9393939393939395e-05,
+ "loss": 1.3372726440429688,
+ "mean_token_accuracy": 0.6762945652008057,
+ "num_tokens": 753664.0,
+ "step": 46
+ },
+ {
+ "entropy": 1.6449471712112427,
+ "epoch": 0.09494949494949495,
+ "grad_norm": 2.2450437545776367,
+ "learning_rate": 1.9380471380471383e-05,
+ "loss": 1.6683790683746338,
+ "mean_token_accuracy": 0.6266487836837769,
+ "num_tokens": 770048.0,
+ "step": 47
+ },
+ {
+ "entropy": 1.2516838312149048,
+ "epoch": 0.09696969696969697,
+ "grad_norm": 2.168203592300415,
+ "learning_rate": 1.9367003367003368e-05,
+ "loss": 1.2885866165161133,
+ "mean_token_accuracy": 0.689667820930481,
+ "num_tokens": 786432.0,
+ "step": 48
+ },
+ {
+ "entropy": 1.7858967781066895,
+ "epoch": 0.09898989898989899,
+ "grad_norm": 2.0490918159484863,
+ "learning_rate": 1.9353535353535356e-05,
+ "loss": 1.8125935792922974,
+ "mean_token_accuracy": 0.5957498550415039,
+ "num_tokens": 802816.0,
+ "step": 49
+ },
+ {
+ "entropy": 1.6833406686782837,
+ "epoch": 0.10101010101010101,
+ "grad_norm": 1.9622541666030884,
+ "learning_rate": 1.934006734006734e-05,
+ "loss": 1.715425729751587,
+ "mean_token_accuracy": 0.606008768081665,
+ "num_tokens": 819200.0,
+ "step": 50
+ },
+ {
+ "entropy": 1.4103842973709106,
+ "epoch": 0.10303030303030303,
+ "grad_norm": 2.015503406524658,
+ "learning_rate": 1.932659932659933e-05,
+ "loss": 1.4374600648880005,
+ "mean_token_accuracy": 0.6615168452262878,
+ "num_tokens": 835584.0,
+ "step": 51
+ },
+ {
+ "entropy": 1.6805938482284546,
+ "epoch": 0.10505050505050505,
+ "grad_norm": 2.139333963394165,
+ "learning_rate": 1.9313131313131314e-05,
+ "loss": 1.684693455696106,
+ "mean_token_accuracy": 0.6270151734352112,
+ "num_tokens": 851968.0,
+ "step": 52
+ },
+ {
+ "entropy": 1.770232915878296,
+ "epoch": 0.10707070707070707,
+ "grad_norm": 2.3331124782562256,
+ "learning_rate": 1.92996632996633e-05,
+ "loss": 1.781606674194336,
+ "mean_token_accuracy": 0.6006350517272949,
+ "num_tokens": 868352.0,
+ "step": 53
+ },
+ {
+ "entropy": 1.676267147064209,
+ "epoch": 0.10909090909090909,
+ "grad_norm": 2.1863765716552734,
+ "learning_rate": 1.9286195286195287e-05,
+ "loss": 1.6873016357421875,
+ "mean_token_accuracy": 0.6182828545570374,
+ "num_tokens": 884736.0,
+ "step": 54
+ },
+ {
+ "entropy": 1.3133333921432495,
+ "epoch": 0.1111111111111111,
+ "grad_norm": 2.455390691757202,
+ "learning_rate": 1.9272727272727275e-05,
+ "loss": 1.336955189704895,
+ "mean_token_accuracy": 0.682584285736084,
+ "num_tokens": 901120.0,
+ "step": 55
+ },
+ {
+ "entropy": 1.8064881563186646,
+ "epoch": 0.11313131313131314,
+ "grad_norm": 2.218869686126709,
+ "learning_rate": 1.925925925925926e-05,
+ "loss": 1.8163663148880005,
+ "mean_token_accuracy": 0.5947728157043457,
+ "num_tokens": 917504.0,
+ "step": 56
+ },
+ {
+ "entropy": 1.495352864265442,
+ "epoch": 0.11515151515151516,
+ "grad_norm": 2.040064573287964,
+ "learning_rate": 1.9245791245791248e-05,
+ "loss": 1.4638073444366455,
+ "mean_token_accuracy": 0.6512579321861267,
+ "num_tokens": 933888.0,
+ "step": 57
+ },
+ {
+ "entropy": 1.5757899284362793,
+ "epoch": 0.11717171717171718,
+ "grad_norm": 1.8449796438217163,
+ "learning_rate": 1.9232323232323233e-05,
+ "loss": 1.5526244640350342,
+ "mean_token_accuracy": 0.6346482634544373,
+ "num_tokens": 950272.0,
+ "step": 58
+ },
+ {
+ "entropy": 1.6834555864334106,
+ "epoch": 0.1191919191919192,
+ "grad_norm": 2.0705320835113525,
+ "learning_rate": 1.921885521885522e-05,
+ "loss": 1.753828763961792,
+ "mean_token_accuracy": 0.6237176060676575,
+ "num_tokens": 966656.0,
+ "step": 59
+ },
+ {
+ "entropy": 1.5231870412826538,
+ "epoch": 0.12121212121212122,
+ "grad_norm": 2.1398122310638428,
+ "learning_rate": 1.9205387205387206e-05,
+ "loss": 1.515265941619873,
+ "mean_token_accuracy": 0.6452125310897827,
+ "num_tokens": 983040.0,
+ "step": 60
+ },
+ {
+ "entropy": 1.6915754079818726,
+ "epoch": 0.12323232323232323,
+ "grad_norm": 2.179158926010132,
+ "learning_rate": 1.9191919191919194e-05,
+ "loss": 1.6949875354766846,
+ "mean_token_accuracy": 0.6109550595283508,
+ "num_tokens": 999424.0,
+ "step": 61
+ },
+ {
+ "entropy": 1.580087661743164,
+ "epoch": 0.12525252525252525,
+ "grad_norm": 2.8643219470977783,
+ "learning_rate": 1.9178451178451182e-05,
+ "loss": 1.6217050552368164,
+ "mean_token_accuracy": 0.6498534679412842,
+ "num_tokens": 1015808.0,
+ "step": 62
+ },
+ {
+ "entropy": 1.7331494092941284,
+ "epoch": 0.12727272727272726,
+ "grad_norm": 2.0613038539886475,
+ "learning_rate": 1.9164983164983167e-05,
+ "loss": 1.7527930736541748,
+ "mean_token_accuracy": 0.59513920545578,
+ "num_tokens": 1032192.0,
+ "step": 63
+ },
+ {
+ "entropy": 1.3311079740524292,
+ "epoch": 0.1292929292929293,
+ "grad_norm": 1.996009111404419,
+ "learning_rate": 1.9151515151515152e-05,
+ "loss": 1.3299833536148071,
+ "mean_token_accuracy": 0.6760503053665161,
+ "num_tokens": 1048576.0,
+ "step": 64
+ },
+ {
+ "entropy": 1.1875317096710205,
+ "epoch": 0.13131313131313133,
+ "grad_norm": 1.9458681344985962,
+ "learning_rate": 1.913804713804714e-05,
+ "loss": 1.1885290145874023,
+ "mean_token_accuracy": 0.6996213793754578,
+ "num_tokens": 1064960.0,
+ "step": 65
+ },
+ {
+ "entropy": 1.0825527906417847,
+ "epoch": 0.13333333333333333,
+ "grad_norm": 1.8164746761322021,
+ "learning_rate": 1.9124579124579125e-05,
+ "loss": 1.0918856859207153,
+ "mean_token_accuracy": 0.7477405667304993,
+ "num_tokens": 1081344.0,
+ "step": 66
+ },
+ {
+ "entropy": 1.5940357446670532,
+ "epoch": 0.13535353535353536,
+ "grad_norm": 1.9322525262832642,
+ "learning_rate": 1.9111111111111113e-05,
+ "loss": 1.6157622337341309,
+ "mean_token_accuracy": 0.6340376138687134,
+ "num_tokens": 1097728.0,
+ "step": 67
+ },
+ {
+ "entropy": 1.683976173400879,
+ "epoch": 0.13737373737373737,
+ "grad_norm": 2.1792430877685547,
+ "learning_rate": 1.9097643097643098e-05,
+ "loss": 1.7643585205078125,
+ "mean_token_accuracy": 0.5998412370681763,
+ "num_tokens": 1114112.0,
+ "step": 68
+ },
+ {
+ "entropy": 1.5753607749938965,
+ "epoch": 0.1393939393939394,
+ "grad_norm": 2.033538818359375,
+ "learning_rate": 1.9084175084175086e-05,
+ "loss": 1.628584861755371,
+ "mean_token_accuracy": 0.6401441097259521,
+ "num_tokens": 1130496.0,
+ "step": 69
+ },
+ {
+ "entropy": 1.4061329364776611,
+ "epoch": 0.1414141414141414,
+ "grad_norm": 1.9239107370376587,
+ "learning_rate": 1.9070707070707074e-05,
+ "loss": 1.4355204105377197,
+ "mean_token_accuracy": 0.6727527976036072,
+ "num_tokens": 1146880.0,
+ "step": 70
+ },
+ {
+ "entropy": 1.1722302436828613,
+ "epoch": 0.14343434343434344,
+ "grad_norm": 2.0911052227020264,
+ "learning_rate": 1.905723905723906e-05,
+ "loss": 1.2008472681045532,
+ "mean_token_accuracy": 0.7064606547355652,
+ "num_tokens": 1163264.0,
+ "step": 71
+ },
+ {
+ "entropy": 1.5291107892990112,
+ "epoch": 0.14545454545454545,
+ "grad_norm": 2.1054158210754395,
+ "learning_rate": 1.9043771043771044e-05,
+ "loss": 1.574873924255371,
+ "mean_token_accuracy": 0.6356253027915955,
+ "num_tokens": 1179648.0,
+ "step": 72
+ },
+ {
+ "entropy": 1.3679379224777222,
+ "epoch": 0.14747474747474748,
+ "grad_norm": 1.928666353225708,
+ "learning_rate": 1.9030303030303032e-05,
+ "loss": 1.3473236560821533,
+ "mean_token_accuracy": 0.679347813129425,
+ "num_tokens": 1196032.0,
+ "step": 73
+ },
+ {
+ "entropy": 1.7593064308166504,
+ "epoch": 0.1494949494949495,
+ "grad_norm": 2.3172972202301025,
+ "learning_rate": 1.9016835016835017e-05,
+ "loss": 1.7953016757965088,
+ "mean_token_accuracy": 0.6107107996940613,
+ "num_tokens": 1212416.0,
+ "step": 74
+ },
+ {
+ "entropy": 1.3841301202774048,
+ "epoch": 0.15151515151515152,
+ "grad_norm": 1.9938782453536987,
+ "learning_rate": 1.9003367003367005e-05,
+ "loss": 1.3923728466033936,
+ "mean_token_accuracy": 0.6686614751815796,
+ "num_tokens": 1228800.0,
+ "step": 75
+ },
+ {
+ "entropy": 1.6030631065368652,
+ "epoch": 0.15353535353535352,
+ "grad_norm": 2.0514845848083496,
+ "learning_rate": 1.8989898989898993e-05,
+ "loss": 1.5902166366577148,
+ "mean_token_accuracy": 0.6376404762268066,
+ "num_tokens": 1245184.0,
+ "step": 76
+ },
+ {
+ "entropy": 1.5902619361877441,
+ "epoch": 0.15555555555555556,
+ "grad_norm": 2.115469217300415,
+ "learning_rate": 1.8976430976430978e-05,
+ "loss": 1.5841395854949951,
+ "mean_token_accuracy": 0.6361138224601746,
+ "num_tokens": 1261568.0,
+ "step": 77
+ },
+ {
+ "entropy": 1.7821043729782104,
+ "epoch": 0.15757575757575756,
+ "grad_norm": 2.008962869644165,
+ "learning_rate": 1.8962962962962966e-05,
+ "loss": 1.7826645374298096,
+ "mean_token_accuracy": 0.6053370833396912,
+ "num_tokens": 1277952.0,
+ "step": 78
+ },
+ {
+ "entropy": 1.833274006843567,
+ "epoch": 0.1595959595959596,
+ "grad_norm": 2.0258517265319824,
+ "learning_rate": 1.894949494949495e-05,
+ "loss": 1.8566474914550781,
+ "mean_token_accuracy": 0.5800561904907227,
+ "num_tokens": 1294336.0,
+ "step": 79
+ },
+ {
+ "entropy": 1.785520315170288,
+ "epoch": 0.16161616161616163,
+ "grad_norm": 2.0918498039245605,
+ "learning_rate": 1.8936026936026936e-05,
+ "loss": 1.775907278060913,
+ "mean_token_accuracy": 0.6063141226768494,
+ "num_tokens": 1310720.0,
+ "step": 80
+ },
+ {
+ "entropy": 1.9341778755187988,
+ "epoch": 0.16363636363636364,
+ "grad_norm": 2.0399975776672363,
+ "learning_rate": 1.8922558922558924e-05,
+ "loss": 1.9579840898513794,
+ "mean_token_accuracy": 0.5619809627532959,
+ "num_tokens": 1327104.0,
+ "step": 81
+ },
+ {
+ "entropy": 1.4945461750030518,
+ "epoch": 0.16565656565656567,
+ "grad_norm": 1.9975999593734741,
+ "learning_rate": 1.8909090909090912e-05,
+ "loss": 1.5436582565307617,
+ "mean_token_accuracy": 0.6423424482345581,
+ "num_tokens": 1343488.0,
+ "step": 82
+ },
+ {
+ "entropy": 1.6639432907104492,
+ "epoch": 0.16767676767676767,
+ "grad_norm": 2.0375208854675293,
+ "learning_rate": 1.8895622895622897e-05,
+ "loss": 1.679243564605713,
+ "mean_token_accuracy": 0.6226184368133545,
+ "num_tokens": 1359872.0,
+ "step": 83
+ },
+ {
+ "entropy": 1.4505668878555298,
+ "epoch": 0.1696969696969697,
+ "grad_norm": 1.8894681930541992,
+ "learning_rate": 1.8882154882154885e-05,
+ "loss": 1.4778342247009277,
+ "mean_token_accuracy": 0.6637151837348938,
+ "num_tokens": 1376256.0,
+ "step": 84
+ },
+ {
+ "entropy": 1.3845125436782837,
+ "epoch": 0.1717171717171717,
+ "grad_norm": 2.1110754013061523,
+ "learning_rate": 1.886868686868687e-05,
+ "loss": 1.4153435230255127,
+ "mean_token_accuracy": 0.6737298369407654,
+ "num_tokens": 1392640.0,
+ "step": 85
+ },
+ {
+ "entropy": 1.8785539865493774,
+ "epoch": 0.17373737373737375,
+ "grad_norm": 2.0894815921783447,
+ "learning_rate": 1.8855218855218858e-05,
+ "loss": 1.8658334016799927,
+ "mean_token_accuracy": 0.5846360325813293,
+ "num_tokens": 1409024.0,
+ "step": 86
+ },
+ {
+ "entropy": 1.6006972789764404,
+ "epoch": 0.17575757575757575,
+ "grad_norm": 2.088477849960327,
+ "learning_rate": 1.8841750841750843e-05,
+ "loss": 1.6257164478302002,
+ "mean_token_accuracy": 0.623900830745697,
+ "num_tokens": 1425408.0,
+ "step": 87
+ },
+ {
+ "entropy": 1.1098109483718872,
+ "epoch": 0.17777777777777778,
+ "grad_norm": 1.934228539466858,
+ "learning_rate": 1.8828282828282827e-05,
+ "loss": 1.0948461294174194,
+ "mean_token_accuracy": 0.7248412370681763,
+ "num_tokens": 1441792.0,
+ "step": 88
+ },
+ {
+ "entropy": 1.4807215929031372,
+ "epoch": 0.1797979797979798,
+ "grad_norm": 2.06062388420105,
+ "learning_rate": 1.8814814814814816e-05,
+ "loss": 1.5115268230438232,
+ "mean_token_accuracy": 0.6463117003440857,
+ "num_tokens": 1458176.0,
+ "step": 89
+ },
+ {
+ "entropy": 1.07956862449646,
+ "epoch": 0.18181818181818182,
+ "grad_norm": 1.8913685083389282,
+ "learning_rate": 1.8801346801346804e-05,
+ "loss": 1.0902754068374634,
+ "mean_token_accuracy": 0.7308256030082703,
+ "num_tokens": 1474560.0,
+ "step": 90
+ },
+ {
+ "entropy": 1.4571733474731445,
+ "epoch": 0.18383838383838383,
+ "grad_norm": 2.2199525833129883,
+ "learning_rate": 1.8787878787878792e-05,
+ "loss": 1.4686330556869507,
+ "mean_token_accuracy": 0.6566927433013916,
+ "num_tokens": 1490944.0,
+ "step": 91
+ },
+ {
+ "entropy": 1.4484740495681763,
+ "epoch": 0.18585858585858586,
+ "grad_norm": 2.0670580863952637,
+ "learning_rate": 1.8774410774410777e-05,
+ "loss": 1.4791338443756104,
+ "mean_token_accuracy": 0.6493649482727051,
+ "num_tokens": 1507328.0,
+ "step": 92
+ },
+ {
+ "entropy": 1.498313546180725,
+ "epoch": 0.18787878787878787,
+ "grad_norm": 2.024329423904419,
+ "learning_rate": 1.876094276094276e-05,
+ "loss": 1.4964686632156372,
+ "mean_token_accuracy": 0.6457620859146118,
+ "num_tokens": 1523712.0,
+ "step": 93
+ },
+ {
+ "entropy": 1.4806159734725952,
+ "epoch": 0.1898989898989899,
+ "grad_norm": 2.1369152069091797,
+ "learning_rate": 1.874747474747475e-05,
+ "loss": 1.4587206840515137,
+ "mean_token_accuracy": 0.6521739363670349,
+ "num_tokens": 1540096.0,
+ "step": 94
+ },
+ {
+ "entropy": 1.3746109008789062,
+ "epoch": 0.1919191919191919,
+ "grad_norm": 2.216958522796631,
+ "learning_rate": 1.8734006734006735e-05,
+ "loss": 1.3600832223892212,
+ "mean_token_accuracy": 0.671775758266449,
+ "num_tokens": 1556480.0,
+ "step": 95
+ },
+ {
+ "entropy": 1.3492207527160645,
+ "epoch": 0.19393939393939394,
+ "grad_norm": 2.093956708908081,
+ "learning_rate": 1.8720538720538723e-05,
+ "loss": 1.3867243528366089,
+ "mean_token_accuracy": 0.6625549793243408,
+ "num_tokens": 1572864.0,
+ "step": 96
+ },
+ {
+ "entropy": 1.560738205909729,
+ "epoch": 0.19595959595959597,
+ "grad_norm": 2.0691707134246826,
+ "learning_rate": 1.8707070707070707e-05,
+ "loss": 1.5875897407531738,
+ "mean_token_accuracy": 0.6345872282981873,
+ "num_tokens": 1589248.0,
+ "step": 97
+ },
+ {
+ "entropy": 1.46450674533844,
+ "epoch": 0.19797979797979798,
+ "grad_norm": 2.064547538757324,
+ "learning_rate": 1.8693602693602696e-05,
+ "loss": 1.4920742511749268,
+ "mean_token_accuracy": 0.6411822438240051,
+ "num_tokens": 1605632.0,
+ "step": 98
+ },
+ {
+ "entropy": 1.5433454513549805,
+ "epoch": 0.2,
+ "grad_norm": 2.014409303665161,
+ "learning_rate": 1.868013468013468e-05,
+ "loss": 1.5737671852111816,
+ "mean_token_accuracy": 0.6349536180496216,
+ "num_tokens": 1622016.0,
+ "step": 99
+ },
+ {
+ "entropy": 1.358772873878479,
+ "epoch": 0.20202020202020202,
+ "grad_norm": 1.9765456914901733,
+ "learning_rate": 1.866666666666667e-05,
+ "loss": 1.3583091497421265,
+ "mean_token_accuracy": 0.680446982383728,
+ "num_tokens": 1638400.0,
+ "step": 100
+ },
+ {
+ "epoch": 0.20202020202020202,
+ "eval_entropy": 1.5664977690865916,
+ "eval_loss": 1.5804312229156494,
+ "eval_mean_token_accuracy": 0.63647972239602,
+ "eval_num_tokens": 1638400.0,
+ "eval_runtime": 44.1541,
+ "eval_samples_per_second": 22.421,
+ "eval_steps_per_second": 2.808,
+ "step": 100
+ },
+ {
+ "entropy": 1.5879693031311035,
+ "epoch": 0.20404040404040405,
+ "grad_norm": 1.8491015434265137,
+ "learning_rate": 1.8653198653198653e-05,
+ "loss": 1.5892077684402466,
+ "mean_token_accuracy": 0.6665852665901184,
+ "num_tokens": 1654784.0,
+ "step": 101
+ },
+ {
+ "entropy": 1.5789942741394043,
+ "epoch": 0.20606060606060606,
+ "grad_norm": 1.9096908569335938,
+ "learning_rate": 1.863973063973064e-05,
+ "loss": 1.594193458557129,
+ "mean_token_accuracy": 0.6293966770172119,
+ "num_tokens": 1671168.0,
+ "step": 102
+ },
+ {
+ "entropy": 1.5245429277420044,
+ "epoch": 0.2080808080808081,
+ "grad_norm": 1.986904501914978,
+ "learning_rate": 1.8626262626262626e-05,
+ "loss": 1.5463616847991943,
+ "mean_token_accuracy": 0.6354421377182007,
+ "num_tokens": 1687552.0,
+ "step": 103
+ },
+ {
+ "entropy": 2.015364408493042,
+ "epoch": 0.2101010101010101,
+ "grad_norm": 2.3297250270843506,
+ "learning_rate": 1.8612794612794615e-05,
+ "loss": 1.987194538116455,
+ "mean_token_accuracy": 0.5604543089866638,
+ "num_tokens": 1703936.0,
+ "step": 104
+ },
+ {
+ "entropy": 1.770432472229004,
+ "epoch": 0.21212121212121213,
+ "grad_norm": 2.3580551147460938,
+ "learning_rate": 1.8599326599326603e-05,
+ "loss": 1.7715529203414917,
+ "mean_token_accuracy": 0.5786516666412354,
+ "num_tokens": 1720320.0,
+ "step": 105
+ },
+ {
+ "entropy": 1.3456813097000122,
+ "epoch": 0.21414141414141413,
+ "grad_norm": 1.9782105684280396,
+ "learning_rate": 1.8585858585858588e-05,
+ "loss": 1.342590570449829,
+ "mean_token_accuracy": 0.6779433488845825,
+ "num_tokens": 1736704.0,
+ "step": 106
+ },
+ {
+ "entropy": 1.4614694118499756,
+ "epoch": 0.21616161616161617,
+ "grad_norm": 1.9486815929412842,
+ "learning_rate": 1.8572390572390572e-05,
+ "loss": 1.4493391513824463,
+ "mean_token_accuracy": 0.6618832349777222,
+ "num_tokens": 1753088.0,
+ "step": 107
+ },
+ {
+ "entropy": 2.007932186126709,
+ "epoch": 0.21818181818181817,
+ "grad_norm": 2.0571465492248535,
+ "learning_rate": 1.855892255892256e-05,
+ "loss": 2.033851385116577,
+ "mean_token_accuracy": 0.5746213793754578,
+ "num_tokens": 1769472.0,
+ "step": 108
+ },
+ {
+ "entropy": 1.506022572517395,
+ "epoch": 0.2202020202020202,
+ "grad_norm": 1.9980531930923462,
+ "learning_rate": 1.8545454545454545e-05,
+ "loss": 1.5305016040802002,
+ "mean_token_accuracy": 0.6416096687316895,
+ "num_tokens": 1785856.0,
+ "step": 109
+ },
+ {
+ "entropy": 1.479506492614746,
+ "epoch": 0.2222222222222222,
+ "grad_norm": 2.411241292953491,
+ "learning_rate": 1.8531986531986533e-05,
+ "loss": 1.5501817464828491,
+ "mean_token_accuracy": 0.6364802122116089,
+ "num_tokens": 1802240.0,
+ "step": 110
+ },
+ {
+ "entropy": 1.7731081247329712,
+ "epoch": 0.22424242424242424,
+ "grad_norm": 2.1088449954986572,
+ "learning_rate": 1.851851851851852e-05,
+ "loss": 1.7582051753997803,
+ "mean_token_accuracy": 0.5985588431358337,
+ "num_tokens": 1818624.0,
+ "step": 111
+ },
+ {
+ "entropy": 1.7963190078735352,
+ "epoch": 0.22626262626262628,
+ "grad_norm": 2.180325508117676,
+ "learning_rate": 1.8505050505050506e-05,
+ "loss": 1.8436882495880127,
+ "mean_token_accuracy": 0.5850635170936584,
+ "num_tokens": 1835008.0,
+ "step": 112
+ },
+ {
+ "entropy": 1.513211727142334,
+ "epoch": 0.22828282828282828,
+ "grad_norm": 2.1563429832458496,
+ "learning_rate": 1.8491582491582495e-05,
+ "loss": 1.547263741493225,
+ "mean_token_accuracy": 0.6399609446525574,
+ "num_tokens": 1851392.0,
+ "step": 113
+ },
+ {
+ "entropy": 1.8904774188995361,
+ "epoch": 0.23030303030303031,
+ "grad_norm": 2.2191243171691895,
+ "learning_rate": 1.847811447811448e-05,
+ "loss": 1.9237418174743652,
+ "mean_token_accuracy": 0.5852466821670532,
+ "num_tokens": 1867776.0,
+ "step": 114
+ },
+ {
+ "entropy": 1.706113338470459,
+ "epoch": 0.23232323232323232,
+ "grad_norm": 2.1411335468292236,
+ "learning_rate": 1.8464646464646464e-05,
+ "loss": 1.7145740985870361,
+ "mean_token_accuracy": 0.6138861775398254,
+ "num_tokens": 1884160.0,
+ "step": 115
+ },
+ {
+ "entropy": 1.726441502571106,
+ "epoch": 0.23434343434343435,
+ "grad_norm": 2.212477922439575,
+ "learning_rate": 1.8451178451178452e-05,
+ "loss": 1.7427434921264648,
+ "mean_token_accuracy": 0.6087567210197449,
+ "num_tokens": 1900544.0,
+ "step": 116
+ },
+ {
+ "entropy": 1.49800443649292,
+ "epoch": 0.23636363636363636,
+ "grad_norm": 2.003248929977417,
+ "learning_rate": 1.8437710437710437e-05,
+ "loss": 1.4677059650421143,
+ "mean_token_accuracy": 0.6617000699043274,
+ "num_tokens": 1916928.0,
+ "step": 117
+ },
+ {
+ "entropy": 1.5079678297042847,
+ "epoch": 0.2383838383838384,
+ "grad_norm": 1.9702556133270264,
+ "learning_rate": 1.8424242424242425e-05,
+ "loss": 1.5194823741912842,
+ "mean_token_accuracy": 0.6593796014785767,
+ "num_tokens": 1933312.0,
+ "step": 118
+ },
+ {
+ "entropy": 1.760438323020935,
+ "epoch": 0.2404040404040404,
+ "grad_norm": 1.9484418630599976,
+ "learning_rate": 1.8410774410774414e-05,
+ "loss": 1.756869912147522,
+ "mean_token_accuracy": 0.6193209290504456,
+ "num_tokens": 1949696.0,
+ "step": 119
+ },
+ {
+ "entropy": 1.3706170320510864,
+ "epoch": 0.24242424242424243,
+ "grad_norm": 2.078357458114624,
+ "learning_rate": 1.83973063973064e-05,
+ "loss": 1.3532414436340332,
+ "mean_token_accuracy": 0.6680508255958557,
+ "num_tokens": 1966080.0,
+ "step": 120
+ },
+ {
+ "entropy": 1.2994399070739746,
+ "epoch": 0.24444444444444444,
+ "grad_norm": 1.9178324937820435,
+ "learning_rate": 1.8383838383838387e-05,
+ "loss": 1.298889398574829,
+ "mean_token_accuracy": 0.6831949353218079,
+ "num_tokens": 1982464.0,
+ "step": 121
+ },
+ {
+ "entropy": 1.6806142330169678,
+ "epoch": 0.24646464646464647,
+ "grad_norm": 2.287158966064453,
+ "learning_rate": 1.837037037037037e-05,
+ "loss": 1.728592038154602,
+ "mean_token_accuracy": 0.6161455512046814,
+ "num_tokens": 1998848.0,
+ "step": 122
+ },
+ {
+ "entropy": 1.2894387245178223,
+ "epoch": 0.24848484848484848,
+ "grad_norm": 1.9998525381088257,
+ "learning_rate": 1.8356902356902356e-05,
+ "loss": 1.3326170444488525,
+ "mean_token_accuracy": 0.6693331599235535,
+ "num_tokens": 2015232.0,
+ "step": 123
+ },
+ {
+ "entropy": 1.5466209650039673,
+ "epoch": 0.2505050505050505,
+ "grad_norm": 1.8439521789550781,
+ "learning_rate": 1.8343434343434344e-05,
+ "loss": 1.5606887340545654,
+ "mean_token_accuracy": 0.6571201682090759,
+ "num_tokens": 2031616.0,
+ "step": 124
+ },
+ {
+ "entropy": 1.2692567110061646,
+ "epoch": 0.25252525252525254,
+ "grad_norm": 1.9568793773651123,
+ "learning_rate": 1.8329966329966332e-05,
+ "loss": 1.308868408203125,
+ "mean_token_accuracy": 0.6954689621925354,
+ "num_tokens": 2048000.0,
+ "step": 125
+ },
+ {
+ "entropy": 1.5780351161956787,
+ "epoch": 0.2545454545454545,
+ "grad_norm": 2.151250123977661,
+ "learning_rate": 1.831649831649832e-05,
+ "loss": 1.60304856300354,
+ "mean_token_accuracy": 0.6258549094200134,
+ "num_tokens": 2064384.0,
+ "step": 126
+ },
+ {
+ "entropy": 1.487567663192749,
+ "epoch": 0.25656565656565655,
+ "grad_norm": 1.9134294986724854,
+ "learning_rate": 1.8303030303030305e-05,
+ "loss": 1.514064073562622,
+ "mean_token_accuracy": 0.6348925232887268,
+ "num_tokens": 2080768.0,
+ "step": 127
+ },
+ {
+ "entropy": 1.6072919368743896,
+ "epoch": 0.2585858585858586,
+ "grad_norm": 2.119290351867676,
+ "learning_rate": 1.828956228956229e-05,
+ "loss": 1.6062684059143066,
+ "mean_token_accuracy": 0.6332437992095947,
+ "num_tokens": 2097152.0,
+ "step": 128
+ },
+ {
+ "entropy": 1.1168174743652344,
+ "epoch": 0.2606060606060606,
+ "grad_norm": 1.8520931005477905,
+ "learning_rate": 1.827609427609428e-05,
+ "loss": 1.1421526670455933,
+ "mean_token_accuracy": 0.7078651785850525,
+ "num_tokens": 2113536.0,
+ "step": 129
+ },
+ {
+ "entropy": 1.349792718887329,
+ "epoch": 0.26262626262626265,
+ "grad_norm": 1.8463630676269531,
+ "learning_rate": 1.8262626262626263e-05,
+ "loss": 1.3712577819824219,
+ "mean_token_accuracy": 0.6736077070236206,
+ "num_tokens": 2129920.0,
+ "step": 130
+ },
+ {
+ "entropy": 1.8531655073165894,
+ "epoch": 0.26464646464646463,
+ "grad_norm": 2.0503969192504883,
+ "learning_rate": 1.824915824915825e-05,
+ "loss": 1.8971941471099854,
+ "mean_token_accuracy": 0.5845749974250793,
+ "num_tokens": 2146304.0,
+ "step": 131
+ },
+ {
+ "entropy": 1.436646819114685,
+ "epoch": 0.26666666666666666,
+ "grad_norm": 1.8517117500305176,
+ "learning_rate": 1.8235690235690236e-05,
+ "loss": 1.4255880117416382,
+ "mean_token_accuracy": 0.6653639674186707,
+ "num_tokens": 2162688.0,
+ "step": 132
+ },
+ {
+ "entropy": 1.9219673871994019,
+ "epoch": 0.2686868686868687,
+ "grad_norm": 2.1424307823181152,
+ "learning_rate": 1.8222222222222224e-05,
+ "loss": 1.9088413715362549,
+ "mean_token_accuracy": 0.5868954658508301,
+ "num_tokens": 2179072.0,
+ "step": 133
+ },
+ {
+ "entropy": 1.4034194946289062,
+ "epoch": 0.27070707070707073,
+ "grad_norm": 1.9987515211105347,
+ "learning_rate": 1.8208754208754212e-05,
+ "loss": 1.3612333536148071,
+ "mean_token_accuracy": 0.6721421480178833,
+ "num_tokens": 2195456.0,
+ "step": 134
+ },
+ {
+ "entropy": 2.064425230026245,
+ "epoch": 0.2727272727272727,
+ "grad_norm": 2.0800976753234863,
+ "learning_rate": 1.8195286195286197e-05,
+ "loss": 2.0595853328704834,
+ "mean_token_accuracy": 0.5694919228553772,
+ "num_tokens": 2211840.0,
+ "step": 135
+ },
+ {
+ "entropy": 1.2883177995681763,
+ "epoch": 0.27474747474747474,
+ "grad_norm": 1.9042892456054688,
+ "learning_rate": 1.8181818181818182e-05,
+ "loss": 1.3220078945159912,
+ "mean_token_accuracy": 0.6805080771446228,
+ "num_tokens": 2228224.0,
+ "step": 136
+ },
+ {
+ "entropy": 1.101396918296814,
+ "epoch": 0.2767676767676768,
+ "grad_norm": 1.857073187828064,
+ "learning_rate": 1.816835016835017e-05,
+ "loss": 1.1156296730041504,
+ "mean_token_accuracy": 0.7214826345443726,
+ "num_tokens": 2244608.0,
+ "step": 137
+ },
+ {
+ "entropy": 1.376597285270691,
+ "epoch": 0.2787878787878788,
+ "grad_norm": 1.9103689193725586,
+ "learning_rate": 1.8154882154882155e-05,
+ "loss": 1.3915824890136719,
+ "mean_token_accuracy": 0.6824010610580444,
+ "num_tokens": 2260992.0,
+ "step": 138
+ },
+ {
+ "entropy": 1.4654942750930786,
+ "epoch": 0.2808080808080808,
+ "grad_norm": 1.9513856172561646,
+ "learning_rate": 1.8141414141414143e-05,
+ "loss": 1.5000076293945312,
+ "mean_token_accuracy": 0.6457620859146118,
+ "num_tokens": 2277376.0,
+ "step": 139
+ },
+ {
+ "entropy": 1.1999047994613647,
+ "epoch": 0.2828282828282828,
+ "grad_norm": 1.824987530708313,
+ "learning_rate": 1.812794612794613e-05,
+ "loss": 1.2217535972595215,
+ "mean_token_accuracy": 0.7034685015678406,
+ "num_tokens": 2293760.0,
+ "step": 140
+ },
+ {
+ "entropy": 1.2822942733764648,
+ "epoch": 0.28484848484848485,
+ "grad_norm": 2.0979108810424805,
+ "learning_rate": 1.8114478114478116e-05,
+ "loss": 1.309725046157837,
+ "mean_token_accuracy": 0.6855764389038086,
+ "num_tokens": 2310144.0,
+ "step": 141
+ },
+ {
+ "entropy": 1.5589981079101562,
+ "epoch": 0.2868686868686869,
+ "grad_norm": 2.030648708343506,
+ "learning_rate": 1.8101010101010104e-05,
+ "loss": 1.5918951034545898,
+ "mean_token_accuracy": 0.6286638975143433,
+ "num_tokens": 2326528.0,
+ "step": 142
+ },
+ {
+ "entropy": 1.5600453615188599,
+ "epoch": 0.28888888888888886,
+ "grad_norm": 2.023409605026245,
+ "learning_rate": 1.808754208754209e-05,
+ "loss": 1.5725011825561523,
+ "mean_token_accuracy": 0.6384342908859253,
+ "num_tokens": 2342912.0,
+ "step": 143
+ },
+ {
+ "entropy": 1.5066077709197998,
+ "epoch": 0.2909090909090909,
+ "grad_norm": 1.9963997602462769,
+ "learning_rate": 1.8074074074074074e-05,
+ "loss": 1.5066845417022705,
+ "mean_token_accuracy": 0.6474108695983887,
+ "num_tokens": 2359296.0,
+ "step": 144
+ },
+ {
+ "entropy": 1.8313122987747192,
+ "epoch": 0.29292929292929293,
+ "grad_norm": 2.0791213512420654,
+ "learning_rate": 1.8060606060606062e-05,
+ "loss": 1.8148126602172852,
+ "mean_token_accuracy": 0.5820102691650391,
+ "num_tokens": 2375680.0,
+ "step": 145
+ },
+ {
+ "entropy": 1.8828215599060059,
+ "epoch": 0.29494949494949496,
+ "grad_norm": 2.1346335411071777,
+ "learning_rate": 1.804713804713805e-05,
+ "loss": 1.905554175376892,
+ "mean_token_accuracy": 0.6041768193244934,
+ "num_tokens": 2392064.0,
+ "step": 146
+ },
+ {
+ "entropy": 1.1984127759933472,
+ "epoch": 0.296969696969697,
+ "grad_norm": 1.6585179567337036,
+ "learning_rate": 1.8033670033670035e-05,
+ "loss": 1.2255632877349854,
+ "mean_token_accuracy": 0.712506115436554,
+ "num_tokens": 2408448.0,
+ "step": 147
+ },
+ {
+ "entropy": 1.4248595237731934,
+ "epoch": 0.298989898989899,
+ "grad_norm": 2.468965530395508,
+ "learning_rate": 1.8020202020202023e-05,
+ "loss": 1.4362621307373047,
+ "mean_token_accuracy": 0.6464948654174805,
+ "num_tokens": 2424832.0,
+ "step": 148
+ },
+ {
+ "entropy": 1.7295368909835815,
+ "epoch": 0.301010101010101,
+ "grad_norm": 1.819359302520752,
+ "learning_rate": 1.8006734006734008e-05,
+ "loss": 1.7290053367614746,
+ "mean_token_accuracy": 0.6312897205352783,
+ "num_tokens": 2441216.0,
+ "step": 149
+ },
+ {
+ "entropy": 1.7334001064300537,
+ "epoch": 0.30303030303030304,
+ "grad_norm": 2.149543285369873,
+ "learning_rate": 1.7993265993265993e-05,
+ "loss": 1.7014907598495483,
+ "mean_token_accuracy": 0.5981314182281494,
+ "num_tokens": 2457600.0,
+ "step": 150
+ },
+ {
+ "entropy": 2.0550050735473633,
+ "epoch": 0.30505050505050507,
+ "grad_norm": 1.9880189895629883,
+ "learning_rate": 1.797979797979798e-05,
+ "loss": 2.001784324645996,
+ "mean_token_accuracy": 0.5552638173103333,
+ "num_tokens": 2473984.0,
+ "step": 151
+ },
+ {
+ "entropy": 1.5352036952972412,
+ "epoch": 0.30707070707070705,
+ "grad_norm": 2.031998634338379,
+ "learning_rate": 1.7966329966329966e-05,
+ "loss": 1.5262815952301025,
+ "mean_token_accuracy": 0.6328163146972656,
+ "num_tokens": 2490368.0,
+ "step": 152
+ },
+ {
+ "entropy": 1.4278677701950073,
+ "epoch": 0.3090909090909091,
+ "grad_norm": 2.101398229598999,
+ "learning_rate": 1.7952861952861954e-05,
+ "loss": 1.4288041591644287,
+ "mean_token_accuracy": 0.6647533178329468,
+ "num_tokens": 2506752.0,
+ "step": 153
+ },
+ {
+ "entropy": 1.7469983100891113,
+ "epoch": 0.3111111111111111,
+ "grad_norm": 2.0824313163757324,
+ "learning_rate": 1.7939393939393942e-05,
+ "loss": 1.770402193069458,
+ "mean_token_accuracy": 0.5997191071510315,
+ "num_tokens": 2523136.0,
+ "step": 154
+ },
+ {
+ "entropy": 1.6565595865249634,
+ "epoch": 0.31313131313131315,
+ "grad_norm": 2.0815961360931396,
+ "learning_rate": 1.7925925925925927e-05,
+ "loss": 1.7024259567260742,
+ "mean_token_accuracy": 0.6109550595283508,
+ "num_tokens": 2539520.0,
+ "step": 155
+ },
+ {
+ "entropy": 1.4104127883911133,
+ "epoch": 0.3151515151515151,
+ "grad_norm": 2.129694700241089,
+ "learning_rate": 1.7912457912457915e-05,
+ "loss": 1.4378812313079834,
+ "mean_token_accuracy": 0.6489985585212708,
+ "num_tokens": 2555904.0,
+ "step": 156
+ },
+ {
+ "entropy": 1.5129127502441406,
+ "epoch": 0.31717171717171716,
+ "grad_norm": 2.180267095565796,
+ "learning_rate": 1.78989898989899e-05,
+ "loss": 1.5564348697662354,
+ "mean_token_accuracy": 0.6327552795410156,
+ "num_tokens": 2572288.0,
+ "step": 157
+ },
+ {
+ "entropy": 1.5240321159362793,
+ "epoch": 0.3191919191919192,
+ "grad_norm": 1.968405842781067,
+ "learning_rate": 1.7885521885521885e-05,
+ "loss": 1.585756778717041,
+ "mean_token_accuracy": 0.629885196685791,
+ "num_tokens": 2588672.0,
+ "step": 158
+ },
+ {
+ "entropy": 1.3169562816619873,
+ "epoch": 0.3212121212121212,
+ "grad_norm": 2.0452873706817627,
+ "learning_rate": 1.7872053872053873e-05,
+ "loss": 1.3569687604904175,
+ "mean_token_accuracy": 0.671775758266449,
+ "num_tokens": 2605056.0,
+ "step": 159
+ },
+ {
+ "entropy": 1.3630796670913696,
+ "epoch": 0.32323232323232326,
+ "grad_norm": 1.9721522331237793,
+ "learning_rate": 1.785858585858586e-05,
+ "loss": 1.3965554237365723,
+ "mean_token_accuracy": 0.6626160144805908,
+ "num_tokens": 2621440.0,
+ "step": 160
+ },
+ {
+ "entropy": 1.178250789642334,
+ "epoch": 0.32525252525252524,
+ "grad_norm": 2.0231528282165527,
+ "learning_rate": 1.7845117845117846e-05,
+ "loss": 1.2126985788345337,
+ "mean_token_accuracy": 0.6977894306182861,
+ "num_tokens": 2637824.0,
+ "step": 161
+ },
+ {
+ "entropy": 1.8399475812911987,
+ "epoch": 0.32727272727272727,
+ "grad_norm": 2.0615484714508057,
+ "learning_rate": 1.7831649831649834e-05,
+ "loss": 1.868000864982605,
+ "mean_token_accuracy": 0.5822545289993286,
+ "num_tokens": 2654208.0,
+ "step": 162
+ },
+ {
+ "entropy": 1.6604139804840088,
+ "epoch": 0.3292929292929293,
+ "grad_norm": 1.9838379621505737,
+ "learning_rate": 1.781818181818182e-05,
+ "loss": 1.6677067279815674,
+ "mean_token_accuracy": 0.621275007724762,
+ "num_tokens": 2670592.0,
+ "step": 163
+ },
+ {
+ "entropy": 1.3761085271835327,
+ "epoch": 0.33131313131313134,
+ "grad_norm": 2.0093271732330322,
+ "learning_rate": 1.7804713804713807e-05,
+ "loss": 1.3829190731048584,
+ "mean_token_accuracy": 0.6655471324920654,
+ "num_tokens": 2686976.0,
+ "step": 164
+ },
+ {
+ "entropy": 1.578093409538269,
+ "epoch": 0.3333333333333333,
+ "grad_norm": 1.9710773229599,
+ "learning_rate": 1.7791245791245792e-05,
+ "loss": 1.560583472251892,
+ "mean_token_accuracy": 0.6344650983810425,
+ "num_tokens": 2703360.0,
+ "step": 165
+ },
+ {
+ "entropy": 1.0447157621383667,
+ "epoch": 0.33535353535353535,
+ "grad_norm": 1.68338942527771,
+ "learning_rate": 1.7777777777777777e-05,
+ "loss": 1.0116058588027954,
+ "mean_token_accuracy": 0.7449315786361694,
+ "num_tokens": 2719744.0,
+ "step": 166
+ },
+ {
+ "entropy": 1.3687469959259033,
+ "epoch": 0.3373737373737374,
+ "grad_norm": 2.062744617462158,
+ "learning_rate": 1.7764309764309765e-05,
+ "loss": 1.3179712295532227,
+ "mean_token_accuracy": 0.6761724352836609,
+ "num_tokens": 2736128.0,
+ "step": 167
+ },
+ {
+ "entropy": 1.8639556169509888,
+ "epoch": 0.3393939393939394,
+ "grad_norm": 2.041898250579834,
+ "learning_rate": 1.7750841750841753e-05,
+ "loss": 1.8531380891799927,
+ "mean_token_accuracy": 0.5920859575271606,
+ "num_tokens": 2752512.0,
+ "step": 168
+ },
+ {
+ "entropy": 1.6900824308395386,
+ "epoch": 0.3414141414141414,
+ "grad_norm": 1.9403504133224487,
+ "learning_rate": 1.773737373737374e-05,
+ "loss": 1.701782464981079,
+ "mean_token_accuracy": 0.6102222800254822,
+ "num_tokens": 2768896.0,
+ "step": 169
+ },
+ {
+ "entropy": 1.6202301979064941,
+ "epoch": 0.3434343434343434,
+ "grad_norm": 2.027329683303833,
+ "learning_rate": 1.7723905723905726e-05,
+ "loss": 1.6471121311187744,
+ "mean_token_accuracy": 0.6310454607009888,
+ "num_tokens": 2785280.0,
+ "step": 170
+ },
+ {
+ "entropy": 1.2701244354248047,
+ "epoch": 0.34545454545454546,
+ "grad_norm": 1.8625534772872925,
+ "learning_rate": 1.771043771043771e-05,
+ "loss": 1.291702151298523,
+ "mean_token_accuracy": 0.67659991979599,
+ "num_tokens": 2801664.0,
+ "step": 171
+ },
+ {
+ "entropy": 1.5759357213974,
+ "epoch": 0.3474747474747475,
+ "grad_norm": 1.945141077041626,
+ "learning_rate": 1.76969696969697e-05,
+ "loss": 1.6189839839935303,
+ "mean_token_accuracy": 0.6225574016571045,
+ "num_tokens": 2818048.0,
+ "step": 172
+ },
+ {
+ "entropy": 1.3319392204284668,
+ "epoch": 0.34949494949494947,
+ "grad_norm": 1.8485770225524902,
+ "learning_rate": 1.7683501683501684e-05,
+ "loss": 1.3553425073623657,
+ "mean_token_accuracy": 0.6759281754493713,
+ "num_tokens": 2834432.0,
+ "step": 173
+ },
+ {
+ "entropy": 1.3251745700836182,
+ "epoch": 0.3515151515151515,
+ "grad_norm": 2.0564401149749756,
+ "learning_rate": 1.7670033670033672e-05,
+ "loss": 1.359522819519043,
+ "mean_token_accuracy": 0.6678065657615662,
+ "num_tokens": 2850816.0,
+ "step": 174
+ },
+ {
+ "entropy": 1.2764136791229248,
+ "epoch": 0.35353535353535354,
+ "grad_norm": 2.1749041080474854,
+ "learning_rate": 1.765656565656566e-05,
+ "loss": 1.3260173797607422,
+ "mean_token_accuracy": 0.6710429787635803,
+ "num_tokens": 2867200.0,
+ "step": 175
+ },
+ {
+ "entropy": 1.6382335424423218,
+ "epoch": 0.35555555555555557,
+ "grad_norm": 1.923366904258728,
+ "learning_rate": 1.7643097643097645e-05,
+ "loss": 1.6868517398834229,
+ "mean_token_accuracy": 0.6268930435180664,
+ "num_tokens": 2883584.0,
+ "step": 176
+ },
+ {
+ "entropy": 1.2399169206619263,
+ "epoch": 0.3575757575757576,
+ "grad_norm": 1.795649766921997,
+ "learning_rate": 1.7629629629629633e-05,
+ "loss": 1.2266355752944946,
+ "mean_token_accuracy": 0.689667820930481,
+ "num_tokens": 2899968.0,
+ "step": 177
+ },
+ {
+ "entropy": 1.5761929750442505,
+ "epoch": 0.3595959595959596,
+ "grad_norm": 2.0536038875579834,
+ "learning_rate": 1.7616161616161618e-05,
+ "loss": 1.5698940753936768,
+ "mean_token_accuracy": 0.6263434290885925,
+ "num_tokens": 2916352.0,
+ "step": 178
+ },
+ {
+ "entropy": 1.3612396717071533,
+ "epoch": 0.3616161616161616,
+ "grad_norm": 2.003840684890747,
+ "learning_rate": 1.7602693602693603e-05,
+ "loss": 1.359879732131958,
+ "mean_token_accuracy": 0.6798974275588989,
+ "num_tokens": 2932736.0,
+ "step": 179
+ },
+ {
+ "entropy": 1.0548239946365356,
+ "epoch": 0.36363636363636365,
+ "grad_norm": 1.6967436075210571,
+ "learning_rate": 1.758922558922559e-05,
+ "loss": 1.040539264678955,
+ "mean_token_accuracy": 0.7433438897132874,
+ "num_tokens": 2949120.0,
+ "step": 180
+ },
+ {
+ "entropy": 1.16105055809021,
+ "epoch": 0.3656565656565657,
+ "grad_norm": 1.9793111085891724,
+ "learning_rate": 1.7575757575757576e-05,
+ "loss": 1.1532450914382935,
+ "mean_token_accuracy": 0.7135441899299622,
+ "num_tokens": 2965504.0,
+ "step": 181
+ },
+ {
+ "entropy": 1.4421862363815308,
+ "epoch": 0.36767676767676766,
+ "grad_norm": 1.8337509632110596,
+ "learning_rate": 1.7562289562289564e-05,
+ "loss": 1.419133186340332,
+ "mean_token_accuracy": 0.659807026386261,
+ "num_tokens": 2981888.0,
+ "step": 182
+ },
+ {
+ "entropy": 1.6186654567718506,
+ "epoch": 0.3696969696969697,
+ "grad_norm": 2.0585362911224365,
+ "learning_rate": 1.7548821548821552e-05,
+ "loss": 1.5880110263824463,
+ "mean_token_accuracy": 0.6322056651115417,
+ "num_tokens": 2998272.0,
+ "step": 183
+ },
+ {
+ "entropy": 1.76968252658844,
+ "epoch": 0.3717171717171717,
+ "grad_norm": 2.139742374420166,
+ "learning_rate": 1.7535353535353537e-05,
+ "loss": 1.797312617301941,
+ "mean_token_accuracy": 0.6096116304397583,
+ "num_tokens": 3014656.0,
+ "step": 184
+ },
+ {
+ "entropy": 1.0575193166732788,
+ "epoch": 0.37373737373737376,
+ "grad_norm": 1.8547794818878174,
+ "learning_rate": 1.7521885521885525e-05,
+ "loss": 1.0575863122940063,
+ "mean_token_accuracy": 0.7325354218482971,
+ "num_tokens": 3031040.0,
+ "step": 185
+ },
+ {
+ "entropy": 1.2937911748886108,
+ "epoch": 0.37575757575757573,
+ "grad_norm": 2.3985140323638916,
+ "learning_rate": 1.750841750841751e-05,
+ "loss": 1.3277016878128052,
+ "mean_token_accuracy": 0.6654860973358154,
+ "num_tokens": 3047424.0,
+ "step": 186
+ },
+ {
+ "entropy": 1.1829862594604492,
+ "epoch": 0.37777777777777777,
+ "grad_norm": 1.948241114616394,
+ "learning_rate": 1.7494949494949494e-05,
+ "loss": 1.2146751880645752,
+ "mean_token_accuracy": 0.6964460015296936,
+ "num_tokens": 3063808.0,
+ "step": 187
+ },
+ {
+ "entropy": 1.4858520030975342,
+ "epoch": 0.3797979797979798,
+ "grad_norm": 2.2712836265563965,
+ "learning_rate": 1.7481481481481483e-05,
+ "loss": 1.5523631572723389,
+ "mean_token_accuracy": 0.6392281651496887,
+ "num_tokens": 3080192.0,
+ "step": 188
+ },
+ {
+ "entropy": 1.7905362844467163,
+ "epoch": 0.38181818181818183,
+ "grad_norm": 1.890699863433838,
+ "learning_rate": 1.746801346801347e-05,
+ "loss": 1.8139519691467285,
+ "mean_token_accuracy": 0.6015510559082031,
+ "num_tokens": 3096576.0,
+ "step": 189
+ },
+ {
+ "entropy": 1.3264763355255127,
+ "epoch": 0.3838383838383838,
+ "grad_norm": 2.1567318439483643,
+ "learning_rate": 1.7454545454545456e-05,
+ "loss": 1.3780615329742432,
+ "mean_token_accuracy": 0.6636541485786438,
+ "num_tokens": 3112960.0,
+ "step": 190
+ },
+ {
+ "entropy": 1.9898782968521118,
+ "epoch": 0.38585858585858585,
+ "grad_norm": 2.008547306060791,
+ "learning_rate": 1.7441077441077444e-05,
+ "loss": 2.0208630561828613,
+ "mean_token_accuracy": 0.5498290061950684,
+ "num_tokens": 3129344.0,
+ "step": 191
+ },
+ {
+ "entropy": 1.2579785585403442,
+ "epoch": 0.3878787878787879,
+ "grad_norm": 1.921314001083374,
+ "learning_rate": 1.742760942760943e-05,
+ "loss": 1.271723985671997,
+ "mean_token_accuracy": 0.6882632970809937,
+ "num_tokens": 3145728.0,
+ "step": 192
+ },
+ {
+ "entropy": 1.3366814851760864,
+ "epoch": 0.3898989898989899,
+ "grad_norm": 1.9525961875915527,
+ "learning_rate": 1.7414141414141413e-05,
+ "loss": 1.3561700582504272,
+ "mean_token_accuracy": 0.6822789311408997,
+ "num_tokens": 3162112.0,
+ "step": 193
+ },
+ {
+ "entropy": 1.5505253076553345,
+ "epoch": 0.39191919191919194,
+ "grad_norm": 2.044635057449341,
+ "learning_rate": 1.74006734006734e-05,
+ "loss": 1.5198928117752075,
+ "mean_token_accuracy": 0.6326331496238708,
+ "num_tokens": 3178496.0,
+ "step": 194
+ },
+ {
+ "entropy": 1.4198534488677979,
+ "epoch": 0.3939393939393939,
+ "grad_norm": 1.9955503940582275,
+ "learning_rate": 1.738720538720539e-05,
+ "loss": 1.4072260856628418,
+ "mean_token_accuracy": 0.6631045341491699,
+ "num_tokens": 3194880.0,
+ "step": 195
+ },
+ {
+ "entropy": 2.0099895000457764,
+ "epoch": 0.39595959595959596,
+ "grad_norm": 2.110247850418091,
+ "learning_rate": 1.7373737373737375e-05,
+ "loss": 2.0012335777282715,
+ "mean_token_accuracy": 0.5668661594390869,
+ "num_tokens": 3211264.0,
+ "step": 196
+ },
+ {
+ "entropy": 2.1619439125061035,
+ "epoch": 0.397979797979798,
+ "grad_norm": 2.0869383811950684,
+ "learning_rate": 1.7360269360269363e-05,
+ "loss": 2.1793670654296875,
+ "mean_token_accuracy": 0.5569125413894653,
+ "num_tokens": 3227648.0,
+ "step": 197
+ },
+ {
+ "entropy": 1.6383775472640991,
+ "epoch": 0.4,
+ "grad_norm": 1.7467572689056396,
+ "learning_rate": 1.7346801346801347e-05,
+ "loss": 1.656137228012085,
+ "mean_token_accuracy": 0.6395334601402283,
+ "num_tokens": 3244032.0,
+ "step": 198
+ },
+ {
+ "entropy": 1.5911450386047363,
+ "epoch": 0.402020202020202,
+ "grad_norm": 1.9362678527832031,
+ "learning_rate": 1.7333333333333336e-05,
+ "loss": 1.5865838527679443,
+ "mean_token_accuracy": 0.621275007724762,
+ "num_tokens": 3260416.0,
+ "step": 199
+ },
+ {
+ "entropy": 1.3983819484710693,
+ "epoch": 0.40404040404040403,
+ "grad_norm": 1.925863265991211,
+ "learning_rate": 1.731986531986532e-05,
+ "loss": 1.3835841417312622,
+ "mean_token_accuracy": 0.6618221998214722,
+ "num_tokens": 3276800.0,
+ "step": 200
+ },
+ {
+ "epoch": 0.40404040404040403,
+ "eval_entropy": 1.5366105552642577,
+ "eval_loss": 1.5502140522003174,
+ "eval_mean_token_accuracy": 0.6411546406246,
+ "eval_num_tokens": 3276800.0,
+ "eval_runtime": 43.8218,
+ "eval_samples_per_second": 22.592,
+ "eval_steps_per_second": 2.83,
+ "step": 200
+ },
+ {
+ "entropy": 1.291393756866455,
+ "epoch": 0.40606060606060607,
+ "grad_norm": 1.9057531356811523,
+ "learning_rate": 1.7306397306397305e-05,
+ "loss": 1.2847933769226074,
+ "mean_token_accuracy": 0.6901563405990601,
+ "num_tokens": 3293184.0,
+ "step": 201
+ },
+ {
+ "entropy": 1.1511558294296265,
+ "epoch": 0.4080808080808081,
+ "grad_norm": 1.849436640739441,
+ "learning_rate": 1.7292929292929293e-05,
+ "loss": 1.1661030054092407,
+ "mean_token_accuracy": 0.7095139026641846,
+ "num_tokens": 3309568.0,
+ "step": 202
+ },
+ {
+ "entropy": 1.5591504573822021,
+ "epoch": 0.4101010101010101,
+ "grad_norm": 1.9354233741760254,
+ "learning_rate": 1.727946127946128e-05,
+ "loss": 1.6045410633087158,
+ "mean_token_accuracy": 0.6226795315742493,
+ "num_tokens": 3325952.0,
+ "step": 203
+ },
+ {
+ "entropy": 1.7634649276733398,
+ "epoch": 0.4121212121212121,
+ "grad_norm": 2.414632797241211,
+ "learning_rate": 1.726599326599327e-05,
+ "loss": 1.8021953105926514,
+ "mean_token_accuracy": 0.6099780201911926,
+ "num_tokens": 3342336.0,
+ "step": 204
+ },
+ {
+ "entropy": 1.6080671548843384,
+ "epoch": 0.41414141414141414,
+ "grad_norm": 1.9936028718948364,
+ "learning_rate": 1.7252525252525255e-05,
+ "loss": 1.630000352859497,
+ "mean_token_accuracy": 0.6256106495857239,
+ "num_tokens": 3358720.0,
+ "step": 205
+ },
+ {
+ "entropy": 1.6845343112945557,
+ "epoch": 0.4161616161616162,
+ "grad_norm": 1.9131947755813599,
+ "learning_rate": 1.723905723905724e-05,
+ "loss": 1.6912329196929932,
+ "mean_token_accuracy": 0.6263434290885925,
+ "num_tokens": 3375104.0,
+ "step": 206
+ },
+ {
+ "entropy": 1.5946446657180786,
+ "epoch": 0.41818181818181815,
+ "grad_norm": 2.0423905849456787,
+ "learning_rate": 1.7225589225589228e-05,
+ "loss": 1.647271752357483,
+ "mean_token_accuracy": 0.6296409368515015,
+ "num_tokens": 3391488.0,
+ "step": 207
+ },
+ {
+ "entropy": 1.3124741315841675,
+ "epoch": 0.4202020202020202,
+ "grad_norm": 1.8896982669830322,
+ "learning_rate": 1.7212121212121212e-05,
+ "loss": 1.3216124773025513,
+ "mean_token_accuracy": 0.687469482421875,
+ "num_tokens": 3407872.0,
+ "step": 208
+ },
+ {
+ "entropy": 1.3931281566619873,
+ "epoch": 0.4222222222222222,
+ "grad_norm": 1.7883074283599854,
+ "learning_rate": 1.71986531986532e-05,
+ "loss": 1.4264600276947021,
+ "mean_token_accuracy": 0.6568148732185364,
+ "num_tokens": 3424256.0,
+ "step": 209
+ },
+ {
+ "entropy": 1.1427687406539917,
+ "epoch": 0.42424242424242425,
+ "grad_norm": 1.9367003440856934,
+ "learning_rate": 1.7185185185185185e-05,
+ "loss": 1.1611723899841309,
+ "mean_token_accuracy": 0.7058500051498413,
+ "num_tokens": 3440640.0,
+ "step": 210
+ },
+ {
+ "entropy": 1.7226167917251587,
+ "epoch": 0.4262626262626263,
+ "grad_norm": 2.175487518310547,
+ "learning_rate": 1.7171717171717173e-05,
+ "loss": 1.7383192777633667,
+ "mean_token_accuracy": 0.6075964570045471,
+ "num_tokens": 3457024.0,
+ "step": 211
+ },
+ {
+ "entropy": 1.3780062198638916,
+ "epoch": 0.42828282828282827,
+ "grad_norm": 1.9817614555358887,
+ "learning_rate": 1.715824915824916e-05,
+ "loss": 1.3755927085876465,
+ "mean_token_accuracy": 0.6750732660293579,
+ "num_tokens": 3473408.0,
+ "step": 212
+ },
+ {
+ "entropy": 1.637842059135437,
+ "epoch": 0.4303030303030303,
+ "grad_norm": 2.2326242923736572,
+ "learning_rate": 1.7144781144781146e-05,
+ "loss": 1.635596752166748,
+ "mean_token_accuracy": 0.6110771894454956,
+ "num_tokens": 3489792.0,
+ "step": 213
+ },
+ {
+ "entropy": 1.9970118999481201,
+ "epoch": 0.43232323232323233,
+ "grad_norm": 2.1303606033325195,
+ "learning_rate": 1.713131313131313e-05,
+ "loss": 2.0099592208862305,
+ "mean_token_accuracy": 0.5781631469726562,
+ "num_tokens": 3506176.0,
+ "step": 214
+ },
+ {
+ "entropy": 1.4759869575500488,
+ "epoch": 0.43434343434343436,
+ "grad_norm": 2.043727397918701,
+ "learning_rate": 1.711784511784512e-05,
+ "loss": 1.4756600856781006,
+ "mean_token_accuracy": 0.6483879089355469,
+ "num_tokens": 3522560.0,
+ "step": 215
+ },
+ {
+ "entropy": 1.2833726406097412,
+ "epoch": 0.43636363636363634,
+ "grad_norm": 1.9260340929031372,
+ "learning_rate": 1.7104377104377104e-05,
+ "loss": 1.2887630462646484,
+ "mean_token_accuracy": 0.6900342106819153,
+ "num_tokens": 3538944.0,
+ "step": 216
+ },
+ {
+ "entropy": 1.3936302661895752,
+ "epoch": 0.4383838383838384,
+ "grad_norm": 1.8590774536132812,
+ "learning_rate": 1.7090909090909092e-05,
+ "loss": 1.3784823417663574,
+ "mean_token_accuracy": 0.6736077070236206,
+ "num_tokens": 3555328.0,
+ "step": 217
+ },
+ {
+ "entropy": 1.2413936853408813,
+ "epoch": 0.4404040404040404,
+ "grad_norm": 2.1111643314361572,
+ "learning_rate": 1.707744107744108e-05,
+ "loss": 1.2740635871887207,
+ "mean_token_accuracy": 0.6883854269981384,
+ "num_tokens": 3571712.0,
+ "step": 218
+ },
+ {
+ "entropy": 1.4879416227340698,
+ "epoch": 0.44242424242424244,
+ "grad_norm": 2.127614736557007,
+ "learning_rate": 1.7063973063973065e-05,
+ "loss": 1.48406183719635,
+ "mean_token_accuracy": 0.649975597858429,
+ "num_tokens": 3588096.0,
+ "step": 219
+ },
+ {
+ "entropy": 1.4226471185684204,
+ "epoch": 0.4444444444444444,
+ "grad_norm": 2.164325475692749,
+ "learning_rate": 1.7050505050505054e-05,
+ "loss": 1.466269850730896,
+ "mean_token_accuracy": 0.6502808928489685,
+ "num_tokens": 3604480.0,
+ "step": 220
+ },
+ {
+ "entropy": 1.6183370351791382,
+ "epoch": 0.44646464646464645,
+ "grad_norm": 2.018846273422241,
+ "learning_rate": 1.7037037037037038e-05,
+ "loss": 1.6348307132720947,
+ "mean_token_accuracy": 0.627198338508606,
+ "num_tokens": 3620864.0,
+ "step": 221
+ },
+ {
+ "entropy": 1.6979204416275024,
+ "epoch": 0.4484848484848485,
+ "grad_norm": 2.1574854850769043,
+ "learning_rate": 1.7023569023569023e-05,
+ "loss": 1.7019602060317993,
+ "mean_token_accuracy": 0.6157181262969971,
+ "num_tokens": 3637248.0,
+ "step": 222
+ },
+ {
+ "entropy": 1.753490686416626,
+ "epoch": 0.4505050505050505,
+ "grad_norm": 2.0253705978393555,
+ "learning_rate": 1.701010101010101e-05,
+ "loss": 1.7466539144515991,
+ "mean_token_accuracy": 0.6008793115615845,
+ "num_tokens": 3653632.0,
+ "step": 223
+ },
+ {
+ "entropy": 1.717929720878601,
+ "epoch": 0.45252525252525255,
+ "grad_norm": 1.9840974807739258,
+ "learning_rate": 1.6996632996633e-05,
+ "loss": 1.703988790512085,
+ "mean_token_accuracy": 0.6099780201911926,
+ "num_tokens": 3670016.0,
+ "step": 224
+ },
+ {
+ "entropy": 1.377287745475769,
+ "epoch": 0.45454545454545453,
+ "grad_norm": 2.1027839183807373,
+ "learning_rate": 1.6983164983164984e-05,
+ "loss": 1.3902864456176758,
+ "mean_token_accuracy": 0.660906195640564,
+ "num_tokens": 3686400.0,
+ "step": 225
+ },
+ {
+ "entropy": 1.237000584602356,
+ "epoch": 0.45656565656565656,
+ "grad_norm": 1.9116249084472656,
+ "learning_rate": 1.6969696969696972e-05,
+ "loss": 1.221006155014038,
+ "mean_token_accuracy": 0.7045676708221436,
+ "num_tokens": 3702784.0,
+ "step": 226
+ },
+ {
+ "entropy": 1.4198882579803467,
+ "epoch": 0.4585858585858586,
+ "grad_norm": 2.099358558654785,
+ "learning_rate": 1.6956228956228957e-05,
+ "loss": 1.4211962223052979,
+ "mean_token_accuracy": 0.6549829244613647,
+ "num_tokens": 3719168.0,
+ "step": 227
+ },
+ {
+ "entropy": 1.4647245407104492,
+ "epoch": 0.46060606060606063,
+ "grad_norm": 1.8873655796051025,
+ "learning_rate": 1.6942760942760945e-05,
+ "loss": 1.4848006963729858,
+ "mean_token_accuracy": 0.6534562706947327,
+ "num_tokens": 3735552.0,
+ "step": 228
+ },
+ {
+ "entropy": 1.348371982574463,
+ "epoch": 0.4626262626262626,
+ "grad_norm": 2.0487565994262695,
+ "learning_rate": 1.692929292929293e-05,
+ "loss": 1.355329155921936,
+ "mean_token_accuracy": 0.6709208488464355,
+ "num_tokens": 3751936.0,
+ "step": 229
+ },
+ {
+ "entropy": 1.6201715469360352,
+ "epoch": 0.46464646464646464,
+ "grad_norm": 1.9640510082244873,
+ "learning_rate": 1.6915824915824915e-05,
+ "loss": 1.6176421642303467,
+ "mean_token_accuracy": 0.6221910119056702,
+ "num_tokens": 3768320.0,
+ "step": 230
+ },
+ {
+ "entropy": 1.461037039756775,
+ "epoch": 0.4666666666666667,
+ "grad_norm": 1.9843394756317139,
+ "learning_rate": 1.6902356902356903e-05,
+ "loss": 1.4502949714660645,
+ "mean_token_accuracy": 0.6463727355003357,
+ "num_tokens": 3784704.0,
+ "step": 231
+ },
+ {
+ "entropy": 1.4964231252670288,
+ "epoch": 0.4686868686868687,
+ "grad_norm": 2.0087051391601562,
+ "learning_rate": 1.688888888888889e-05,
+ "loss": 1.5107879638671875,
+ "mean_token_accuracy": 0.6536394953727722,
+ "num_tokens": 3801088.0,
+ "step": 232
+ },
+ {
+ "entropy": 1.3013520240783691,
+ "epoch": 0.4707070707070707,
+ "grad_norm": 2.1042368412017822,
+ "learning_rate": 1.6875420875420876e-05,
+ "loss": 1.312164306640625,
+ "mean_token_accuracy": 0.673363447189331,
+ "num_tokens": 3817472.0,
+ "step": 233
+ },
+ {
+ "entropy": 1.6242352724075317,
+ "epoch": 0.4727272727272727,
+ "grad_norm": 1.8541103601455688,
+ "learning_rate": 1.6861952861952864e-05,
+ "loss": 1.6327502727508545,
+ "mean_token_accuracy": 0.6376404762268066,
+ "num_tokens": 3833856.0,
+ "step": 234
+ },
+ {
+ "entropy": 1.4227592945098877,
+ "epoch": 0.47474747474747475,
+ "grad_norm": 1.8749339580535889,
+ "learning_rate": 1.684848484848485e-05,
+ "loss": 1.436142921447754,
+ "mean_token_accuracy": 0.6705544590950012,
+ "num_tokens": 3850240.0,
+ "step": 235
+ },
+ {
+ "entropy": 1.4807487726211548,
+ "epoch": 0.4767676767676768,
+ "grad_norm": 2.1475114822387695,
+ "learning_rate": 1.6835016835016837e-05,
+ "loss": 1.4670445919036865,
+ "mean_token_accuracy": 0.6385564208030701,
+ "num_tokens": 3866624.0,
+ "step": 236
+ },
+ {
+ "entropy": 2.0623066425323486,
+ "epoch": 0.47878787878787876,
+ "grad_norm": 2.1698546409606934,
+ "learning_rate": 1.6821548821548822e-05,
+ "loss": 2.08683180809021,
+ "mean_token_accuracy": 0.5531876087188721,
+ "num_tokens": 3883008.0,
+ "step": 237
+ },
+ {
+ "entropy": 1.6894460916519165,
+ "epoch": 0.4808080808080808,
+ "grad_norm": 2.158062219619751,
+ "learning_rate": 1.680808080808081e-05,
+ "loss": 1.7244186401367188,
+ "mean_token_accuracy": 0.6055813431739807,
+ "num_tokens": 3899392.0,
+ "step": 238
+ },
+ {
+ "entropy": 1.2741345167160034,
+ "epoch": 0.48282828282828283,
+ "grad_norm": 1.918306827545166,
+ "learning_rate": 1.67946127946128e-05,
+ "loss": 1.2937202453613281,
+ "mean_token_accuracy": 0.6878969073295593,
+ "num_tokens": 3915776.0,
+ "step": 239
+ },
+ {
+ "entropy": 1.556649088859558,
+ "epoch": 0.48484848484848486,
+ "grad_norm": 2.0560667514801025,
+ "learning_rate": 1.6781144781144783e-05,
+ "loss": 1.5751030445098877,
+ "mean_token_accuracy": 0.6324499249458313,
+ "num_tokens": 3932160.0,
+ "step": 240
+ },
+ {
+ "entropy": 1.7632875442504883,
+ "epoch": 0.4868686868686869,
+ "grad_norm": 1.8408738374710083,
+ "learning_rate": 1.6767676767676768e-05,
+ "loss": 1.7745842933654785,
+ "mean_token_accuracy": 0.6091231107711792,
+ "num_tokens": 3948544.0,
+ "step": 241
+ },
+ {
+ "entropy": 1.4793967008590698,
+ "epoch": 0.4888888888888889,
+ "grad_norm": 2.0134575366973877,
+ "learning_rate": 1.6754208754208756e-05,
+ "loss": 1.514477014541626,
+ "mean_token_accuracy": 0.6414875388145447,
+ "num_tokens": 3964928.0,
+ "step": 242
+ },
+ {
+ "entropy": 1.4762561321258545,
+ "epoch": 0.4909090909090909,
+ "grad_norm": 2.087501287460327,
+ "learning_rate": 1.674074074074074e-05,
+ "loss": 1.5195722579956055,
+ "mean_token_accuracy": 0.6289081573486328,
+ "num_tokens": 3981312.0,
+ "step": 243
+ },
+ {
+ "entropy": 1.6629496812820435,
+ "epoch": 0.49292929292929294,
+ "grad_norm": 1.7386047840118408,
+ "learning_rate": 1.672727272727273e-05,
+ "loss": 1.6932936906814575,
+ "mean_token_accuracy": 0.6289692521095276,
+ "num_tokens": 3997696.0,
+ "step": 244
+ },
+ {
+ "entropy": 1.4506888389587402,
+ "epoch": 0.494949494949495,
+ "grad_norm": 1.8268823623657227,
+ "learning_rate": 1.6713804713804714e-05,
+ "loss": 1.4257543087005615,
+ "mean_token_accuracy": 0.6678065657615662,
+ "num_tokens": 4014080.0,
+ "step": 245
+ },
+ {
+ "entropy": 1.6132714748382568,
+ "epoch": 0.49696969696969695,
+ "grad_norm": 2.168206214904785,
+ "learning_rate": 1.6700336700336702e-05,
+ "loss": 1.602414608001709,
+ "mean_token_accuracy": 0.6202979683876038,
+ "num_tokens": 4030464.0,
+ "step": 246
+ },
+ {
+ "entropy": 1.2860945463180542,
+ "epoch": 0.498989898989899,
+ "grad_norm": 1.830817699432373,
+ "learning_rate": 1.668686868686869e-05,
+ "loss": 1.284934163093567,
+ "mean_token_accuracy": 0.6854543089866638,
+ "num_tokens": 4046848.0,
+ "step": 247
+ },
+ {
+ "entropy": 1.4948641061782837,
+ "epoch": 0.501010101010101,
+ "grad_norm": 1.9775508642196655,
+ "learning_rate": 1.6673400673400675e-05,
+ "loss": 1.497206449508667,
+ "mean_token_accuracy": 0.6471055150032043,
+ "num_tokens": 4063232.0,
+ "step": 248
+ },
+ {
+ "entropy": 1.490235447883606,
+ "epoch": 0.503030303030303,
+ "grad_norm": 2.207184076309204,
+ "learning_rate": 1.665993265993266e-05,
+ "loss": 1.5091016292572021,
+ "mean_token_accuracy": 0.6482657790184021,
+ "num_tokens": 4079616.0,
+ "step": 249
+ },
+ {
+ "entropy": 1.5752832889556885,
+ "epoch": 0.5050505050505051,
+ "grad_norm": 2.0718111991882324,
+ "learning_rate": 1.6646464646464648e-05,
+ "loss": 1.576171875,
+ "mean_token_accuracy": 0.6375793814659119,
+ "num_tokens": 4096000.0,
+ "step": 250
+ },
+ {
+ "entropy": 1.3935530185699463,
+ "epoch": 0.5070707070707071,
+ "grad_norm": 1.9379569292068481,
+ "learning_rate": 1.6632996632996633e-05,
+ "loss": 1.4019020795822144,
+ "mean_token_accuracy": 0.6742794513702393,
+ "num_tokens": 4112384.0,
+ "step": 251
+ },
+ {
+ "entropy": 1.3622194528579712,
+ "epoch": 0.509090909090909,
+ "grad_norm": 1.9684133529663086,
+ "learning_rate": 1.661952861952862e-05,
+ "loss": 1.3581968545913696,
+ "mean_token_accuracy": 0.6631045341491699,
+ "num_tokens": 4128768.0,
+ "step": 252
+ },
+ {
+ "entropy": 2.1161346435546875,
+ "epoch": 0.5111111111111111,
+ "grad_norm": 1.9295378923416138,
+ "learning_rate": 1.660606060606061e-05,
+ "loss": 2.1316990852355957,
+ "mean_token_accuracy": 0.5643624663352966,
+ "num_tokens": 4145152.0,
+ "step": 253
+ },
+ {
+ "entropy": 1.4322566986083984,
+ "epoch": 0.5131313131313131,
+ "grad_norm": 2.0198557376861572,
+ "learning_rate": 1.6592592592592594e-05,
+ "loss": 1.4390883445739746,
+ "mean_token_accuracy": 0.6648143529891968,
+ "num_tokens": 4161536.0,
+ "step": 254
+ },
+ {
+ "entropy": 1.5892291069030762,
+ "epoch": 0.5151515151515151,
+ "grad_norm": 2.0127851963043213,
+ "learning_rate": 1.6579124579124582e-05,
+ "loss": 1.6437480449676514,
+ "mean_token_accuracy": 0.6223131418228149,
+ "num_tokens": 4177920.0,
+ "step": 255
+ },
+ {
+ "entropy": 1.445371150970459,
+ "epoch": 0.5171717171717172,
+ "grad_norm": 1.9758318662643433,
+ "learning_rate": 1.6565656565656567e-05,
+ "loss": 1.4749128818511963,
+ "mean_token_accuracy": 0.6457010507583618,
+ "num_tokens": 4194304.0,
+ "step": 256
+ },
+ {
+ "entropy": 1.5587478876113892,
+ "epoch": 0.5191919191919192,
+ "grad_norm": 1.918809413909912,
+ "learning_rate": 1.6552188552188552e-05,
+ "loss": 1.5739175081253052,
+ "mean_token_accuracy": 0.6334269642829895,
+ "num_tokens": 4210688.0,
+ "step": 257
+ },
+ {
+ "entropy": 1.5112392902374268,
+ "epoch": 0.5212121212121212,
+ "grad_norm": 2.0176963806152344,
+ "learning_rate": 1.653872053872054e-05,
+ "loss": 1.5279680490493774,
+ "mean_token_accuracy": 0.640754759311676,
+ "num_tokens": 4227072.0,
+ "step": 258
+ },
+ {
+ "entropy": 1.4898114204406738,
+ "epoch": 0.5232323232323233,
+ "grad_norm": 2.0637850761413574,
+ "learning_rate": 1.6525252525252528e-05,
+ "loss": 1.5081419944763184,
+ "mean_token_accuracy": 0.6375183463096619,
+ "num_tokens": 4243456.0,
+ "step": 259
+ },
+ {
+ "entropy": 1.4768201112747192,
+ "epoch": 0.5252525252525253,
+ "grad_norm": 1.835053563117981,
+ "learning_rate": 1.6511784511784513e-05,
+ "loss": 1.494248390197754,
+ "mean_token_accuracy": 0.6496092081069946,
+ "num_tokens": 4259840.0,
+ "step": 260
+ },
+ {
+ "entropy": 1.655643105506897,
+ "epoch": 0.5272727272727272,
+ "grad_norm": 1.9655805826187134,
+ "learning_rate": 1.64983164983165e-05,
+ "loss": 1.6360158920288086,
+ "mean_token_accuracy": 0.6209086179733276,
+ "num_tokens": 4276224.0,
+ "step": 261
+ },
+ {
+ "entropy": 1.5621604919433594,
+ "epoch": 0.5292929292929293,
+ "grad_norm": 1.9304734468460083,
+ "learning_rate": 1.6484848484848486e-05,
+ "loss": 1.5850739479064941,
+ "mean_token_accuracy": 0.6422203183174133,
+ "num_tokens": 4292608.0,
+ "step": 262
+ },
+ {
+ "entropy": 1.592617392539978,
+ "epoch": 0.5313131313131313,
+ "grad_norm": 1.9590504169464111,
+ "learning_rate": 1.6471380471380474e-05,
+ "loss": 1.583874225616455,
+ "mean_token_accuracy": 0.6230459213256836,
+ "num_tokens": 4308992.0,
+ "step": 263
+ },
+ {
+ "entropy": 1.5268265008926392,
+ "epoch": 0.5333333333333333,
+ "grad_norm": 2.0704433917999268,
+ "learning_rate": 1.645791245791246e-05,
+ "loss": 1.5327314138412476,
+ "mean_token_accuracy": 0.6398388147354126,
+ "num_tokens": 4325376.0,
+ "step": 264
+ },
+ {
+ "entropy": 1.1762311458587646,
+ "epoch": 0.5353535353535354,
+ "grad_norm": 2.0408453941345215,
+ "learning_rate": 1.6444444444444444e-05,
+ "loss": 1.1851191520690918,
+ "mean_token_accuracy": 0.7027967572212219,
+ "num_tokens": 4341760.0,
+ "step": 265
+ },
+ {
+ "entropy": 1.567710518836975,
+ "epoch": 0.5373737373737374,
+ "grad_norm": 1.8334625959396362,
+ "learning_rate": 1.6430976430976432e-05,
+ "loss": 1.564115047454834,
+ "mean_token_accuracy": 0.6340376138687134,
+ "num_tokens": 4358144.0,
+ "step": 266
+ },
+ {
+ "entropy": 1.2294297218322754,
+ "epoch": 0.5393939393939394,
+ "grad_norm": 3.5099127292633057,
+ "learning_rate": 1.641750841750842e-05,
+ "loss": 1.2887773513793945,
+ "mean_token_accuracy": 0.688629686832428,
+ "num_tokens": 4374528.0,
+ "step": 267
+ },
+ {
+ "entropy": 1.5130258798599243,
+ "epoch": 0.5414141414141415,
+ "grad_norm": 1.8295154571533203,
+ "learning_rate": 1.6404040404040405e-05,
+ "loss": 1.581455945968628,
+ "mean_token_accuracy": 0.639655590057373,
+ "num_tokens": 4390912.0,
+ "step": 268
+ },
+ {
+ "entropy": 1.6612389087677002,
+ "epoch": 0.5434343434343434,
+ "grad_norm": 2.1676483154296875,
+ "learning_rate": 1.6390572390572393e-05,
+ "loss": 1.6873081922531128,
+ "mean_token_accuracy": 0.6049095988273621,
+ "num_tokens": 4407296.0,
+ "step": 269
+ },
+ {
+ "entropy": 1.416680932044983,
+ "epoch": 0.5454545454545454,
+ "grad_norm": 2.044541358947754,
+ "learning_rate": 1.6377104377104378e-05,
+ "loss": 1.4432693719863892,
+ "mean_token_accuracy": 0.6580972075462341,
+ "num_tokens": 4423680.0,
+ "step": 270
+ },
+ {
+ "entropy": 1.6253215074539185,
+ "epoch": 0.5474747474747474,
+ "grad_norm": 2.1488161087036133,
+ "learning_rate": 1.6363636363636366e-05,
+ "loss": 1.6667883396148682,
+ "mean_token_accuracy": 0.6122373938560486,
+ "num_tokens": 4440064.0,
+ "step": 271
+ },
+ {
+ "entropy": 1.492803931236267,
+ "epoch": 0.5494949494949495,
+ "grad_norm": 2.0591108798980713,
+ "learning_rate": 1.635016835016835e-05,
+ "loss": 1.5023317337036133,
+ "mean_token_accuracy": 0.6420371532440186,
+ "num_tokens": 4456448.0,
+ "step": 272
+ },
+ {
+ "entropy": 1.5818493366241455,
+ "epoch": 0.5515151515151515,
+ "grad_norm": 1.9004875421524048,
+ "learning_rate": 1.633670033670034e-05,
+ "loss": 1.5762417316436768,
+ "mean_token_accuracy": 0.6375793814659119,
+ "num_tokens": 4472832.0,
+ "step": 273
+ },
+ {
+ "entropy": 1.2263695001602173,
+ "epoch": 0.5535353535353535,
+ "grad_norm": 1.8976677656173706,
+ "learning_rate": 1.6323232323232324e-05,
+ "loss": 1.2397428750991821,
+ "mean_token_accuracy": 0.70658278465271,
+ "num_tokens": 4489216.0,
+ "step": 274
+ },
+ {
+ "entropy": 1.370070219039917,
+ "epoch": 0.5555555555555556,
+ "grad_norm": 1.875389575958252,
+ "learning_rate": 1.6309764309764312e-05,
+ "loss": 1.3603096008300781,
+ "mean_token_accuracy": 0.6843551397323608,
+ "num_tokens": 4505600.0,
+ "step": 275
+ },
+ {
+ "entropy": 1.9219107627868652,
+ "epoch": 0.5575757575757576,
+ "grad_norm": 2.057508945465088,
+ "learning_rate": 1.6296296296296297e-05,
+ "loss": 1.9082988500595093,
+ "mean_token_accuracy": 0.5834758281707764,
+ "num_tokens": 4521984.0,
+ "step": 276
+ },
+ {
+ "entropy": 1.6965564489364624,
+ "epoch": 0.5595959595959596,
+ "grad_norm": 1.9217735528945923,
+ "learning_rate": 1.6282828282828285e-05,
+ "loss": 1.7024450302124023,
+ "mean_token_accuracy": 0.6050928235054016,
+ "num_tokens": 4538368.0,
+ "step": 277
+ },
+ {
+ "entropy": 1.7517896890640259,
+ "epoch": 0.5616161616161616,
+ "grad_norm": 2.029672145843506,
+ "learning_rate": 1.626936026936027e-05,
+ "loss": 1.7533857822418213,
+ "mean_token_accuracy": 0.5986199378967285,
+ "num_tokens": 4554752.0,
+ "step": 278
+ },
+ {
+ "entropy": 1.6086634397506714,
+ "epoch": 0.5636363636363636,
+ "grad_norm": 1.980076551437378,
+ "learning_rate": 1.6255892255892258e-05,
+ "loss": 1.5998293161392212,
+ "mean_token_accuracy": 0.6469223499298096,
+ "num_tokens": 4571136.0,
+ "step": 279
+ },
+ {
+ "entropy": 1.5670934915542603,
+ "epoch": 0.5656565656565656,
+ "grad_norm": 1.9693994522094727,
+ "learning_rate": 1.6242424242424243e-05,
+ "loss": 1.583737850189209,
+ "mean_token_accuracy": 0.6189545392990112,
+ "num_tokens": 4587520.0,
+ "step": 280
+ },
+ {
+ "entropy": 1.5290220975875854,
+ "epoch": 0.5676767676767677,
+ "grad_norm": 1.9464402198791504,
+ "learning_rate": 1.622895622895623e-05,
+ "loss": 1.523187518119812,
+ "mean_token_accuracy": 0.6289692521095276,
+ "num_tokens": 4603904.0,
+ "step": 281
+ },
+ {
+ "entropy": 1.2070591449737549,
+ "epoch": 0.5696969696969697,
+ "grad_norm": 2.1152102947235107,
+ "learning_rate": 1.621548821548822e-05,
+ "loss": 1.2107012271881104,
+ "mean_token_accuracy": 0.695652186870575,
+ "num_tokens": 4620288.0,
+ "step": 282
+ },
+ {
+ "entropy": 1.4039427042007446,
+ "epoch": 0.5717171717171717,
+ "grad_norm": 1.936285376548767,
+ "learning_rate": 1.6202020202020204e-05,
+ "loss": 1.425846815109253,
+ "mean_token_accuracy": 0.663593053817749,
+ "num_tokens": 4636672.0,
+ "step": 283
+ },
+ {
+ "entropy": 1.2910168170928955,
+ "epoch": 0.5737373737373738,
+ "grad_norm": 1.8882776498794556,
+ "learning_rate": 1.618855218855219e-05,
+ "loss": 1.3109780550003052,
+ "mean_token_accuracy": 0.6896067261695862,
+ "num_tokens": 4653056.0,
+ "step": 284
+ },
+ {
+ "entropy": 1.7089474201202393,
+ "epoch": 0.5757575757575758,
+ "grad_norm": 2.0113184452056885,
+ "learning_rate": 1.6175084175084177e-05,
+ "loss": 1.7272329330444336,
+ "mean_token_accuracy": 0.600024402141571,
+ "num_tokens": 4669440.0,
+ "step": 285
+ },
+ {
+ "entropy": 1.12257719039917,
+ "epoch": 0.5777777777777777,
+ "grad_norm": 2.0191409587860107,
+ "learning_rate": 1.616161616161616e-05,
+ "loss": 1.1193959712982178,
+ "mean_token_accuracy": 0.7199560403823853,
+ "num_tokens": 4685824.0,
+ "step": 286
+ },
+ {
+ "entropy": 1.2052935361862183,
+ "epoch": 0.5797979797979798,
+ "grad_norm": 1.9647032022476196,
+ "learning_rate": 1.614814814814815e-05,
+ "loss": 1.2192071676254272,
+ "mean_token_accuracy": 0.6977894306182861,
+ "num_tokens": 4702208.0,
+ "step": 287
+ },
+ {
+ "entropy": 1.5017658472061157,
+ "epoch": 0.5818181818181818,
+ "grad_norm": 2.0078749656677246,
+ "learning_rate": 1.6134680134680138e-05,
+ "loss": 1.5069187879562378,
+ "mean_token_accuracy": 0.6404494643211365,
+ "num_tokens": 4718592.0,
+ "step": 288
+ },
+ {
+ "entropy": 1.4429490566253662,
+ "epoch": 0.5838383838383838,
+ "grad_norm": 1.8666913509368896,
+ "learning_rate": 1.6121212121212123e-05,
+ "loss": 1.4498789310455322,
+ "mean_token_accuracy": 0.6577919125556946,
+ "num_tokens": 4734976.0,
+ "step": 289
+ },
+ {
+ "entropy": 1.216312050819397,
+ "epoch": 0.5858585858585859,
+ "grad_norm": 1.8042014837265015,
+ "learning_rate": 1.610774410774411e-05,
+ "loss": 1.2346259355545044,
+ "mean_token_accuracy": 0.7015144228935242,
+ "num_tokens": 4751360.0,
+ "step": 290
+ },
+ {
+ "entropy": 1.1411411762237549,
+ "epoch": 0.5878787878787879,
+ "grad_norm": 1.7743948698043823,
+ "learning_rate": 1.6094276094276096e-05,
+ "loss": 1.158776044845581,
+ "mean_token_accuracy": 0.7111626863479614,
+ "num_tokens": 4767744.0,
+ "step": 291
+ },
+ {
+ "entropy": 1.5691558122634888,
+ "epoch": 0.5898989898989899,
+ "grad_norm": 2.0790789127349854,
+ "learning_rate": 1.608080808080808e-05,
+ "loss": 1.5900537967681885,
+ "mean_token_accuracy": 0.6259770393371582,
+ "num_tokens": 4784128.0,
+ "step": 292
+ },
+ {
+ "entropy": 1.5600252151489258,
+ "epoch": 0.591919191919192,
+ "grad_norm": 1.8288682699203491,
+ "learning_rate": 1.606734006734007e-05,
+ "loss": 1.5642855167388916,
+ "mean_token_accuracy": 0.6497313380241394,
+ "num_tokens": 4800512.0,
+ "step": 293
+ },
+ {
+ "entropy": 1.316757082939148,
+ "epoch": 0.593939393939394,
+ "grad_norm": 2.051280975341797,
+ "learning_rate": 1.6053872053872053e-05,
+ "loss": 1.3361237049102783,
+ "mean_token_accuracy": 0.6642647981643677,
+ "num_tokens": 4816896.0,
+ "step": 294
+ },
+ {
+ "entropy": 1.4421989917755127,
+ "epoch": 0.5959595959595959,
+ "grad_norm": 2.257932662963867,
+ "learning_rate": 1.604040404040404e-05,
+ "loss": 1.4325928688049316,
+ "mean_token_accuracy": 0.6571201682090759,
+ "num_tokens": 4833280.0,
+ "step": 295
+ },
+ {
+ "entropy": 1.4986436367034912,
+ "epoch": 0.597979797979798,
+ "grad_norm": 2.0747647285461426,
+ "learning_rate": 1.602693602693603e-05,
+ "loss": 1.508068323135376,
+ "mean_token_accuracy": 0.6551050543785095,
+ "num_tokens": 4849664.0,
+ "step": 296
+ },
+ {
+ "entropy": 1.4758206605911255,
+ "epoch": 0.6,
+ "grad_norm": 1.874723196029663,
+ "learning_rate": 1.6013468013468014e-05,
+ "loss": 1.4831781387329102,
+ "mean_token_accuracy": 0.6479604244232178,
+ "num_tokens": 4866048.0,
+ "step": 297
+ },
+ {
+ "entropy": 1.8049858808517456,
+ "epoch": 0.602020202020202,
+ "grad_norm": 2.089683771133423,
+ "learning_rate": 1.6000000000000003e-05,
+ "loss": 1.8325893878936768,
+ "mean_token_accuracy": 0.5932462215423584,
+ "num_tokens": 4882432.0,
+ "step": 298
+ },
+ {
+ "entropy": 1.6346150636672974,
+ "epoch": 0.604040404040404,
+ "grad_norm": 1.9526349306106567,
+ "learning_rate": 1.5986531986531987e-05,
+ "loss": 1.6568968296051025,
+ "mean_token_accuracy": 0.6246336102485657,
+ "num_tokens": 4898816.0,
+ "step": 299
+ },
+ {
+ "entropy": 1.431725025177002,
+ "epoch": 0.6060606060606061,
+ "grad_norm": 2.109987497329712,
+ "learning_rate": 1.5973063973063972e-05,
+ "loss": 1.4448115825653076,
+ "mean_token_accuracy": 0.6481436491012573,
+ "num_tokens": 4915200.0,
+ "step": 300
+ },
+ {
+ "epoch": 0.6060606060606061,
+ "eval_entropy": 1.524387352889584,
+ "eval_loss": 1.5324952602386475,
+ "eval_mean_token_accuracy": 0.6443250953189789,
+ "eval_num_tokens": 4915200.0,
+ "eval_runtime": 43.7714,
+ "eval_samples_per_second": 22.618,
+ "eval_steps_per_second": 2.833,
+ "step": 300
+ },
+ {
+ "entropy": 1.4816409349441528,
+ "epoch": 0.6080808080808081,
+ "grad_norm": 3.3732845783233643,
+ "learning_rate": 1.595959595959596e-05,
+ "loss": 1.4913065433502197,
+ "mean_token_accuracy": 0.658707857131958,
+ "num_tokens": 4931584.0,
+ "step": 301
+ },
+ {
+ "entropy": 1.4020944833755493,
+ "epoch": 0.6101010101010101,
+ "grad_norm": 2.1152141094207764,
+ "learning_rate": 1.594612794612795e-05,
+ "loss": 1.4000526666641235,
+ "mean_token_accuracy": 0.656509518623352,
+ "num_tokens": 4947968.0,
+ "step": 302
+ },
+ {
+ "entropy": 1.4965202808380127,
+ "epoch": 0.6121212121212121,
+ "grad_norm": 2.1385467052459717,
+ "learning_rate": 1.5932659932659933e-05,
+ "loss": 1.5221803188323975,
+ "mean_token_accuracy": 0.6567537784576416,
+ "num_tokens": 4964352.0,
+ "step": 303
+ },
+ {
+ "entropy": 1.4024577140808105,
+ "epoch": 0.6141414141414141,
+ "grad_norm": 1.9162694215774536,
+ "learning_rate": 1.591919191919192e-05,
+ "loss": 1.4050084352493286,
+ "mean_token_accuracy": 0.6670737862586975,
+ "num_tokens": 4980736.0,
+ "step": 304
+ },
+ {
+ "entropy": 1.430273413658142,
+ "epoch": 0.6161616161616161,
+ "grad_norm": 2.0312552452087402,
+ "learning_rate": 1.5905723905723906e-05,
+ "loss": 1.418628454208374,
+ "mean_token_accuracy": 0.6585857272148132,
+ "num_tokens": 4997120.0,
+ "step": 305
+ },
+ {
+ "entropy": 1.700549602508545,
+ "epoch": 0.6181818181818182,
+ "grad_norm": 2.080810546875,
+ "learning_rate": 1.5892255892255895e-05,
+ "loss": 1.740492820739746,
+ "mean_token_accuracy": 0.6042989492416382,
+ "num_tokens": 5013504.0,
+ "step": 306
+ },
+ {
+ "entropy": 1.6465320587158203,
+ "epoch": 0.6202020202020202,
+ "grad_norm": 1.8810964822769165,
+ "learning_rate": 1.587878787878788e-05,
+ "loss": 1.6414886713027954,
+ "mean_token_accuracy": 0.629824161529541,
+ "num_tokens": 5029888.0,
+ "step": 307
+ },
+ {
+ "entropy": 1.3717931509017944,
+ "epoch": 0.6222222222222222,
+ "grad_norm": 1.921536922454834,
+ "learning_rate": 1.5865319865319868e-05,
+ "loss": 1.3700361251831055,
+ "mean_token_accuracy": 0.6630434989929199,
+ "num_tokens": 5046272.0,
+ "step": 308
+ },
+ {
+ "entropy": 1.4906446933746338,
+ "epoch": 0.6242424242424243,
+ "grad_norm": 2.1185555458068848,
+ "learning_rate": 1.5851851851851852e-05,
+ "loss": 1.4850339889526367,
+ "mean_token_accuracy": 0.6402662396430969,
+ "num_tokens": 5062656.0,
+ "step": 309
+ },
+ {
+ "entropy": 1.4417473077774048,
+ "epoch": 0.6262626262626263,
+ "grad_norm": 2.0196895599365234,
+ "learning_rate": 1.583838383838384e-05,
+ "loss": 1.4567816257476807,
+ "mean_token_accuracy": 0.6760503053665161,
+ "num_tokens": 5079040.0,
+ "step": 310
+ },
+ {
+ "entropy": 1.4662094116210938,
+ "epoch": 0.6282828282828283,
+ "grad_norm": 1.8859882354736328,
+ "learning_rate": 1.5824915824915825e-05,
+ "loss": 1.4870874881744385,
+ "mean_token_accuracy": 0.6618221998214722,
+ "num_tokens": 5095424.0,
+ "step": 311
+ },
+ {
+ "entropy": 1.8497077226638794,
+ "epoch": 0.6303030303030303,
+ "grad_norm": 1.974537968635559,
+ "learning_rate": 1.5811447811447813e-05,
+ "loss": 1.840538501739502,
+ "mean_token_accuracy": 0.589093804359436,
+ "num_tokens": 5111808.0,
+ "step": 312
+ },
+ {
+ "entropy": 1.4185130596160889,
+ "epoch": 0.6323232323232323,
+ "grad_norm": 1.8612359762191772,
+ "learning_rate": 1.5797979797979798e-05,
+ "loss": 1.4404246807098389,
+ "mean_token_accuracy": 0.664631187915802,
+ "num_tokens": 5128192.0,
+ "step": 313
+ },
+ {
+ "entropy": 1.8270463943481445,
+ "epoch": 0.6343434343434343,
+ "grad_norm": 2.1050970554351807,
+ "learning_rate": 1.5784511784511786e-05,
+ "loss": 1.8682940006256104,
+ "mean_token_accuracy": 0.5893380641937256,
+ "num_tokens": 5144576.0,
+ "step": 314
+ },
+ {
+ "entropy": 1.5919326543807983,
+ "epoch": 0.6363636363636364,
+ "grad_norm": 1.8464620113372803,
+ "learning_rate": 1.577104377104377e-05,
+ "loss": 1.6087713241577148,
+ "mean_token_accuracy": 0.6350146532058716,
+ "num_tokens": 5160960.0,
+ "step": 315
+ },
+ {
+ "entropy": 1.4776946306228638,
+ "epoch": 0.6383838383838384,
+ "grad_norm": 2.031834602355957,
+ "learning_rate": 1.575757575757576e-05,
+ "loss": 1.5114507675170898,
+ "mean_token_accuracy": 0.6498534679412842,
+ "num_tokens": 5177344.0,
+ "step": 316
+ },
+ {
+ "entropy": 1.3789927959442139,
+ "epoch": 0.6404040404040404,
+ "grad_norm": 1.8604289293289185,
+ "learning_rate": 1.5744107744107748e-05,
+ "loss": 1.3792052268981934,
+ "mean_token_accuracy": 0.6712262034416199,
+ "num_tokens": 5193728.0,
+ "step": 317
+ },
+ {
+ "entropy": 1.5859184265136719,
+ "epoch": 0.6424242424242425,
+ "grad_norm": 2.1134088039398193,
+ "learning_rate": 1.5730639730639732e-05,
+ "loss": 1.595099925994873,
+ "mean_token_accuracy": 0.6351367831230164,
+ "num_tokens": 5210112.0,
+ "step": 318
+ },
+ {
+ "entropy": 1.3681901693344116,
+ "epoch": 0.6444444444444445,
+ "grad_norm": 1.9200235605239868,
+ "learning_rate": 1.5717171717171717e-05,
+ "loss": 1.3700852394104004,
+ "mean_token_accuracy": 0.6671959161758423,
+ "num_tokens": 5226496.0,
+ "step": 319
+ },
+ {
+ "entropy": 1.594885230064392,
+ "epoch": 0.6464646464646465,
+ "grad_norm": 1.7683591842651367,
+ "learning_rate": 1.5703703703703705e-05,
+ "loss": 1.5906047821044922,
+ "mean_token_accuracy": 0.6350757479667664,
+ "num_tokens": 5242880.0,
+ "step": 320
+ },
+ {
+ "entropy": 1.240937352180481,
+ "epoch": 0.6484848484848484,
+ "grad_norm": 2.117145299911499,
+ "learning_rate": 1.569023569023569e-05,
+ "loss": 1.2188963890075684,
+ "mean_token_accuracy": 0.6958353519439697,
+ "num_tokens": 5259264.0,
+ "step": 321
+ },
+ {
+ "entropy": 1.244801640510559,
+ "epoch": 0.6505050505050505,
+ "grad_norm": 1.983111023902893,
+ "learning_rate": 1.5676767676767678e-05,
+ "loss": 1.246530294418335,
+ "mean_token_accuracy": 0.6966902613639832,
+ "num_tokens": 5275648.0,
+ "step": 322
+ },
+ {
+ "entropy": 1.5355960130691528,
+ "epoch": 0.6525252525252525,
+ "grad_norm": 2.126024007797241,
+ "learning_rate": 1.5663299663299666e-05,
+ "loss": 1.576147198677063,
+ "mean_token_accuracy": 0.6414265036582947,
+ "num_tokens": 5292032.0,
+ "step": 323
+ },
+ {
+ "entropy": 1.13125741481781,
+ "epoch": 0.6545454545454545,
+ "grad_norm": 1.8933371305465698,
+ "learning_rate": 1.564983164983165e-05,
+ "loss": 1.1167645454406738,
+ "mean_token_accuracy": 0.7080483436584473,
+ "num_tokens": 5308416.0,
+ "step": 324
+ },
+ {
+ "entropy": 1.6553086042404175,
+ "epoch": 0.6565656565656566,
+ "grad_norm": 1.9253427982330322,
+ "learning_rate": 1.563636363636364e-05,
+ "loss": 1.6746933460235596,
+ "mean_token_accuracy": 0.6138250827789307,
+ "num_tokens": 5324800.0,
+ "step": 325
+ },
+ {
+ "entropy": 1.7187142372131348,
+ "epoch": 0.6585858585858586,
+ "grad_norm": 1.8655303716659546,
+ "learning_rate": 1.5622895622895624e-05,
+ "loss": 1.7368483543395996,
+ "mean_token_accuracy": 0.6073521971702576,
+ "num_tokens": 5341184.0,
+ "step": 326
+ },
+ {
+ "entropy": 1.4150934219360352,
+ "epoch": 0.6606060606060606,
+ "grad_norm": 1.8396921157836914,
+ "learning_rate": 1.560942760942761e-05,
+ "loss": 1.450613260269165,
+ "mean_token_accuracy": 0.6628602743148804,
+ "num_tokens": 5357568.0,
+ "step": 327
+ },
+ {
+ "entropy": 1.6463909149169922,
+ "epoch": 0.6626262626262627,
+ "grad_norm": 2.0179622173309326,
+ "learning_rate": 1.5595959595959597e-05,
+ "loss": 1.6767137050628662,
+ "mean_token_accuracy": 0.6147410869598389,
+ "num_tokens": 5373952.0,
+ "step": 328
+ },
+ {
+ "entropy": 1.2962068319320679,
+ "epoch": 0.6646464646464646,
+ "grad_norm": 1.9322903156280518,
+ "learning_rate": 1.5582491582491582e-05,
+ "loss": 1.288381814956665,
+ "mean_token_accuracy": 0.6877137422561646,
+ "num_tokens": 5390336.0,
+ "step": 329
+ },
+ {
+ "entropy": 1.4675830602645874,
+ "epoch": 0.6666666666666666,
+ "grad_norm": 1.7735666036605835,
+ "learning_rate": 1.556902356902357e-05,
+ "loss": 1.4585455656051636,
+ "mean_token_accuracy": 0.671775758266449,
+ "num_tokens": 5406720.0,
+ "step": 330
+ },
+ {
+ "entropy": 1.55418062210083,
+ "epoch": 0.6686868686868687,
+ "grad_norm": 1.9255602359771729,
+ "learning_rate": 1.555555555555556e-05,
+ "loss": 1.6033880710601807,
+ "mean_token_accuracy": 0.6486321687698364,
+ "num_tokens": 5423104.0,
+ "step": 331
+ },
+ {
+ "entropy": 1.4288218021392822,
+ "epoch": 0.6707070707070707,
+ "grad_norm": 1.8075612783432007,
+ "learning_rate": 1.5542087542087543e-05,
+ "loss": 1.457895278930664,
+ "mean_token_accuracy": 0.6725085377693176,
+ "num_tokens": 5439488.0,
+ "step": 332
+ },
+ {
+ "entropy": 1.3419297933578491,
+ "epoch": 0.6727272727272727,
+ "grad_norm": 2.1730353832244873,
+ "learning_rate": 1.552861952861953e-05,
+ "loss": 1.3681403398513794,
+ "mean_token_accuracy": 0.6664020419120789,
+ "num_tokens": 5455872.0,
+ "step": 333
+ },
+ {
+ "entropy": 1.706923007965088,
+ "epoch": 0.6747474747474748,
+ "grad_norm": 1.9297505617141724,
+ "learning_rate": 1.5515151515151516e-05,
+ "loss": 1.7304211854934692,
+ "mean_token_accuracy": 0.6020395755767822,
+ "num_tokens": 5472256.0,
+ "step": 334
+ },
+ {
+ "entropy": 1.117404818534851,
+ "epoch": 0.6767676767676768,
+ "grad_norm": 1.7363555431365967,
+ "learning_rate": 1.55016835016835e-05,
+ "loss": 1.1002353429794312,
+ "mean_token_accuracy": 0.7285661697387695,
+ "num_tokens": 5488640.0,
+ "step": 335
+ },
+ {
+ "entropy": 1.409253478050232,
+ "epoch": 0.6787878787878788,
+ "grad_norm": 2.029304265975952,
+ "learning_rate": 1.548821548821549e-05,
+ "loss": 1.4001437425613403,
+ "mean_token_accuracy": 0.6604176759719849,
+ "num_tokens": 5505024.0,
+ "step": 336
+ },
+ {
+ "entropy": 1.0427494049072266,
+ "epoch": 0.6808080808080809,
+ "grad_norm": 2.0266847610473633,
+ "learning_rate": 1.5474747474747477e-05,
+ "loss": 1.0636063814163208,
+ "mean_token_accuracy": 0.7298485636711121,
+ "num_tokens": 5521408.0,
+ "step": 337
+ },
+ {
+ "entropy": 1.479506254196167,
+ "epoch": 0.6828282828282828,
+ "grad_norm": 1.937677264213562,
+ "learning_rate": 1.5461279461279462e-05,
+ "loss": 1.4711894989013672,
+ "mean_token_accuracy": 0.6436248421669006,
+ "num_tokens": 5537792.0,
+ "step": 338
+ },
+ {
+ "entropy": 1.636492371559143,
+ "epoch": 0.6848484848484848,
+ "grad_norm": 1.9746829271316528,
+ "learning_rate": 1.544781144781145e-05,
+ "loss": 1.638418436050415,
+ "mean_token_accuracy": 0.6204200983047485,
+ "num_tokens": 5554176.0,
+ "step": 339
+ },
+ {
+ "entropy": 1.6094971895217896,
+ "epoch": 0.6868686868686869,
+ "grad_norm": 1.998382806777954,
+ "learning_rate": 1.5434343434343435e-05,
+ "loss": 1.6025879383087158,
+ "mean_token_accuracy": 0.6193209290504456,
+ "num_tokens": 5570560.0,
+ "step": 340
+ },
+ {
+ "entropy": 1.3113542795181274,
+ "epoch": 0.6888888888888889,
+ "grad_norm": 2.059739589691162,
+ "learning_rate": 1.5420875420875423e-05,
+ "loss": 1.32335364818573,
+ "mean_token_accuracy": 0.6776379942893982,
+ "num_tokens": 5586944.0,
+ "step": 341
+ },
+ {
+ "entropy": 1.3761622905731201,
+ "epoch": 0.6909090909090909,
+ "grad_norm": 1.9774043560028076,
+ "learning_rate": 1.5407407407407408e-05,
+ "loss": 1.3731458187103271,
+ "mean_token_accuracy": 0.681546151638031,
+ "num_tokens": 5603328.0,
+ "step": 342
+ },
+ {
+ "entropy": 1.3822734355926514,
+ "epoch": 0.692929292929293,
+ "grad_norm": 5.41459321975708,
+ "learning_rate": 1.5393939393939393e-05,
+ "loss": 1.4077341556549072,
+ "mean_token_accuracy": 0.6688446402549744,
+ "num_tokens": 5619712.0,
+ "step": 343
+ },
+ {
+ "entropy": 1.4885843992233276,
+ "epoch": 0.694949494949495,
+ "grad_norm": 1.9256173372268677,
+ "learning_rate": 1.538047138047138e-05,
+ "loss": 1.5017898082733154,
+ "mean_token_accuracy": 0.6411211490631104,
+ "num_tokens": 5636096.0,
+ "step": 344
+ },
+ {
+ "entropy": 1.4558689594268799,
+ "epoch": 0.696969696969697,
+ "grad_norm": 1.923056721687317,
+ "learning_rate": 1.536700336700337e-05,
+ "loss": 1.4409637451171875,
+ "mean_token_accuracy": 0.6607840657234192,
+ "num_tokens": 5652480.0,
+ "step": 345
+ },
+ {
+ "entropy": 1.3396257162094116,
+ "epoch": 0.6989898989898989,
+ "grad_norm": 1.9321191310882568,
+ "learning_rate": 1.5353535353535354e-05,
+ "loss": 1.3465441465377808,
+ "mean_token_accuracy": 0.6682339906692505,
+ "num_tokens": 5668864.0,
+ "step": 346
+ },
+ {
+ "entropy": 1.931525468826294,
+ "epoch": 0.701010101010101,
+ "grad_norm": 2.11252498626709,
+ "learning_rate": 1.5340067340067342e-05,
+ "loss": 1.947523832321167,
+ "mean_token_accuracy": 0.5751709938049316,
+ "num_tokens": 5685248.0,
+ "step": 347
+ },
+ {
+ "entropy": 1.4246045351028442,
+ "epoch": 0.703030303030303,
+ "grad_norm": 2.017690658569336,
+ "learning_rate": 1.5326599326599327e-05,
+ "loss": 1.431095004081726,
+ "mean_token_accuracy": 0.6621274948120117,
+ "num_tokens": 5701632.0,
+ "step": 348
+ },
+ {
+ "entropy": 1.2840481996536255,
+ "epoch": 0.705050505050505,
+ "grad_norm": 1.897916555404663,
+ "learning_rate": 1.5313131313131315e-05,
+ "loss": 1.2944300174713135,
+ "mean_token_accuracy": 0.7013311982154846,
+ "num_tokens": 5718016.0,
+ "step": 349
+ },
+ {
+ "entropy": 1.1308226585388184,
+ "epoch": 0.7070707070707071,
+ "grad_norm": 1.8326802253723145,
+ "learning_rate": 1.52996632996633e-05,
+ "loss": 1.1564277410507202,
+ "mean_token_accuracy": 0.714154839515686,
+ "num_tokens": 5734400.0,
+ "step": 350
+ },
+ {
+ "entropy": 1.4157475233078003,
+ "epoch": 0.7090909090909091,
+ "grad_norm": 1.9305709600448608,
+ "learning_rate": 1.5286195286195288e-05,
+ "loss": 1.421119213104248,
+ "mean_token_accuracy": 0.6651197075843811,
+ "num_tokens": 5750784.0,
+ "step": 351
+ },
+ {
+ "entropy": 1.353966474533081,
+ "epoch": 0.7111111111111111,
+ "grad_norm": 2.058542251586914,
+ "learning_rate": 1.5272727272727276e-05,
+ "loss": 1.3575413227081299,
+ "mean_token_accuracy": 0.6724475026130676,
+ "num_tokens": 5767168.0,
+ "step": 352
+ },
+ {
+ "entropy": 1.7946882247924805,
+ "epoch": 0.7131313131313132,
+ "grad_norm": 1.9397060871124268,
+ "learning_rate": 1.525925925925926e-05,
+ "loss": 1.7882269620895386,
+ "mean_token_accuracy": 0.6064972877502441,
+ "num_tokens": 5783552.0,
+ "step": 353
+ },
+ {
+ "entropy": 1.4986019134521484,
+ "epoch": 0.7151515151515152,
+ "grad_norm": 1.9084810018539429,
+ "learning_rate": 1.5245791245791246e-05,
+ "loss": 1.5188498497009277,
+ "mean_token_accuracy": 0.6610283255577087,
+ "num_tokens": 5799936.0,
+ "step": 354
+ },
+ {
+ "entropy": 1.9081071615219116,
+ "epoch": 0.7171717171717171,
+ "grad_norm": 2.1528689861297607,
+ "learning_rate": 1.5232323232323234e-05,
+ "loss": 1.8967041969299316,
+ "mean_token_accuracy": 0.5691255331039429,
+ "num_tokens": 5816320.0,
+ "step": 355
+ },
+ {
+ "entropy": 1.641337513923645,
+ "epoch": 0.7191919191919192,
+ "grad_norm": 1.9197942018508911,
+ "learning_rate": 1.521885521885522e-05,
+ "loss": 1.6193110942840576,
+ "mean_token_accuracy": 0.6286638975143433,
+ "num_tokens": 5832704.0,
+ "step": 356
+ },
+ {
+ "entropy": 1.348304271697998,
+ "epoch": 0.7212121212121212,
+ "grad_norm": 1.9729533195495605,
+ "learning_rate": 1.5205387205387207e-05,
+ "loss": 1.3781111240386963,
+ "mean_token_accuracy": 0.6654250025749207,
+ "num_tokens": 5849088.0,
+ "step": 357
+ },
+ {
+ "entropy": 1.5427544116973877,
+ "epoch": 0.7232323232323232,
+ "grad_norm": 2.0730769634246826,
+ "learning_rate": 1.5191919191919193e-05,
+ "loss": 1.5770654678344727,
+ "mean_token_accuracy": 0.6273815631866455,
+ "num_tokens": 5865472.0,
+ "step": 358
+ },
+ {
+ "entropy": 1.5560660362243652,
+ "epoch": 0.7252525252525253,
+ "grad_norm": 2.0436644554138184,
+ "learning_rate": 1.5178451178451178e-05,
+ "loss": 1.5465143918991089,
+ "mean_token_accuracy": 0.6335490942001343,
+ "num_tokens": 5881856.0,
+ "step": 359
+ },
+ {
+ "entropy": 1.5396223068237305,
+ "epoch": 0.7272727272727273,
+ "grad_norm": 1.9955254793167114,
+ "learning_rate": 1.5164983164983166e-05,
+ "loss": 1.5742223262786865,
+ "mean_token_accuracy": 0.6344650983810425,
+ "num_tokens": 5898240.0,
+ "step": 360
+ },
+ {
+ "entropy": 1.1664644479751587,
+ "epoch": 0.7292929292929293,
+ "grad_norm": 1.6942992210388184,
+ "learning_rate": 1.5151515151515153e-05,
+ "loss": 1.159040927886963,
+ "mean_token_accuracy": 0.7144601941108704,
+ "num_tokens": 5914624.0,
+ "step": 361
+ },
+ {
+ "entropy": 1.356544017791748,
+ "epoch": 0.7313131313131314,
+ "grad_norm": 1.890787124633789,
+ "learning_rate": 1.5138047138047138e-05,
+ "loss": 1.347895622253418,
+ "mean_token_accuracy": 0.6813629865646362,
+ "num_tokens": 5931008.0,
+ "step": 362
+ },
+ {
+ "entropy": 1.6837173700332642,
+ "epoch": 0.7333333333333333,
+ "grad_norm": 2.1780381202697754,
+ "learning_rate": 1.5124579124579126e-05,
+ "loss": 1.6997990608215332,
+ "mean_token_accuracy": 0.6098558902740479,
+ "num_tokens": 5947392.0,
+ "step": 363
+ },
+ {
+ "entropy": 1.1412957906723022,
+ "epoch": 0.7353535353535353,
+ "grad_norm": 1.801152229309082,
+ "learning_rate": 1.5111111111111112e-05,
+ "loss": 1.161649227142334,
+ "mean_token_accuracy": 0.705483615398407,
+ "num_tokens": 5963776.0,
+ "step": 364
+ },
+ {
+ "entropy": 1.2495859861373901,
+ "epoch": 0.7373737373737373,
+ "grad_norm": 1.8535690307617188,
+ "learning_rate": 1.50976430976431e-05,
+ "loss": 1.2595994472503662,
+ "mean_token_accuracy": 0.6926599740982056,
+ "num_tokens": 5980160.0,
+ "step": 365
+ },
+ {
+ "entropy": 1.8037965297698975,
+ "epoch": 0.7393939393939394,
+ "grad_norm": 2.0506303310394287,
+ "learning_rate": 1.5084175084175085e-05,
+ "loss": 1.8499953746795654,
+ "mean_token_accuracy": 0.5861626863479614,
+ "num_tokens": 5996544.0,
+ "step": 366
+ },
+ {
+ "entropy": 1.2877405881881714,
+ "epoch": 0.7414141414141414,
+ "grad_norm": 1.8159914016723633,
+ "learning_rate": 1.5070707070707072e-05,
+ "loss": 1.3165576457977295,
+ "mean_token_accuracy": 0.685271143913269,
+ "num_tokens": 6012928.0,
+ "step": 367
+ },
+ {
+ "entropy": 1.345953106880188,
+ "epoch": 0.7434343434343434,
+ "grad_norm": 2.003962516784668,
+ "learning_rate": 1.505723905723906e-05,
+ "loss": 1.3270621299743652,
+ "mean_token_accuracy": 0.6671348214149475,
+ "num_tokens": 6029312.0,
+ "step": 368
+ },
+ {
+ "entropy": 1.408361792564392,
+ "epoch": 0.7454545454545455,
+ "grad_norm": 1.9815948009490967,
+ "learning_rate": 1.5043771043771045e-05,
+ "loss": 1.4097518920898438,
+ "mean_token_accuracy": 0.652662456035614,
+ "num_tokens": 6045696.0,
+ "step": 369
+ },
+ {
+ "entropy": 1.4007829427719116,
+ "epoch": 0.7474747474747475,
+ "grad_norm": 1.928751826286316,
+ "learning_rate": 1.5030303030303031e-05,
+ "loss": 1.4142816066741943,
+ "mean_token_accuracy": 0.671775758266449,
+ "num_tokens": 6062080.0,
+ "step": 370
+ },
+ {
+ "entropy": 1.7548134326934814,
+ "epoch": 0.7494949494949495,
+ "grad_norm": 2.081939458847046,
+ "learning_rate": 1.5016835016835018e-05,
+ "loss": 1.792219877243042,
+ "mean_token_accuracy": 0.6044821739196777,
+ "num_tokens": 6078464.0,
+ "step": 371
+ },
+ {
+ "entropy": 1.6162793636322021,
+ "epoch": 0.7515151515151515,
+ "grad_norm": 1.8544763326644897,
+ "learning_rate": 1.5003367003367004e-05,
+ "loss": 1.650557041168213,
+ "mean_token_accuracy": 0.6289692521095276,
+ "num_tokens": 6094848.0,
+ "step": 372
+ },
+ {
+ "entropy": 1.6329439878463745,
+ "epoch": 0.7535353535353535,
+ "grad_norm": 1.8025282621383667,
+ "learning_rate": 1.4989898989898992e-05,
+ "loss": 1.6812629699707031,
+ "mean_token_accuracy": 0.6405104994773865,
+ "num_tokens": 6111232.0,
+ "step": 373
+ },
+ {
+ "entropy": 1.2201604843139648,
+ "epoch": 0.7555555555555555,
+ "grad_norm": 1.9814586639404297,
+ "learning_rate": 1.4976430976430977e-05,
+ "loss": 1.2086223363876343,
+ "mean_token_accuracy": 0.7020029425621033,
+ "num_tokens": 6127616.0,
+ "step": 374
+ },
+ {
+ "entropy": 1.8105695247650146,
+ "epoch": 0.7575757575757576,
+ "grad_norm": 2.0217790603637695,
+ "learning_rate": 1.4962962962962964e-05,
+ "loss": 1.8443559408187866,
+ "mean_token_accuracy": 0.5897654891014099,
+ "num_tokens": 6144000.0,
+ "step": 375
+ },
+ {
+ "entropy": 1.197646141052246,
+ "epoch": 0.7595959595959596,
+ "grad_norm": 1.9286304712295532,
+ "learning_rate": 1.4949494949494952e-05,
+ "loss": 1.2183899879455566,
+ "mean_token_accuracy": 0.6925989389419556,
+ "num_tokens": 6160384.0,
+ "step": 376
+ },
+ {
+ "entropy": 1.4825936555862427,
+ "epoch": 0.7616161616161616,
+ "grad_norm": 1.6694327592849731,
+ "learning_rate": 1.4936026936026937e-05,
+ "loss": 1.4815235137939453,
+ "mean_token_accuracy": 0.6572422981262207,
+ "num_tokens": 6176768.0,
+ "step": 377
+ },
+ {
+ "entropy": 1.3497473001480103,
+ "epoch": 0.7636363636363637,
+ "grad_norm": 2.0366406440734863,
+ "learning_rate": 1.4922558922558923e-05,
+ "loss": 1.3637127876281738,
+ "mean_token_accuracy": 0.6690889000892639,
+ "num_tokens": 6193152.0,
+ "step": 378
+ },
+ {
+ "entropy": 1.314116358757019,
+ "epoch": 0.7656565656565657,
+ "grad_norm": 1.742874026298523,
+ "learning_rate": 1.4909090909090911e-05,
+ "loss": 1.300222396850586,
+ "mean_token_accuracy": 0.6977283954620361,
+ "num_tokens": 6209536.0,
+ "step": 379
+ },
+ {
+ "entropy": 1.4347912073135376,
+ "epoch": 0.7676767676767676,
+ "grad_norm": 1.9317418336868286,
+ "learning_rate": 1.4895622895622896e-05,
+ "loss": 1.429826021194458,
+ "mean_token_accuracy": 0.6650586128234863,
+ "num_tokens": 6225920.0,
+ "step": 380
+ },
+ {
+ "entropy": 1.448641061782837,
+ "epoch": 0.7696969696969697,
+ "grad_norm": 1.8417843580245972,
+ "learning_rate": 1.4882154882154884e-05,
+ "loss": 1.4439187049865723,
+ "mean_token_accuracy": 0.6615168452262878,
+ "num_tokens": 6242304.0,
+ "step": 381
+ },
+ {
+ "entropy": 1.2918578386306763,
+ "epoch": 0.7717171717171717,
+ "grad_norm": 1.8720030784606934,
+ "learning_rate": 1.486868686868687e-05,
+ "loss": 1.3158597946166992,
+ "mean_token_accuracy": 0.6865534782409668,
+ "num_tokens": 6258688.0,
+ "step": 382
+ },
+ {
+ "entropy": 1.5822116136550903,
+ "epoch": 0.7737373737373737,
+ "grad_norm": 1.9301713705062866,
+ "learning_rate": 1.4855218855218856e-05,
+ "loss": 1.6046046018600464,
+ "mean_token_accuracy": 0.6361138224601746,
+ "num_tokens": 6275072.0,
+ "step": 383
+ },
+ {
+ "entropy": 1.6837408542633057,
+ "epoch": 0.7757575757575758,
+ "grad_norm": 2.270636796951294,
+ "learning_rate": 1.4841750841750844e-05,
+ "loss": 1.7323150634765625,
+ "mean_token_accuracy": 0.6006350517272949,
+ "num_tokens": 6291456.0,
+ "step": 384
+ },
+ {
+ "entropy": 1.4837889671325684,
+ "epoch": 0.7777777777777778,
+ "grad_norm": 1.908146619796753,
+ "learning_rate": 1.482828282828283e-05,
+ "loss": 1.4938652515411377,
+ "mean_token_accuracy": 0.64667809009552,
+ "num_tokens": 6307840.0,
+ "step": 385
+ },
+ {
+ "entropy": 1.680192232131958,
+ "epoch": 0.7797979797979798,
+ "grad_norm": 2.2369024753570557,
+ "learning_rate": 1.4814814814814815e-05,
+ "loss": 1.692443609237671,
+ "mean_token_accuracy": 0.6279922127723694,
+ "num_tokens": 6324224.0,
+ "step": 386
+ },
+ {
+ "entropy": 1.686415195465088,
+ "epoch": 0.7818181818181819,
+ "grad_norm": 1.8252373933792114,
+ "learning_rate": 1.4801346801346803e-05,
+ "loss": 1.7022712230682373,
+ "mean_token_accuracy": 0.6302515864372253,
+ "num_tokens": 6340608.0,
+ "step": 387
+ },
+ {
+ "entropy": 1.6967185735702515,
+ "epoch": 0.7838383838383839,
+ "grad_norm": 2.102458953857422,
+ "learning_rate": 1.478787878787879e-05,
+ "loss": 1.6931922435760498,
+ "mean_token_accuracy": 0.6149853467941284,
+ "num_tokens": 6356992.0,
+ "step": 388
+ },
+ {
+ "entropy": 1.201438307762146,
+ "epoch": 0.7858585858585858,
+ "grad_norm": 1.8396239280700684,
+ "learning_rate": 1.4774410774410774e-05,
+ "loss": 1.2081820964813232,
+ "mean_token_accuracy": 0.6987664699554443,
+ "num_tokens": 6373376.0,
+ "step": 389
+ },
+ {
+ "entropy": 1.5559548139572144,
+ "epoch": 0.7878787878787878,
+ "grad_norm": 2.063498020172119,
+ "learning_rate": 1.4760942760942763e-05,
+ "loss": 1.545433759689331,
+ "mean_token_accuracy": 0.6267709136009216,
+ "num_tokens": 6389760.0,
+ "step": 390
+ },
+ {
+ "entropy": 1.7186503410339355,
+ "epoch": 0.7898989898989899,
+ "grad_norm": 2.0129787921905518,
+ "learning_rate": 1.4747474747474747e-05,
+ "loss": 1.717472791671753,
+ "mean_token_accuracy": 0.6540058851242065,
+ "num_tokens": 6406144.0,
+ "step": 391
+ },
+ {
+ "entropy": 1.643048882484436,
+ "epoch": 0.7919191919191919,
+ "grad_norm": 1.9443073272705078,
+ "learning_rate": 1.4734006734006736e-05,
+ "loss": 1.6441841125488281,
+ "mean_token_accuracy": 0.6458231806755066,
+ "num_tokens": 6422528.0,
+ "step": 392
+ },
+ {
+ "entropy": 1.3627581596374512,
+ "epoch": 0.793939393939394,
+ "grad_norm": 1.9852598905563354,
+ "learning_rate": 1.4720538720538722e-05,
+ "loss": 1.3558071851730347,
+ "mean_token_accuracy": 0.6720200181007385,
+ "num_tokens": 6438912.0,
+ "step": 393
+ },
+ {
+ "entropy": 0.8946540951728821,
+ "epoch": 0.795959595959596,
+ "grad_norm": 1.54441237449646,
+ "learning_rate": 1.4707070707070707e-05,
+ "loss": 0.8874151706695557,
+ "mean_token_accuracy": 0.7685027122497559,
+ "num_tokens": 6455296.0,
+ "step": 394
+ },
+ {
+ "entropy": 1.35861074924469,
+ "epoch": 0.797979797979798,
+ "grad_norm": 2.158553123474121,
+ "learning_rate": 1.4693602693602695e-05,
+ "loss": 1.4291828870773315,
+ "mean_token_accuracy": 0.6647533178329468,
+ "num_tokens": 6471680.0,
+ "step": 395
+ },
+ {
+ "entropy": 1.7047442197799683,
+ "epoch": 0.8,
+ "grad_norm": 2.1242175102233887,
+ "learning_rate": 1.4680134680134681e-05,
+ "loss": 1.7065966129302979,
+ "mean_token_accuracy": 0.6058256030082703,
+ "num_tokens": 6488064.0,
+ "step": 396
+ },
+ {
+ "entropy": 1.7153913974761963,
+ "epoch": 0.802020202020202,
+ "grad_norm": 1.9085419178009033,
+ "learning_rate": 1.4666666666666666e-05,
+ "loss": 1.7373592853546143,
+ "mean_token_accuracy": 0.6020395755767822,
+ "num_tokens": 6504448.0,
+ "step": 397
+ },
+ {
+ "entropy": 1.1062475442886353,
+ "epoch": 0.804040404040404,
+ "grad_norm": 1.9173986911773682,
+ "learning_rate": 1.4653198653198654e-05,
+ "loss": 1.0991405248641968,
+ "mean_token_accuracy": 0.7195896506309509,
+ "num_tokens": 6520832.0,
+ "step": 398
+ },
+ {
+ "entropy": 1.5717406272888184,
+ "epoch": 0.806060606060606,
+ "grad_norm": 2.0077219009399414,
+ "learning_rate": 1.4639730639730641e-05,
+ "loss": 1.595954418182373,
+ "mean_token_accuracy": 0.6270151734352112,
+ "num_tokens": 6537216.0,
+ "step": 399
+ },
+ {
+ "entropy": 1.3960795402526855,
+ "epoch": 0.8080808080808081,
+ "grad_norm": 2.2864415645599365,
+ "learning_rate": 1.4626262626262629e-05,
+ "loss": 1.416130781173706,
+ "mean_token_accuracy": 0.6631656289100647,
+ "num_tokens": 6553600.0,
+ "step": 400
+ },
+ {
+ "epoch": 0.8080808080808081,
+ "eval_entropy": 1.510355769626556,
+ "eval_loss": 1.5199862718582153,
+ "eval_mean_token_accuracy": 0.6461201098657423,
+ "eval_num_tokens": 6553600.0,
+ "eval_runtime": 43.8634,
+ "eval_samples_per_second": 22.57,
+ "eval_steps_per_second": 2.827,
+ "step": 400
+ },
+ {
+ "entropy": 1.5662645101547241,
+ "epoch": 0.8101010101010101,
+ "grad_norm": 2.1806085109710693,
+ "learning_rate": 1.4612794612794614e-05,
+ "loss": 1.5763804912567139,
+ "mean_token_accuracy": 0.6330605745315552,
+ "num_tokens": 6569984.0,
+ "step": 401
+ },
+ {
+ "entropy": 1.4046087265014648,
+ "epoch": 0.8121212121212121,
+ "grad_norm": 1.9650650024414062,
+ "learning_rate": 1.45993265993266e-05,
+ "loss": 1.396510124206543,
+ "mean_token_accuracy": 0.6568148732185364,
+ "num_tokens": 6586368.0,
+ "step": 402
+ },
+ {
+ "entropy": 1.084455966949463,
+ "epoch": 0.8141414141414142,
+ "grad_norm": 1.926537275314331,
+ "learning_rate": 1.4585858585858587e-05,
+ "loss": 1.096333622932434,
+ "mean_token_accuracy": 0.7233145833015442,
+ "num_tokens": 6602752.0,
+ "step": 403
+ },
+ {
+ "entropy": 1.379700779914856,
+ "epoch": 0.8161616161616162,
+ "grad_norm": 1.957202672958374,
+ "learning_rate": 1.4572390572390573e-05,
+ "loss": 1.4035027027130127,
+ "mean_token_accuracy": 0.674462616443634,
+ "num_tokens": 6619136.0,
+ "step": 404
+ },
+ {
+ "entropy": 1.3775221109390259,
+ "epoch": 0.8181818181818182,
+ "grad_norm": 1.9095491170883179,
+ "learning_rate": 1.455892255892256e-05,
+ "loss": 1.3915553092956543,
+ "mean_token_accuracy": 0.6811187267303467,
+ "num_tokens": 6635520.0,
+ "step": 405
+ },
+ {
+ "entropy": 1.4745222330093384,
+ "epoch": 0.8202020202020202,
+ "grad_norm": 2.0017123222351074,
+ "learning_rate": 1.4545454545454546e-05,
+ "loss": 1.4845868349075317,
+ "mean_token_accuracy": 0.6557767391204834,
+ "num_tokens": 6651904.0,
+ "step": 406
+ },
+ {
+ "entropy": 1.5129855871200562,
+ "epoch": 0.8222222222222222,
+ "grad_norm": 2.127979040145874,
+ "learning_rate": 1.4531986531986533e-05,
+ "loss": 1.5081605911254883,
+ "mean_token_accuracy": 0.6425256729125977,
+ "num_tokens": 6668288.0,
+ "step": 407
+ },
+ {
+ "entropy": 1.1438099145889282,
+ "epoch": 0.8242424242424242,
+ "grad_norm": 1.7389986515045166,
+ "learning_rate": 1.4518518518518521e-05,
+ "loss": 1.1446681022644043,
+ "mean_token_accuracy": 0.7180629968643188,
+ "num_tokens": 6684672.0,
+ "step": 408
+ },
+ {
+ "entropy": 1.448157787322998,
+ "epoch": 0.8262626262626263,
+ "grad_norm": 1.8483424186706543,
+ "learning_rate": 1.4505050505050506e-05,
+ "loss": 1.4317030906677246,
+ "mean_token_accuracy": 0.6659746170043945,
+ "num_tokens": 6701056.0,
+ "step": 409
+ },
+ {
+ "entropy": 1.419940710067749,
+ "epoch": 0.8282828282828283,
+ "grad_norm": 1.961125135421753,
+ "learning_rate": 1.4491582491582492e-05,
+ "loss": 1.42207670211792,
+ "mean_token_accuracy": 0.672874927520752,
+ "num_tokens": 6717440.0,
+ "step": 410
+ },
+ {
+ "entropy": 1.5707522630691528,
+ "epoch": 0.8303030303030303,
+ "grad_norm": 2.0862255096435547,
+ "learning_rate": 1.447811447811448e-05,
+ "loss": 1.5721523761749268,
+ "mean_token_accuracy": 0.6305569410324097,
+ "num_tokens": 6733824.0,
+ "step": 411
+ },
+ {
+ "entropy": 1.4308481216430664,
+ "epoch": 0.8323232323232324,
+ "grad_norm": 2.1505820751190186,
+ "learning_rate": 1.4464646464646465e-05,
+ "loss": 1.4522666931152344,
+ "mean_token_accuracy": 0.6502198576927185,
+ "num_tokens": 6750208.0,
+ "step": 412
+ },
+ {
+ "entropy": 1.7424806356430054,
+ "epoch": 0.8343434343434344,
+ "grad_norm": 2.1787710189819336,
+ "learning_rate": 1.4451178451178452e-05,
+ "loss": 1.7719662189483643,
+ "mean_token_accuracy": 0.5897654891014099,
+ "num_tokens": 6766592.0,
+ "step": 413
+ },
+ {
+ "entropy": 1.9777010679244995,
+ "epoch": 0.8363636363636363,
+ "grad_norm": 2.1178369522094727,
+ "learning_rate": 1.443771043771044e-05,
+ "loss": 1.9885730743408203,
+ "mean_token_accuracy": 0.5628358721733093,
+ "num_tokens": 6782976.0,
+ "step": 414
+ },
+ {
+ "entropy": 1.701235055923462,
+ "epoch": 0.8383838383838383,
+ "grad_norm": 1.9038937091827393,
+ "learning_rate": 1.4424242424242425e-05,
+ "loss": 1.6970818042755127,
+ "mean_token_accuracy": 0.6209086179733276,
+ "num_tokens": 6799360.0,
+ "step": 415
+ },
+ {
+ "entropy": 1.5118876695632935,
+ "epoch": 0.8404040404040404,
+ "grad_norm": 1.8924365043640137,
+ "learning_rate": 1.4410774410774413e-05,
+ "loss": 1.5288946628570557,
+ "mean_token_accuracy": 0.6480215191841125,
+ "num_tokens": 6815744.0,
+ "step": 416
+ },
+ {
+ "entropy": 1.5526877641677856,
+ "epoch": 0.8424242424242424,
+ "grad_norm": 2.109499216079712,
+ "learning_rate": 1.43973063973064e-05,
+ "loss": 1.5581820011138916,
+ "mean_token_accuracy": 0.6351978778839111,
+ "num_tokens": 6832128.0,
+ "step": 417
+ },
+ {
+ "entropy": 1.8132506608963013,
+ "epoch": 0.8444444444444444,
+ "grad_norm": 2.0389890670776367,
+ "learning_rate": 1.4383838383838384e-05,
+ "loss": 1.8378231525421143,
+ "mean_token_accuracy": 0.592391312122345,
+ "num_tokens": 6848512.0,
+ "step": 418
+ },
+ {
+ "entropy": 1.3766752481460571,
+ "epoch": 0.8464646464646465,
+ "grad_norm": 1.8586320877075195,
+ "learning_rate": 1.4370370370370372e-05,
+ "loss": 1.3741214275360107,
+ "mean_token_accuracy": 0.6796531677246094,
+ "num_tokens": 6864896.0,
+ "step": 419
+ },
+ {
+ "entropy": 1.1854407787322998,
+ "epoch": 0.8484848484848485,
+ "grad_norm": 1.875899314880371,
+ "learning_rate": 1.4356902356902359e-05,
+ "loss": 1.2393877506256104,
+ "mean_token_accuracy": 0.704323410987854,
+ "num_tokens": 6881280.0,
+ "step": 420
+ },
+ {
+ "entropy": 1.498558759689331,
+ "epoch": 0.8505050505050505,
+ "grad_norm": 1.898848295211792,
+ "learning_rate": 1.4343434343434344e-05,
+ "loss": 1.4963322877883911,
+ "mean_token_accuracy": 0.6542501449584961,
+ "num_tokens": 6897664.0,
+ "step": 421
+ },
+ {
+ "entropy": 1.5070384740829468,
+ "epoch": 0.8525252525252526,
+ "grad_norm": 2.095853090286255,
+ "learning_rate": 1.4329966329966332e-05,
+ "loss": 1.5026702880859375,
+ "mean_token_accuracy": 0.6561431288719177,
+ "num_tokens": 6914048.0,
+ "step": 422
+ },
+ {
+ "entropy": 1.7848025560379028,
+ "epoch": 0.8545454545454545,
+ "grad_norm": 1.9265769720077515,
+ "learning_rate": 1.4316498316498317e-05,
+ "loss": 1.8066134452819824,
+ "mean_token_accuracy": 0.5936736464500427,
+ "num_tokens": 6930432.0,
+ "step": 423
+ },
+ {
+ "entropy": 1.4278018474578857,
+ "epoch": 0.8565656565656565,
+ "grad_norm": 1.900985836982727,
+ "learning_rate": 1.4303030303030305e-05,
+ "loss": 1.4045631885528564,
+ "mean_token_accuracy": 0.6630434989929199,
+ "num_tokens": 6946816.0,
+ "step": 424
+ },
+ {
+ "entropy": 1.728216290473938,
+ "epoch": 0.8585858585858586,
+ "grad_norm": 1.952841877937317,
+ "learning_rate": 1.4289562289562291e-05,
+ "loss": 1.728804349899292,
+ "mean_token_accuracy": 0.613031268119812,
+ "num_tokens": 6963200.0,
+ "step": 425
+ },
+ {
+ "entropy": 1.6632241010665894,
+ "epoch": 0.8606060606060606,
+ "grad_norm": 2.187973737716675,
+ "learning_rate": 1.4276094276094276e-05,
+ "loss": 1.6703932285308838,
+ "mean_token_accuracy": 0.6150463819503784,
+ "num_tokens": 6979584.0,
+ "step": 426
+ },
+ {
+ "entropy": 1.761826992034912,
+ "epoch": 0.8626262626262626,
+ "grad_norm": 2.203636646270752,
+ "learning_rate": 1.4262626262626264e-05,
+ "loss": 1.727257490158081,
+ "mean_token_accuracy": 0.5861626863479614,
+ "num_tokens": 6995968.0,
+ "step": 427
+ },
+ {
+ "entropy": 1.4528838396072388,
+ "epoch": 0.8646464646464647,
+ "grad_norm": 1.7967053651809692,
+ "learning_rate": 1.424915824915825e-05,
+ "loss": 1.4786958694458008,
+ "mean_token_accuracy": 0.6589521169662476,
+ "num_tokens": 7012352.0,
+ "step": 428
+ },
+ {
+ "entropy": 1.2221835851669312,
+ "epoch": 0.8666666666666667,
+ "grad_norm": 1.8118345737457275,
+ "learning_rate": 1.4235690235690235e-05,
+ "loss": 1.247727632522583,
+ "mean_token_accuracy": 0.7024303674697876,
+ "num_tokens": 7028736.0,
+ "step": 429
+ },
+ {
+ "entropy": 1.5066863298416138,
+ "epoch": 0.8686868686868687,
+ "grad_norm": 1.8390358686447144,
+ "learning_rate": 1.4222222222222224e-05,
+ "loss": 1.5283207893371582,
+ "mean_token_accuracy": 0.6553493142127991,
+ "num_tokens": 7045120.0,
+ "step": 430
+ },
+ {
+ "entropy": 1.346379041671753,
+ "epoch": 0.8707070707070707,
+ "grad_norm": 2.070962905883789,
+ "learning_rate": 1.420875420875421e-05,
+ "loss": 1.386117935180664,
+ "mean_token_accuracy": 0.6690889000892639,
+ "num_tokens": 7061504.0,
+ "step": 431
+ },
+ {
+ "entropy": 1.8691747188568115,
+ "epoch": 0.8727272727272727,
+ "grad_norm": 1.7868962287902832,
+ "learning_rate": 1.4195286195286198e-05,
+ "loss": 1.9285637140274048,
+ "mean_token_accuracy": 0.5680874586105347,
+ "num_tokens": 7077888.0,
+ "step": 432
+ },
+ {
+ "entropy": 1.8500115871429443,
+ "epoch": 0.8747474747474747,
+ "grad_norm": 1.8853468894958496,
+ "learning_rate": 1.4181818181818183e-05,
+ "loss": 1.8733386993408203,
+ "mean_token_accuracy": 0.5956888198852539,
+ "num_tokens": 7094272.0,
+ "step": 433
+ },
+ {
+ "entropy": 1.3737305402755737,
+ "epoch": 0.8767676767676768,
+ "grad_norm": 1.8363661766052246,
+ "learning_rate": 1.416835016835017e-05,
+ "loss": 1.4020421504974365,
+ "mean_token_accuracy": 0.6723253726959229,
+ "num_tokens": 7110656.0,
+ "step": 434
+ },
+ {
+ "entropy": 1.5069265365600586,
+ "epoch": 0.8787878787878788,
+ "grad_norm": 2.1154301166534424,
+ "learning_rate": 1.4154882154882156e-05,
+ "loss": 1.5137630701065063,
+ "mean_token_accuracy": 0.628724992275238,
+ "num_tokens": 7127040.0,
+ "step": 435
+ },
+ {
+ "entropy": 1.6280715465545654,
+ "epoch": 0.8808080808080808,
+ "grad_norm": 1.9827117919921875,
+ "learning_rate": 1.4141414141414143e-05,
+ "loss": 1.6629055738449097,
+ "mean_token_accuracy": 0.6300684213638306,
+ "num_tokens": 7143424.0,
+ "step": 436
+ },
+ {
+ "entropy": 1.0767987966537476,
+ "epoch": 0.8828282828282829,
+ "grad_norm": 1.8503392934799194,
+ "learning_rate": 1.4127946127946129e-05,
+ "loss": 1.0964105129241943,
+ "mean_token_accuracy": 0.7118954658508301,
+ "num_tokens": 7159808.0,
+ "step": 437
+ },
+ {
+ "entropy": 1.6979929208755493,
+ "epoch": 0.8848484848484849,
+ "grad_norm": 2.114170551300049,
+ "learning_rate": 1.4114478114478116e-05,
+ "loss": 1.7339307069778442,
+ "mean_token_accuracy": 0.6071690320968628,
+ "num_tokens": 7176192.0,
+ "step": 438
+ },
+ {
+ "entropy": 1.840633511543274,
+ "epoch": 0.8868686868686869,
+ "grad_norm": 2.1661946773529053,
+ "learning_rate": 1.4101010101010102e-05,
+ "loss": 1.866344690322876,
+ "mean_token_accuracy": 0.5988031029701233,
+ "num_tokens": 7192576.0,
+ "step": 439
+ },
+ {
+ "entropy": 1.8278590440750122,
+ "epoch": 0.8888888888888888,
+ "grad_norm": 1.9278390407562256,
+ "learning_rate": 1.4087542087542087e-05,
+ "loss": 1.81584632396698,
+ "mean_token_accuracy": 0.5878114104270935,
+ "num_tokens": 7208960.0,
+ "step": 440
+ },
+ {
+ "entropy": 1.405017375946045,
+ "epoch": 0.8909090909090909,
+ "grad_norm": 1.9446384906768799,
+ "learning_rate": 1.4074074074074075e-05,
+ "loss": 1.4040675163269043,
+ "mean_token_accuracy": 0.6647533178329468,
+ "num_tokens": 7225344.0,
+ "step": 441
+ },
+ {
+ "entropy": 1.0967023372650146,
+ "epoch": 0.8929292929292929,
+ "grad_norm": 1.8880785703659058,
+ "learning_rate": 1.4060606060606061e-05,
+ "loss": 1.0798790454864502,
+ "mean_token_accuracy": 0.7167806625366211,
+ "num_tokens": 7241728.0,
+ "step": 442
+ },
+ {
+ "entropy": 1.6126327514648438,
+ "epoch": 0.8949494949494949,
+ "grad_norm": 2.2159016132354736,
+ "learning_rate": 1.404713804713805e-05,
+ "loss": 1.620417833328247,
+ "mean_token_accuracy": 0.6215192675590515,
+ "num_tokens": 7258112.0,
+ "step": 443
+ },
+ {
+ "entropy": 1.4352757930755615,
+ "epoch": 0.896969696969697,
+ "grad_norm": 2.070910692214966,
+ "learning_rate": 1.4033670033670034e-05,
+ "loss": 1.409663200378418,
+ "mean_token_accuracy": 0.658646821975708,
+ "num_tokens": 7274496.0,
+ "step": 444
+ },
+ {
+ "entropy": 1.350129246711731,
+ "epoch": 0.898989898989899,
+ "grad_norm": 1.8672778606414795,
+ "learning_rate": 1.4020202020202021e-05,
+ "loss": 1.3485581874847412,
+ "mean_token_accuracy": 0.6743404865264893,
+ "num_tokens": 7290880.0,
+ "step": 445
+ },
+ {
+ "entropy": 1.0900049209594727,
+ "epoch": 0.901010101010101,
+ "grad_norm": 1.8460185527801514,
+ "learning_rate": 1.4006734006734009e-05,
+ "loss": 1.074802279472351,
+ "mean_token_accuracy": 0.727161705493927,
+ "num_tokens": 7307264.0,
+ "step": 446
+ },
+ {
+ "entropy": 1.6430293321609497,
+ "epoch": 0.9030303030303031,
+ "grad_norm": 2.0190517902374268,
+ "learning_rate": 1.3993265993265994e-05,
+ "loss": 1.6505115032196045,
+ "mean_token_accuracy": 0.6073521971702576,
+ "num_tokens": 7323648.0,
+ "step": 447
+ },
+ {
+ "entropy": 1.465686559677124,
+ "epoch": 0.9050505050505051,
+ "grad_norm": 2.0139927864074707,
+ "learning_rate": 1.397979797979798e-05,
+ "loss": 1.480743408203125,
+ "mean_token_accuracy": 0.6593796014785767,
+ "num_tokens": 7340032.0,
+ "step": 448
+ },
+ {
+ "entropy": 2.152183771133423,
+ "epoch": 0.907070707070707,
+ "grad_norm": 2.2376039028167725,
+ "learning_rate": 1.3966329966329969e-05,
+ "loss": 2.117891788482666,
+ "mean_token_accuracy": 0.5404250025749207,
+ "num_tokens": 7356416.0,
+ "step": 449
+ },
+ {
+ "entropy": 1.6066724061965942,
+ "epoch": 0.9090909090909091,
+ "grad_norm": 1.895409107208252,
+ "learning_rate": 1.3952861952861953e-05,
+ "loss": 1.6247167587280273,
+ "mean_token_accuracy": 0.6281143426895142,
+ "num_tokens": 7372800.0,
+ "step": 450
+ },
+ {
+ "entropy": 1.7240369319915771,
+ "epoch": 0.9111111111111111,
+ "grad_norm": 1.9780583381652832,
+ "learning_rate": 1.3939393939393942e-05,
+ "loss": 1.7521916627883911,
+ "mean_token_accuracy": 0.6022838354110718,
+ "num_tokens": 7389184.0,
+ "step": 451
+ },
+ {
+ "entropy": 1.168492317199707,
+ "epoch": 0.9131313131313131,
+ "grad_norm": 1.676292896270752,
+ "learning_rate": 1.3925925925925928e-05,
+ "loss": 1.1678345203399658,
+ "mean_token_accuracy": 0.7136663198471069,
+ "num_tokens": 7405568.0,
+ "step": 452
+ },
+ {
+ "entropy": 1.4993922710418701,
+ "epoch": 0.9151515151515152,
+ "grad_norm": 2.051370620727539,
+ "learning_rate": 1.3912457912457913e-05,
+ "loss": 1.5273159742355347,
+ "mean_token_accuracy": 0.6378847360610962,
+ "num_tokens": 7421952.0,
+ "step": 453
+ },
+ {
+ "entropy": 1.3478108644485474,
+ "epoch": 0.9171717171717172,
+ "grad_norm": 1.9196240901947021,
+ "learning_rate": 1.3898989898989901e-05,
+ "loss": 1.3547455072402954,
+ "mean_token_accuracy": 0.666829526424408,
+ "num_tokens": 7438336.0,
+ "step": 454
+ },
+ {
+ "entropy": 1.1411141157150269,
+ "epoch": 0.9191919191919192,
+ "grad_norm": 1.9581170082092285,
+ "learning_rate": 1.3885521885521886e-05,
+ "loss": 1.1604664325714111,
+ "mean_token_accuracy": 0.7018197178840637,
+ "num_tokens": 7454720.0,
+ "step": 455
+ },
+ {
+ "entropy": 1.1814258098602295,
+ "epoch": 0.9212121212121213,
+ "grad_norm": 2.005451202392578,
+ "learning_rate": 1.3872053872053872e-05,
+ "loss": 1.1768745183944702,
+ "mean_token_accuracy": 0.7044455409049988,
+ "num_tokens": 7471104.0,
+ "step": 456
+ },
+ {
+ "entropy": 1.335093379020691,
+ "epoch": 0.9232323232323232,
+ "grad_norm": 1.928747534751892,
+ "learning_rate": 1.385858585858586e-05,
+ "loss": 1.3504598140716553,
+ "mean_token_accuracy": 0.6774548292160034,
+ "num_tokens": 7487488.0,
+ "step": 457
+ },
+ {
+ "entropy": 1.6866769790649414,
+ "epoch": 0.9252525252525252,
+ "grad_norm": 1.9078223705291748,
+ "learning_rate": 1.3845117845117845e-05,
+ "loss": 1.688689947128296,
+ "mean_token_accuracy": 0.6077796816825867,
+ "num_tokens": 7503872.0,
+ "step": 458
+ },
+ {
+ "entropy": 1.2842025756835938,
+ "epoch": 0.9272727272727272,
+ "grad_norm": 1.909472942352295,
+ "learning_rate": 1.3831649831649833e-05,
+ "loss": 1.3128941059112549,
+ "mean_token_accuracy": 0.6872252225875854,
+ "num_tokens": 7520256.0,
+ "step": 459
+ },
+ {
+ "entropy": 1.7797539234161377,
+ "epoch": 0.9292929292929293,
+ "grad_norm": 2.246411085128784,
+ "learning_rate": 1.381818181818182e-05,
+ "loss": 1.8016175031661987,
+ "mean_token_accuracy": 0.6049095988273621,
+ "num_tokens": 7536640.0,
+ "step": 460
+ },
+ {
+ "entropy": 1.3464831113815308,
+ "epoch": 0.9313131313131313,
+ "grad_norm": 1.9999858140945435,
+ "learning_rate": 1.3804713804713805e-05,
+ "loss": 1.3523657321929932,
+ "mean_token_accuracy": 0.6787371635437012,
+ "num_tokens": 7553024.0,
+ "step": 461
+ },
+ {
+ "entropy": 1.5901066064834595,
+ "epoch": 0.9333333333333333,
+ "grad_norm": 1.9632524251937866,
+ "learning_rate": 1.3791245791245793e-05,
+ "loss": 1.596254587173462,
+ "mean_token_accuracy": 0.6428309679031372,
+ "num_tokens": 7569408.0,
+ "step": 462
+ },
+ {
+ "entropy": 1.2511993646621704,
+ "epoch": 0.9353535353535354,
+ "grad_norm": 1.7678464651107788,
+ "learning_rate": 1.377777777777778e-05,
+ "loss": 1.254664659500122,
+ "mean_token_accuracy": 0.6981558203697205,
+ "num_tokens": 7585792.0,
+ "step": 463
+ },
+ {
+ "entropy": 1.546912431716919,
+ "epoch": 0.9373737373737374,
+ "grad_norm": 2.0353212356567383,
+ "learning_rate": 1.3764309764309764e-05,
+ "loss": 1.54923677444458,
+ "mean_token_accuracy": 0.6403273344039917,
+ "num_tokens": 7602176.0,
+ "step": 464
+ },
+ {
+ "entropy": 1.6535273790359497,
+ "epoch": 0.9393939393939394,
+ "grad_norm": 2.1584854125976562,
+ "learning_rate": 1.3750841750841752e-05,
+ "loss": 1.6464964151382446,
+ "mean_token_accuracy": 0.6218246221542358,
+ "num_tokens": 7618560.0,
+ "step": 465
+ },
+ {
+ "entropy": 1.1885185241699219,
+ "epoch": 0.9414141414141414,
+ "grad_norm": 1.8068963289260864,
+ "learning_rate": 1.3737373737373739e-05,
+ "loss": 1.1980109214782715,
+ "mean_token_accuracy": 0.6990718245506287,
+ "num_tokens": 7634944.0,
+ "step": 466
+ },
+ {
+ "entropy": 1.2306157350540161,
+ "epoch": 0.9434343434343434,
+ "grad_norm": 1.9044586420059204,
+ "learning_rate": 1.3723905723905725e-05,
+ "loss": 1.2291686534881592,
+ "mean_token_accuracy": 0.6996824741363525,
+ "num_tokens": 7651328.0,
+ "step": 467
+ },
+ {
+ "entropy": 1.2496092319488525,
+ "epoch": 0.9454545454545454,
+ "grad_norm": 1.7264868021011353,
+ "learning_rate": 1.3710437710437712e-05,
+ "loss": 1.2440049648284912,
+ "mean_token_accuracy": 0.6877137422561646,
+ "num_tokens": 7667712.0,
+ "step": 468
+ },
+ {
+ "entropy": 1.8231102228164673,
+ "epoch": 0.9474747474747475,
+ "grad_norm": 2.099104166030884,
+ "learning_rate": 1.3696969696969698e-05,
+ "loss": 1.84971284866333,
+ "mean_token_accuracy": 0.5869565010070801,
+ "num_tokens": 7684096.0,
+ "step": 469
+ },
+ {
+ "entropy": 1.537786841392517,
+ "epoch": 0.9494949494949495,
+ "grad_norm": 1.872962474822998,
+ "learning_rate": 1.3683501683501685e-05,
+ "loss": 1.5448193550109863,
+ "mean_token_accuracy": 0.6465559601783752,
+ "num_tokens": 7700480.0,
+ "step": 470
+ },
+ {
+ "entropy": 1.8270875215530396,
+ "epoch": 0.9515151515151515,
+ "grad_norm": 2.2747695446014404,
+ "learning_rate": 1.3670033670033671e-05,
+ "loss": 1.854095697402954,
+ "mean_token_accuracy": 0.5834758281707764,
+ "num_tokens": 7716864.0,
+ "step": 471
+ },
+ {
+ "entropy": 1.505604863166809,
+ "epoch": 0.9535353535353536,
+ "grad_norm": 1.9903687238693237,
+ "learning_rate": 1.3656565656565656e-05,
+ "loss": 1.5269451141357422,
+ "mean_token_accuracy": 0.6475940346717834,
+ "num_tokens": 7733248.0,
+ "step": 472
+ },
+ {
+ "entropy": 1.8228932619094849,
+ "epoch": 0.9555555555555556,
+ "grad_norm": 2.4003078937530518,
+ "learning_rate": 1.3643097643097644e-05,
+ "loss": 1.846016764640808,
+ "mean_token_accuracy": 0.5900097489356995,
+ "num_tokens": 7749632.0,
+ "step": 473
+ },
+ {
+ "entropy": 1.2786308526992798,
+ "epoch": 0.9575757575757575,
+ "grad_norm": 1.9355131387710571,
+ "learning_rate": 1.362962962962963e-05,
+ "loss": 1.3152062892913818,
+ "mean_token_accuracy": 0.6703712940216064,
+ "num_tokens": 7766016.0,
+ "step": 474
+ },
+ {
+ "entropy": 1.1032856702804565,
+ "epoch": 0.9595959595959596,
+ "grad_norm": 1.7720186710357666,
+ "learning_rate": 1.3616161616161619e-05,
+ "loss": 1.1242918968200684,
+ "mean_token_accuracy": 0.7250854969024658,
+ "num_tokens": 7782400.0,
+ "step": 475
+ },
+ {
+ "entropy": 1.5261021852493286,
+ "epoch": 0.9616161616161616,
+ "grad_norm": 1.8472843170166016,
+ "learning_rate": 1.3602693602693604e-05,
+ "loss": 1.54013991355896,
+ "mean_token_accuracy": 0.6471055150032043,
+ "num_tokens": 7798784.0,
+ "step": 476
+ },
+ {
+ "entropy": 0.9697252511978149,
+ "epoch": 0.9636363636363636,
+ "grad_norm": 1.8871296644210815,
+ "learning_rate": 1.358922558922559e-05,
+ "loss": 0.9874091148376465,
+ "mean_token_accuracy": 0.7430996298789978,
+ "num_tokens": 7815168.0,
+ "step": 477
+ },
+ {
+ "entropy": 1.6003227233886719,
+ "epoch": 0.9656565656565657,
+ "grad_norm": 2.371385097503662,
+ "learning_rate": 1.3575757575757578e-05,
+ "loss": 1.580587387084961,
+ "mean_token_accuracy": 0.6273815631866455,
+ "num_tokens": 7831552.0,
+ "step": 478
+ },
+ {
+ "entropy": 1.556900978088379,
+ "epoch": 0.9676767676767677,
+ "grad_norm": 1.839432954788208,
+ "learning_rate": 1.3562289562289563e-05,
+ "loss": 1.5154083967208862,
+ "mean_token_accuracy": 0.628724992275238,
+ "num_tokens": 7847936.0,
+ "step": 479
+ },
+ {
+ "entropy": 1.2394448518753052,
+ "epoch": 0.9696969696969697,
+ "grad_norm": 1.6638867855072021,
+ "learning_rate": 1.354882154882155e-05,
+ "loss": 1.2016921043395996,
+ "mean_token_accuracy": 0.6991939544677734,
+ "num_tokens": 7864320.0,
+ "step": 480
+ },
+ {
+ "entropy": 1.4200247526168823,
+ "epoch": 0.9717171717171718,
+ "grad_norm": 1.8742305040359497,
+ "learning_rate": 1.3535353535353538e-05,
+ "loss": 1.4267700910568237,
+ "mean_token_accuracy": 0.6604176759719849,
+ "num_tokens": 7880704.0,
+ "step": 481
+ },
+ {
+ "entropy": 2.0156707763671875,
+ "epoch": 0.9737373737373738,
+ "grad_norm": 2.1772451400756836,
+ "learning_rate": 1.3521885521885523e-05,
+ "loss": 2.0306758880615234,
+ "mean_token_accuracy": 0.5579506754875183,
+ "num_tokens": 7897088.0,
+ "step": 482
+ },
+ {
+ "entropy": 1.2802858352661133,
+ "epoch": 0.9757575757575757,
+ "grad_norm": 1.8951549530029297,
+ "learning_rate": 1.3508417508417509e-05,
+ "loss": 1.2761621475219727,
+ "mean_token_accuracy": 0.6825231909751892,
+ "num_tokens": 7913472.0,
+ "step": 483
+ },
+ {
+ "entropy": 1.3758989572525024,
+ "epoch": 0.9777777777777777,
+ "grad_norm": 1.8072963953018188,
+ "learning_rate": 1.3494949494949497e-05,
+ "loss": 1.3532235622406006,
+ "mean_token_accuracy": 0.6749511361122131,
+ "num_tokens": 7929856.0,
+ "step": 484
+ },
+ {
+ "entropy": 1.4488792419433594,
+ "epoch": 0.9797979797979798,
+ "grad_norm": 1.7842215299606323,
+ "learning_rate": 1.3481481481481482e-05,
+ "loss": 1.4513285160064697,
+ "mean_token_accuracy": 0.6539447903633118,
+ "num_tokens": 7946240.0,
+ "step": 485
+ },
+ {
+ "entropy": 1.2957613468170166,
+ "epoch": 0.9818181818181818,
+ "grad_norm": 1.8612068891525269,
+ "learning_rate": 1.346801346801347e-05,
+ "loss": 1.3016514778137207,
+ "mean_token_accuracy": 0.6848436594009399,
+ "num_tokens": 7962624.0,
+ "step": 486
+ },
+ {
+ "entropy": 1.4321255683898926,
+ "epoch": 0.9838383838383838,
+ "grad_norm": 1.7746661901474,
+ "learning_rate": 1.3454545454545455e-05,
+ "loss": 1.4230387210845947,
+ "mean_token_accuracy": 0.664692223072052,
+ "num_tokens": 7979008.0,
+ "step": 487
+ },
+ {
+ "entropy": 1.7615760564804077,
+ "epoch": 0.9858585858585859,
+ "grad_norm": 1.9986366033554077,
+ "learning_rate": 1.3441077441077441e-05,
+ "loss": 1.7871575355529785,
+ "mean_token_accuracy": 0.5994748473167419,
+ "num_tokens": 7995392.0,
+ "step": 488
+ },
+ {
+ "entropy": 1.349948763847351,
+ "epoch": 0.9878787878787879,
+ "grad_norm": 2.0217247009277344,
+ "learning_rate": 1.342760942760943e-05,
+ "loss": 1.352069616317749,
+ "mean_token_accuracy": 0.6762335300445557,
+ "num_tokens": 8011776.0,
+ "step": 489
+ },
+ {
+ "entropy": 1.6317073106765747,
+ "epoch": 0.98989898989899,
+ "grad_norm": 1.9479029178619385,
+ "learning_rate": 1.3414141414141414e-05,
+ "loss": 1.6507500410079956,
+ "mean_token_accuracy": 0.6186492443084717,
+ "num_tokens": 8028160.0,
+ "step": 490
+ },
+ {
+ "entropy": 2.0288875102996826,
+ "epoch": 0.9919191919191919,
+ "grad_norm": 2.1520888805389404,
+ "learning_rate": 1.3400673400673401e-05,
+ "loss": 2.06215500831604,
+ "mean_token_accuracy": 0.5530043840408325,
+ "num_tokens": 8044544.0,
+ "step": 491
+ },
+ {
+ "entropy": 1.8297137022018433,
+ "epoch": 0.9939393939393939,
+ "grad_norm": 1.949404001235962,
+ "learning_rate": 1.3387205387205389e-05,
+ "loss": 1.8859628438949585,
+ "mean_token_accuracy": 0.5876282453536987,
+ "num_tokens": 8060928.0,
+ "step": 492
+ },
+ {
+ "entropy": 1.587094783782959,
+ "epoch": 0.9959595959595959,
+ "grad_norm": 1.9205344915390015,
+ "learning_rate": 1.3373737373737374e-05,
+ "loss": 1.6053454875946045,
+ "mean_token_accuracy": 0.6257938742637634,
+ "num_tokens": 8077312.0,
+ "step": 493
+ },
+ {
+ "entropy": 1.6583648920059204,
+ "epoch": 0.997979797979798,
+ "grad_norm": 1.976946234703064,
+ "learning_rate": 1.3360269360269362e-05,
+ "loss": 1.6656805276870728,
+ "mean_token_accuracy": 0.6186492443084717,
+ "num_tokens": 8093696.0,
+ "step": 494
+ },
+ {
+ "entropy": 1.1685364246368408,
+ "epoch": 1.0,
+ "grad_norm": 1.8258967399597168,
+ "learning_rate": 1.3346801346801349e-05,
+ "loss": 1.165276288986206,
+ "mean_token_accuracy": 0.7082315683364868,
+ "num_tokens": 8110080.0,
+ "step": 495
+ },
+ {
+ "entropy": 1.1211013793945312,
+ "epoch": 1.002020202020202,
+ "grad_norm": 2.27962327003479,
+ "learning_rate": 1.3333333333333333e-05,
+ "loss": 0.9540231227874756,
+ "mean_token_accuracy": 0.747801661491394,
+ "num_tokens": 8126464.0,
+ "step": 496
+ },
+ {
+ "entropy": 1.370996356010437,
+ "epoch": 1.004040404040404,
+ "grad_norm": 2.3193519115448,
+ "learning_rate": 1.3319865319865321e-05,
+ "loss": 1.240065097808838,
+ "mean_token_accuracy": 0.6976062655448914,
+ "num_tokens": 8142848.0,
+ "step": 497
+ },
+ {
+ "entropy": 1.4673359394073486,
+ "epoch": 1.006060606060606,
+ "grad_norm": 2.071213483810425,
+ "learning_rate": 1.3306397306397308e-05,
+ "loss": 1.3351866006851196,
+ "mean_token_accuracy": 0.6921104192733765,
+ "num_tokens": 8159232.0,
+ "step": 498
+ },
+ {
+ "entropy": 1.1151381731033325,
+ "epoch": 1.0080808080808081,
+ "grad_norm": 2.1310431957244873,
+ "learning_rate": 1.3292929292929293e-05,
+ "loss": 1.0092531442642212,
+ "mean_token_accuracy": 0.7261846661567688,
+ "num_tokens": 8175616.0,
+ "step": 499
+ },
+ {
+ "entropy": 1.4734766483306885,
+ "epoch": 1.0101010101010102,
+ "grad_norm": 2.015787124633789,
+ "learning_rate": 1.3279461279461281e-05,
+ "loss": 1.3748562335968018,
+ "mean_token_accuracy": 0.6682339906692505,
+ "num_tokens": 8192000.0,
+ "step": 500
+ },
+ {
+ "epoch": 1.0101010101010102,
+ "eval_entropy": 1.3614274196086391,
+ "eval_loss": 1.5201176404953003,
+ "eval_mean_token_accuracy": 0.647282806134993,
+ "eval_num_tokens": 8192000.0,
+ "eval_runtime": 43.8232,
+ "eval_samples_per_second": 22.591,
+ "eval_steps_per_second": 2.83,
+ "step": 500
+ },
+ {
+ "entropy": 1.3656656742095947,
+ "epoch": 1.0121212121212122,
+ "grad_norm": 2.102975845336914,
+ "learning_rate": 1.3265993265993267e-05,
+ "loss": 1.2834689617156982,
+ "mean_token_accuracy": 0.6831338405609131,
+ "num_tokens": 8208384.0,
+ "step": 501
+ },
+ {
+ "entropy": 1.0829076766967773,
+ "epoch": 1.0141414141414142,
+ "grad_norm": 1.8827977180480957,
+ "learning_rate": 1.3252525252525254e-05,
+ "loss": 1.070042610168457,
+ "mean_token_accuracy": 0.72007817029953,
+ "num_tokens": 8224768.0,
+ "step": 502
+ },
+ {
+ "entropy": 1.2298822402954102,
+ "epoch": 1.0161616161616163,
+ "grad_norm": 1.9913817644119263,
+ "learning_rate": 1.323905723905724e-05,
+ "loss": 1.2619516849517822,
+ "mean_token_accuracy": 0.7024303674697876,
+ "num_tokens": 8241152.0,
+ "step": 503
+ },
+ {
+ "entropy": 0.9216552972793579,
+ "epoch": 1.018181818181818,
+ "grad_norm": 1.8755308389663696,
+ "learning_rate": 1.3225589225589225e-05,
+ "loss": 0.9175798892974854,
+ "mean_token_accuracy": 0.7650830745697021,
+ "num_tokens": 8257536.0,
+ "step": 504
+ },
+ {
+ "entropy": 0.7169107794761658,
+ "epoch": 1.02020202020202,
+ "grad_norm": 2.100609064102173,
+ "learning_rate": 1.3212121212121213e-05,
+ "loss": 0.7609860897064209,
+ "mean_token_accuracy": 0.7913410067558289,
+ "num_tokens": 8273920.0,
+ "step": 505
+ },
+ {
+ "entropy": 1.5552464723587036,
+ "epoch": 1.0222222222222221,
+ "grad_norm": 2.5406126976013184,
+ "learning_rate": 1.31986531986532e-05,
+ "loss": 1.6440303325653076,
+ "mean_token_accuracy": 0.6193820238113403,
+ "num_tokens": 8290304.0,
+ "step": 506
+ },
+ {
+ "entropy": 0.9679869413375854,
+ "epoch": 1.0242424242424242,
+ "grad_norm": 2.2049074172973633,
+ "learning_rate": 1.3185185185185185e-05,
+ "loss": 1.0273747444152832,
+ "mean_token_accuracy": 0.729421079158783,
+ "num_tokens": 8306688.0,
+ "step": 507
+ },
+ {
+ "entropy": 1.5491999387741089,
+ "epoch": 1.0262626262626262,
+ "grad_norm": 2.2690296173095703,
+ "learning_rate": 1.3171717171717173e-05,
+ "loss": 1.6104233264923096,
+ "mean_token_accuracy": 0.6261602640151978,
+ "num_tokens": 8323072.0,
+ "step": 508
+ },
+ {
+ "entropy": 1.157227635383606,
+ "epoch": 1.0282828282828282,
+ "grad_norm": 2.5081140995025635,
+ "learning_rate": 1.315824915824916e-05,
+ "loss": 1.2283058166503906,
+ "mean_token_accuracy": 0.7013311982154846,
+ "num_tokens": 8339456.0,
+ "step": 509
+ },
+ {
+ "entropy": 1.2934587001800537,
+ "epoch": 1.0303030303030303,
+ "grad_norm": 2.0315630435943604,
+ "learning_rate": 1.3144781144781147e-05,
+ "loss": 1.313960075378418,
+ "mean_token_accuracy": 0.6683561205863953,
+ "num_tokens": 8355840.0,
+ "step": 510
+ },
+ {
+ "entropy": 1.058363437652588,
+ "epoch": 1.0323232323232323,
+ "grad_norm": 2.0326106548309326,
+ "learning_rate": 1.3131313131313132e-05,
+ "loss": 1.060279130935669,
+ "mean_token_accuracy": 0.7263678312301636,
+ "num_tokens": 8372224.0,
+ "step": 511
+ },
+ {
+ "entropy": 1.166068196296692,
+ "epoch": 1.0343434343434343,
+ "grad_norm": 1.9384857416152954,
+ "learning_rate": 1.3117845117845119e-05,
+ "loss": 1.1466734409332275,
+ "mean_token_accuracy": 0.7182462215423584,
+ "num_tokens": 8388608.0,
+ "step": 512
+ },
+ {
+ "entropy": 1.1295984983444214,
+ "epoch": 1.0363636363636364,
+ "grad_norm": 2.0500752925872803,
+ "learning_rate": 1.3104377104377107e-05,
+ "loss": 1.1119377613067627,
+ "mean_token_accuracy": 0.7249022722244263,
+ "num_tokens": 8404992.0,
+ "step": 513
+ },
+ {
+ "entropy": 1.2471498250961304,
+ "epoch": 1.0383838383838384,
+ "grad_norm": 2.057642698287964,
+ "learning_rate": 1.3090909090909092e-05,
+ "loss": 1.2348787784576416,
+ "mean_token_accuracy": 0.69840008020401,
+ "num_tokens": 8421376.0,
+ "step": 514
+ },
+ {
+ "entropy": 1.2146048545837402,
+ "epoch": 1.0404040404040404,
+ "grad_norm": 1.8724642992019653,
+ "learning_rate": 1.3077441077441078e-05,
+ "loss": 1.1992437839508057,
+ "mean_token_accuracy": 0.7179408669471741,
+ "num_tokens": 8437760.0,
+ "step": 515
+ },
+ {
+ "entropy": 1.119163990020752,
+ "epoch": 1.0424242424242425,
+ "grad_norm": 2.013307571411133,
+ "learning_rate": 1.3063973063973066e-05,
+ "loss": 1.0880366563796997,
+ "mean_token_accuracy": 0.7274059653282166,
+ "num_tokens": 8454144.0,
+ "step": 516
+ },
+ {
+ "entropy": 1.055690884590149,
+ "epoch": 1.0444444444444445,
+ "grad_norm": 2.9534032344818115,
+ "learning_rate": 1.3050505050505051e-05,
+ "loss": 1.0625638961791992,
+ "mean_token_accuracy": 0.7302759885787964,
+ "num_tokens": 8470528.0,
+ "step": 517
+ },
+ {
+ "entropy": 1.629905343055725,
+ "epoch": 1.0464646464646465,
+ "grad_norm": 1.9111285209655762,
+ "learning_rate": 1.303703703703704e-05,
+ "loss": 1.642975926399231,
+ "mean_token_accuracy": 0.638067901134491,
+ "num_tokens": 8486912.0,
+ "step": 518
+ },
+ {
+ "entropy": 1.2197786569595337,
+ "epoch": 1.0484848484848486,
+ "grad_norm": 2.0819573402404785,
+ "learning_rate": 1.3023569023569024e-05,
+ "loss": 1.178088903427124,
+ "mean_token_accuracy": 0.7040180563926697,
+ "num_tokens": 8503296.0,
+ "step": 519
+ },
+ {
+ "entropy": 1.4044541120529175,
+ "epoch": 1.0505050505050506,
+ "grad_norm": 9.860201835632324,
+ "learning_rate": 1.301010101010101e-05,
+ "loss": 1.6388373374938965,
+ "mean_token_accuracy": 0.6572422981262207,
+ "num_tokens": 8519680.0,
+ "step": 520
+ },
+ {
+ "entropy": 0.9115698337554932,
+ "epoch": 1.0525252525252524,
+ "grad_norm": 2.1228489875793457,
+ "learning_rate": 1.2996632996632999e-05,
+ "loss": 0.8868257999420166,
+ "mean_token_accuracy": 0.7558622360229492,
+ "num_tokens": 8536064.0,
+ "step": 521
+ },
+ {
+ "entropy": 1.0348355770111084,
+ "epoch": 1.0545454545454545,
+ "grad_norm": 1.8459810018539429,
+ "learning_rate": 1.2983164983164984e-05,
+ "loss": 1.0149322748184204,
+ "mean_token_accuracy": 0.7421836853027344,
+ "num_tokens": 8552448.0,
+ "step": 522
+ },
+ {
+ "entropy": 1.0639865398406982,
+ "epoch": 1.0565656565656565,
+ "grad_norm": 2.0999417304992676,
+ "learning_rate": 1.296969696969697e-05,
+ "loss": 1.0582343339920044,
+ "mean_token_accuracy": 0.7242916226387024,
+ "num_tokens": 8568832.0,
+ "step": 523
+ },
+ {
+ "entropy": 1.2032114267349243,
+ "epoch": 1.0585858585858585,
+ "grad_norm": 2.0500640869140625,
+ "learning_rate": 1.2956228956228958e-05,
+ "loss": 1.2051773071289062,
+ "mean_token_accuracy": 0.7076819539070129,
+ "num_tokens": 8585216.0,
+ "step": 524
+ },
+ {
+ "entropy": 1.2425649166107178,
+ "epoch": 1.0606060606060606,
+ "grad_norm": 2.1104960441589355,
+ "learning_rate": 1.2942760942760943e-05,
+ "loss": 1.2576614618301392,
+ "mean_token_accuracy": 0.6907669901847839,
+ "num_tokens": 8601600.0,
+ "step": 525
+ },
+ {
+ "entropy": 1.2280266284942627,
+ "epoch": 1.0626262626262626,
+ "grad_norm": 2.0641212463378906,
+ "learning_rate": 1.2929292929292931e-05,
+ "loss": 1.2378020286560059,
+ "mean_token_accuracy": 0.6943697929382324,
+ "num_tokens": 8617984.0,
+ "step": 526
+ },
+ {
+ "entropy": 0.9312539100646973,
+ "epoch": 1.0646464646464646,
+ "grad_norm": 2.0006070137023926,
+ "learning_rate": 1.2915824915824918e-05,
+ "loss": 0.952136754989624,
+ "mean_token_accuracy": 0.7538471221923828,
+ "num_tokens": 8634368.0,
+ "step": 527
+ },
+ {
+ "entropy": 1.5708260536193848,
+ "epoch": 1.0666666666666667,
+ "grad_norm": 1.873006820678711,
+ "learning_rate": 1.2902356902356902e-05,
+ "loss": 1.5705666542053223,
+ "mean_token_accuracy": 0.6449071764945984,
+ "num_tokens": 8650752.0,
+ "step": 528
+ },
+ {
+ "entropy": 0.9064179062843323,
+ "epoch": 1.0686868686868687,
+ "grad_norm": 2.109816551208496,
+ "learning_rate": 1.288888888888889e-05,
+ "loss": 0.9165636301040649,
+ "mean_token_accuracy": 0.754824161529541,
+ "num_tokens": 8667136.0,
+ "step": 529
+ },
+ {
+ "entropy": 1.4197264909744263,
+ "epoch": 1.0707070707070707,
+ "grad_norm": 2.2213833332061768,
+ "learning_rate": 1.2875420875420877e-05,
+ "loss": 1.4330552816390991,
+ "mean_token_accuracy": 0.6589521169662476,
+ "num_tokens": 8683520.0,
+ "step": 530
+ },
+ {
+ "entropy": 1.3674145936965942,
+ "epoch": 1.0727272727272728,
+ "grad_norm": 2.2046334743499756,
+ "learning_rate": 1.2861952861952862e-05,
+ "loss": 1.3838258981704712,
+ "mean_token_accuracy": 0.6660356521606445,
+ "num_tokens": 8699904.0,
+ "step": 531
+ },
+ {
+ "entropy": 1.0396678447723389,
+ "epoch": 1.0747474747474748,
+ "grad_norm": 1.9246002435684204,
+ "learning_rate": 1.284848484848485e-05,
+ "loss": 1.0304534435272217,
+ "mean_token_accuracy": 0.7421836853027344,
+ "num_tokens": 8716288.0,
+ "step": 532
+ },
+ {
+ "entropy": 0.8541752696037292,
+ "epoch": 1.0767676767676768,
+ "grad_norm": 1.9644383192062378,
+ "learning_rate": 1.2835016835016837e-05,
+ "loss": 0.8437563180923462,
+ "mean_token_accuracy": 0.7683805823326111,
+ "num_tokens": 8732672.0,
+ "step": 533
+ },
+ {
+ "entropy": 0.962359607219696,
+ "epoch": 1.0787878787878789,
+ "grad_norm": 1.8767060041427612,
+ "learning_rate": 1.2821548821548821e-05,
+ "loss": 0.9354047775268555,
+ "mean_token_accuracy": 0.7493893504142761,
+ "num_tokens": 8749056.0,
+ "step": 534
+ },
+ {
+ "entropy": 1.1609500646591187,
+ "epoch": 1.0808080808080809,
+ "grad_norm": 2.316633939743042,
+ "learning_rate": 1.280808080808081e-05,
+ "loss": 1.1338696479797363,
+ "mean_token_accuracy": 0.7132388949394226,
+ "num_tokens": 8765440.0,
+ "step": 535
+ },
+ {
+ "entropy": 0.9635956883430481,
+ "epoch": 1.082828282828283,
+ "grad_norm": 1.8842763900756836,
+ "learning_rate": 1.2794612794612794e-05,
+ "loss": 0.9779557585716248,
+ "mean_token_accuracy": 0.7515876889228821,
+ "num_tokens": 8781824.0,
+ "step": 536
+ },
+ {
+ "entropy": 1.5283102989196777,
+ "epoch": 1.084848484848485,
+ "grad_norm": 2.2277283668518066,
+ "learning_rate": 1.2781144781144783e-05,
+ "loss": 1.5118470191955566,
+ "mean_token_accuracy": 0.6464338302612305,
+ "num_tokens": 8798208.0,
+ "step": 537
+ },
+ {
+ "entropy": 1.1301655769348145,
+ "epoch": 1.0868686868686868,
+ "grad_norm": 2.0282225608825684,
+ "learning_rate": 1.2767676767676769e-05,
+ "loss": 1.0959392786026,
+ "mean_token_accuracy": 0.7204445600509644,
+ "num_tokens": 8814592.0,
+ "step": 538
+ },
+ {
+ "entropy": 1.3107094764709473,
+ "epoch": 1.0888888888888888,
+ "grad_norm": 1.9527665376663208,
+ "learning_rate": 1.2754208754208754e-05,
+ "loss": 1.2717593908309937,
+ "mean_token_accuracy": 0.6916218996047974,
+ "num_tokens": 8830976.0,
+ "step": 539
+ },
+ {
+ "entropy": 1.3232251405715942,
+ "epoch": 1.0909090909090908,
+ "grad_norm": 2.134962320327759,
+ "learning_rate": 1.2740740740740742e-05,
+ "loss": 1.3589959144592285,
+ "mean_token_accuracy": 0.6701880693435669,
+ "num_tokens": 8847360.0,
+ "step": 540
+ },
+ {
+ "entropy": 1.5372718572616577,
+ "epoch": 1.0929292929292929,
+ "grad_norm": 1.8864489793777466,
+ "learning_rate": 1.2727272727272728e-05,
+ "loss": 1.5291733741760254,
+ "mean_token_accuracy": 0.6487542986869812,
+ "num_tokens": 8863744.0,
+ "step": 541
+ },
+ {
+ "entropy": 1.2037636041641235,
+ "epoch": 1.094949494949495,
+ "grad_norm": 1.8530166149139404,
+ "learning_rate": 1.2713804713804713e-05,
+ "loss": 1.1905983686447144,
+ "mean_token_accuracy": 0.6986443400382996,
+ "num_tokens": 8880128.0,
+ "step": 542
+ },
+ {
+ "entropy": 1.256689429283142,
+ "epoch": 1.096969696969697,
+ "grad_norm": 1.9364465475082397,
+ "learning_rate": 1.2700336700336701e-05,
+ "loss": 1.275427222251892,
+ "mean_token_accuracy": 0.689728856086731,
+ "num_tokens": 8896512.0,
+ "step": 543
+ },
+ {
+ "entropy": 1.154582142829895,
+ "epoch": 1.098989898989899,
+ "grad_norm": 2.097226619720459,
+ "learning_rate": 1.2686868686868688e-05,
+ "loss": 1.1638195514678955,
+ "mean_token_accuracy": 0.7220322489738464,
+ "num_tokens": 8912896.0,
+ "step": 544
+ },
+ {
+ "entropy": 1.3060903549194336,
+ "epoch": 1.101010101010101,
+ "grad_norm": 2.1401753425598145,
+ "learning_rate": 1.2673400673400676e-05,
+ "loss": 1.337110996246338,
+ "mean_token_accuracy": 0.6781876087188721,
+ "num_tokens": 8929280.0,
+ "step": 545
+ },
+ {
+ "entropy": 1.2550262212753296,
+ "epoch": 1.103030303030303,
+ "grad_norm": 1.8216264247894287,
+ "learning_rate": 1.2659932659932661e-05,
+ "loss": 1.2674927711486816,
+ "mean_token_accuracy": 0.6860649585723877,
+ "num_tokens": 8945664.0,
+ "step": 546
+ },
+ {
+ "entropy": 1.1487771272659302,
+ "epoch": 1.105050505050505,
+ "grad_norm": 1.8050950765609741,
+ "learning_rate": 1.2646464646464647e-05,
+ "loss": 1.1384658813476562,
+ "mean_token_accuracy": 0.734245240688324,
+ "num_tokens": 8962048.0,
+ "step": 547
+ },
+ {
+ "entropy": 1.2411508560180664,
+ "epoch": 1.107070707070707,
+ "grad_norm": 2.0532772541046143,
+ "learning_rate": 1.2632996632996634e-05,
+ "loss": 1.2501752376556396,
+ "mean_token_accuracy": 0.7154982686042786,
+ "num_tokens": 8978432.0,
+ "step": 548
+ },
+ {
+ "entropy": 1.383638620376587,
+ "epoch": 1.1090909090909091,
+ "grad_norm": 1.9422906637191772,
+ "learning_rate": 1.261952861952862e-05,
+ "loss": 1.4019615650177002,
+ "mean_token_accuracy": 0.677760124206543,
+ "num_tokens": 8994816.0,
+ "step": 549
+ },
+ {
+ "entropy": 1.302299976348877,
+ "epoch": 1.1111111111111112,
+ "grad_norm": 2.008962869644165,
+ "learning_rate": 1.2606060606060607e-05,
+ "loss": 1.2934541702270508,
+ "mean_token_accuracy": 0.7078040838241577,
+ "num_tokens": 9011200.0,
+ "step": 550
+ },
+ {
+ "entropy": 1.2423378229141235,
+ "epoch": 1.1131313131313132,
+ "grad_norm": 1.9461158514022827,
+ "learning_rate": 1.2592592592592593e-05,
+ "loss": 1.241358995437622,
+ "mean_token_accuracy": 0.7046898007392883,
+ "num_tokens": 9027584.0,
+ "step": 551
+ },
+ {
+ "entropy": 1.3034340143203735,
+ "epoch": 1.1151515151515152,
+ "grad_norm": 1.8627954721450806,
+ "learning_rate": 1.257912457912458e-05,
+ "loss": 1.3082997798919678,
+ "mean_token_accuracy": 0.6942476630210876,
+ "num_tokens": 9043968.0,
+ "step": 552
+ },
+ {
+ "entropy": 1.2468732595443726,
+ "epoch": 1.1171717171717173,
+ "grad_norm": 1.8033503293991089,
+ "learning_rate": 1.2565656565656568e-05,
+ "loss": 1.2398372888565063,
+ "mean_token_accuracy": 0.7059721350669861,
+ "num_tokens": 9060352.0,
+ "step": 553
+ },
+ {
+ "entropy": 1.3763928413391113,
+ "epoch": 1.1191919191919193,
+ "grad_norm": 2.109393358230591,
+ "learning_rate": 1.2552188552188553e-05,
+ "loss": 1.3908042907714844,
+ "mean_token_accuracy": 0.6643258333206177,
+ "num_tokens": 9076736.0,
+ "step": 554
+ },
+ {
+ "entropy": 1.0741946697235107,
+ "epoch": 1.121212121212121,
+ "grad_norm": 1.9498950242996216,
+ "learning_rate": 1.253872053872054e-05,
+ "loss": 1.0534818172454834,
+ "mean_token_accuracy": 0.7283830046653748,
+ "num_tokens": 9093120.0,
+ "step": 555
+ },
+ {
+ "entropy": 1.2464348077774048,
+ "epoch": 1.1232323232323231,
+ "grad_norm": 1.7962212562561035,
+ "learning_rate": 1.2525252525252527e-05,
+ "loss": 1.225571632385254,
+ "mean_token_accuracy": 0.695652186870575,
+ "num_tokens": 9109504.0,
+ "step": 556
+ },
+ {
+ "entropy": 1.1979366540908813,
+ "epoch": 1.1252525252525252,
+ "grad_norm": 2.0588653087615967,
+ "learning_rate": 1.2511784511784512e-05,
+ "loss": 1.1866135597229004,
+ "mean_token_accuracy": 0.705422580242157,
+ "num_tokens": 9125888.0,
+ "step": 557
+ },
+ {
+ "entropy": 0.9303950071334839,
+ "epoch": 1.1272727272727272,
+ "grad_norm": 1.8577561378479004,
+ "learning_rate": 1.2498316498316499e-05,
+ "loss": 0.9317413568496704,
+ "mean_token_accuracy": 0.7565950155258179,
+ "num_tokens": 9142272.0,
+ "step": 558
+ },
+ {
+ "entropy": 0.857255756855011,
+ "epoch": 1.1292929292929292,
+ "grad_norm": 1.8342596292495728,
+ "learning_rate": 1.2484848484848487e-05,
+ "loss": 0.8612891435623169,
+ "mean_token_accuracy": 0.7796775698661804,
+ "num_tokens": 9158656.0,
+ "step": 559
+ },
+ {
+ "entropy": 0.928502082824707,
+ "epoch": 1.1313131313131313,
+ "grad_norm": 1.9290180206298828,
+ "learning_rate": 1.2471380471380472e-05,
+ "loss": 0.9316297769546509,
+ "mean_token_accuracy": 0.7509160041809082,
+ "num_tokens": 9175040.0,
+ "step": 560
+ },
+ {
+ "entropy": 1.4019330739974976,
+ "epoch": 1.1333333333333333,
+ "grad_norm": 2.1189591884613037,
+ "learning_rate": 1.245791245791246e-05,
+ "loss": 1.3917508125305176,
+ "mean_token_accuracy": 0.6691499948501587,
+ "num_tokens": 9191424.0,
+ "step": 561
+ },
+ {
+ "entropy": 1.4660407304763794,
+ "epoch": 1.1353535353535353,
+ "grad_norm": 2.417506217956543,
+ "learning_rate": 1.2444444444444446e-05,
+ "loss": 1.4893054962158203,
+ "mean_token_accuracy": 0.6543111801147461,
+ "num_tokens": 9207808.0,
+ "step": 562
+ },
+ {
+ "entropy": 1.4580096006393433,
+ "epoch": 1.1373737373737374,
+ "grad_norm": 2.1689300537109375,
+ "learning_rate": 1.2430976430976431e-05,
+ "loss": 1.498686671257019,
+ "mean_token_accuracy": 0.6479604244232178,
+ "num_tokens": 9224192.0,
+ "step": 563
+ },
+ {
+ "entropy": 1.5781100988388062,
+ "epoch": 1.1393939393939394,
+ "grad_norm": 1.9947389364242554,
+ "learning_rate": 1.241750841750842e-05,
+ "loss": 1.604151725769043,
+ "mean_token_accuracy": 0.6561431288719177,
+ "num_tokens": 9240576.0,
+ "step": 564
+ },
+ {
+ "entropy": 0.898691713809967,
+ "epoch": 1.1414141414141414,
+ "grad_norm": 2.0886003971099854,
+ "learning_rate": 1.2404040404040406e-05,
+ "loss": 0.9051169157028198,
+ "mean_token_accuracy": 0.7479848265647888,
+ "num_tokens": 9256960.0,
+ "step": 565
+ },
+ {
+ "entropy": 1.2837234735488892,
+ "epoch": 1.1434343434343435,
+ "grad_norm": 2.1896820068359375,
+ "learning_rate": 1.239057239057239e-05,
+ "loss": 1.2775177955627441,
+ "mean_token_accuracy": 0.6811187267303467,
+ "num_tokens": 9273344.0,
+ "step": 566
+ },
+ {
+ "entropy": 1.1134051084518433,
+ "epoch": 1.1454545454545455,
+ "grad_norm": 1.883119821548462,
+ "learning_rate": 1.2377104377104379e-05,
+ "loss": 1.1263504028320312,
+ "mean_token_accuracy": 0.7154982686042786,
+ "num_tokens": 9289728.0,
+ "step": 567
+ },
+ {
+ "entropy": 1.1212451457977295,
+ "epoch": 1.1474747474747475,
+ "grad_norm": 2.0665414333343506,
+ "learning_rate": 1.2363636363636364e-05,
+ "loss": 1.1123074293136597,
+ "mean_token_accuracy": 0.7117122411727905,
+ "num_tokens": 9306112.0,
+ "step": 568
+ },
+ {
+ "entropy": 1.1619912385940552,
+ "epoch": 1.1494949494949496,
+ "grad_norm": 1.986931562423706,
+ "learning_rate": 1.2350168350168352e-05,
+ "loss": 1.133022665977478,
+ "mean_token_accuracy": 0.7123839855194092,
+ "num_tokens": 9322496.0,
+ "step": 569
+ },
+ {
+ "entropy": 1.433761477470398,
+ "epoch": 1.1515151515151516,
+ "grad_norm": 1.8984476327896118,
+ "learning_rate": 1.2336700336700338e-05,
+ "loss": 1.434887170791626,
+ "mean_token_accuracy": 0.6634098887443542,
+ "num_tokens": 9338880.0,
+ "step": 570
+ },
+ {
+ "entropy": 1.2688939571380615,
+ "epoch": 1.1535353535353536,
+ "grad_norm": 1.9783724546432495,
+ "learning_rate": 1.2323232323232323e-05,
+ "loss": 1.2475486993789673,
+ "mean_token_accuracy": 0.7003541588783264,
+ "num_tokens": 9355264.0,
+ "step": 571
+ },
+ {
+ "entropy": 1.3158199787139893,
+ "epoch": 1.1555555555555554,
+ "grad_norm": 2.1993207931518555,
+ "learning_rate": 1.2309764309764311e-05,
+ "loss": 1.2641775608062744,
+ "mean_token_accuracy": 0.6900342106819153,
+ "num_tokens": 9371648.0,
+ "step": 572
+ },
+ {
+ "entropy": 1.5717110633850098,
+ "epoch": 1.1575757575757575,
+ "grad_norm": 2.0973942279815674,
+ "learning_rate": 1.2296296296296298e-05,
+ "loss": 1.5607821941375732,
+ "mean_token_accuracy": 0.6434416174888611,
+ "num_tokens": 9388032.0,
+ "step": 573
+ },
+ {
+ "entropy": 1.1339603662490845,
+ "epoch": 1.1595959595959595,
+ "grad_norm": 2.1611239910125732,
+ "learning_rate": 1.2282828282828282e-05,
+ "loss": 1.1485066413879395,
+ "mean_token_accuracy": 0.7079262137413025,
+ "num_tokens": 9404416.0,
+ "step": 574
+ },
+ {
+ "entropy": 0.7215157151222229,
+ "epoch": 1.1616161616161615,
+ "grad_norm": 1.7199064493179321,
+ "learning_rate": 1.226936026936027e-05,
+ "loss": 0.6996407508850098,
+ "mean_token_accuracy": 0.8110039234161377,
+ "num_tokens": 9420800.0,
+ "step": 575
+ },
+ {
+ "entropy": 1.207773208618164,
+ "epoch": 1.1636363636363636,
+ "grad_norm": 1.9363049268722534,
+ "learning_rate": 1.2255892255892257e-05,
+ "loss": 1.2100518941879272,
+ "mean_token_accuracy": 0.6953468322753906,
+ "num_tokens": 9437184.0,
+ "step": 576
+ },
+ {
+ "entropy": 1.1832739114761353,
+ "epoch": 1.1656565656565656,
+ "grad_norm": 1.9463727474212646,
+ "learning_rate": 1.2242424242424242e-05,
+ "loss": 1.1800014972686768,
+ "mean_token_accuracy": 0.7148265838623047,
+ "num_tokens": 9453568.0,
+ "step": 577
+ },
+ {
+ "entropy": 0.969874918460846,
+ "epoch": 1.1676767676767676,
+ "grad_norm": 1.9776639938354492,
+ "learning_rate": 1.222895622895623e-05,
+ "loss": 0.9835254549980164,
+ "mean_token_accuracy": 0.7401074767112732,
+ "num_tokens": 9469952.0,
+ "step": 578
+ },
+ {
+ "entropy": 1.2477227449417114,
+ "epoch": 1.1696969696969697,
+ "grad_norm": 2.054093599319458,
+ "learning_rate": 1.2215488215488217e-05,
+ "loss": 1.2632465362548828,
+ "mean_token_accuracy": 0.6811797618865967,
+ "num_tokens": 9486336.0,
+ "step": 579
+ },
+ {
+ "entropy": 1.3652271032333374,
+ "epoch": 1.1717171717171717,
+ "grad_norm": 2.033511161804199,
+ "learning_rate": 1.2202020202020203e-05,
+ "loss": 1.3719289302825928,
+ "mean_token_accuracy": 0.6808744668960571,
+ "num_tokens": 9502720.0,
+ "step": 580
+ },
+ {
+ "entropy": 1.0066897869110107,
+ "epoch": 1.1737373737373737,
+ "grad_norm": 1.906139612197876,
+ "learning_rate": 1.218855218855219e-05,
+ "loss": 1.0317155122756958,
+ "mean_token_accuracy": 0.7470688819885254,
+ "num_tokens": 9519104.0,
+ "step": 581
+ },
+ {
+ "entropy": 1.1423532962799072,
+ "epoch": 1.1757575757575758,
+ "grad_norm": 2.0951437950134277,
+ "learning_rate": 1.2175084175084176e-05,
+ "loss": 1.1328619718551636,
+ "mean_token_accuracy": 0.7098802924156189,
+ "num_tokens": 9535488.0,
+ "step": 582
+ },
+ {
+ "entropy": 0.9396458864212036,
+ "epoch": 1.1777777777777778,
+ "grad_norm": 2.001469373703003,
+ "learning_rate": 1.2161616161616162e-05,
+ "loss": 0.9508752822875977,
+ "mean_token_accuracy": 0.7453590631484985,
+ "num_tokens": 9551872.0,
+ "step": 583
+ },
+ {
+ "entropy": 1.2478508949279785,
+ "epoch": 1.1797979797979798,
+ "grad_norm": 2.1304337978363037,
+ "learning_rate": 1.2148148148148149e-05,
+ "loss": 1.2533063888549805,
+ "mean_token_accuracy": 0.6894235610961914,
+ "num_tokens": 9568256.0,
+ "step": 584
+ },
+ {
+ "entropy": 0.8429924249649048,
+ "epoch": 1.1818181818181819,
+ "grad_norm": 1.891184687614441,
+ "learning_rate": 1.2134680134680134e-05,
+ "loss": 0.8334753513336182,
+ "mean_token_accuracy": 0.7737542986869812,
+ "num_tokens": 9584640.0,
+ "step": 585
+ },
+ {
+ "entropy": 1.1487921476364136,
+ "epoch": 1.183838383838384,
+ "grad_norm": 2.100118637084961,
+ "learning_rate": 1.2121212121212122e-05,
+ "loss": 1.1393694877624512,
+ "mean_token_accuracy": 0.7046287059783936,
+ "num_tokens": 9601024.0,
+ "step": 586
+ },
+ {
+ "entropy": 1.406325101852417,
+ "epoch": 1.185858585858586,
+ "grad_norm": 2.0391998291015625,
+ "learning_rate": 1.2107744107744108e-05,
+ "loss": 1.395864725112915,
+ "mean_token_accuracy": 0.6802638173103333,
+ "num_tokens": 9617408.0,
+ "step": 587
+ },
+ {
+ "entropy": 1.192980170249939,
+ "epoch": 1.187878787878788,
+ "grad_norm": 2.105341911315918,
+ "learning_rate": 1.2094276094276097e-05,
+ "loss": 1.2203993797302246,
+ "mean_token_accuracy": 0.6992549896240234,
+ "num_tokens": 9633792.0,
+ "step": 588
+ },
+ {
+ "entropy": 1.4770102500915527,
+ "epoch": 1.1898989898989898,
+ "grad_norm": 1.959194302558899,
+ "learning_rate": 1.2080808080808081e-05,
+ "loss": 1.4826602935791016,
+ "mean_token_accuracy": 0.6475329995155334,
+ "num_tokens": 9650176.0,
+ "step": 589
+ },
+ {
+ "entropy": 1.440915584564209,
+ "epoch": 1.1919191919191918,
+ "grad_norm": 2.2827632427215576,
+ "learning_rate": 1.2067340067340068e-05,
+ "loss": 1.4349164962768555,
+ "mean_token_accuracy": 0.6570591330528259,
+ "num_tokens": 9666560.0,
+ "step": 590
+ },
+ {
+ "entropy": 0.9503017067909241,
+ "epoch": 1.1939393939393939,
+ "grad_norm": 1.8947645425796509,
+ "learning_rate": 1.2053872053872056e-05,
+ "loss": 0.9562087655067444,
+ "mean_token_accuracy": 0.7511602640151978,
+ "num_tokens": 9682944.0,
+ "step": 591
+ },
+ {
+ "entropy": 0.935599148273468,
+ "epoch": 1.195959595959596,
+ "grad_norm": 1.8788244724273682,
+ "learning_rate": 1.2040404040404041e-05,
+ "loss": 0.9235643148422241,
+ "mean_token_accuracy": 0.7525647282600403,
+ "num_tokens": 9699328.0,
+ "step": 592
+ },
+ {
+ "entropy": 1.3421721458435059,
+ "epoch": 1.197979797979798,
+ "grad_norm": 2.038698673248291,
+ "learning_rate": 1.2026936026936027e-05,
+ "loss": 1.3459725379943848,
+ "mean_token_accuracy": 0.6803248524665833,
+ "num_tokens": 9715712.0,
+ "step": 593
+ },
+ {
+ "entropy": 1.4244537353515625,
+ "epoch": 1.2,
+ "grad_norm": 2.0669443607330322,
+ "learning_rate": 1.2013468013468016e-05,
+ "loss": 1.462540626525879,
+ "mean_token_accuracy": 0.6539447903633118,
+ "num_tokens": 9732096.0,
+ "step": 594
+ },
+ {
+ "entropy": 1.3264589309692383,
+ "epoch": 1.202020202020202,
+ "grad_norm": 2.2563624382019043,
+ "learning_rate": 1.2e-05,
+ "loss": 1.3387104272842407,
+ "mean_token_accuracy": 0.6822789311408997,
+ "num_tokens": 9748480.0,
+ "step": 595
+ },
+ {
+ "entropy": 1.250544548034668,
+ "epoch": 1.204040404040404,
+ "grad_norm": 1.9611469507217407,
+ "learning_rate": 1.1986531986531988e-05,
+ "loss": 1.2569901943206787,
+ "mean_token_accuracy": 0.6947361826896667,
+ "num_tokens": 9764864.0,
+ "step": 596
+ },
+ {
+ "entropy": 1.3189572095870972,
+ "epoch": 1.206060606060606,
+ "grad_norm": 1.9136087894439697,
+ "learning_rate": 1.1973063973063975e-05,
+ "loss": 1.3251599073410034,
+ "mean_token_accuracy": 0.6856985688209534,
+ "num_tokens": 9781248.0,
+ "step": 597
+ },
+ {
+ "entropy": 1.1400903463363647,
+ "epoch": 1.208080808080808,
+ "grad_norm": 1.9812469482421875,
+ "learning_rate": 1.195959595959596e-05,
+ "loss": 1.1514993906021118,
+ "mean_token_accuracy": 0.7175744771957397,
+ "num_tokens": 9797632.0,
+ "step": 598
+ },
+ {
+ "entropy": 1.263106346130371,
+ "epoch": 1.2101010101010101,
+ "grad_norm": 2.1461827754974365,
+ "learning_rate": 1.1946127946127948e-05,
+ "loss": 1.2711405754089355,
+ "mean_token_accuracy": 0.6880801320075989,
+ "num_tokens": 9814016.0,
+ "step": 599
+ },
+ {
+ "entropy": 1.1541659832000732,
+ "epoch": 1.2121212121212122,
+ "grad_norm": 1.8964859247207642,
+ "learning_rate": 1.1932659932659933e-05,
+ "loss": 1.1726963520050049,
+ "mean_token_accuracy": 0.7143380641937256,
+ "num_tokens": 9830400.0,
+ "step": 600
+ },
+ {
+ "epoch": 1.2121212121212122,
+ "eval_entropy": 1.3044754386909547,
+ "eval_loss": 1.5363746881484985,
+ "eval_mean_token_accuracy": 0.6458704312962871,
+ "eval_num_tokens": 9830400.0,
+ "eval_runtime": 43.9189,
+ "eval_samples_per_second": 22.542,
+ "eval_steps_per_second": 2.823,
+ "step": 600
+ },
+ {
+ "entropy": 1.4692459106445312,
+ "epoch": 1.2141414141414142,
+ "grad_norm": 1.9165432453155518,
+ "learning_rate": 1.191919191919192e-05,
+ "loss": 1.444916009902954,
+ "mean_token_accuracy": 0.6709208488464355,
+ "num_tokens": 9846784.0,
+ "step": 601
+ },
+ {
+ "entropy": 1.2140837907791138,
+ "epoch": 1.2161616161616162,
+ "grad_norm": 2.184677839279175,
+ "learning_rate": 1.1905723905723907e-05,
+ "loss": 1.1810932159423828,
+ "mean_token_accuracy": 0.7119565010070801,
+ "num_tokens": 9863168.0,
+ "step": 602
+ },
+ {
+ "entropy": 1.2266523838043213,
+ "epoch": 1.2181818181818183,
+ "grad_norm": 2.0835864543914795,
+ "learning_rate": 1.1892255892255892e-05,
+ "loss": 1.216700792312622,
+ "mean_token_accuracy": 0.702125072479248,
+ "num_tokens": 9879552.0,
+ "step": 603
+ },
+ {
+ "entropy": 1.153336524963379,
+ "epoch": 1.2202020202020203,
+ "grad_norm": 1.959340214729309,
+ "learning_rate": 1.187878787878788e-05,
+ "loss": 1.1497607231140137,
+ "mean_token_accuracy": 0.7254518866539001,
+ "num_tokens": 9895936.0,
+ "step": 604
+ },
+ {
+ "entropy": 1.1328232288360596,
+ "epoch": 1.2222222222222223,
+ "grad_norm": 2.165703773498535,
+ "learning_rate": 1.1865319865319867e-05,
+ "loss": 1.1248265504837036,
+ "mean_token_accuracy": 0.7068881392478943,
+ "num_tokens": 9912320.0,
+ "step": 605
+ },
+ {
+ "entropy": 1.0859829187393188,
+ "epoch": 1.2242424242424241,
+ "grad_norm": 2.0244224071502686,
+ "learning_rate": 1.1851851851851852e-05,
+ "loss": 1.0981967449188232,
+ "mean_token_accuracy": 0.717391312122345,
+ "num_tokens": 9928704.0,
+ "step": 606
+ },
+ {
+ "entropy": 0.9204480648040771,
+ "epoch": 1.2262626262626264,
+ "grad_norm": 1.7957704067230225,
+ "learning_rate": 1.183838383838384e-05,
+ "loss": 0.9045253992080688,
+ "mean_token_accuracy": 0.7639839053153992,
+ "num_tokens": 9945088.0,
+ "step": 607
+ },
+ {
+ "entropy": 1.1566129922866821,
+ "epoch": 1.2282828282828282,
+ "grad_norm": 2.0805907249450684,
+ "learning_rate": 1.1824915824915826e-05,
+ "loss": 1.1485098600387573,
+ "mean_token_accuracy": 0.7118343710899353,
+ "num_tokens": 9961472.0,
+ "step": 608
+ },
+ {
+ "entropy": 1.108127236366272,
+ "epoch": 1.2303030303030302,
+ "grad_norm": 1.977317214012146,
+ "learning_rate": 1.1811447811447811e-05,
+ "loss": 1.1131000518798828,
+ "mean_token_accuracy": 0.7167806625366211,
+ "num_tokens": 9977856.0,
+ "step": 609
+ },
+ {
+ "entropy": 0.8749650716781616,
+ "epoch": 1.2323232323232323,
+ "grad_norm": 2.009488344192505,
+ "learning_rate": 1.17979797979798e-05,
+ "loss": 0.9132063388824463,
+ "mean_token_accuracy": 0.7530532479286194,
+ "num_tokens": 9994240.0,
+ "step": 610
+ },
+ {
+ "entropy": 1.2267354726791382,
+ "epoch": 1.2343434343434343,
+ "grad_norm": 2.075202465057373,
+ "learning_rate": 1.1784511784511786e-05,
+ "loss": 1.2091751098632812,
+ "mean_token_accuracy": 0.70652174949646,
+ "num_tokens": 10010624.0,
+ "step": 611
+ },
+ {
+ "entropy": 1.3990167379379272,
+ "epoch": 1.2363636363636363,
+ "grad_norm": 2.2843780517578125,
+ "learning_rate": 1.1771043771043772e-05,
+ "loss": 1.3952152729034424,
+ "mean_token_accuracy": 0.6685393452644348,
+ "num_tokens": 10027008.0,
+ "step": 612
+ },
+ {
+ "entropy": 1.3222941160202026,
+ "epoch": 1.2383838383838384,
+ "grad_norm": 2.1916584968566895,
+ "learning_rate": 1.1757575757575759e-05,
+ "loss": 1.322625994682312,
+ "mean_token_accuracy": 0.6761114001274109,
+ "num_tokens": 10043392.0,
+ "step": 613
+ },
+ {
+ "entropy": 1.3163132667541504,
+ "epoch": 1.2404040404040404,
+ "grad_norm": 2.1066598892211914,
+ "learning_rate": 1.1744107744107745e-05,
+ "loss": 1.3350703716278076,
+ "mean_token_accuracy": 0.6811797618865967,
+ "num_tokens": 10059776.0,
+ "step": 614
+ },
+ {
+ "entropy": 1.3258253335952759,
+ "epoch": 1.2424242424242424,
+ "grad_norm": 2.16098952293396,
+ "learning_rate": 1.1730639730639732e-05,
+ "loss": 1.3305881023406982,
+ "mean_token_accuracy": 0.6830117106437683,
+ "num_tokens": 10076160.0,
+ "step": 615
+ },
+ {
+ "entropy": 0.9164132475852966,
+ "epoch": 1.2444444444444445,
+ "grad_norm": 1.950881838798523,
+ "learning_rate": 1.1717171717171718e-05,
+ "loss": 0.9203709959983826,
+ "mean_token_accuracy": 0.7569614052772522,
+ "num_tokens": 10092544.0,
+ "step": 616
+ },
+ {
+ "entropy": 1.1378085613250732,
+ "epoch": 1.2464646464646465,
+ "grad_norm": 1.99736487865448,
+ "learning_rate": 1.1703703703703703e-05,
+ "loss": 1.1409945487976074,
+ "mean_token_accuracy": 0.7130556702613831,
+ "num_tokens": 10108928.0,
+ "step": 617
+ },
+ {
+ "entropy": 1.010982871055603,
+ "epoch": 1.2484848484848485,
+ "grad_norm": 1.8678277730941772,
+ "learning_rate": 1.1690235690235691e-05,
+ "loss": 1.0040092468261719,
+ "mean_token_accuracy": 0.7382755279541016,
+ "num_tokens": 10125312.0,
+ "step": 618
+ },
+ {
+ "entropy": 1.3451189994812012,
+ "epoch": 1.2505050505050506,
+ "grad_norm": 2.1611146926879883,
+ "learning_rate": 1.1676767676767678e-05,
+ "loss": 1.3347704410552979,
+ "mean_token_accuracy": 0.6760503053665161,
+ "num_tokens": 10141696.0,
+ "step": 619
+ },
+ {
+ "entropy": 1.5893058776855469,
+ "epoch": 1.2525252525252526,
+ "grad_norm": 2.5862345695495605,
+ "learning_rate": 1.1663299663299666e-05,
+ "loss": 1.6102404594421387,
+ "mean_token_accuracy": 0.6231069564819336,
+ "num_tokens": 10158080.0,
+ "step": 620
+ },
+ {
+ "entropy": 1.4471327066421509,
+ "epoch": 1.2545454545454544,
+ "grad_norm": 1.8857513666152954,
+ "learning_rate": 1.164983164983165e-05,
+ "loss": 1.4646904468536377,
+ "mean_token_accuracy": 0.6591353416442871,
+ "num_tokens": 10174464.0,
+ "step": 621
+ },
+ {
+ "entropy": 1.1457897424697876,
+ "epoch": 1.2565656565656567,
+ "grad_norm": 1.9036853313446045,
+ "learning_rate": 1.1636363636363637e-05,
+ "loss": 1.136209487915039,
+ "mean_token_accuracy": 0.7179408669471741,
+ "num_tokens": 10190848.0,
+ "step": 622
+ },
+ {
+ "entropy": 1.3230067491531372,
+ "epoch": 1.2585858585858585,
+ "grad_norm": 1.969241738319397,
+ "learning_rate": 1.1622895622895625e-05,
+ "loss": 1.3203729391098022,
+ "mean_token_accuracy": 0.6909501552581787,
+ "num_tokens": 10207232.0,
+ "step": 623
+ },
+ {
+ "entropy": 0.9331954717636108,
+ "epoch": 1.2606060606060607,
+ "grad_norm": 1.8972336053848267,
+ "learning_rate": 1.160942760942761e-05,
+ "loss": 0.9346438050270081,
+ "mean_token_accuracy": 0.7507327795028687,
+ "num_tokens": 10223616.0,
+ "step": 624
+ },
+ {
+ "entropy": 1.0203893184661865,
+ "epoch": 1.2626262626262625,
+ "grad_norm": 1.9246903657913208,
+ "learning_rate": 1.1595959595959597e-05,
+ "loss": 1.0318937301635742,
+ "mean_token_accuracy": 0.7396799921989441,
+ "num_tokens": 10240000.0,
+ "step": 625
+ },
+ {
+ "entropy": 1.0469106435775757,
+ "epoch": 1.2646464646464646,
+ "grad_norm": 2.0860235691070557,
+ "learning_rate": 1.1582491582491585e-05,
+ "loss": 1.0661770105361938,
+ "mean_token_accuracy": 0.7215437293052673,
+ "num_tokens": 10256384.0,
+ "step": 626
+ },
+ {
+ "entropy": 0.9044442772865295,
+ "epoch": 1.2666666666666666,
+ "grad_norm": 1.922624111175537,
+ "learning_rate": 1.156902356902357e-05,
+ "loss": 0.8972492218017578,
+ "mean_token_accuracy": 0.7624572515487671,
+ "num_tokens": 10272768.0,
+ "step": 627
+ },
+ {
+ "entropy": 1.7082630395889282,
+ "epoch": 1.2686868686868686,
+ "grad_norm": 2.0335583686828613,
+ "learning_rate": 1.1555555555555556e-05,
+ "loss": 1.7194339036941528,
+ "mean_token_accuracy": 0.6278089880943298,
+ "num_tokens": 10289152.0,
+ "step": 628
+ },
+ {
+ "entropy": 1.1749809980392456,
+ "epoch": 1.2707070707070707,
+ "grad_norm": 2.3157317638397217,
+ "learning_rate": 1.1542087542087544e-05,
+ "loss": 1.1912015676498413,
+ "mean_token_accuracy": 0.6991939544677734,
+ "num_tokens": 10305536.0,
+ "step": 629
+ },
+ {
+ "entropy": 1.0230133533477783,
+ "epoch": 1.2727272727272727,
+ "grad_norm": 1.9861854314804077,
+ "learning_rate": 1.1528619528619529e-05,
+ "loss": 1.030585765838623,
+ "mean_token_accuracy": 0.7328407168388367,
+ "num_tokens": 10321920.0,
+ "step": 630
+ },
+ {
+ "entropy": 0.9128814935684204,
+ "epoch": 1.2747474747474747,
+ "grad_norm": 5.766819000244141,
+ "learning_rate": 1.1515151515151517e-05,
+ "loss": 0.9959099292755127,
+ "mean_token_accuracy": 0.7520151734352112,
+ "num_tokens": 10338304.0,
+ "step": 631
+ },
+ {
+ "entropy": 1.3987287282943726,
+ "epoch": 1.2767676767676768,
+ "grad_norm": 1.934733510017395,
+ "learning_rate": 1.1501683501683502e-05,
+ "loss": 1.3910452127456665,
+ "mean_token_accuracy": 0.675561785697937,
+ "num_tokens": 10354688.0,
+ "step": 632
+ },
+ {
+ "entropy": 1.7590066194534302,
+ "epoch": 1.2787878787878788,
+ "grad_norm": 2.003899574279785,
+ "learning_rate": 1.1488215488215488e-05,
+ "loss": 1.771543264389038,
+ "mean_token_accuracy": 0.6064972877502441,
+ "num_tokens": 10371072.0,
+ "step": 633
+ },
+ {
+ "entropy": 1.1135497093200684,
+ "epoch": 1.2808080808080808,
+ "grad_norm": 1.9084051847457886,
+ "learning_rate": 1.1474747474747477e-05,
+ "loss": 1.1299227476119995,
+ "mean_token_accuracy": 0.7198339104652405,
+ "num_tokens": 10387456.0,
+ "step": 634
+ },
+ {
+ "entropy": 1.4671945571899414,
+ "epoch": 1.2828282828282829,
+ "grad_norm": 1.9825108051300049,
+ "learning_rate": 1.1461279461279461e-05,
+ "loss": 1.486473560333252,
+ "mean_token_accuracy": 0.6673180460929871,
+ "num_tokens": 10403840.0,
+ "step": 635
+ },
+ {
+ "entropy": 1.2157530784606934,
+ "epoch": 1.284848484848485,
+ "grad_norm": 2.048701286315918,
+ "learning_rate": 1.1447811447811448e-05,
+ "loss": 1.196000099182129,
+ "mean_token_accuracy": 0.7078040838241577,
+ "num_tokens": 10420224.0,
+ "step": 636
+ },
+ {
+ "entropy": 0.8687402009963989,
+ "epoch": 1.286868686868687,
+ "grad_norm": 1.8956520557403564,
+ "learning_rate": 1.1434343434343436e-05,
+ "loss": 0.86002117395401,
+ "mean_token_accuracy": 0.7663654088973999,
+ "num_tokens": 10436608.0,
+ "step": 637
+ },
+ {
+ "entropy": 1.1118111610412598,
+ "epoch": 1.2888888888888888,
+ "grad_norm": 1.9835058450698853,
+ "learning_rate": 1.142087542087542e-05,
+ "loss": 1.1115164756774902,
+ "mean_token_accuracy": 0.7170859575271606,
+ "num_tokens": 10452992.0,
+ "step": 638
+ },
+ {
+ "entropy": 1.1080268621444702,
+ "epoch": 1.290909090909091,
+ "grad_norm": 1.9650659561157227,
+ "learning_rate": 1.1407407407407409e-05,
+ "loss": 1.0900099277496338,
+ "mean_token_accuracy": 0.7143990993499756,
+ "num_tokens": 10469376.0,
+ "step": 639
+ },
+ {
+ "entropy": 0.9909329414367676,
+ "epoch": 1.2929292929292928,
+ "grad_norm": 2.1029200553894043,
+ "learning_rate": 1.1393939393939395e-05,
+ "loss": 0.9947470426559448,
+ "mean_token_accuracy": 0.7403517365455627,
+ "num_tokens": 10485760.0,
+ "step": 640
+ },
+ {
+ "entropy": 1.1272226572036743,
+ "epoch": 1.294949494949495,
+ "grad_norm": 1.9888865947723389,
+ "learning_rate": 1.138047138047138e-05,
+ "loss": 1.1474848985671997,
+ "mean_token_accuracy": 0.7138495445251465,
+ "num_tokens": 10502144.0,
+ "step": 641
+ },
+ {
+ "entropy": 1.5329686403274536,
+ "epoch": 1.2969696969696969,
+ "grad_norm": 2.046448230743408,
+ "learning_rate": 1.1367003367003368e-05,
+ "loss": 1.556037425994873,
+ "mean_token_accuracy": 0.642953097820282,
+ "num_tokens": 10518528.0,
+ "step": 642
+ },
+ {
+ "entropy": 1.177565336227417,
+ "epoch": 1.298989898989899,
+ "grad_norm": 2.1602399349212646,
+ "learning_rate": 1.1353535353535355e-05,
+ "loss": 1.1836936473846436,
+ "mean_token_accuracy": 0.6971787810325623,
+ "num_tokens": 10534912.0,
+ "step": 643
+ },
+ {
+ "entropy": 0.8429449796676636,
+ "epoch": 1.301010101010101,
+ "grad_norm": 1.7227115631103516,
+ "learning_rate": 1.134006734006734e-05,
+ "loss": 0.8446363806724548,
+ "mean_token_accuracy": 0.7757083773612976,
+ "num_tokens": 10551296.0,
+ "step": 644
+ },
+ {
+ "entropy": 1.397851824760437,
+ "epoch": 1.303030303030303,
+ "grad_norm": 2.137099504470825,
+ "learning_rate": 1.1326599326599328e-05,
+ "loss": 1.3887293338775635,
+ "mean_token_accuracy": 0.6610283255577087,
+ "num_tokens": 10567680.0,
+ "step": 645
+ },
+ {
+ "entropy": 1.2101221084594727,
+ "epoch": 1.305050505050505,
+ "grad_norm": 2.073007583618164,
+ "learning_rate": 1.1313131313131314e-05,
+ "loss": 1.1941102743148804,
+ "mean_token_accuracy": 0.7100635170936584,
+ "num_tokens": 10584064.0,
+ "step": 646
+ },
+ {
+ "entropy": 1.626240849494934,
+ "epoch": 1.307070707070707,
+ "grad_norm": 2.073517084121704,
+ "learning_rate": 1.1299663299663301e-05,
+ "loss": 1.6281862258911133,
+ "mean_token_accuracy": 0.6289692521095276,
+ "num_tokens": 10600448.0,
+ "step": 647
+ },
+ {
+ "entropy": 1.0951712131500244,
+ "epoch": 1.309090909090909,
+ "grad_norm": 2.126551866531372,
+ "learning_rate": 1.1286195286195287e-05,
+ "loss": 1.105857014656067,
+ "mean_token_accuracy": 0.7147044539451599,
+ "num_tokens": 10616832.0,
+ "step": 648
+ },
+ {
+ "entropy": 1.2107304334640503,
+ "epoch": 1.3111111111111111,
+ "grad_norm": 2.067368507385254,
+ "learning_rate": 1.1272727272727272e-05,
+ "loss": 1.2249386310577393,
+ "mean_token_accuracy": 0.7076819539070129,
+ "num_tokens": 10633216.0,
+ "step": 649
+ },
+ {
+ "entropy": 1.4364724159240723,
+ "epoch": 1.3131313131313131,
+ "grad_norm": 1.9724938869476318,
+ "learning_rate": 1.125925925925926e-05,
+ "loss": 1.4194120168685913,
+ "mean_token_accuracy": 0.6655471324920654,
+ "num_tokens": 10649600.0,
+ "step": 650
+ },
+ {
+ "entropy": 0.9386561512947083,
+ "epoch": 1.3151515151515152,
+ "grad_norm": 1.9386829137802124,
+ "learning_rate": 1.1245791245791247e-05,
+ "loss": 0.9313067197799683,
+ "mean_token_accuracy": 0.7560454607009888,
+ "num_tokens": 10665984.0,
+ "step": 651
+ },
+ {
+ "entropy": 1.3809468746185303,
+ "epoch": 1.3171717171717172,
+ "grad_norm": 2.1187713146209717,
+ "learning_rate": 1.1232323232323232e-05,
+ "loss": 1.3900315761566162,
+ "mean_token_accuracy": 0.6675012111663818,
+ "num_tokens": 10682368.0,
+ "step": 652
+ },
+ {
+ "entropy": 1.1068683862686157,
+ "epoch": 1.3191919191919192,
+ "grad_norm": 1.9972929954528809,
+ "learning_rate": 1.121885521885522e-05,
+ "loss": 1.1015276908874512,
+ "mean_token_accuracy": 0.7225818037986755,
+ "num_tokens": 10698752.0,
+ "step": 653
+ },
+ {
+ "entropy": 0.8208921551704407,
+ "epoch": 1.3212121212121213,
+ "grad_norm": 1.9667426347732544,
+ "learning_rate": 1.1205387205387206e-05,
+ "loss": 0.8475431799888611,
+ "mean_token_accuracy": 0.773937463760376,
+ "num_tokens": 10715136.0,
+ "step": 654
+ },
+ {
+ "entropy": 1.2287651300430298,
+ "epoch": 1.3232323232323233,
+ "grad_norm": 2.038112163543701,
+ "learning_rate": 1.1191919191919194e-05,
+ "loss": 1.2165236473083496,
+ "mean_token_accuracy": 0.7084147334098816,
+ "num_tokens": 10731520.0,
+ "step": 655
+ },
+ {
+ "entropy": 1.3068023920059204,
+ "epoch": 1.3252525252525253,
+ "grad_norm": 2.001093626022339,
+ "learning_rate": 1.117845117845118e-05,
+ "loss": 1.3044464588165283,
+ "mean_token_accuracy": 0.6809965968132019,
+ "num_tokens": 10747904.0,
+ "step": 656
+ },
+ {
+ "entropy": 1.0627591609954834,
+ "epoch": 1.3272727272727272,
+ "grad_norm": 2.204498529434204,
+ "learning_rate": 1.1164983164983166e-05,
+ "loss": 1.07448410987854,
+ "mean_token_accuracy": 0.7211162447929382,
+ "num_tokens": 10764288.0,
+ "step": 657
+ },
+ {
+ "entropy": 1.1071605682373047,
+ "epoch": 1.3292929292929294,
+ "grad_norm": 2.2397496700286865,
+ "learning_rate": 1.1151515151515154e-05,
+ "loss": 1.1061029434204102,
+ "mean_token_accuracy": 0.7100635170936584,
+ "num_tokens": 10780672.0,
+ "step": 658
+ },
+ {
+ "entropy": 1.2299960851669312,
+ "epoch": 1.3313131313131312,
+ "grad_norm": 2.0838606357574463,
+ "learning_rate": 1.1138047138047139e-05,
+ "loss": 1.2077662944793701,
+ "mean_token_accuracy": 0.6886907815933228,
+ "num_tokens": 10797056.0,
+ "step": 659
+ },
+ {
+ "entropy": 1.2082552909851074,
+ "epoch": 1.3333333333333333,
+ "grad_norm": 2.0905637741088867,
+ "learning_rate": 1.1124579124579125e-05,
+ "loss": 1.2045608758926392,
+ "mean_token_accuracy": 0.6978505253791809,
+ "num_tokens": 10813440.0,
+ "step": 660
+ },
+ {
+ "entropy": 1.4011980295181274,
+ "epoch": 1.3353535353535353,
+ "grad_norm": 2.227360725402832,
+ "learning_rate": 1.1111111111111113e-05,
+ "loss": 1.4221925735473633,
+ "mean_token_accuracy": 0.6541890501976013,
+ "num_tokens": 10829824.0,
+ "step": 661
+ },
+ {
+ "entropy": 0.7984166145324707,
+ "epoch": 1.3373737373737373,
+ "grad_norm": 1.7253413200378418,
+ "learning_rate": 1.1097643097643098e-05,
+ "loss": 0.7818700671195984,
+ "mean_token_accuracy": 0.7938446402549744,
+ "num_tokens": 10846208.0,
+ "step": 662
+ },
+ {
+ "entropy": 1.0456970930099487,
+ "epoch": 1.3393939393939394,
+ "grad_norm": 2.098661422729492,
+ "learning_rate": 1.1084175084175086e-05,
+ "loss": 1.084232211112976,
+ "mean_token_accuracy": 0.735405445098877,
+ "num_tokens": 10862592.0,
+ "step": 663
+ },
+ {
+ "entropy": 1.1313530206680298,
+ "epoch": 1.3414141414141414,
+ "grad_norm": 1.8916199207305908,
+ "learning_rate": 1.1070707070707071e-05,
+ "loss": 1.1359466314315796,
+ "mean_token_accuracy": 0.7338177561759949,
+ "num_tokens": 10878976.0,
+ "step": 664
+ },
+ {
+ "entropy": 0.9603059887886047,
+ "epoch": 1.3434343434343434,
+ "grad_norm": 2.061540365219116,
+ "learning_rate": 1.1057239057239058e-05,
+ "loss": 0.9516211748123169,
+ "mean_token_accuracy": 0.7465803623199463,
+ "num_tokens": 10895360.0,
+ "step": 665
+ },
+ {
+ "entropy": 0.9865843057632446,
+ "epoch": 1.3454545454545455,
+ "grad_norm": 1.9756824970245361,
+ "learning_rate": 1.1043771043771046e-05,
+ "loss": 0.995586633682251,
+ "mean_token_accuracy": 0.7349780201911926,
+ "num_tokens": 10911744.0,
+ "step": 666
+ },
+ {
+ "entropy": 1.098454475402832,
+ "epoch": 1.3474747474747475,
+ "grad_norm": 1.936257243156433,
+ "learning_rate": 1.103030303030303e-05,
+ "loss": 1.0999541282653809,
+ "mean_token_accuracy": 0.7220322489738464,
+ "num_tokens": 10928128.0,
+ "step": 667
+ },
+ {
+ "entropy": 1.1791200637817383,
+ "epoch": 1.3494949494949495,
+ "grad_norm": 2.2319350242614746,
+ "learning_rate": 1.1016835016835017e-05,
+ "loss": 1.1725637912750244,
+ "mean_token_accuracy": 0.6991328597068787,
+ "num_tokens": 10944512.0,
+ "step": 668
+ },
+ {
+ "entropy": 1.2307621240615845,
+ "epoch": 1.3515151515151516,
+ "grad_norm": 1.9910095930099487,
+ "learning_rate": 1.1003367003367005e-05,
+ "loss": 1.2371525764465332,
+ "mean_token_accuracy": 0.6916218996047974,
+ "num_tokens": 10960896.0,
+ "step": 669
+ },
+ {
+ "entropy": 1.2578529119491577,
+ "epoch": 1.3535353535353536,
+ "grad_norm": 2.1415979862213135,
+ "learning_rate": 1.098989898989899e-05,
+ "loss": 1.2753688097000122,
+ "mean_token_accuracy": 0.6794699430465698,
+ "num_tokens": 10977280.0,
+ "step": 670
+ },
+ {
+ "entropy": 1.0493848323822021,
+ "epoch": 1.3555555555555556,
+ "grad_norm": 1.8233927488327026,
+ "learning_rate": 1.0976430976430978e-05,
+ "loss": 1.0482254028320312,
+ "mean_token_accuracy": 0.744076669216156,
+ "num_tokens": 10993664.0,
+ "step": 671
+ },
+ {
+ "entropy": 0.8162504434585571,
+ "epoch": 1.3575757575757577,
+ "grad_norm": 1.7477539777755737,
+ "learning_rate": 1.0962962962962965e-05,
+ "loss": 0.8021665811538696,
+ "mean_token_accuracy": 0.7877381443977356,
+ "num_tokens": 11010048.0,
+ "step": 672
+ },
+ {
+ "entropy": 1.5074793100357056,
+ "epoch": 1.3595959595959597,
+ "grad_norm": 2.096250057220459,
+ "learning_rate": 1.094949494949495e-05,
+ "loss": 1.4992775917053223,
+ "mean_token_accuracy": 0.6537005305290222,
+ "num_tokens": 11026432.0,
+ "step": 673
+ },
+ {
+ "entropy": 0.866222620010376,
+ "epoch": 1.3616161616161615,
+ "grad_norm": 1.9504317045211792,
+ "learning_rate": 1.0936026936026938e-05,
+ "loss": 0.8860467672348022,
+ "mean_token_accuracy": 0.7655715942382812,
+ "num_tokens": 11042816.0,
+ "step": 674
+ },
+ {
+ "entropy": 1.0089516639709473,
+ "epoch": 1.3636363636363638,
+ "grad_norm": 2.218254566192627,
+ "learning_rate": 1.0922558922558924e-05,
+ "loss": 1.005429983139038,
+ "mean_token_accuracy": 0.7360160946846008,
+ "num_tokens": 11059200.0,
+ "step": 675
+ },
+ {
+ "entropy": 1.2176142930984497,
+ "epoch": 1.3656565656565656,
+ "grad_norm": 2.1257565021514893,
+ "learning_rate": 1.0909090909090909e-05,
+ "loss": 1.286620020866394,
+ "mean_token_accuracy": 0.7042012810707092,
+ "num_tokens": 11075584.0,
+ "step": 676
+ },
+ {
+ "entropy": 1.1085315942764282,
+ "epoch": 1.3676767676767676,
+ "grad_norm": 2.065206527709961,
+ "learning_rate": 1.0895622895622897e-05,
+ "loss": 1.1294854879379272,
+ "mean_token_accuracy": 0.7205055952072144,
+ "num_tokens": 11091968.0,
+ "step": 677
+ },
+ {
+ "entropy": 1.2821027040481567,
+ "epoch": 1.3696969696969696,
+ "grad_norm": 2.126866340637207,
+ "learning_rate": 1.0882154882154884e-05,
+ "loss": 1.2661428451538086,
+ "mean_token_accuracy": 0.6976673007011414,
+ "num_tokens": 11108352.0,
+ "step": 678
+ },
+ {
+ "entropy": 0.9285033941268921,
+ "epoch": 1.3717171717171717,
+ "grad_norm": 1.9017925262451172,
+ "learning_rate": 1.0868686868686868e-05,
+ "loss": 0.9141640663146973,
+ "mean_token_accuracy": 0.7670371532440186,
+ "num_tokens": 11124736.0,
+ "step": 679
+ },
+ {
+ "entropy": 0.7374900579452515,
+ "epoch": 1.3737373737373737,
+ "grad_norm": 1.645703911781311,
+ "learning_rate": 1.0855218855218857e-05,
+ "loss": 0.7550391554832458,
+ "mean_token_accuracy": 0.7958598136901855,
+ "num_tokens": 11141120.0,
+ "step": 680
+ },
+ {
+ "entropy": 1.2956714630126953,
+ "epoch": 1.3757575757575757,
+ "grad_norm": 1.9934611320495605,
+ "learning_rate": 1.0841750841750841e-05,
+ "loss": 1.318368673324585,
+ "mean_token_accuracy": 0.686370313167572,
+ "num_tokens": 11157504.0,
+ "step": 681
+ },
+ {
+ "entropy": 1.0948729515075684,
+ "epoch": 1.3777777777777778,
+ "grad_norm": 1.8784940242767334,
+ "learning_rate": 1.082828282828283e-05,
+ "loss": 1.0897748470306396,
+ "mean_token_accuracy": 0.7231314182281494,
+ "num_tokens": 11173888.0,
+ "step": 682
+ },
+ {
+ "entropy": 1.5263433456420898,
+ "epoch": 1.3797979797979798,
+ "grad_norm": 1.9812215566635132,
+ "learning_rate": 1.0814814814814816e-05,
+ "loss": 1.535595178604126,
+ "mean_token_accuracy": 0.6683561205863953,
+ "num_tokens": 11190272.0,
+ "step": 683
+ },
+ {
+ "entropy": 1.3359065055847168,
+ "epoch": 1.3818181818181818,
+ "grad_norm": 2.1247689723968506,
+ "learning_rate": 1.08013468013468e-05,
+ "loss": 1.3298604488372803,
+ "mean_token_accuracy": 0.6813629865646362,
+ "num_tokens": 11206656.0,
+ "step": 684
+ },
+ {
+ "entropy": 1.2030925750732422,
+ "epoch": 1.3838383838383839,
+ "grad_norm": 2.178311586380005,
+ "learning_rate": 1.0787878787878789e-05,
+ "loss": 1.1808526515960693,
+ "mean_token_accuracy": 0.7067660093307495,
+ "num_tokens": 11223040.0,
+ "step": 685
+ },
+ {
+ "entropy": 0.954181969165802,
+ "epoch": 1.385858585858586,
+ "grad_norm": 1.9475486278533936,
+ "learning_rate": 1.0774410774410775e-05,
+ "loss": 0.9589706063270569,
+ "mean_token_accuracy": 0.7527479529380798,
+ "num_tokens": 11239424.0,
+ "step": 686
+ },
+ {
+ "entropy": 1.310365080833435,
+ "epoch": 1.387878787878788,
+ "grad_norm": 2.1845083236694336,
+ "learning_rate": 1.076094276094276e-05,
+ "loss": 1.2851444482803345,
+ "mean_token_accuracy": 0.6789203882217407,
+ "num_tokens": 11255808.0,
+ "step": 687
+ },
+ {
+ "entropy": 1.1187219619750977,
+ "epoch": 1.38989898989899,
+ "grad_norm": 1.9923648834228516,
+ "learning_rate": 1.0747474747474748e-05,
+ "loss": 1.1112818717956543,
+ "mean_token_accuracy": 0.7198339104652405,
+ "num_tokens": 11272192.0,
+ "step": 688
+ },
+ {
+ "entropy": 1.1317063570022583,
+ "epoch": 1.391919191919192,
+ "grad_norm": 2.003026008605957,
+ "learning_rate": 1.0734006734006735e-05,
+ "loss": 1.1108404397964478,
+ "mean_token_accuracy": 0.7223986387252808,
+ "num_tokens": 11288576.0,
+ "step": 689
+ },
+ {
+ "entropy": 1.1476070880889893,
+ "epoch": 1.393939393939394,
+ "grad_norm": 2.190676689147949,
+ "learning_rate": 1.0720538720538723e-05,
+ "loss": 1.1344139575958252,
+ "mean_token_accuracy": 0.7170249223709106,
+ "num_tokens": 11304960.0,
+ "step": 690
+ },
+ {
+ "entropy": 0.8959853053092957,
+ "epoch": 1.3959595959595958,
+ "grad_norm": 2.078281879425049,
+ "learning_rate": 1.0707070707070708e-05,
+ "loss": 0.9233289361000061,
+ "mean_token_accuracy": 0.7490229606628418,
+ "num_tokens": 11321344.0,
+ "step": 691
+ },
+ {
+ "entropy": 0.8892031311988831,
+ "epoch": 1.397979797979798,
+ "grad_norm": 1.8343734741210938,
+ "learning_rate": 1.0693602693602694e-05,
+ "loss": 0.8952107429504395,
+ "mean_token_accuracy": 0.7662432789802551,
+ "num_tokens": 11337728.0,
+ "step": 692
+ },
+ {
+ "entropy": 1.0275429487228394,
+ "epoch": 1.4,
+ "grad_norm": 1.8822777271270752,
+ "learning_rate": 1.0680134680134683e-05,
+ "loss": 1.0251365900039673,
+ "mean_token_accuracy": 0.7409623861312866,
+ "num_tokens": 11354112.0,
+ "step": 693
+ },
+ {
+ "entropy": 1.2872905731201172,
+ "epoch": 1.402020202020202,
+ "grad_norm": 1.9260203838348389,
+ "learning_rate": 1.0666666666666667e-05,
+ "loss": 1.3024024963378906,
+ "mean_token_accuracy": 0.6938202381134033,
+ "num_tokens": 11370496.0,
+ "step": 694
+ },
+ {
+ "entropy": 1.0969563722610474,
+ "epoch": 1.404040404040404,
+ "grad_norm": 2.07882022857666,
+ "learning_rate": 1.0653198653198654e-05,
+ "loss": 1.086198091506958,
+ "mean_token_accuracy": 0.7195285558700562,
+ "num_tokens": 11386880.0,
+ "step": 695
+ },
+ {
+ "entropy": 1.4206339120864868,
+ "epoch": 1.406060606060606,
+ "grad_norm": 2.110828399658203,
+ "learning_rate": 1.063973063973064e-05,
+ "loss": 1.4270105361938477,
+ "mean_token_accuracy": 0.6843551397323608,
+ "num_tokens": 11403264.0,
+ "step": 696
+ },
+ {
+ "entropy": 1.3813765048980713,
+ "epoch": 1.408080808080808,
+ "grad_norm": 1.8829882144927979,
+ "learning_rate": 1.0626262626262627e-05,
+ "loss": 1.4026310443878174,
+ "mean_token_accuracy": 0.663593053817749,
+ "num_tokens": 11419648.0,
+ "step": 697
+ },
+ {
+ "entropy": 0.8580052256584167,
+ "epoch": 1.41010101010101,
+ "grad_norm": 2.0236165523529053,
+ "learning_rate": 1.0612794612794615e-05,
+ "loss": 0.8655292391777039,
+ "mean_token_accuracy": 0.7634953856468201,
+ "num_tokens": 11436032.0,
+ "step": 698
+ },
+ {
+ "entropy": 1.0694427490234375,
+ "epoch": 1.412121212121212,
+ "grad_norm": 1.8688758611679077,
+ "learning_rate": 1.05993265993266e-05,
+ "loss": 1.089691162109375,
+ "mean_token_accuracy": 0.7183072566986084,
+ "num_tokens": 11452416.0,
+ "step": 699
+ },
+ {
+ "entropy": 1.2054415941238403,
+ "epoch": 1.4141414141414141,
+ "grad_norm": 2.0692546367645264,
+ "learning_rate": 1.0585858585858586e-05,
+ "loss": 1.2053213119506836,
+ "mean_token_accuracy": 0.7134220600128174,
+ "num_tokens": 11468800.0,
+ "step": 700
+ },
+ {
+ "epoch": 1.4141414141414141,
+ "eval_entropy": 1.2882747703021573,
+ "eval_loss": 1.535590410232544,
+ "eval_mean_token_accuracy": 0.6466120765093835,
+ "eval_num_tokens": 11468800.0,
+ "eval_runtime": 43.9264,
+ "eval_samples_per_second": 22.538,
+ "eval_steps_per_second": 2.823,
+ "step": 700
+ },
+ {
+ "entropy": 1.3370676040649414,
+ "epoch": 1.4161616161616162,
+ "grad_norm": 2.075853109359741,
+ "learning_rate": 1.0572390572390574e-05,
+ "loss": 1.3726792335510254,
+ "mean_token_accuracy": 0.6731802821159363,
+ "num_tokens": 11485184.0,
+ "step": 701
+ },
+ {
+ "entropy": 1.1095269918441772,
+ "epoch": 1.4181818181818182,
+ "grad_norm": 2.0868141651153564,
+ "learning_rate": 1.055892255892256e-05,
+ "loss": 1.124086856842041,
+ "mean_token_accuracy": 0.7162310481071472,
+ "num_tokens": 11501568.0,
+ "step": 702
+ },
+ {
+ "entropy": 1.1948339939117432,
+ "epoch": 1.4202020202020202,
+ "grad_norm": 2.091768741607666,
+ "learning_rate": 1.0545454545454546e-05,
+ "loss": 1.208143949508667,
+ "mean_token_accuracy": 0.7008426785469055,
+ "num_tokens": 11517952.0,
+ "step": 703
+ },
+ {
+ "entropy": 0.8022822141647339,
+ "epoch": 1.4222222222222223,
+ "grad_norm": 1.799911379814148,
+ "learning_rate": 1.0531986531986534e-05,
+ "loss": 0.7870017290115356,
+ "mean_token_accuracy": 0.789631187915802,
+ "num_tokens": 11534336.0,
+ "step": 704
+ },
+ {
+ "entropy": 1.0242470502853394,
+ "epoch": 1.4242424242424243,
+ "grad_norm": 1.8621245622634888,
+ "learning_rate": 1.0518518518518519e-05,
+ "loss": 1.031789779663086,
+ "mean_token_accuracy": 0.7375427484512329,
+ "num_tokens": 11550720.0,
+ "step": 705
+ },
+ {
+ "entropy": 0.9582471251487732,
+ "epoch": 1.4262626262626263,
+ "grad_norm": 2.0054924488067627,
+ "learning_rate": 1.0505050505050507e-05,
+ "loss": 0.9522819519042969,
+ "mean_token_accuracy": 0.739130437374115,
+ "num_tokens": 11567104.0,
+ "step": 706
+ },
+ {
+ "entropy": 1.172863483428955,
+ "epoch": 1.4282828282828284,
+ "grad_norm": 1.8495111465454102,
+ "learning_rate": 1.0491582491582493e-05,
+ "loss": 1.1783076524734497,
+ "mean_token_accuracy": 0.7132999300956726,
+ "num_tokens": 11583488.0,
+ "step": 707
+ },
+ {
+ "entropy": 1.2278543710708618,
+ "epoch": 1.4303030303030302,
+ "grad_norm": 2.095710277557373,
+ "learning_rate": 1.0478114478114478e-05,
+ "loss": 1.2355749607086182,
+ "mean_token_accuracy": 0.6905227303504944,
+ "num_tokens": 11599872.0,
+ "step": 708
+ },
+ {
+ "entropy": 0.9415866732597351,
+ "epoch": 1.4323232323232324,
+ "grad_norm": 1.9720549583435059,
+ "learning_rate": 1.0464646464646466e-05,
+ "loss": 0.9524072408676147,
+ "mean_token_accuracy": 0.7501221299171448,
+ "num_tokens": 11616256.0,
+ "step": 709
+ },
+ {
+ "entropy": 1.371761441230774,
+ "epoch": 1.4343434343434343,
+ "grad_norm": 2.2934458255767822,
+ "learning_rate": 1.0451178451178453e-05,
+ "loss": 1.3502682447433472,
+ "mean_token_accuracy": 0.665730357170105,
+ "num_tokens": 11632640.0,
+ "step": 710
+ },
+ {
+ "entropy": 1.1605411767959595,
+ "epoch": 1.4363636363636363,
+ "grad_norm": 1.978593349456787,
+ "learning_rate": 1.0437710437710438e-05,
+ "loss": 1.166311264038086,
+ "mean_token_accuracy": 0.6971177458763123,
+ "num_tokens": 11649024.0,
+ "step": 711
+ },
+ {
+ "entropy": 1.2717126607894897,
+ "epoch": 1.4383838383838383,
+ "grad_norm": 2.077765703201294,
+ "learning_rate": 1.0424242424242426e-05,
+ "loss": 1.2650127410888672,
+ "mean_token_accuracy": 0.6827064156532288,
+ "num_tokens": 11665408.0,
+ "step": 712
+ },
+ {
+ "entropy": 1.42205810546875,
+ "epoch": 1.4404040404040404,
+ "grad_norm": 2.152578592300415,
+ "learning_rate": 1.041077441077441e-05,
+ "loss": 1.4107255935668945,
+ "mean_token_accuracy": 0.6676233410835266,
+ "num_tokens": 11681792.0,
+ "step": 713
+ },
+ {
+ "entropy": 1.8077080249786377,
+ "epoch": 1.4424242424242424,
+ "grad_norm": 2.0739355087280273,
+ "learning_rate": 1.0397306397306399e-05,
+ "loss": 1.83225417137146,
+ "mean_token_accuracy": 0.6088177561759949,
+ "num_tokens": 11698176.0,
+ "step": 714
+ },
+ {
+ "entropy": 1.3185032606124878,
+ "epoch": 1.4444444444444444,
+ "grad_norm": 2.1207802295684814,
+ "learning_rate": 1.0383838383838385e-05,
+ "loss": 1.3216420412063599,
+ "mean_token_accuracy": 0.6762335300445557,
+ "num_tokens": 11714560.0,
+ "step": 715
+ },
+ {
+ "entropy": 1.1990821361541748,
+ "epoch": 1.4464646464646465,
+ "grad_norm": 1.9028881788253784,
+ "learning_rate": 1.037037037037037e-05,
+ "loss": 1.2123594284057617,
+ "mean_token_accuracy": 0.7231314182281494,
+ "num_tokens": 11730944.0,
+ "step": 716
+ },
+ {
+ "entropy": 1.1971170902252197,
+ "epoch": 1.4484848484848485,
+ "grad_norm": 2.071993827819824,
+ "learning_rate": 1.0356902356902358e-05,
+ "loss": 1.1753935813903809,
+ "mean_token_accuracy": 0.7173302173614502,
+ "num_tokens": 11747328.0,
+ "step": 717
+ },
+ {
+ "entropy": 1.16183602809906,
+ "epoch": 1.4505050505050505,
+ "grad_norm": 2.0574443340301514,
+ "learning_rate": 1.0343434343434345e-05,
+ "loss": 1.171074390411377,
+ "mean_token_accuracy": 0.7076819539070129,
+ "num_tokens": 11763712.0,
+ "step": 718
+ },
+ {
+ "entropy": 1.1637240648269653,
+ "epoch": 1.4525252525252526,
+ "grad_norm": 1.9361884593963623,
+ "learning_rate": 1.032996632996633e-05,
+ "loss": 1.1374025344848633,
+ "mean_token_accuracy": 0.7241694927215576,
+ "num_tokens": 11780096.0,
+ "step": 719
+ },
+ {
+ "entropy": 1.0883492231369019,
+ "epoch": 1.4545454545454546,
+ "grad_norm": 2.0204899311065674,
+ "learning_rate": 1.0316498316498318e-05,
+ "loss": 1.0666342973709106,
+ "mean_token_accuracy": 0.7216047644615173,
+ "num_tokens": 11796480.0,
+ "step": 720
+ },
+ {
+ "entropy": 1.3728728294372559,
+ "epoch": 1.4565656565656566,
+ "grad_norm": 1.8690729141235352,
+ "learning_rate": 1.0303030303030304e-05,
+ "loss": 1.3879103660583496,
+ "mean_token_accuracy": 0.6769052147865295,
+ "num_tokens": 11812864.0,
+ "step": 721
+ },
+ {
+ "entropy": 1.1204687356948853,
+ "epoch": 1.4585858585858587,
+ "grad_norm": 2.091113567352295,
+ "learning_rate": 1.0289562289562289e-05,
+ "loss": 1.1190675497055054,
+ "mean_token_accuracy": 0.7191621661186218,
+ "num_tokens": 11829248.0,
+ "step": 722
+ },
+ {
+ "entropy": 1.479430913925171,
+ "epoch": 1.4606060606060607,
+ "grad_norm": 2.194871187210083,
+ "learning_rate": 1.0276094276094277e-05,
+ "loss": 1.4967867136001587,
+ "mean_token_accuracy": 0.6475329995155334,
+ "num_tokens": 11845632.0,
+ "step": 723
+ },
+ {
+ "entropy": 1.083875060081482,
+ "epoch": 1.4626262626262627,
+ "grad_norm": 1.9257639646530151,
+ "learning_rate": 1.0262626262626264e-05,
+ "loss": 1.1053729057312012,
+ "mean_token_accuracy": 0.7168416976928711,
+ "num_tokens": 11862016.0,
+ "step": 724
+ },
+ {
+ "entropy": 1.074636697769165,
+ "epoch": 1.4646464646464645,
+ "grad_norm": 1.956766963005066,
+ "learning_rate": 1.024915824915825e-05,
+ "loss": 1.0775985717773438,
+ "mean_token_accuracy": 0.7348558902740479,
+ "num_tokens": 11878400.0,
+ "step": 725
+ },
+ {
+ "entropy": 1.1595367193222046,
+ "epoch": 1.4666666666666668,
+ "grad_norm": 2.0679080486297607,
+ "learning_rate": 1.0235690235690236e-05,
+ "loss": 1.1976954936981201,
+ "mean_token_accuracy": 0.7014533281326294,
+ "num_tokens": 11894784.0,
+ "step": 726
+ },
+ {
+ "entropy": 1.0546326637268066,
+ "epoch": 1.4686868686868686,
+ "grad_norm": 1.8939472436904907,
+ "learning_rate": 1.0222222222222223e-05,
+ "loss": 1.0483310222625732,
+ "mean_token_accuracy": 0.7339398860931396,
+ "num_tokens": 11911168.0,
+ "step": 727
+ },
+ {
+ "entropy": 1.3418383598327637,
+ "epoch": 1.4707070707070706,
+ "grad_norm": 2.239595413208008,
+ "learning_rate": 1.020875420875421e-05,
+ "loss": 1.35018789768219,
+ "mean_token_accuracy": 0.6685393452644348,
+ "num_tokens": 11927552.0,
+ "step": 728
+ },
+ {
+ "entropy": 0.9122293591499329,
+ "epoch": 1.4727272727272727,
+ "grad_norm": 2.1075761318206787,
+ "learning_rate": 1.0195286195286196e-05,
+ "loss": 0.9050356149673462,
+ "mean_token_accuracy": 0.7633121609687805,
+ "num_tokens": 11943936.0,
+ "step": 729
+ },
+ {
+ "entropy": 0.9527847766876221,
+ "epoch": 1.4747474747474747,
+ "grad_norm": 2.017185688018799,
+ "learning_rate": 1.0181818181818182e-05,
+ "loss": 0.9490136504173279,
+ "mean_token_accuracy": 0.7539081573486328,
+ "num_tokens": 11960320.0,
+ "step": 730
+ },
+ {
+ "entropy": 1.2855827808380127,
+ "epoch": 1.4767676767676767,
+ "grad_norm": 1.9821733236312866,
+ "learning_rate": 1.0168350168350169e-05,
+ "loss": 1.2902567386627197,
+ "mean_token_accuracy": 0.6854543089866638,
+ "num_tokens": 11976704.0,
+ "step": 731
+ },
+ {
+ "entropy": 1.143114686012268,
+ "epoch": 1.4787878787878788,
+ "grad_norm": 2.0281424522399902,
+ "learning_rate": 1.0154882154882155e-05,
+ "loss": 1.1373090744018555,
+ "mean_token_accuracy": 0.7071934342384338,
+ "num_tokens": 11993088.0,
+ "step": 732
+ },
+ {
+ "entropy": 1.1341931819915771,
+ "epoch": 1.4808080808080808,
+ "grad_norm": 2.075814723968506,
+ "learning_rate": 1.0141414141414144e-05,
+ "loss": 1.1449689865112305,
+ "mean_token_accuracy": 0.7180019617080688,
+ "num_tokens": 12009472.0,
+ "step": 733
+ },
+ {
+ "entropy": 0.9200477004051208,
+ "epoch": 1.4828282828282828,
+ "grad_norm": 1.7193031311035156,
+ "learning_rate": 1.0127946127946128e-05,
+ "loss": 0.9225804209709167,
+ "mean_token_accuracy": 0.7621519565582275,
+ "num_tokens": 12025856.0,
+ "step": 734
+ },
+ {
+ "entropy": 1.3266046047210693,
+ "epoch": 1.4848484848484849,
+ "grad_norm": 2.0543408393859863,
+ "learning_rate": 1.0114478114478115e-05,
+ "loss": 1.3158385753631592,
+ "mean_token_accuracy": 0.6758060455322266,
+ "num_tokens": 12042240.0,
+ "step": 735
+ },
+ {
+ "entropy": 1.2785359621047974,
+ "epoch": 1.486868686868687,
+ "grad_norm": 2.0174546241760254,
+ "learning_rate": 1.0101010101010103e-05,
+ "loss": 1.269796371459961,
+ "mean_token_accuracy": 0.7070102691650391,
+ "num_tokens": 12058624.0,
+ "step": 736
+ },
+ {
+ "entropy": 1.3418751955032349,
+ "epoch": 1.488888888888889,
+ "grad_norm": 2.0906383991241455,
+ "learning_rate": 1.0087542087542088e-05,
+ "loss": 1.35174560546875,
+ "mean_token_accuracy": 0.6758060455322266,
+ "num_tokens": 12075008.0,
+ "step": 737
+ },
+ {
+ "entropy": 1.266552448272705,
+ "epoch": 1.490909090909091,
+ "grad_norm": 1.7485158443450928,
+ "learning_rate": 1.0074074074074074e-05,
+ "loss": 1.2538254261016846,
+ "mean_token_accuracy": 0.7070713043212891,
+ "num_tokens": 12091392.0,
+ "step": 738
+ },
+ {
+ "entropy": 0.9562437534332275,
+ "epoch": 1.492929292929293,
+ "grad_norm": 1.9106290340423584,
+ "learning_rate": 1.0060606060606062e-05,
+ "loss": 0.938219428062439,
+ "mean_token_accuracy": 0.7383365631103516,
+ "num_tokens": 12107776.0,
+ "step": 739
+ },
+ {
+ "entropy": 1.3171569108963013,
+ "epoch": 1.494949494949495,
+ "grad_norm": 2.127641439437866,
+ "learning_rate": 1.0047138047138047e-05,
+ "loss": 1.3213756084442139,
+ "mean_token_accuracy": 0.6819125413894653,
+ "num_tokens": 12124160.0,
+ "step": 740
+ },
+ {
+ "entropy": 1.0634039640426636,
+ "epoch": 1.496969696969697,
+ "grad_norm": 2.0710532665252686,
+ "learning_rate": 1.0033670033670035e-05,
+ "loss": 1.0797393321990967,
+ "mean_token_accuracy": 0.7150708436965942,
+ "num_tokens": 12140544.0,
+ "step": 741
+ },
+ {
+ "entropy": 1.3235414028167725,
+ "epoch": 1.4989898989898989,
+ "grad_norm": 2.040738105773926,
+ "learning_rate": 1.0020202020202022e-05,
+ "loss": 1.3383063077926636,
+ "mean_token_accuracy": 0.6737298369407654,
+ "num_tokens": 12156928.0,
+ "step": 742
+ },
+ {
+ "entropy": 1.2156025171279907,
+ "epoch": 1.5010101010101011,
+ "grad_norm": 2.196033239364624,
+ "learning_rate": 1.0006734006734007e-05,
+ "loss": 1.2233551740646362,
+ "mean_token_accuracy": 0.7015144228935242,
+ "num_tokens": 12173312.0,
+ "step": 743
+ },
+ {
+ "entropy": 0.9380976557731628,
+ "epoch": 1.503030303030303,
+ "grad_norm": 1.951636791229248,
+ "learning_rate": 9.993265993265993e-06,
+ "loss": 0.9319983124732971,
+ "mean_token_accuracy": 0.7494503855705261,
+ "num_tokens": 12189696.0,
+ "step": 744
+ },
+ {
+ "entropy": 1.3272504806518555,
+ "epoch": 1.5050505050505052,
+ "grad_norm": 2.045417308807373,
+ "learning_rate": 9.97979797979798e-06,
+ "loss": 1.3266823291778564,
+ "mean_token_accuracy": 0.6848436594009399,
+ "num_tokens": 12206080.0,
+ "step": 745
+ },
+ {
+ "entropy": 0.955014705657959,
+ "epoch": 1.507070707070707,
+ "grad_norm": 1.7301888465881348,
+ "learning_rate": 9.966329966329968e-06,
+ "loss": 0.9721895456314087,
+ "mean_token_accuracy": 0.7561064958572388,
+ "num_tokens": 12222464.0,
+ "step": 746
+ },
+ {
+ "entropy": 1.0614056587219238,
+ "epoch": 1.509090909090909,
+ "grad_norm": 2.235422372817993,
+ "learning_rate": 9.952861952861954e-06,
+ "loss": 1.081586241722107,
+ "mean_token_accuracy": 0.7147044539451599,
+ "num_tokens": 12238848.0,
+ "step": 747
+ },
+ {
+ "entropy": 1.075492024421692,
+ "epoch": 1.511111111111111,
+ "grad_norm": 2.0352933406829834,
+ "learning_rate": 9.939393939393939e-06,
+ "loss": 1.083856463432312,
+ "mean_token_accuracy": 0.7148265838623047,
+ "num_tokens": 12255232.0,
+ "step": 748
+ },
+ {
+ "entropy": 1.1964384317398071,
+ "epoch": 1.513131313131313,
+ "grad_norm": 1.9096226692199707,
+ "learning_rate": 9.925925925925927e-06,
+ "loss": 1.1777125597000122,
+ "mean_token_accuracy": 0.7061553597450256,
+ "num_tokens": 12271616.0,
+ "step": 749
+ },
+ {
+ "entropy": 1.2923556566238403,
+ "epoch": 1.5151515151515151,
+ "grad_norm": 2.1002533435821533,
+ "learning_rate": 9.912457912457914e-06,
+ "loss": 1.2916597127914429,
+ "mean_token_accuracy": 0.6882632970809937,
+ "num_tokens": 12288000.0,
+ "step": 750
+ },
+ {
+ "entropy": 1.008608341217041,
+ "epoch": 1.5171717171717172,
+ "grad_norm": 2.1212828159332275,
+ "learning_rate": 9.8989898989899e-06,
+ "loss": 1.0104637145996094,
+ "mean_token_accuracy": 0.7343673706054688,
+ "num_tokens": 12304384.0,
+ "step": 751
+ },
+ {
+ "entropy": 1.550642967224121,
+ "epoch": 1.5191919191919192,
+ "grad_norm": 1.9693539142608643,
+ "learning_rate": 9.885521885521887e-06,
+ "loss": 1.5664982795715332,
+ "mean_token_accuracy": 0.6830728054046631,
+ "num_tokens": 12320768.0,
+ "step": 752
+ },
+ {
+ "entropy": 1.3540711402893066,
+ "epoch": 1.5212121212121212,
+ "grad_norm": 2.181079149246216,
+ "learning_rate": 9.872053872053873e-06,
+ "loss": 1.3578659296035767,
+ "mean_token_accuracy": 0.666829526424408,
+ "num_tokens": 12337152.0,
+ "step": 753
+ },
+ {
+ "entropy": 1.002195119857788,
+ "epoch": 1.5232323232323233,
+ "grad_norm": 2.051185369491577,
+ "learning_rate": 9.85858585858586e-06,
+ "loss": 0.9798580408096313,
+ "mean_token_accuracy": 0.7434660196304321,
+ "num_tokens": 12353536.0,
+ "step": 754
+ },
+ {
+ "entropy": 1.2805551290512085,
+ "epoch": 1.5252525252525253,
+ "grad_norm": 2.0427937507629395,
+ "learning_rate": 9.845117845117846e-06,
+ "loss": 1.2829053401947021,
+ "mean_token_accuracy": 0.6933317184448242,
+ "num_tokens": 12369920.0,
+ "step": 755
+ },
+ {
+ "entropy": 1.3517152070999146,
+ "epoch": 1.5272727272727273,
+ "grad_norm": 2.0065114498138428,
+ "learning_rate": 9.831649831649833e-06,
+ "loss": 1.3636512756347656,
+ "mean_token_accuracy": 0.6757450103759766,
+ "num_tokens": 12386304.0,
+ "step": 756
+ },
+ {
+ "entropy": 0.7547817826271057,
+ "epoch": 1.5292929292929291,
+ "grad_norm": 3.2289278507232666,
+ "learning_rate": 9.81818181818182e-06,
+ "loss": 0.7594918012619019,
+ "mean_token_accuracy": 0.7917684316635132,
+ "num_tokens": 12402688.0,
+ "step": 757
+ },
+ {
+ "entropy": 1.4882941246032715,
+ "epoch": 1.5313131313131314,
+ "grad_norm": 1.9724371433258057,
+ "learning_rate": 9.804713804713806e-06,
+ "loss": 1.4832470417022705,
+ "mean_token_accuracy": 0.6436858773231506,
+ "num_tokens": 12419072.0,
+ "step": 758
+ },
+ {
+ "entropy": 0.9879036545753479,
+ "epoch": 1.5333333333333332,
+ "grad_norm": 1.9205714464187622,
+ "learning_rate": 9.791245791245792e-06,
+ "loss": 0.9901232123374939,
+ "mean_token_accuracy": 0.7465192675590515,
+ "num_tokens": 12435456.0,
+ "step": 759
+ },
+ {
+ "entropy": 0.9692004919052124,
+ "epoch": 1.5353535353535355,
+ "grad_norm": 1.8762351274490356,
+ "learning_rate": 9.777777777777779e-06,
+ "loss": 0.9544116258621216,
+ "mean_token_accuracy": 0.7518319487571716,
+ "num_tokens": 12451840.0,
+ "step": 760
+ },
+ {
+ "entropy": 0.8084391951560974,
+ "epoch": 1.5373737373737373,
+ "grad_norm": 1.9590117931365967,
+ "learning_rate": 9.764309764309765e-06,
+ "loss": 0.8268290162086487,
+ "mean_token_accuracy": 0.7758305072784424,
+ "num_tokens": 12468224.0,
+ "step": 761
+ },
+ {
+ "entropy": 1.2048877477645874,
+ "epoch": 1.5393939393939395,
+ "grad_norm": 2.1473560333251953,
+ "learning_rate": 9.750841750841752e-06,
+ "loss": 1.1967238187789917,
+ "mean_token_accuracy": 0.7075598239898682,
+ "num_tokens": 12484608.0,
+ "step": 762
+ },
+ {
+ "entropy": 1.330485224723816,
+ "epoch": 1.5414141414141413,
+ "grad_norm": 2.0944108963012695,
+ "learning_rate": 9.737373737373738e-06,
+ "loss": 1.3377408981323242,
+ "mean_token_accuracy": 0.6920493245124817,
+ "num_tokens": 12500992.0,
+ "step": 763
+ },
+ {
+ "entropy": 0.994591474533081,
+ "epoch": 1.5434343434343434,
+ "grad_norm": 2.224015712738037,
+ "learning_rate": 9.723905723905725e-06,
+ "loss": 0.9741864204406738,
+ "mean_token_accuracy": 0.734306275844574,
+ "num_tokens": 12517376.0,
+ "step": 764
+ },
+ {
+ "entropy": 1.3140320777893066,
+ "epoch": 1.5454545454545454,
+ "grad_norm": 2.0910675525665283,
+ "learning_rate": 9.710437710437711e-06,
+ "loss": 1.3168857097625732,
+ "mean_token_accuracy": 0.6908280253410339,
+ "num_tokens": 12533760.0,
+ "step": 765
+ },
+ {
+ "entropy": 0.5885581374168396,
+ "epoch": 1.5474747474747474,
+ "grad_norm": 1.6729995012283325,
+ "learning_rate": 9.696969696969698e-06,
+ "loss": 0.587052583694458,
+ "mean_token_accuracy": 0.8340253829956055,
+ "num_tokens": 12550144.0,
+ "step": 766
+ },
+ {
+ "entropy": 1.1873608827590942,
+ "epoch": 1.5494949494949495,
+ "grad_norm": 2.0278220176696777,
+ "learning_rate": 9.683501683501684e-06,
+ "loss": 1.1988582611083984,
+ "mean_token_accuracy": 0.708109438419342,
+ "num_tokens": 12566528.0,
+ "step": 767
+ },
+ {
+ "entropy": 1.600446343421936,
+ "epoch": 1.5515151515151515,
+ "grad_norm": 2.1766979694366455,
+ "learning_rate": 9.67003367003367e-06,
+ "loss": 1.6514582633972168,
+ "mean_token_accuracy": 0.6134586930274963,
+ "num_tokens": 12582912.0,
+ "step": 768
+ },
+ {
+ "entropy": 1.3398208618164062,
+ "epoch": 1.5535353535353535,
+ "grad_norm": 2.3216426372528076,
+ "learning_rate": 9.656565656565657e-06,
+ "loss": 1.3702855110168457,
+ "mean_token_accuracy": 0.6650586128234863,
+ "num_tokens": 12599296.0,
+ "step": 769
+ },
+ {
+ "entropy": 1.061170220375061,
+ "epoch": 1.5555555555555556,
+ "grad_norm": 1.9703660011291504,
+ "learning_rate": 9.643097643097643e-06,
+ "loss": 1.0738681554794312,
+ "mean_token_accuracy": 0.7376648783683777,
+ "num_tokens": 12615680.0,
+ "step": 770
+ },
+ {
+ "entropy": 1.1564687490463257,
+ "epoch": 1.5575757575757576,
+ "grad_norm": 2.0951433181762695,
+ "learning_rate": 9.62962962962963e-06,
+ "loss": 1.1778810024261475,
+ "mean_token_accuracy": 0.7112237215042114,
+ "num_tokens": 12632064.0,
+ "step": 771
+ },
+ {
+ "entropy": 0.8667913675308228,
+ "epoch": 1.5595959595959596,
+ "grad_norm": 1.9066787958145142,
+ "learning_rate": 9.616161616161616e-06,
+ "loss": 0.8634734153747559,
+ "mean_token_accuracy": 0.769052267074585,
+ "num_tokens": 12648448.0,
+ "step": 772
+ },
+ {
+ "entropy": 1.20845627784729,
+ "epoch": 1.5616161616161617,
+ "grad_norm": 2.0367836952209473,
+ "learning_rate": 9.602693602693603e-06,
+ "loss": 1.2191519737243652,
+ "mean_token_accuracy": 0.69840008020401,
+ "num_tokens": 12664832.0,
+ "step": 773
+ },
+ {
+ "entropy": 1.3325128555297852,
+ "epoch": 1.5636363636363635,
+ "grad_norm": 2.000307083129883,
+ "learning_rate": 9.589225589225591e-06,
+ "loss": 1.3183135986328125,
+ "mean_token_accuracy": 0.6904006004333496,
+ "num_tokens": 12681216.0,
+ "step": 774
+ },
+ {
+ "entropy": 1.3372687101364136,
+ "epoch": 1.5656565656565657,
+ "grad_norm": 2.2447569370269775,
+ "learning_rate": 9.575757575757576e-06,
+ "loss": 1.3026032447814941,
+ "mean_token_accuracy": 0.6769663095474243,
+ "num_tokens": 12697600.0,
+ "step": 775
+ },
+ {
+ "entropy": 1.1477705240249634,
+ "epoch": 1.5676767676767676,
+ "grad_norm": 1.9308643341064453,
+ "learning_rate": 9.562289562289562e-06,
+ "loss": 1.1485618352890015,
+ "mean_token_accuracy": 0.7186126112937927,
+ "num_tokens": 12713984.0,
+ "step": 776
+ },
+ {
+ "entropy": 1.021490454673767,
+ "epoch": 1.5696969696969698,
+ "grad_norm": 1.9314731359481812,
+ "learning_rate": 9.548821548821549e-06,
+ "loss": 1.002140998840332,
+ "mean_token_accuracy": 0.735344409942627,
+ "num_tokens": 12730368.0,
+ "step": 777
+ },
+ {
+ "entropy": 1.1351706981658936,
+ "epoch": 1.5717171717171716,
+ "grad_norm": 1.9402823448181152,
+ "learning_rate": 9.535353535353537e-06,
+ "loss": 1.1265199184417725,
+ "mean_token_accuracy": 0.716292142868042,
+ "num_tokens": 12746752.0,
+ "step": 778
+ },
+ {
+ "entropy": 0.9608347415924072,
+ "epoch": 1.5737373737373739,
+ "grad_norm": 1.9762189388275146,
+ "learning_rate": 9.521885521885522e-06,
+ "loss": 0.9408327341079712,
+ "mean_token_accuracy": 0.7556179761886597,
+ "num_tokens": 12763136.0,
+ "step": 779
+ },
+ {
+ "entropy": 1.2921335697174072,
+ "epoch": 1.5757575757575757,
+ "grad_norm": 1.9008197784423828,
+ "learning_rate": 9.508417508417508e-06,
+ "loss": 1.298734426498413,
+ "mean_token_accuracy": 0.6901563405990601,
+ "num_tokens": 12779520.0,
+ "step": 780
+ },
+ {
+ "entropy": 1.1303656101226807,
+ "epoch": 1.5777777777777777,
+ "grad_norm": 2.368401527404785,
+ "learning_rate": 9.494949494949497e-06,
+ "loss": 1.1172974109649658,
+ "mean_token_accuracy": 0.7112848162651062,
+ "num_tokens": 12795904.0,
+ "step": 781
+ },
+ {
+ "entropy": 1.5801783800125122,
+ "epoch": 1.5797979797979798,
+ "grad_norm": 2.1036503314971924,
+ "learning_rate": 9.481481481481483e-06,
+ "loss": 1.5825071334838867,
+ "mean_token_accuracy": 0.6334269642829895,
+ "num_tokens": 12812288.0,
+ "step": 782
+ },
+ {
+ "entropy": 1.1247191429138184,
+ "epoch": 1.5818181818181818,
+ "grad_norm": 2.0482163429260254,
+ "learning_rate": 9.468013468013468e-06,
+ "loss": 1.1644407510757446,
+ "mean_token_accuracy": 0.7239863276481628,
+ "num_tokens": 12828672.0,
+ "step": 783
+ },
+ {
+ "entropy": 1.6274546384811401,
+ "epoch": 1.5838383838383838,
+ "grad_norm": 2.075209379196167,
+ "learning_rate": 9.454545454545456e-06,
+ "loss": 1.6390056610107422,
+ "mean_token_accuracy": 0.6265266537666321,
+ "num_tokens": 12845056.0,
+ "step": 784
+ },
+ {
+ "entropy": 1.1876834630966187,
+ "epoch": 1.5858585858585859,
+ "grad_norm": 2.3457517623901367,
+ "learning_rate": 9.441077441077442e-06,
+ "loss": 1.229914903640747,
+ "mean_token_accuracy": 0.686431348323822,
+ "num_tokens": 12861440.0,
+ "step": 785
+ },
+ {
+ "entropy": 1.0943474769592285,
+ "epoch": 1.587878787878788,
+ "grad_norm": 2.2637226581573486,
+ "learning_rate": 9.427609427609429e-06,
+ "loss": 1.1246657371520996,
+ "mean_token_accuracy": 0.7178187370300293,
+ "num_tokens": 12877824.0,
+ "step": 786
+ },
+ {
+ "entropy": 1.1606156826019287,
+ "epoch": 1.58989898989899,
+ "grad_norm": 2.1095638275146484,
+ "learning_rate": 9.414141414141414e-06,
+ "loss": 1.146558403968811,
+ "mean_token_accuracy": 0.7142769694328308,
+ "num_tokens": 12894208.0,
+ "step": 787
+ },
+ {
+ "entropy": 0.7914054989814758,
+ "epoch": 1.591919191919192,
+ "grad_norm": 2.0504069328308105,
+ "learning_rate": 9.400673400673402e-06,
+ "loss": 0.7990944385528564,
+ "mean_token_accuracy": 0.7863947153091431,
+ "num_tokens": 12910592.0,
+ "step": 788
+ },
+ {
+ "entropy": 1.2664366960525513,
+ "epoch": 1.593939393939394,
+ "grad_norm": 2.174043893814087,
+ "learning_rate": 9.387205387205388e-06,
+ "loss": 1.263155221939087,
+ "mean_token_accuracy": 0.6905227303504944,
+ "num_tokens": 12926976.0,
+ "step": 789
+ },
+ {
+ "entropy": 1.6059703826904297,
+ "epoch": 1.595959595959596,
+ "grad_norm": 2.0089824199676514,
+ "learning_rate": 9.373737373737375e-06,
+ "loss": 1.628819465637207,
+ "mean_token_accuracy": 0.6278700828552246,
+ "num_tokens": 12943360.0,
+ "step": 790
+ },
+ {
+ "entropy": 1.4796923398971558,
+ "epoch": 1.5979797979797978,
+ "grad_norm": 2.4513726234436035,
+ "learning_rate": 9.360269360269361e-06,
+ "loss": 1.5144509077072144,
+ "mean_token_accuracy": 0.6381289958953857,
+ "num_tokens": 12959744.0,
+ "step": 791
+ },
+ {
+ "entropy": 1.1704087257385254,
+ "epoch": 1.6,
+ "grad_norm": 2.079355478286743,
+ "learning_rate": 9.346801346801348e-06,
+ "loss": 1.1726913452148438,
+ "mean_token_accuracy": 0.709208607673645,
+ "num_tokens": 12976128.0,
+ "step": 792
+ },
+ {
+ "entropy": 1.2573802471160889,
+ "epoch": 1.602020202020202,
+ "grad_norm": 2.087904453277588,
+ "learning_rate": 9.333333333333334e-06,
+ "loss": 1.2652850151062012,
+ "mean_token_accuracy": 0.6868588328361511,
+ "num_tokens": 12992512.0,
+ "step": 793
+ },
+ {
+ "entropy": 1.5265566110610962,
+ "epoch": 1.6040404040404042,
+ "grad_norm": 2.1586968898773193,
+ "learning_rate": 9.31986531986532e-06,
+ "loss": 1.5234975814819336,
+ "mean_token_accuracy": 0.6436248421669006,
+ "num_tokens": 13008896.0,
+ "step": 794
+ },
+ {
+ "entropy": 1.2587038278579712,
+ "epoch": 1.606060606060606,
+ "grad_norm": 2.0685694217681885,
+ "learning_rate": 9.306397306397307e-06,
+ "loss": 1.2585840225219727,
+ "mean_token_accuracy": 0.6930874586105347,
+ "num_tokens": 13025280.0,
+ "step": 795
+ },
+ {
+ "entropy": 1.1857022047042847,
+ "epoch": 1.6080808080808082,
+ "grad_norm": 2.0476908683776855,
+ "learning_rate": 9.292929292929294e-06,
+ "loss": 1.2005798816680908,
+ "mean_token_accuracy": 0.693453848361969,
+ "num_tokens": 13041664.0,
+ "step": 796
+ },
+ {
+ "entropy": 1.5042216777801514,
+ "epoch": 1.61010101010101,
+ "grad_norm": 1.973929762840271,
+ "learning_rate": 9.27946127946128e-06,
+ "loss": 1.5095748901367188,
+ "mean_token_accuracy": 0.6579140424728394,
+ "num_tokens": 13058048.0,
+ "step": 797
+ },
+ {
+ "entropy": 1.5203564167022705,
+ "epoch": 1.612121212121212,
+ "grad_norm": 1.957383155822754,
+ "learning_rate": 9.265993265993267e-06,
+ "loss": 1.4991892576217651,
+ "mean_token_accuracy": 0.6539447903633118,
+ "num_tokens": 13074432.0,
+ "step": 798
+ },
+ {
+ "entropy": 1.3984291553497314,
+ "epoch": 1.614141414141414,
+ "grad_norm": 2.143216848373413,
+ "learning_rate": 9.252525252525253e-06,
+ "loss": 1.3863308429718018,
+ "mean_token_accuracy": 0.6649364829063416,
+ "num_tokens": 13090816.0,
+ "step": 799
+ },
+ {
+ "entropy": 1.662933111190796,
+ "epoch": 1.6161616161616161,
+ "grad_norm": 2.111069440841675,
+ "learning_rate": 9.23905723905724e-06,
+ "loss": 1.6813087463378906,
+ "mean_token_accuracy": 0.6262823939323425,
+ "num_tokens": 13107200.0,
+ "step": 800
+ },
+ {
+ "epoch": 1.6161616161616161,
+ "eval_entropy": 1.3006088190501737,
+ "eval_loss": 1.5314096212387085,
+ "eval_mean_token_accuracy": 0.6469700956536878,
+ "eval_num_tokens": 13107200.0,
+ "eval_runtime": 43.8956,
+ "eval_samples_per_second": 22.554,
+ "eval_steps_per_second": 2.825,
+ "step": 800
+ },
+ {
+ "entropy": 1.709677815437317,
+ "epoch": 1.6181818181818182,
+ "grad_norm": 2.790998935699463,
+ "learning_rate": 9.225589225589226e-06,
+ "loss": 1.740628719329834,
+ "mean_token_accuracy": 0.6136419177055359,
+ "num_tokens": 13123584.0,
+ "step": 801
+ },
+ {
+ "entropy": 1.2373168468475342,
+ "epoch": 1.6202020202020202,
+ "grad_norm": 2.49934458732605,
+ "learning_rate": 9.212121212121213e-06,
+ "loss": 1.2430124282836914,
+ "mean_token_accuracy": 0.6856985688209534,
+ "num_tokens": 13139968.0,
+ "step": 802
+ },
+ {
+ "entropy": 1.4974594116210938,
+ "epoch": 1.6222222222222222,
+ "grad_norm": 2.015254259109497,
+ "learning_rate": 9.1986531986532e-06,
+ "loss": 1.5037147998809814,
+ "mean_token_accuracy": 0.6581583023071289,
+ "num_tokens": 13156352.0,
+ "step": 803
+ },
+ {
+ "entropy": 1.0019752979278564,
+ "epoch": 1.6242424242424243,
+ "grad_norm": 1.9795469045639038,
+ "learning_rate": 9.185185185185186e-06,
+ "loss": 1.0053883790969849,
+ "mean_token_accuracy": 0.7297264337539673,
+ "num_tokens": 13172736.0,
+ "step": 804
+ },
+ {
+ "entropy": 1.2690199613571167,
+ "epoch": 1.6262626262626263,
+ "grad_norm": 2.1013784408569336,
+ "learning_rate": 9.171717171717172e-06,
+ "loss": 1.2500054836273193,
+ "mean_token_accuracy": 0.6856985688209534,
+ "num_tokens": 13189120.0,
+ "step": 805
+ },
+ {
+ "entropy": 1.1282989978790283,
+ "epoch": 1.6282828282828283,
+ "grad_norm": 2.2034029960632324,
+ "learning_rate": 9.15824915824916e-06,
+ "loss": 1.1393691301345825,
+ "mean_token_accuracy": 0.7051783204078674,
+ "num_tokens": 13205504.0,
+ "step": 806
+ },
+ {
+ "entropy": 1.1744720935821533,
+ "epoch": 1.6303030303030304,
+ "grad_norm": 1.9687124490737915,
+ "learning_rate": 9.144781144781145e-06,
+ "loss": 1.1625237464904785,
+ "mean_token_accuracy": 0.7002931237220764,
+ "num_tokens": 13221888.0,
+ "step": 807
+ },
+ {
+ "entropy": 0.9849017858505249,
+ "epoch": 1.6323232323232322,
+ "grad_norm": 1.9646586179733276,
+ "learning_rate": 9.131313131313132e-06,
+ "loss": 0.9992689490318298,
+ "mean_token_accuracy": 0.7460307478904724,
+ "num_tokens": 13238272.0,
+ "step": 808
+ },
+ {
+ "entropy": 1.142819881439209,
+ "epoch": 1.6343434343434344,
+ "grad_norm": 1.975189208984375,
+ "learning_rate": 9.117845117845118e-06,
+ "loss": 1.1389580965042114,
+ "mean_token_accuracy": 0.710368812084198,
+ "num_tokens": 13254656.0,
+ "step": 809
+ },
+ {
+ "entropy": 1.1362377405166626,
+ "epoch": 1.6363636363636362,
+ "grad_norm": 2.0360374450683594,
+ "learning_rate": 9.104377104377106e-06,
+ "loss": 1.15363609790802,
+ "mean_token_accuracy": 0.7113458514213562,
+ "num_tokens": 13271040.0,
+ "step": 810
+ },
+ {
+ "entropy": 1.6209924221038818,
+ "epoch": 1.6383838383838385,
+ "grad_norm": 2.2165496349334717,
+ "learning_rate": 9.090909090909091e-06,
+ "loss": 1.637102484703064,
+ "mean_token_accuracy": 0.6318392753601074,
+ "num_tokens": 13287424.0,
+ "step": 811
+ },
+ {
+ "entropy": 1.050283432006836,
+ "epoch": 1.6404040404040403,
+ "grad_norm": 2.0473668575286865,
+ "learning_rate": 9.077441077441078e-06,
+ "loss": 1.0430940389633179,
+ "mean_token_accuracy": 0.7264899611473083,
+ "num_tokens": 13303808.0,
+ "step": 812
+ },
+ {
+ "entropy": 1.0707964897155762,
+ "epoch": 1.6424242424242426,
+ "grad_norm": 2.013641357421875,
+ "learning_rate": 9.063973063973066e-06,
+ "loss": 1.0929813385009766,
+ "mean_token_accuracy": 0.7231924533843994,
+ "num_tokens": 13320192.0,
+ "step": 813
+ },
+ {
+ "entropy": 0.8777739405632019,
+ "epoch": 1.6444444444444444,
+ "grad_norm": 1.815849781036377,
+ "learning_rate": 9.050505050505052e-06,
+ "loss": 0.868852972984314,
+ "mean_token_accuracy": 0.7692965269088745,
+ "num_tokens": 13336576.0,
+ "step": 814
+ },
+ {
+ "entropy": 1.1157307624816895,
+ "epoch": 1.6464646464646466,
+ "grad_norm": 2.1264355182647705,
+ "learning_rate": 9.037037037037037e-06,
+ "loss": 1.1249959468841553,
+ "mean_token_accuracy": 0.7153151035308838,
+ "num_tokens": 13352960.0,
+ "step": 815
+ },
+ {
+ "entropy": 1.348052740097046,
+ "epoch": 1.6484848484848484,
+ "grad_norm": 1.951642394065857,
+ "learning_rate": 9.023569023569025e-06,
+ "loss": 1.3196947574615479,
+ "mean_token_accuracy": 0.6911333799362183,
+ "num_tokens": 13369344.0,
+ "step": 816
+ },
+ {
+ "entropy": 1.2172725200653076,
+ "epoch": 1.6505050505050505,
+ "grad_norm": 2.2433671951293945,
+ "learning_rate": 9.010101010101012e-06,
+ "loss": 1.1833471059799194,
+ "mean_token_accuracy": 0.6938812732696533,
+ "num_tokens": 13385728.0,
+ "step": 817
+ },
+ {
+ "entropy": 1.5047324895858765,
+ "epoch": 1.6525252525252525,
+ "grad_norm": 2.015489101409912,
+ "learning_rate": 8.996632996632996e-06,
+ "loss": 1.48842453956604,
+ "mean_token_accuracy": 0.6552271842956543,
+ "num_tokens": 13402112.0,
+ "step": 818
+ },
+ {
+ "entropy": 1.3752695322036743,
+ "epoch": 1.6545454545454545,
+ "grad_norm": 2.024895191192627,
+ "learning_rate": 8.983164983164983e-06,
+ "loss": 1.3467098474502563,
+ "mean_token_accuracy": 0.681546151638031,
+ "num_tokens": 13418496.0,
+ "step": 819
+ },
+ {
+ "entropy": 1.1657299995422363,
+ "epoch": 1.6565656565656566,
+ "grad_norm": 2.4099085330963135,
+ "learning_rate": 8.969696969696971e-06,
+ "loss": 1.1612054109573364,
+ "mean_token_accuracy": 0.696140706539154,
+ "num_tokens": 13434880.0,
+ "step": 820
+ },
+ {
+ "entropy": 1.3764209747314453,
+ "epoch": 1.6585858585858586,
+ "grad_norm": 1.9971355199813843,
+ "learning_rate": 8.956228956228958e-06,
+ "loss": 1.3754298686981201,
+ "mean_token_accuracy": 0.6687836050987244,
+ "num_tokens": 13451264.0,
+ "step": 821
+ },
+ {
+ "entropy": 1.2135001420974731,
+ "epoch": 1.6606060606060606,
+ "grad_norm": 1.9414972066879272,
+ "learning_rate": 8.942760942760942e-06,
+ "loss": 1.2148737907409668,
+ "mean_token_accuracy": 0.7018808126449585,
+ "num_tokens": 13467648.0,
+ "step": 822
+ },
+ {
+ "entropy": 1.2993255853652954,
+ "epoch": 1.6626262626262627,
+ "grad_norm": 2.149869203567505,
+ "learning_rate": 8.92929292929293e-06,
+ "loss": 1.2947715520858765,
+ "mean_token_accuracy": 0.6836223602294922,
+ "num_tokens": 13484032.0,
+ "step": 823
+ },
+ {
+ "entropy": 1.2745556831359863,
+ "epoch": 1.6646464646464647,
+ "grad_norm": 2.1189515590667725,
+ "learning_rate": 8.915824915824917e-06,
+ "loss": 1.2992188930511475,
+ "mean_token_accuracy": 0.6871030926704407,
+ "num_tokens": 13500416.0,
+ "step": 824
+ },
+ {
+ "entropy": 1.1959195137023926,
+ "epoch": 1.6666666666666665,
+ "grad_norm": 2.2528860569000244,
+ "learning_rate": 8.902356902356904e-06,
+ "loss": 1.200485110282898,
+ "mean_token_accuracy": 0.6969956159591675,
+ "num_tokens": 13516800.0,
+ "step": 825
+ },
+ {
+ "entropy": 1.358959674835205,
+ "epoch": 1.6686868686868688,
+ "grad_norm": 2.0133697986602783,
+ "learning_rate": 8.888888888888888e-06,
+ "loss": 1.3683624267578125,
+ "mean_token_accuracy": 0.6767830848693848,
+ "num_tokens": 13533184.0,
+ "step": 826
+ },
+ {
+ "entropy": 1.4699946641921997,
+ "epoch": 1.6707070707070706,
+ "grad_norm": 1.9599978923797607,
+ "learning_rate": 8.875420875420876e-06,
+ "loss": 1.4912409782409668,
+ "mean_token_accuracy": 0.663593053817749,
+ "num_tokens": 13549568.0,
+ "step": 827
+ },
+ {
+ "entropy": 1.1315391063690186,
+ "epoch": 1.6727272727272728,
+ "grad_norm": 2.053920269012451,
+ "learning_rate": 8.861952861952863e-06,
+ "loss": 1.140032172203064,
+ "mean_token_accuracy": 0.7090864777565002,
+ "num_tokens": 13565952.0,
+ "step": 828
+ },
+ {
+ "entropy": 1.5379900932312012,
+ "epoch": 1.6747474747474747,
+ "grad_norm": 2.007784128189087,
+ "learning_rate": 8.84848484848485e-06,
+ "loss": 1.5630019903182983,
+ "mean_token_accuracy": 0.6397777199745178,
+ "num_tokens": 13582336.0,
+ "step": 829
+ },
+ {
+ "entropy": 1.241722822189331,
+ "epoch": 1.676767676767677,
+ "grad_norm": 2.060314416885376,
+ "learning_rate": 8.835016835016836e-06,
+ "loss": 1.24226975440979,
+ "mean_token_accuracy": 0.691316545009613,
+ "num_tokens": 13598720.0,
+ "step": 830
+ },
+ {
+ "entropy": 1.0310786962509155,
+ "epoch": 1.6787878787878787,
+ "grad_norm": 1.9201405048370361,
+ "learning_rate": 8.821548821548822e-06,
+ "loss": 1.0244829654693604,
+ "mean_token_accuracy": 0.7352222800254822,
+ "num_tokens": 13615104.0,
+ "step": 831
+ },
+ {
+ "entropy": 0.8319131731987,
+ "epoch": 1.680808080808081,
+ "grad_norm": 1.8874962329864502,
+ "learning_rate": 8.808080808080809e-06,
+ "loss": 0.8486998081207275,
+ "mean_token_accuracy": 0.7804714441299438,
+ "num_tokens": 13631488.0,
+ "step": 832
+ },
+ {
+ "entropy": 1.3878363370895386,
+ "epoch": 1.6828282828282828,
+ "grad_norm": 2.034135580062866,
+ "learning_rate": 8.794612794612795e-06,
+ "loss": 1.389417052268982,
+ "mean_token_accuracy": 0.6738519668579102,
+ "num_tokens": 13647872.0,
+ "step": 833
+ },
+ {
+ "entropy": 1.2692970037460327,
+ "epoch": 1.6848484848484848,
+ "grad_norm": 2.251734972000122,
+ "learning_rate": 8.781144781144782e-06,
+ "loss": 1.3026533126831055,
+ "mean_token_accuracy": 0.6725085377693176,
+ "num_tokens": 13664256.0,
+ "step": 834
+ },
+ {
+ "entropy": 1.030187726020813,
+ "epoch": 1.6868686868686869,
+ "grad_norm": 2.072669506072998,
+ "learning_rate": 8.767676767676768e-06,
+ "loss": 1.0316928625106812,
+ "mean_token_accuracy": 0.7219711542129517,
+ "num_tokens": 13680640.0,
+ "step": 835
+ },
+ {
+ "entropy": 0.9987007975578308,
+ "epoch": 1.6888888888888889,
+ "grad_norm": 1.9975026845932007,
+ "learning_rate": 8.754208754208755e-06,
+ "loss": 0.9982929229736328,
+ "mean_token_accuracy": 0.7407181262969971,
+ "num_tokens": 13697024.0,
+ "step": 836
+ },
+ {
+ "entropy": 1.335033655166626,
+ "epoch": 1.690909090909091,
+ "grad_norm": 2.9397995471954346,
+ "learning_rate": 8.740740740740741e-06,
+ "loss": 1.3791459798812866,
+ "mean_token_accuracy": 0.6587689518928528,
+ "num_tokens": 13713408.0,
+ "step": 837
+ },
+ {
+ "entropy": 1.064363718032837,
+ "epoch": 1.692929292929293,
+ "grad_norm": 1.859521508216858,
+ "learning_rate": 8.727272727272728e-06,
+ "loss": 1.0654652118682861,
+ "mean_token_accuracy": 0.7314362525939941,
+ "num_tokens": 13729792.0,
+ "step": 838
+ },
+ {
+ "entropy": 1.2016503810882568,
+ "epoch": 1.694949494949495,
+ "grad_norm": 1.9492405652999878,
+ "learning_rate": 8.713804713804714e-06,
+ "loss": 1.2206454277038574,
+ "mean_token_accuracy": 0.6963849663734436,
+ "num_tokens": 13746176.0,
+ "step": 839
+ },
+ {
+ "entropy": 1.0941510200500488,
+ "epoch": 1.696969696969697,
+ "grad_norm": 1.9697571992874146,
+ "learning_rate": 8.7003367003367e-06,
+ "loss": 1.0997231006622314,
+ "mean_token_accuracy": 0.7194675207138062,
+ "num_tokens": 13762560.0,
+ "step": 840
+ },
+ {
+ "entropy": 1.2805196046829224,
+ "epoch": 1.698989898989899,
+ "grad_norm": 2.183673620223999,
+ "learning_rate": 8.686868686868687e-06,
+ "loss": 1.2889196872711182,
+ "mean_token_accuracy": 0.6751953959465027,
+ "num_tokens": 13778944.0,
+ "step": 841
+ },
+ {
+ "entropy": 1.3549253940582275,
+ "epoch": 1.7010101010101009,
+ "grad_norm": 2.1223721504211426,
+ "learning_rate": 8.673400673400674e-06,
+ "loss": 1.351933240890503,
+ "mean_token_accuracy": 0.6756228804588318,
+ "num_tokens": 13795328.0,
+ "step": 842
+ },
+ {
+ "entropy": 1.333280324935913,
+ "epoch": 1.7030303030303031,
+ "grad_norm": 2.0345280170440674,
+ "learning_rate": 8.65993265993266e-06,
+ "loss": 1.3359044790267944,
+ "mean_token_accuracy": 0.6783707737922668,
+ "num_tokens": 13811712.0,
+ "step": 843
+ },
+ {
+ "entropy": 1.0658303499221802,
+ "epoch": 1.705050505050505,
+ "grad_norm": 1.926890254020691,
+ "learning_rate": 8.646464646464647e-06,
+ "loss": 1.0638974905014038,
+ "mean_token_accuracy": 0.7286272644996643,
+ "num_tokens": 13828096.0,
+ "step": 844
+ },
+ {
+ "entropy": 0.9549879431724548,
+ "epoch": 1.7070707070707072,
+ "grad_norm": 2.124664306640625,
+ "learning_rate": 8.632996632996635e-06,
+ "loss": 0.9564298987388611,
+ "mean_token_accuracy": 0.7528700828552246,
+ "num_tokens": 13844480.0,
+ "step": 845
+ },
+ {
+ "entropy": 1.6893866062164307,
+ "epoch": 1.709090909090909,
+ "grad_norm": 2.070680856704712,
+ "learning_rate": 8.61952861952862e-06,
+ "loss": 1.6929473876953125,
+ "mean_token_accuracy": 0.6165730357170105,
+ "num_tokens": 13860864.0,
+ "step": 846
+ },
+ {
+ "entropy": 0.8587391972541809,
+ "epoch": 1.7111111111111112,
+ "grad_norm": 1.9683139324188232,
+ "learning_rate": 8.606060606060606e-06,
+ "loss": 0.8467673659324646,
+ "mean_token_accuracy": 0.7707620859146118,
+ "num_tokens": 13877248.0,
+ "step": 847
+ },
+ {
+ "entropy": 1.6370538473129272,
+ "epoch": 1.713131313131313,
+ "grad_norm": 2.1608047485351562,
+ "learning_rate": 8.592592592592593e-06,
+ "loss": 1.6496617794036865,
+ "mean_token_accuracy": 0.6143136024475098,
+ "num_tokens": 13893632.0,
+ "step": 848
+ },
+ {
+ "entropy": 1.23484206199646,
+ "epoch": 1.7151515151515153,
+ "grad_norm": 2.0296223163604736,
+ "learning_rate": 8.57912457912458e-06,
+ "loss": 1.2227973937988281,
+ "mean_token_accuracy": 0.689667820930481,
+ "num_tokens": 13910016.0,
+ "step": 849
+ },
+ {
+ "entropy": 1.2687101364135742,
+ "epoch": 1.7171717171717171,
+ "grad_norm": 2.204806089401245,
+ "learning_rate": 8.565656565656566e-06,
+ "loss": 1.2694804668426514,
+ "mean_token_accuracy": 0.6865534782409668,
+ "num_tokens": 13926400.0,
+ "step": 850
+ },
+ {
+ "entropy": 1.2225878238677979,
+ "epoch": 1.7191919191919192,
+ "grad_norm": 2.1541168689727783,
+ "learning_rate": 8.552188552188552e-06,
+ "loss": 1.2527751922607422,
+ "mean_token_accuracy": 0.6800195574760437,
+ "num_tokens": 13942784.0,
+ "step": 851
+ },
+ {
+ "entropy": 1.2106293439865112,
+ "epoch": 1.7212121212121212,
+ "grad_norm": 2.0455424785614014,
+ "learning_rate": 8.53872053872054e-06,
+ "loss": 1.2005270719528198,
+ "mean_token_accuracy": 0.6949804425239563,
+ "num_tokens": 13959168.0,
+ "step": 852
+ },
+ {
+ "entropy": 1.169908046722412,
+ "epoch": 1.7232323232323232,
+ "grad_norm": 2.0055043697357178,
+ "learning_rate": 8.525252525252527e-06,
+ "loss": 1.1640398502349854,
+ "mean_token_accuracy": 0.7149487137794495,
+ "num_tokens": 13975552.0,
+ "step": 853
+ },
+ {
+ "entropy": 1.1458942890167236,
+ "epoch": 1.7252525252525253,
+ "grad_norm": 2.0365331172943115,
+ "learning_rate": 8.511784511784512e-06,
+ "loss": 1.1413549184799194,
+ "mean_token_accuracy": 0.7126282453536987,
+ "num_tokens": 13991936.0,
+ "step": 854
+ },
+ {
+ "entropy": 0.8760596513748169,
+ "epoch": 1.7272727272727273,
+ "grad_norm": 1.9858993291854858,
+ "learning_rate": 8.4983164983165e-06,
+ "loss": 0.8733371496200562,
+ "mean_token_accuracy": 0.760869562625885,
+ "num_tokens": 14008320.0,
+ "step": 855
+ },
+ {
+ "entropy": 1.3949522972106934,
+ "epoch": 1.7292929292929293,
+ "grad_norm": 2.6730244159698486,
+ "learning_rate": 8.484848484848486e-06,
+ "loss": 1.4128756523132324,
+ "mean_token_accuracy": 0.6792256832122803,
+ "num_tokens": 14024704.0,
+ "step": 856
+ },
+ {
+ "entropy": 1.4345626831054688,
+ "epoch": 1.7313131313131314,
+ "grad_norm": 1.9446519613265991,
+ "learning_rate": 8.471380471380473e-06,
+ "loss": 1.450282096862793,
+ "mean_token_accuracy": 0.664631187915802,
+ "num_tokens": 14041088.0,
+ "step": 857
+ },
+ {
+ "entropy": 1.1180377006530762,
+ "epoch": 1.7333333333333334,
+ "grad_norm": 2.051783323287964,
+ "learning_rate": 8.457912457912457e-06,
+ "loss": 1.1588772535324097,
+ "mean_token_accuracy": 0.7128114104270935,
+ "num_tokens": 14057472.0,
+ "step": 858
+ },
+ {
+ "entropy": 1.2605851888656616,
+ "epoch": 1.7353535353535352,
+ "grad_norm": 2.0577232837677,
+ "learning_rate": 8.444444444444446e-06,
+ "loss": 1.2173504829406738,
+ "mean_token_accuracy": 0.6938812732696533,
+ "num_tokens": 14073856.0,
+ "step": 859
+ },
+ {
+ "entropy": 1.2957148551940918,
+ "epoch": 1.7373737373737375,
+ "grad_norm": 2.14068603515625,
+ "learning_rate": 8.430976430976432e-06,
+ "loss": 1.301836609840393,
+ "mean_token_accuracy": 0.6832559704780579,
+ "num_tokens": 14090240.0,
+ "step": 860
+ },
+ {
+ "entropy": 0.9871225953102112,
+ "epoch": 1.7393939393939393,
+ "grad_norm": 1.9262301921844482,
+ "learning_rate": 8.417508417508419e-06,
+ "loss": 1.0081474781036377,
+ "mean_token_accuracy": 0.7432828545570374,
+ "num_tokens": 14106624.0,
+ "step": 861
+ },
+ {
+ "entropy": 1.2774608135223389,
+ "epoch": 1.7414141414141415,
+ "grad_norm": 1.8633641004562378,
+ "learning_rate": 8.404040404040405e-06,
+ "loss": 1.2942487001419067,
+ "mean_token_accuracy": 0.6910722851753235,
+ "num_tokens": 14123008.0,
+ "step": 862
+ },
+ {
+ "entropy": 1.3229460716247559,
+ "epoch": 1.7434343434343433,
+ "grad_norm": 2.1861679553985596,
+ "learning_rate": 8.390572390572392e-06,
+ "loss": 1.3244147300720215,
+ "mean_token_accuracy": 0.6719589829444885,
+ "num_tokens": 14139392.0,
+ "step": 863
+ },
+ {
+ "entropy": 1.306138277053833,
+ "epoch": 1.7454545454545456,
+ "grad_norm": 2.1364035606384277,
+ "learning_rate": 8.377104377104378e-06,
+ "loss": 1.3266233205795288,
+ "mean_token_accuracy": 0.6864924430847168,
+ "num_tokens": 14155776.0,
+ "step": 864
+ },
+ {
+ "entropy": 0.9926152229309082,
+ "epoch": 1.7474747474747474,
+ "grad_norm": 1.784975528717041,
+ "learning_rate": 8.363636363636365e-06,
+ "loss": 1.0243797302246094,
+ "mean_token_accuracy": 0.7546409368515015,
+ "num_tokens": 14172160.0,
+ "step": 865
+ },
+ {
+ "entropy": 1.7003151178359985,
+ "epoch": 1.7494949494949497,
+ "grad_norm": 2.081186056137085,
+ "learning_rate": 8.350168350168351e-06,
+ "loss": 1.699352502822876,
+ "mean_token_accuracy": 0.6183438897132874,
+ "num_tokens": 14188544.0,
+ "step": 866
+ },
+ {
+ "entropy": 1.7050426006317139,
+ "epoch": 1.7515151515151515,
+ "grad_norm": 1.8780465126037598,
+ "learning_rate": 8.336700336700338e-06,
+ "loss": 1.7067642211914062,
+ "mean_token_accuracy": 0.6159623861312866,
+ "num_tokens": 14204928.0,
+ "step": 867
+ },
+ {
+ "entropy": 1.0717660188674927,
+ "epoch": 1.7535353535353535,
+ "grad_norm": 2.0461418628692627,
+ "learning_rate": 8.323232323232324e-06,
+ "loss": 1.0874615907669067,
+ "mean_token_accuracy": 0.7315583825111389,
+ "num_tokens": 14221312.0,
+ "step": 868
+ },
+ {
+ "entropy": 1.3650007247924805,
+ "epoch": 1.7555555555555555,
+ "grad_norm": 1.9097316265106201,
+ "learning_rate": 8.30976430976431e-06,
+ "loss": 1.3483806848526,
+ "mean_token_accuracy": 0.6889350414276123,
+ "num_tokens": 14237696.0,
+ "step": 869
+ },
+ {
+ "entropy": 0.9856852293014526,
+ "epoch": 1.7575757575757576,
+ "grad_norm": 1.9712227582931519,
+ "learning_rate": 8.296296296296297e-06,
+ "loss": 0.975986897945404,
+ "mean_token_accuracy": 0.7417562007904053,
+ "num_tokens": 14254080.0,
+ "step": 870
+ },
+ {
+ "entropy": 0.8876434564590454,
+ "epoch": 1.7595959595959596,
+ "grad_norm": 1.8536908626556396,
+ "learning_rate": 8.282828282828283e-06,
+ "loss": 0.8868429660797119,
+ "mean_token_accuracy": 0.7714338302612305,
+ "num_tokens": 14270464.0,
+ "step": 871
+ },
+ {
+ "entropy": 0.7506260871887207,
+ "epoch": 1.7616161616161616,
+ "grad_norm": 1.7068337202072144,
+ "learning_rate": 8.26936026936027e-06,
+ "loss": 0.7422510385513306,
+ "mean_token_accuracy": 0.8030654788017273,
+ "num_tokens": 14286848.0,
+ "step": 872
+ },
+ {
+ "entropy": 1.726669192314148,
+ "epoch": 1.7636363636363637,
+ "grad_norm": 2.157275676727295,
+ "learning_rate": 8.255892255892256e-06,
+ "loss": 1.7225112915039062,
+ "mean_token_accuracy": 0.6207864880561829,
+ "num_tokens": 14303232.0,
+ "step": 873
+ },
+ {
+ "entropy": 0.8383756279945374,
+ "epoch": 1.7656565656565657,
+ "grad_norm": 1.8378020524978638,
+ "learning_rate": 8.242424242424243e-06,
+ "loss": 0.8363397717475891,
+ "mean_token_accuracy": 0.772838294506073,
+ "num_tokens": 14319616.0,
+ "step": 874
+ },
+ {
+ "entropy": 1.1146882772445679,
+ "epoch": 1.7676767676767677,
+ "grad_norm": 2.144796371459961,
+ "learning_rate": 8.22895622895623e-06,
+ "loss": 1.124634027481079,
+ "mean_token_accuracy": 0.711467981338501,
+ "num_tokens": 14336000.0,
+ "step": 875
+ },
+ {
+ "entropy": 1.4330589771270752,
+ "epoch": 1.7696969696969695,
+ "grad_norm": 2.268897533416748,
+ "learning_rate": 8.215488215488216e-06,
+ "loss": 1.4430513381958008,
+ "mean_token_accuracy": 0.6437469720840454,
+ "num_tokens": 14352384.0,
+ "step": 876
+ },
+ {
+ "entropy": 1.1955921649932861,
+ "epoch": 1.7717171717171718,
+ "grad_norm": 2.109701156616211,
+ "learning_rate": 8.202020202020202e-06,
+ "loss": 1.1778371334075928,
+ "mean_token_accuracy": 0.6951025724411011,
+ "num_tokens": 14368768.0,
+ "step": 877
+ },
+ {
+ "entropy": 1.0419657230377197,
+ "epoch": 1.7737373737373736,
+ "grad_norm": 2.002164840698242,
+ "learning_rate": 8.188552188552189e-06,
+ "loss": 1.0472073554992676,
+ "mean_token_accuracy": 0.7313751578330994,
+ "num_tokens": 14385152.0,
+ "step": 878
+ },
+ {
+ "entropy": 1.324011206626892,
+ "epoch": 1.7757575757575759,
+ "grad_norm": 2.233525514602661,
+ "learning_rate": 8.175084175084175e-06,
+ "loss": 1.3433442115783691,
+ "mean_token_accuracy": 0.666890561580658,
+ "num_tokens": 14401536.0,
+ "step": 879
+ },
+ {
+ "entropy": 1.0424224138259888,
+ "epoch": 1.7777777777777777,
+ "grad_norm": 1.8213649988174438,
+ "learning_rate": 8.161616161616162e-06,
+ "loss": 1.0525646209716797,
+ "mean_token_accuracy": 0.7383976578712463,
+ "num_tokens": 14417920.0,
+ "step": 880
+ },
+ {
+ "entropy": 1.3322721719741821,
+ "epoch": 1.77979797979798,
+ "grad_norm": 2.0028865337371826,
+ "learning_rate": 8.148148148148148e-06,
+ "loss": 1.356465458869934,
+ "mean_token_accuracy": 0.6836223602294922,
+ "num_tokens": 14434304.0,
+ "step": 881
+ },
+ {
+ "entropy": 1.2495578527450562,
+ "epoch": 1.7818181818181817,
+ "grad_norm": 2.021883249282837,
+ "learning_rate": 8.134680134680135e-06,
+ "loss": 1.2500100135803223,
+ "mean_token_accuracy": 0.7239863276481628,
+ "num_tokens": 14450688.0,
+ "step": 882
+ },
+ {
+ "entropy": 1.3212776184082031,
+ "epoch": 1.783838383838384,
+ "grad_norm": 1.932453989982605,
+ "learning_rate": 8.121212121212121e-06,
+ "loss": 1.3274552822113037,
+ "mean_token_accuracy": 0.6889350414276123,
+ "num_tokens": 14467072.0,
+ "step": 883
+ },
+ {
+ "entropy": 1.1536792516708374,
+ "epoch": 1.7858585858585858,
+ "grad_norm": 1.944832682609558,
+ "learning_rate": 8.10774410774411e-06,
+ "loss": 1.1310882568359375,
+ "mean_token_accuracy": 0.7239863276481628,
+ "num_tokens": 14483456.0,
+ "step": 884
+ },
+ {
+ "entropy": 1.335244059562683,
+ "epoch": 1.7878787878787878,
+ "grad_norm": 1.9081436395645142,
+ "learning_rate": 8.094276094276094e-06,
+ "loss": 1.3430235385894775,
+ "mean_token_accuracy": 0.6779433488845825,
+ "num_tokens": 14499840.0,
+ "step": 885
+ },
+ {
+ "entropy": 0.9776304364204407,
+ "epoch": 1.7898989898989899,
+ "grad_norm": 2.077418088912964,
+ "learning_rate": 8.08080808080808e-06,
+ "loss": 0.9834706783294678,
+ "mean_token_accuracy": 0.7418172955513,
+ "num_tokens": 14516224.0,
+ "step": 886
+ },
+ {
+ "entropy": 0.9367865920066833,
+ "epoch": 1.791919191919192,
+ "grad_norm": 1.9657868146896362,
+ "learning_rate": 8.067340067340069e-06,
+ "loss": 0.9443397521972656,
+ "mean_token_accuracy": 0.7449926733970642,
+ "num_tokens": 14532608.0,
+ "step": 887
+ },
+ {
+ "entropy": 1.352593183517456,
+ "epoch": 1.793939393939394,
+ "grad_norm": 2.039900779724121,
+ "learning_rate": 8.053872053872055e-06,
+ "loss": 1.3500186204910278,
+ "mean_token_accuracy": 0.6840498447418213,
+ "num_tokens": 14548992.0,
+ "step": 888
+ },
+ {
+ "entropy": 1.21983802318573,
+ "epoch": 1.795959595959596,
+ "grad_norm": 2.04494309425354,
+ "learning_rate": 8.04040404040404e-06,
+ "loss": 1.2253628969192505,
+ "mean_token_accuracy": 0.6975451707839966,
+ "num_tokens": 14565376.0,
+ "step": 889
+ },
+ {
+ "entropy": 1.1911543607711792,
+ "epoch": 1.797979797979798,
+ "grad_norm": 2.09802508354187,
+ "learning_rate": 8.026936026936027e-06,
+ "loss": 1.2186518907546997,
+ "mean_token_accuracy": 0.6941866278648376,
+ "num_tokens": 14581760.0,
+ "step": 890
+ },
+ {
+ "entropy": 1.2681410312652588,
+ "epoch": 1.8,
+ "grad_norm": 2.242945432662964,
+ "learning_rate": 8.013468013468015e-06,
+ "loss": 1.2717292308807373,
+ "mean_token_accuracy": 0.6806302070617676,
+ "num_tokens": 14598144.0,
+ "step": 891
+ },
+ {
+ "entropy": 1.0779279470443726,
+ "epoch": 1.802020202020202,
+ "grad_norm": 2.0271153450012207,
+ "learning_rate": 8.000000000000001e-06,
+ "loss": 1.070134162902832,
+ "mean_token_accuracy": 0.7261846661567688,
+ "num_tokens": 14614528.0,
+ "step": 892
+ },
+ {
+ "entropy": 1.9025505781173706,
+ "epoch": 1.8040404040404039,
+ "grad_norm": 2.221864938735962,
+ "learning_rate": 7.986531986531986e-06,
+ "loss": 1.9073364734649658,
+ "mean_token_accuracy": 0.5933683514595032,
+ "num_tokens": 14630912.0,
+ "step": 893
+ },
+ {
+ "entropy": 1.0104752779006958,
+ "epoch": 1.8060606060606061,
+ "grad_norm": 2.165552854537964,
+ "learning_rate": 7.973063973063974e-06,
+ "loss": 1.0037627220153809,
+ "mean_token_accuracy": 0.7404738664627075,
+ "num_tokens": 14647296.0,
+ "step": 894
+ },
+ {
+ "entropy": 1.5633832216262817,
+ "epoch": 1.808080808080808,
+ "grad_norm": 2.1008827686309814,
+ "learning_rate": 7.95959595959596e-06,
+ "loss": 1.5352282524108887,
+ "mean_token_accuracy": 0.6398388147354126,
+ "num_tokens": 14663680.0,
+ "step": 895
+ },
+ {
+ "entropy": 1.080839991569519,
+ "epoch": 1.8101010101010102,
+ "grad_norm": 1.9256923198699951,
+ "learning_rate": 7.946127946127947e-06,
+ "loss": 1.0921295881271362,
+ "mean_token_accuracy": 0.7247801423072815,
+ "num_tokens": 14680064.0,
+ "step": 896
+ },
+ {
+ "entropy": 0.9788808226585388,
+ "epoch": 1.812121212121212,
+ "grad_norm": 1.9602711200714111,
+ "learning_rate": 7.932659932659934e-06,
+ "loss": 0.9705313444137573,
+ "mean_token_accuracy": 0.7442598938941956,
+ "num_tokens": 14696448.0,
+ "step": 897
+ },
+ {
+ "entropy": 1.1658309698104858,
+ "epoch": 1.8141414141414143,
+ "grad_norm": 1.8840819597244263,
+ "learning_rate": 7.91919191919192e-06,
+ "loss": 1.1346904039382935,
+ "mean_token_accuracy": 0.7229481935501099,
+ "num_tokens": 14712832.0,
+ "step": 898
+ },
+ {
+ "entropy": 0.8824312686920166,
+ "epoch": 1.816161616161616,
+ "grad_norm": 2.1427361965179443,
+ "learning_rate": 7.905723905723907e-06,
+ "loss": 0.9055237770080566,
+ "mean_token_accuracy": 0.759770393371582,
+ "num_tokens": 14729216.0,
+ "step": 899
+ },
+ {
+ "entropy": 1.0914777517318726,
+ "epoch": 1.8181818181818183,
+ "grad_norm": 2.282011032104492,
+ "learning_rate": 7.892255892255893e-06,
+ "loss": 1.101895809173584,
+ "mean_token_accuracy": 0.7096360325813293,
+ "num_tokens": 14745600.0,
+ "step": 900
+ },
+ {
+ "epoch": 1.8181818181818183,
+ "eval_entropy": 1.303592009890464,
+ "eval_loss": 1.5282857418060303,
+ "eval_mean_token_accuracy": 0.6474241422068688,
+ "eval_num_tokens": 14745600.0,
+ "eval_runtime": 43.8637,
+ "eval_samples_per_second": 22.57,
+ "eval_steps_per_second": 2.827,
+ "step": 900
+ },
+ {
+ "entropy": 1.4693570137023926,
+ "epoch": 1.8202020202020202,
+ "grad_norm": 2.0938942432403564,
+ "learning_rate": 7.87878787878788e-06,
+ "loss": 1.4499380588531494,
+ "mean_token_accuracy": 0.6446018815040588,
+ "num_tokens": 14761984.0,
+ "step": 901
+ },
+ {
+ "entropy": 1.0810022354125977,
+ "epoch": 1.8222222222222222,
+ "grad_norm": 2.0279886722564697,
+ "learning_rate": 7.865319865319866e-06,
+ "loss": 1.0581843852996826,
+ "mean_token_accuracy": 0.723375678062439,
+ "num_tokens": 14778368.0,
+ "step": 902
+ },
+ {
+ "entropy": 1.0680485963821411,
+ "epoch": 1.8242424242424242,
+ "grad_norm": 2.015047788619995,
+ "learning_rate": 7.851851851851853e-06,
+ "loss": 1.0612249374389648,
+ "mean_token_accuracy": 0.7264289259910583,
+ "num_tokens": 14794752.0,
+ "step": 903
+ },
+ {
+ "entropy": 0.981645941734314,
+ "epoch": 1.8262626262626263,
+ "grad_norm": 2.0122671127319336,
+ "learning_rate": 7.838383838383839e-06,
+ "loss": 0.9650060534477234,
+ "mean_token_accuracy": 0.7337567210197449,
+ "num_tokens": 14811136.0,
+ "step": 904
+ },
+ {
+ "entropy": 1.0827676057815552,
+ "epoch": 1.8282828282828283,
+ "grad_norm": 2.1086010932922363,
+ "learning_rate": 7.824915824915826e-06,
+ "loss": 1.097700595855713,
+ "mean_token_accuracy": 0.7087811231613159,
+ "num_tokens": 14827520.0,
+ "step": 905
+ },
+ {
+ "entropy": 0.9498234391212463,
+ "epoch": 1.8303030303030303,
+ "grad_norm": 1.9012964963912964,
+ "learning_rate": 7.811447811447812e-06,
+ "loss": 0.9704345464706421,
+ "mean_token_accuracy": 0.7472520470619202,
+ "num_tokens": 14843904.0,
+ "step": 906
+ },
+ {
+ "entropy": 0.8871595859527588,
+ "epoch": 1.8323232323232324,
+ "grad_norm": 1.9970269203186035,
+ "learning_rate": 7.797979797979799e-06,
+ "loss": 0.8876000642776489,
+ "mean_token_accuracy": 0.7578163146972656,
+ "num_tokens": 14860288.0,
+ "step": 907
+ },
+ {
+ "entropy": 1.1324913501739502,
+ "epoch": 1.8343434343434344,
+ "grad_norm": 2.1133675575256348,
+ "learning_rate": 7.784511784511785e-06,
+ "loss": 1.142140507698059,
+ "mean_token_accuracy": 0.7180629968643188,
+ "num_tokens": 14876672.0,
+ "step": 908
+ },
+ {
+ "entropy": 1.1514050960540771,
+ "epoch": 1.8363636363636364,
+ "grad_norm": 2.073002338409424,
+ "learning_rate": 7.771043771043772e-06,
+ "loss": 1.1670506000518799,
+ "mean_token_accuracy": 0.7009037733078003,
+ "num_tokens": 14893056.0,
+ "step": 909
+ },
+ {
+ "entropy": 1.17017662525177,
+ "epoch": 1.8383838383838382,
+ "grad_norm": 2.17962384223938,
+ "learning_rate": 7.757575757575758e-06,
+ "loss": 1.1800150871276855,
+ "mean_token_accuracy": 0.7028578519821167,
+ "num_tokens": 14909440.0,
+ "step": 910
+ },
+ {
+ "entropy": 0.9377864599227905,
+ "epoch": 1.8404040404040405,
+ "grad_norm": 2.2141757011413574,
+ "learning_rate": 7.744107744107745e-06,
+ "loss": 0.9712649583816528,
+ "mean_token_accuracy": 0.7405349016189575,
+ "num_tokens": 14925824.0,
+ "step": 911
+ },
+ {
+ "entropy": 1.2093497514724731,
+ "epoch": 1.8424242424242423,
+ "grad_norm": 2.0677497386932373,
+ "learning_rate": 7.730639730639731e-06,
+ "loss": 1.210886001586914,
+ "mean_token_accuracy": 0.7037737965583801,
+ "num_tokens": 14942208.0,
+ "step": 912
+ },
+ {
+ "entropy": 0.8966845273971558,
+ "epoch": 1.8444444444444446,
+ "grad_norm": 2.0689940452575684,
+ "learning_rate": 7.717171717171717e-06,
+ "loss": 0.9142885208129883,
+ "mean_token_accuracy": 0.7523815631866455,
+ "num_tokens": 14958592.0,
+ "step": 913
+ },
+ {
+ "entropy": 1.4647231101989746,
+ "epoch": 1.8464646464646464,
+ "grad_norm": 2.1815598011016846,
+ "learning_rate": 7.703703703703704e-06,
+ "loss": 1.4676027297973633,
+ "mean_token_accuracy": 0.6506472826004028,
+ "num_tokens": 14974976.0,
+ "step": 914
+ },
+ {
+ "entropy": 1.5831035375595093,
+ "epoch": 1.8484848484848486,
+ "grad_norm": 2.028407096862793,
+ "learning_rate": 7.69023569023569e-06,
+ "loss": 1.5933328866958618,
+ "mean_token_accuracy": 0.6368466019630432,
+ "num_tokens": 14991360.0,
+ "step": 915
+ },
+ {
+ "entropy": 1.3269871473312378,
+ "epoch": 1.8505050505050504,
+ "grad_norm": 2.173727035522461,
+ "learning_rate": 7.676767676767677e-06,
+ "loss": 1.3404830694198608,
+ "mean_token_accuracy": 0.6750122308731079,
+ "num_tokens": 15007744.0,
+ "step": 916
+ },
+ {
+ "entropy": 1.378677487373352,
+ "epoch": 1.8525252525252527,
+ "grad_norm": 2.0357983112335205,
+ "learning_rate": 7.663299663299663e-06,
+ "loss": 1.3752634525299072,
+ "mean_token_accuracy": 0.6711651086807251,
+ "num_tokens": 15024128.0,
+ "step": 917
+ },
+ {
+ "entropy": 1.5462778806686401,
+ "epoch": 1.8545454545454545,
+ "grad_norm": 1.9574400186538696,
+ "learning_rate": 7.64983164983165e-06,
+ "loss": 1.5374811887741089,
+ "mean_token_accuracy": 0.6375183463096619,
+ "num_tokens": 15040512.0,
+ "step": 918
+ },
+ {
+ "entropy": 1.2211424112319946,
+ "epoch": 1.8565656565656565,
+ "grad_norm": 2.0203700065612793,
+ "learning_rate": 7.636363636363638e-06,
+ "loss": 1.2375779151916504,
+ "mean_token_accuracy": 0.6878358721733093,
+ "num_tokens": 15056896.0,
+ "step": 919
+ },
+ {
+ "entropy": 1.493086338043213,
+ "epoch": 1.8585858585858586,
+ "grad_norm": 2.1047544479370117,
+ "learning_rate": 7.622895622895623e-06,
+ "loss": 1.471447229385376,
+ "mean_token_accuracy": 0.6543111801147461,
+ "num_tokens": 15073280.0,
+ "step": 920
+ },
+ {
+ "entropy": 1.351884126663208,
+ "epoch": 1.8606060606060606,
+ "grad_norm": 2.2803215980529785,
+ "learning_rate": 7.60942760942761e-06,
+ "loss": 1.3451658487319946,
+ "mean_token_accuracy": 0.6725696325302124,
+ "num_tokens": 15089664.0,
+ "step": 921
+ },
+ {
+ "entropy": 1.4343167543411255,
+ "epoch": 1.8626262626262626,
+ "grad_norm": 1.9821034669876099,
+ "learning_rate": 7.595959595959597e-06,
+ "loss": 1.4694747924804688,
+ "mean_token_accuracy": 0.6622496247291565,
+ "num_tokens": 15106048.0,
+ "step": 922
+ },
+ {
+ "entropy": 1.3977091312408447,
+ "epoch": 1.8646464646464647,
+ "grad_norm": 2.0600125789642334,
+ "learning_rate": 7.582491582491583e-06,
+ "loss": 1.4120681285858154,
+ "mean_token_accuracy": 0.6642037034034729,
+ "num_tokens": 15122432.0,
+ "step": 923
+ },
+ {
+ "entropy": 1.4182765483856201,
+ "epoch": 1.8666666666666667,
+ "grad_norm": 1.9010684490203857,
+ "learning_rate": 7.569023569023569e-06,
+ "loss": 1.4160431623458862,
+ "mean_token_accuracy": 0.6696995496749878,
+ "num_tokens": 15138816.0,
+ "step": 924
+ },
+ {
+ "entropy": 1.1520206928253174,
+ "epoch": 1.8686868686868687,
+ "grad_norm": 1.8414452075958252,
+ "learning_rate": 7.555555555555556e-06,
+ "loss": 1.154735803604126,
+ "mean_token_accuracy": 0.7159867882728577,
+ "num_tokens": 15155200.0,
+ "step": 925
+ },
+ {
+ "entropy": 0.8958664536476135,
+ "epoch": 1.8707070707070708,
+ "grad_norm": 1.82782781124115,
+ "learning_rate": 7.542087542087543e-06,
+ "loss": 0.8718663454055786,
+ "mean_token_accuracy": 0.7771739363670349,
+ "num_tokens": 15171584.0,
+ "step": 926
+ },
+ {
+ "entropy": 1.4068299531936646,
+ "epoch": 1.8727272727272726,
+ "grad_norm": 1.9482192993164062,
+ "learning_rate": 7.52861952861953e-06,
+ "loss": 1.3947927951812744,
+ "mean_token_accuracy": 0.6699438095092773,
+ "num_tokens": 15187968.0,
+ "step": 927
+ },
+ {
+ "entropy": 0.8980593085289001,
+ "epoch": 1.8747474747474748,
+ "grad_norm": 2.040997266769409,
+ "learning_rate": 7.515151515151516e-06,
+ "loss": 0.9092239141464233,
+ "mean_token_accuracy": 0.7550684213638306,
+ "num_tokens": 15204352.0,
+ "step": 928
+ },
+ {
+ "entropy": 1.2259284257888794,
+ "epoch": 1.8767676767676766,
+ "grad_norm": 2.1932485103607178,
+ "learning_rate": 7.501683501683502e-06,
+ "loss": 1.2324566841125488,
+ "mean_token_accuracy": 0.6918050646781921,
+ "num_tokens": 15220736.0,
+ "step": 929
+ },
+ {
+ "entropy": 1.3737105131149292,
+ "epoch": 1.878787878787879,
+ "grad_norm": 1.9829115867614746,
+ "learning_rate": 7.4882154882154886e-06,
+ "loss": 1.4132641553878784,
+ "mean_token_accuracy": 0.6854543089866638,
+ "num_tokens": 15237120.0,
+ "step": 930
+ },
+ {
+ "entropy": 1.1767487525939941,
+ "epoch": 1.8808080808080807,
+ "grad_norm": 2.279439926147461,
+ "learning_rate": 7.474747474747476e-06,
+ "loss": 1.1658971309661865,
+ "mean_token_accuracy": 0.711467981338501,
+ "num_tokens": 15253504.0,
+ "step": 931
+ },
+ {
+ "entropy": 1.0110876560211182,
+ "epoch": 1.882828282828283,
+ "grad_norm": 2.144711494445801,
+ "learning_rate": 7.4612794612794615e-06,
+ "loss": 1.0287975072860718,
+ "mean_token_accuracy": 0.7346726655960083,
+ "num_tokens": 15269888.0,
+ "step": 932
+ },
+ {
+ "entropy": 1.0826390981674194,
+ "epoch": 1.8848484848484848,
+ "grad_norm": 2.1858105659484863,
+ "learning_rate": 7.447811447811448e-06,
+ "loss": 1.1194939613342285,
+ "mean_token_accuracy": 0.7099413871765137,
+ "num_tokens": 15286272.0,
+ "step": 933
+ },
+ {
+ "entropy": 1.430822491645813,
+ "epoch": 1.886868686868687,
+ "grad_norm": 1.9496047496795654,
+ "learning_rate": 7.434343434343435e-06,
+ "loss": 1.452394723892212,
+ "mean_token_accuracy": 0.6591353416442871,
+ "num_tokens": 15302656.0,
+ "step": 934
+ },
+ {
+ "entropy": 0.9242729544639587,
+ "epoch": 1.8888888888888888,
+ "grad_norm": 2.1464316844940186,
+ "learning_rate": 7.420875420875422e-06,
+ "loss": 0.9562792778015137,
+ "mean_token_accuracy": 0.7407791614532471,
+ "num_tokens": 15319040.0,
+ "step": 935
+ },
+ {
+ "entropy": 1.5215003490447998,
+ "epoch": 1.8909090909090909,
+ "grad_norm": 2.105729818344116,
+ "learning_rate": 7.4074074074074075e-06,
+ "loss": 1.5162043571472168,
+ "mean_token_accuracy": 0.6430141925811768,
+ "num_tokens": 15335424.0,
+ "step": 936
+ },
+ {
+ "entropy": 1.1809922456741333,
+ "epoch": 1.892929292929293,
+ "grad_norm": 2.1825973987579346,
+ "learning_rate": 7.393939393939395e-06,
+ "loss": 1.192020058631897,
+ "mean_token_accuracy": 0.693453848361969,
+ "num_tokens": 15351808.0,
+ "step": 937
+ },
+ {
+ "entropy": 1.0707145929336548,
+ "epoch": 1.894949494949495,
+ "grad_norm": 2.033658027648926,
+ "learning_rate": 7.380471380471381e-06,
+ "loss": 1.0794811248779297,
+ "mean_token_accuracy": 0.7291157841682434,
+ "num_tokens": 15368192.0,
+ "step": 938
+ },
+ {
+ "entropy": 0.8133494257926941,
+ "epoch": 1.896969696969697,
+ "grad_norm": 1.9038894176483154,
+ "learning_rate": 7.367003367003368e-06,
+ "loss": 0.817105233669281,
+ "mean_token_accuracy": 0.785906195640564,
+ "num_tokens": 15384576.0,
+ "step": 939
+ },
+ {
+ "entropy": 1.1366889476776123,
+ "epoch": 1.898989898989899,
+ "grad_norm": 2.3373923301696777,
+ "learning_rate": 7.353535353535353e-06,
+ "loss": 1.1590774059295654,
+ "mean_token_accuracy": 0.7004152536392212,
+ "num_tokens": 15400960.0,
+ "step": 940
+ },
+ {
+ "entropy": 1.0165143013000488,
+ "epoch": 1.901010101010101,
+ "grad_norm": 1.8290495872497559,
+ "learning_rate": 7.340067340067341e-06,
+ "loss": 0.9955418109893799,
+ "mean_token_accuracy": 0.7454811930656433,
+ "num_tokens": 15417344.0,
+ "step": 941
+ },
+ {
+ "entropy": 1.536665916442871,
+ "epoch": 1.903030303030303,
+ "grad_norm": 2.2636616230010986,
+ "learning_rate": 7.326599326599327e-06,
+ "loss": 1.5516541004180908,
+ "mean_token_accuracy": 0.6290913820266724,
+ "num_tokens": 15433728.0,
+ "step": 942
+ },
+ {
+ "entropy": 1.0885748863220215,
+ "epoch": 1.905050505050505,
+ "grad_norm": 2.104462146759033,
+ "learning_rate": 7.3131313131313146e-06,
+ "loss": 1.096163272857666,
+ "mean_token_accuracy": 0.7129946351051331,
+ "num_tokens": 15450112.0,
+ "step": 943
+ },
+ {
+ "entropy": 1.0774754285812378,
+ "epoch": 1.907070707070707,
+ "grad_norm": 2.006735324859619,
+ "learning_rate": 7.2996632996633e-06,
+ "loss": 1.0466490983963013,
+ "mean_token_accuracy": 0.7385197877883911,
+ "num_tokens": 15466496.0,
+ "step": 944
+ },
+ {
+ "entropy": 1.3489689826965332,
+ "epoch": 1.9090909090909092,
+ "grad_norm": 2.148700714111328,
+ "learning_rate": 7.286195286195287e-06,
+ "loss": 1.317432165145874,
+ "mean_token_accuracy": 0.6951025724411011,
+ "num_tokens": 15482880.0,
+ "step": 945
+ },
+ {
+ "entropy": 0.957065999507904,
+ "epoch": 1.911111111111111,
+ "grad_norm": 2.118453025817871,
+ "learning_rate": 7.272727272727273e-06,
+ "loss": 0.9499804377555847,
+ "mean_token_accuracy": 0.7590376138687134,
+ "num_tokens": 15499264.0,
+ "step": 946
+ },
+ {
+ "entropy": 1.1132235527038574,
+ "epoch": 1.9131313131313132,
+ "grad_norm": 1.8254663944244385,
+ "learning_rate": 7.2592592592592605e-06,
+ "loss": 1.0975847244262695,
+ "mean_token_accuracy": 0.7224596738815308,
+ "num_tokens": 15515648.0,
+ "step": 947
+ },
+ {
+ "entropy": 1.1259424686431885,
+ "epoch": 1.915151515151515,
+ "grad_norm": 2.105797529220581,
+ "learning_rate": 7.245791245791246e-06,
+ "loss": 1.1171300411224365,
+ "mean_token_accuracy": 0.7150097489356995,
+ "num_tokens": 15532032.0,
+ "step": 948
+ },
+ {
+ "entropy": 1.3454829454421997,
+ "epoch": 1.9171717171717173,
+ "grad_norm": 2.2715842723846436,
+ "learning_rate": 7.232323232323233e-06,
+ "loss": 1.359604835510254,
+ "mean_token_accuracy": 0.6568759083747864,
+ "num_tokens": 15548416.0,
+ "step": 949
+ },
+ {
+ "entropy": 1.2873634099960327,
+ "epoch": 1.9191919191919191,
+ "grad_norm": 1.8735847473144531,
+ "learning_rate": 7.21885521885522e-06,
+ "loss": 1.2842814922332764,
+ "mean_token_accuracy": 0.6951025724411011,
+ "num_tokens": 15564800.0,
+ "step": 950
+ },
+ {
+ "entropy": 1.154240369796753,
+ "epoch": 1.9212121212121214,
+ "grad_norm": 1.9821181297302246,
+ "learning_rate": 7.2053872053872064e-06,
+ "loss": 1.1434863805770874,
+ "mean_token_accuracy": 0.7203834652900696,
+ "num_tokens": 15581184.0,
+ "step": 951
+ },
+ {
+ "entropy": 1.05129075050354,
+ "epoch": 1.9232323232323232,
+ "grad_norm": 1.990182638168335,
+ "learning_rate": 7.191919191919192e-06,
+ "loss": 1.0694432258605957,
+ "mean_token_accuracy": 0.734245240688324,
+ "num_tokens": 15597568.0,
+ "step": 952
+ },
+ {
+ "entropy": 1.0232638120651245,
+ "epoch": 1.9252525252525252,
+ "grad_norm": 2.017350673675537,
+ "learning_rate": 7.178451178451179e-06,
+ "loss": 1.0090606212615967,
+ "mean_token_accuracy": 0.7305202484130859,
+ "num_tokens": 15613952.0,
+ "step": 953
+ },
+ {
+ "entropy": 0.8146723508834839,
+ "epoch": 1.9272727272727272,
+ "grad_norm": 1.8920353651046753,
+ "learning_rate": 7.164983164983166e-06,
+ "loss": 0.8227875232696533,
+ "mean_token_accuracy": 0.778761625289917,
+ "num_tokens": 15630336.0,
+ "step": 954
+ },
+ {
+ "entropy": 0.6415520906448364,
+ "epoch": 1.9292929292929293,
+ "grad_norm": 1.7636535167694092,
+ "learning_rate": 7.151515151515152e-06,
+ "loss": 0.6381626725196838,
+ "mean_token_accuracy": 0.8171104192733765,
+ "num_tokens": 15646720.0,
+ "step": 955
+ },
+ {
+ "entropy": 1.4309028387069702,
+ "epoch": 1.9313131313131313,
+ "grad_norm": 1.8857070207595825,
+ "learning_rate": 7.138047138047138e-06,
+ "loss": 1.4779207706451416,
+ "mean_token_accuracy": 0.6556546092033386,
+ "num_tokens": 15663104.0,
+ "step": 956
+ },
+ {
+ "entropy": 1.1201391220092773,
+ "epoch": 1.9333333333333333,
+ "grad_norm": 1.833791732788086,
+ "learning_rate": 7.124579124579125e-06,
+ "loss": 1.0972872972488403,
+ "mean_token_accuracy": 0.7187347412109375,
+ "num_tokens": 15679488.0,
+ "step": 957
+ },
+ {
+ "entropy": 1.1922115087509155,
+ "epoch": 1.9353535353535354,
+ "grad_norm": 1.8989986181259155,
+ "learning_rate": 7.111111111111112e-06,
+ "loss": 1.2037092447280884,
+ "mean_token_accuracy": 0.7104299068450928,
+ "num_tokens": 15695872.0,
+ "step": 958
+ },
+ {
+ "entropy": 0.9312219023704529,
+ "epoch": 1.9373737373737374,
+ "grad_norm": 2.0068769454956055,
+ "learning_rate": 7.097643097643099e-06,
+ "loss": 0.9126096963882446,
+ "mean_token_accuracy": 0.7556790709495544,
+ "num_tokens": 15712256.0,
+ "step": 959
+ },
+ {
+ "entropy": 1.1126971244812012,
+ "epoch": 1.9393939393939394,
+ "grad_norm": 1.9915603399276733,
+ "learning_rate": 7.084175084175085e-06,
+ "loss": 1.1312336921691895,
+ "mean_token_accuracy": 0.7178187370300293,
+ "num_tokens": 15728640.0,
+ "step": 960
+ },
+ {
+ "entropy": 1.487212896347046,
+ "epoch": 1.9414141414141413,
+ "grad_norm": 2.068739414215088,
+ "learning_rate": 7.070707070707071e-06,
+ "loss": 1.4876431226730347,
+ "mean_token_accuracy": 0.6568759083747864,
+ "num_tokens": 15745024.0,
+ "step": 961
+ },
+ {
+ "entropy": 1.207465648651123,
+ "epoch": 1.9434343434343435,
+ "grad_norm": 1.9962066411972046,
+ "learning_rate": 7.057239057239058e-06,
+ "loss": 1.2003669738769531,
+ "mean_token_accuracy": 0.7024914622306824,
+ "num_tokens": 15761408.0,
+ "step": 962
+ },
+ {
+ "entropy": 1.7148067951202393,
+ "epoch": 1.9454545454545453,
+ "grad_norm": 2.3441736698150635,
+ "learning_rate": 7.043771043771043e-06,
+ "loss": 1.7310147285461426,
+ "mean_token_accuracy": 0.6288471221923828,
+ "num_tokens": 15777792.0,
+ "step": 963
+ },
+ {
+ "entropy": 1.3103407621383667,
+ "epoch": 1.9474747474747476,
+ "grad_norm": 1.915368676185608,
+ "learning_rate": 7.030303030303031e-06,
+ "loss": 1.3412322998046875,
+ "mean_token_accuracy": 0.6845383644104004,
+ "num_tokens": 15794176.0,
+ "step": 964
+ },
+ {
+ "entropy": 0.9835378527641296,
+ "epoch": 1.9494949494949494,
+ "grad_norm": 1.9611473083496094,
+ "learning_rate": 7.016835016835017e-06,
+ "loss": 0.9903290271759033,
+ "mean_token_accuracy": 0.7569003701210022,
+ "num_tokens": 15810560.0,
+ "step": 965
+ },
+ {
+ "entropy": 1.4645164012908936,
+ "epoch": 1.9515151515151516,
+ "grad_norm": 2.017500162124634,
+ "learning_rate": 7.0033670033670045e-06,
+ "loss": 1.4612789154052734,
+ "mean_token_accuracy": 0.6558378338813782,
+ "num_tokens": 15826944.0,
+ "step": 966
+ },
+ {
+ "entropy": 1.2586326599121094,
+ "epoch": 1.9535353535353535,
+ "grad_norm": 2.0969791412353516,
+ "learning_rate": 6.98989898989899e-06,
+ "loss": 1.224534273147583,
+ "mean_token_accuracy": 0.6958353519439697,
+ "num_tokens": 15843328.0,
+ "step": 967
+ },
+ {
+ "entropy": 0.8170226216316223,
+ "epoch": 1.9555555555555557,
+ "grad_norm": 1.9889802932739258,
+ "learning_rate": 6.976430976430977e-06,
+ "loss": 0.8214236497879028,
+ "mean_token_accuracy": 0.7777845859527588,
+ "num_tokens": 15859712.0,
+ "step": 968
+ },
+ {
+ "entropy": 0.8966978788375854,
+ "epoch": 1.9575757575757575,
+ "grad_norm": 1.8261324167251587,
+ "learning_rate": 6.962962962962964e-06,
+ "loss": 0.8880574107170105,
+ "mean_token_accuracy": 0.7681363224983215,
+ "num_tokens": 15876096.0,
+ "step": 969
+ },
+ {
+ "entropy": 1.4782947301864624,
+ "epoch": 1.9595959595959596,
+ "grad_norm": 2.2666432857513428,
+ "learning_rate": 6.9494949494949505e-06,
+ "loss": 1.5416191816329956,
+ "mean_token_accuracy": 0.6491206884384155,
+ "num_tokens": 15892480.0,
+ "step": 970
+ },
+ {
+ "entropy": 1.6055350303649902,
+ "epoch": 1.9616161616161616,
+ "grad_norm": 2.0585474967956543,
+ "learning_rate": 6.936026936026936e-06,
+ "loss": 1.627528190612793,
+ "mean_token_accuracy": 0.6376404762268066,
+ "num_tokens": 15908864.0,
+ "step": 971
+ },
+ {
+ "entropy": 1.092020869255066,
+ "epoch": 1.9636363636363636,
+ "grad_norm": 2.002554416656494,
+ "learning_rate": 6.922558922558923e-06,
+ "loss": 1.0792224407196045,
+ "mean_token_accuracy": 0.7230092883110046,
+ "num_tokens": 15925248.0,
+ "step": 972
+ },
+ {
+ "entropy": 0.7485284805297852,
+ "epoch": 1.9656565656565657,
+ "grad_norm": 1.9477611780166626,
+ "learning_rate": 6.90909090909091e-06,
+ "loss": 0.7471826076507568,
+ "mean_token_accuracy": 0.7961040735244751,
+ "num_tokens": 15941632.0,
+ "step": 973
+ },
+ {
+ "entropy": 1.3997598886489868,
+ "epoch": 1.9676767676767677,
+ "grad_norm": 2.1589887142181396,
+ "learning_rate": 6.895622895622896e-06,
+ "loss": 1.3947563171386719,
+ "mean_token_accuracy": 0.6626160144805908,
+ "num_tokens": 15958016.0,
+ "step": 974
+ },
+ {
+ "entropy": 1.2053040266036987,
+ "epoch": 1.9696969696969697,
+ "grad_norm": 2.1021888256073,
+ "learning_rate": 6.882154882154882e-06,
+ "loss": 1.2196800708770752,
+ "mean_token_accuracy": 0.6948583126068115,
+ "num_tokens": 15974400.0,
+ "step": 975
+ },
+ {
+ "entropy": 1.1462217569351196,
+ "epoch": 1.9717171717171718,
+ "grad_norm": 2.1295032501220703,
+ "learning_rate": 6.868686868686869e-06,
+ "loss": 1.1645277738571167,
+ "mean_token_accuracy": 0.717391312122345,
+ "num_tokens": 15990784.0,
+ "step": 976
+ },
+ {
+ "entropy": 0.9189468026161194,
+ "epoch": 1.9737373737373738,
+ "grad_norm": 1.9384510517120361,
+ "learning_rate": 6.855218855218856e-06,
+ "loss": 0.8912248015403748,
+ "mean_token_accuracy": 0.7561064958572388,
+ "num_tokens": 16007168.0,
+ "step": 977
+ },
+ {
+ "entropy": 1.2767856121063232,
+ "epoch": 1.9757575757575756,
+ "grad_norm": 2.086789608001709,
+ "learning_rate": 6.841750841750842e-06,
+ "loss": 1.2951477766036987,
+ "mean_token_accuracy": 0.6822178959846497,
+ "num_tokens": 16023552.0,
+ "step": 978
+ },
+ {
+ "entropy": 1.4691493511199951,
+ "epoch": 1.9777777777777779,
+ "grad_norm": 2.081007719039917,
+ "learning_rate": 6.828282828282828e-06,
+ "loss": 1.4899260997772217,
+ "mean_token_accuracy": 0.6618832349777222,
+ "num_tokens": 16039936.0,
+ "step": 979
+ },
+ {
+ "entropy": 1.5689647197723389,
+ "epoch": 1.9797979797979797,
+ "grad_norm": 2.168604850769043,
+ "learning_rate": 6.814814814814815e-06,
+ "loss": 1.57144296169281,
+ "mean_token_accuracy": 0.6405715942382812,
+ "num_tokens": 16056320.0,
+ "step": 980
+ },
+ {
+ "entropy": 1.4429019689559937,
+ "epoch": 1.981818181818182,
+ "grad_norm": 1.9658228158950806,
+ "learning_rate": 6.801346801346802e-06,
+ "loss": 1.473623275756836,
+ "mean_token_accuracy": 0.6576697826385498,
+ "num_tokens": 16072704.0,
+ "step": 981
+ },
+ {
+ "entropy": 1.0032570362091064,
+ "epoch": 1.9838383838383837,
+ "grad_norm": 1.9377433061599731,
+ "learning_rate": 6.787878787878789e-06,
+ "loss": 0.9838194847106934,
+ "mean_token_accuracy": 0.7490839958190918,
+ "num_tokens": 16089088.0,
+ "step": 982
+ },
+ {
+ "entropy": 0.9308913946151733,
+ "epoch": 1.985858585858586,
+ "grad_norm": 1.9042673110961914,
+ "learning_rate": 6.774410774410775e-06,
+ "loss": 0.9097878932952881,
+ "mean_token_accuracy": 0.7631289958953857,
+ "num_tokens": 16105472.0,
+ "step": 983
+ },
+ {
+ "entropy": 1.0642609596252441,
+ "epoch": 1.9878787878787878,
+ "grad_norm": 2.1934077739715576,
+ "learning_rate": 6.760942760942761e-06,
+ "loss": 1.0470092296600342,
+ "mean_token_accuracy": 0.7263067960739136,
+ "num_tokens": 16121856.0,
+ "step": 984
+ },
+ {
+ "entropy": 1.2610602378845215,
+ "epoch": 1.98989898989899,
+ "grad_norm": 2.2924704551696777,
+ "learning_rate": 6.747474747474749e-06,
+ "loss": 1.246216058731079,
+ "mean_token_accuracy": 0.6873473525047302,
+ "num_tokens": 16138240.0,
+ "step": 985
+ },
+ {
+ "entropy": 1.4189343452453613,
+ "epoch": 1.9919191919191919,
+ "grad_norm": 1.906319260597229,
+ "learning_rate": 6.734006734006735e-06,
+ "loss": 1.4098241329193115,
+ "mean_token_accuracy": 0.6734855771064758,
+ "num_tokens": 16154624.0,
+ "step": 986
+ },
+ {
+ "entropy": 1.339056134223938,
+ "epoch": 1.993939393939394,
+ "grad_norm": 2.034353733062744,
+ "learning_rate": 6.720538720538721e-06,
+ "loss": 1.3571393489837646,
+ "mean_token_accuracy": 0.678859293460846,
+ "num_tokens": 16171008.0,
+ "step": 987
+ },
+ {
+ "entropy": 1.057302474975586,
+ "epoch": 1.995959595959596,
+ "grad_norm": 2.1716647148132324,
+ "learning_rate": 6.707070707070707e-06,
+ "loss": 1.042845606803894,
+ "mean_token_accuracy": 0.7322300672531128,
+ "num_tokens": 16187392.0,
+ "step": 988
+ },
+ {
+ "entropy": 1.2057303190231323,
+ "epoch": 1.997979797979798,
+ "grad_norm": 1.9809064865112305,
+ "learning_rate": 6.6936026936026945e-06,
+ "loss": 1.2091717720031738,
+ "mean_token_accuracy": 0.7068270444869995,
+ "num_tokens": 16203776.0,
+ "step": 989
+ },
+ {
+ "entropy": 1.3678697347640991,
+ "epoch": 2.0,
+ "grad_norm": 2.1474292278289795,
+ "learning_rate": 6.680134680134681e-06,
+ "loss": 1.359403133392334,
+ "mean_token_accuracy": 0.6702491641044617,
+ "num_tokens": 16220160.0,
+ "step": 990
+ },
+ {
+ "entropy": 1.6815240383148193,
+ "epoch": 2.002020202020202,
+ "grad_norm": 2.209095001220703,
+ "learning_rate": 6.666666666666667e-06,
+ "loss": 1.5768096446990967,
+ "mean_token_accuracy": 0.6408768892288208,
+ "num_tokens": 16236544.0,
+ "step": 991
+ },
+ {
+ "entropy": 1.1142786741256714,
+ "epoch": 2.004040404040404,
+ "grad_norm": 2.8133344650268555,
+ "learning_rate": 6.653198653198654e-06,
+ "loss": 0.937615692615509,
+ "mean_token_accuracy": 0.7556179761886597,
+ "num_tokens": 16252928.0,
+ "step": 992
+ },
+ {
+ "entropy": 1.2004083395004272,
+ "epoch": 2.006060606060606,
+ "grad_norm": 2.753380060195923,
+ "learning_rate": 6.6397306397306405e-06,
+ "loss": 1.0454245805740356,
+ "mean_token_accuracy": 0.7423058152198792,
+ "num_tokens": 16269312.0,
+ "step": 993
+ },
+ {
+ "entropy": 0.9945010542869568,
+ "epoch": 2.008080808080808,
+ "grad_norm": 2.3262150287628174,
+ "learning_rate": 6.626262626262627e-06,
+ "loss": 0.8623301982879639,
+ "mean_token_accuracy": 0.7791890501976013,
+ "num_tokens": 16285696.0,
+ "step": 994
+ },
+ {
+ "entropy": 1.0330396890640259,
+ "epoch": 2.01010101010101,
+ "grad_norm": 2.3145525455474854,
+ "learning_rate": 6.612794612794613e-06,
+ "loss": 0.9156472682952881,
+ "mean_token_accuracy": 0.7564118504524231,
+ "num_tokens": 16302080.0,
+ "step": 995
+ },
+ {
+ "entropy": 1.109296202659607,
+ "epoch": 2.012121212121212,
+ "grad_norm": 1.9910767078399658,
+ "learning_rate": 6.5993265993266e-06,
+ "loss": 1.0271074771881104,
+ "mean_token_accuracy": 0.7449315786361694,
+ "num_tokens": 16318464.0,
+ "step": 996
+ },
+ {
+ "entropy": 1.3861018419265747,
+ "epoch": 2.014141414141414,
+ "grad_norm": 2.2054574489593506,
+ "learning_rate": 6.585858585858586e-06,
+ "loss": 1.3088997602462769,
+ "mean_token_accuracy": 0.6818515062332153,
+ "num_tokens": 16334848.0,
+ "step": 997
+ },
+ {
+ "entropy": 1.1170406341552734,
+ "epoch": 2.0161616161616163,
+ "grad_norm": 2.3239619731903076,
+ "learning_rate": 6.572390572390574e-06,
+ "loss": 1.036240577697754,
+ "mean_token_accuracy": 0.735405445098877,
+ "num_tokens": 16351232.0,
+ "step": 998
+ },
+ {
+ "entropy": 0.8306671977043152,
+ "epoch": 2.018181818181818,
+ "grad_norm": 1.844927430152893,
+ "learning_rate": 6.558922558922559e-06,
+ "loss": 0.7926205396652222,
+ "mean_token_accuracy": 0.790730357170105,
+ "num_tokens": 16367616.0,
+ "step": 999
+ },
+ {
+ "entropy": 1.0606129169464111,
+ "epoch": 2.0202020202020203,
+ "grad_norm": 1.884960412979126,
+ "learning_rate": 6.545454545454546e-06,
+ "loss": 1.0186505317687988,
+ "mean_token_accuracy": 0.7497557401657104,
+ "num_tokens": 16384000.0,
+ "step": 1000
+ },
+ {
+ "epoch": 2.0202020202020203,
+ "eval_entropy": 1.1442755407864047,
+ "eval_loss": 1.5800968408584595,
+ "eval_mean_token_accuracy": 0.6456370079709638,
+ "eval_num_tokens": 16384000.0,
+ "eval_runtime": 43.8308,
+ "eval_samples_per_second": 22.587,
+ "eval_steps_per_second": 2.829,
+ "step": 1000
+ },
+ {
+ "entropy": 1.1903036832809448,
+ "epoch": 2.022222222222222,
+ "grad_norm": 2.1258959770202637,
+ "learning_rate": 6.531986531986533e-06,
+ "loss": 1.1637260913848877,
+ "mean_token_accuracy": 0.7187347412109375,
+ "num_tokens": 16400384.0,
+ "step": 1001
+ },
+ {
+ "entropy": 0.6594457626342773,
+ "epoch": 2.0242424242424244,
+ "grad_norm": 1.934865951538086,
+ "learning_rate": 6.51851851851852e-06,
+ "loss": 0.6422234773635864,
+ "mean_token_accuracy": 0.8183317184448242,
+ "num_tokens": 16416768.0,
+ "step": 1002
+ },
+ {
+ "entropy": 1.4628888368606567,
+ "epoch": 2.026262626262626,
+ "grad_norm": 2.2606966495513916,
+ "learning_rate": 6.505050505050505e-06,
+ "loss": 1.4868237972259521,
+ "mean_token_accuracy": 0.6523571014404297,
+ "num_tokens": 16433152.0,
+ "step": 1003
+ },
+ {
+ "entropy": 0.7812108397483826,
+ "epoch": 2.0282828282828285,
+ "grad_norm": 2.222006320953369,
+ "learning_rate": 6.491582491582492e-06,
+ "loss": 0.7988066673278809,
+ "mean_token_accuracy": 0.7873717546463013,
+ "num_tokens": 16449536.0,
+ "step": 1004
+ },
+ {
+ "entropy": 0.8421438336372375,
+ "epoch": 2.0303030303030303,
+ "grad_norm": 2.264430284500122,
+ "learning_rate": 6.478114478114479e-06,
+ "loss": 0.8724686503410339,
+ "mean_token_accuracy": 0.7742428183555603,
+ "num_tokens": 16465920.0,
+ "step": 1005
+ },
+ {
+ "entropy": 0.8786948919296265,
+ "epoch": 2.0323232323232325,
+ "grad_norm": 2.0655956268310547,
+ "learning_rate": 6.464646464646466e-06,
+ "loss": 0.8711583614349365,
+ "mean_token_accuracy": 0.7763800621032715,
+ "num_tokens": 16482304.0,
+ "step": 1006
+ },
+ {
+ "entropy": 0.7819579839706421,
+ "epoch": 2.0343434343434343,
+ "grad_norm": 2.360408067703247,
+ "learning_rate": 6.451178451178451e-06,
+ "loss": 0.8174005150794983,
+ "mean_token_accuracy": 0.7810820937156677,
+ "num_tokens": 16498688.0,
+ "step": 1007
+ },
+ {
+ "entropy": 1.3011908531188965,
+ "epoch": 2.036363636363636,
+ "grad_norm": 2.417349338531494,
+ "learning_rate": 6.4377104377104386e-06,
+ "loss": 1.3457268476486206,
+ "mean_token_accuracy": 0.6885075569152832,
+ "num_tokens": 16515072.0,
+ "step": 1008
+ },
+ {
+ "entropy": 0.5790199041366577,
+ "epoch": 2.0383838383838384,
+ "grad_norm": 2.0773134231567383,
+ "learning_rate": 6.424242424242425e-06,
+ "loss": 0.5803444981575012,
+ "mean_token_accuracy": 0.8409257531166077,
+ "num_tokens": 16531456.0,
+ "step": 1009
+ },
+ {
+ "entropy": 1.3786348104476929,
+ "epoch": 2.04040404040404,
+ "grad_norm": 2.4630050659179688,
+ "learning_rate": 6.410774410774411e-06,
+ "loss": 1.413762092590332,
+ "mean_token_accuracy": 0.6840498447418213,
+ "num_tokens": 16547840.0,
+ "step": 1010
+ },
+ {
+ "entropy": 1.0889877080917358,
+ "epoch": 2.0424242424242425,
+ "grad_norm": 2.0827176570892334,
+ "learning_rate": 6.397306397306397e-06,
+ "loss": 1.1012202501296997,
+ "mean_token_accuracy": 0.7329018115997314,
+ "num_tokens": 16564224.0,
+ "step": 1011
+ },
+ {
+ "entropy": 1.1251029968261719,
+ "epoch": 2.0444444444444443,
+ "grad_norm": 2.2465784549713135,
+ "learning_rate": 6.3838383838383845e-06,
+ "loss": 1.1805188655853271,
+ "mean_token_accuracy": 0.7307034730911255,
+ "num_tokens": 16580608.0,
+ "step": 1012
+ },
+ {
+ "entropy": 1.0284773111343384,
+ "epoch": 2.0464646464646465,
+ "grad_norm": 2.208253860473633,
+ "learning_rate": 6.370370370370371e-06,
+ "loss": 1.0413470268249512,
+ "mean_token_accuracy": 0.7445651888847351,
+ "num_tokens": 16596992.0,
+ "step": 1013
+ },
+ {
+ "entropy": 0.9029683470726013,
+ "epoch": 2.0484848484848484,
+ "grad_norm": 2.5125656127929688,
+ "learning_rate": 6.356902356902357e-06,
+ "loss": 0.9083307981491089,
+ "mean_token_accuracy": 0.7583048343658447,
+ "num_tokens": 16613376.0,
+ "step": 1014
+ },
+ {
+ "entropy": 1.0634405612945557,
+ "epoch": 2.0505050505050506,
+ "grad_norm": 2.4080238342285156,
+ "learning_rate": 6.343434343434344e-06,
+ "loss": 1.0854010581970215,
+ "mean_token_accuracy": 0.7280166149139404,
+ "num_tokens": 16629760.0,
+ "step": 1015
+ },
+ {
+ "entropy": 1.095357894897461,
+ "epoch": 2.0525252525252524,
+ "grad_norm": 2.3696935176849365,
+ "learning_rate": 6.3299663299663304e-06,
+ "loss": 1.1008697748184204,
+ "mean_token_accuracy": 0.7330239415168762,
+ "num_tokens": 16646144.0,
+ "step": 1016
+ },
+ {
+ "entropy": 0.8473373651504517,
+ "epoch": 2.0545454545454547,
+ "grad_norm": 2.2760303020477295,
+ "learning_rate": 6.316498316498317e-06,
+ "loss": 0.8324838876724243,
+ "mean_token_accuracy": 0.7822422981262207,
+ "num_tokens": 16662528.0,
+ "step": 1017
+ },
+ {
+ "entropy": 1.4589424133300781,
+ "epoch": 2.0565656565656565,
+ "grad_norm": 2.2009267807006836,
+ "learning_rate": 6.303030303030303e-06,
+ "loss": 1.4676311016082764,
+ "mean_token_accuracy": 0.6726306676864624,
+ "num_tokens": 16678912.0,
+ "step": 1018
+ },
+ {
+ "entropy": 1.1835463047027588,
+ "epoch": 2.0585858585858587,
+ "grad_norm": 2.264815330505371,
+ "learning_rate": 6.28956228956229e-06,
+ "loss": 1.172943353652954,
+ "mean_token_accuracy": 0.7181851267814636,
+ "num_tokens": 16695296.0,
+ "step": 1019
+ },
+ {
+ "entropy": 1.0366170406341553,
+ "epoch": 2.0606060606060606,
+ "grad_norm": 2.040250539779663,
+ "learning_rate": 6.276094276094276e-06,
+ "loss": 1.0314842462539673,
+ "mean_token_accuracy": 0.745175838470459,
+ "num_tokens": 16711680.0,
+ "step": 1020
+ },
+ {
+ "entropy": 1.0229740142822266,
+ "epoch": 2.062626262626263,
+ "grad_norm": 2.3892908096313477,
+ "learning_rate": 6.262626262626264e-06,
+ "loss": 0.9976632595062256,
+ "mean_token_accuracy": 0.751099169254303,
+ "num_tokens": 16728064.0,
+ "step": 1021
+ },
+ {
+ "entropy": 1.0881341695785522,
+ "epoch": 2.0646464646464646,
+ "grad_norm": 2.459239959716797,
+ "learning_rate": 6.249158249158249e-06,
+ "loss": 1.0147933959960938,
+ "mean_token_accuracy": 0.7465803623199463,
+ "num_tokens": 16744448.0,
+ "step": 1022
+ },
+ {
+ "entropy": 1.3220701217651367,
+ "epoch": 2.066666666666667,
+ "grad_norm": 2.4171364307403564,
+ "learning_rate": 6.235690235690236e-06,
+ "loss": 1.3207433223724365,
+ "mean_token_accuracy": 0.7020029425621033,
+ "num_tokens": 16760832.0,
+ "step": 1023
+ },
+ {
+ "entropy": 1.1744258403778076,
+ "epoch": 2.0686868686868687,
+ "grad_norm": 2.1978869438171387,
+ "learning_rate": 6.222222222222223e-06,
+ "loss": 1.1690993309020996,
+ "mean_token_accuracy": 0.7153761386871338,
+ "num_tokens": 16777216.0,
+ "step": 1024
+ },
+ {
+ "entropy": 0.8706873059272766,
+ "epoch": 2.0707070707070705,
+ "grad_norm": 2.0721466541290283,
+ "learning_rate": 6.20875420875421e-06,
+ "loss": 0.8407986760139465,
+ "mean_token_accuracy": 0.7790669202804565,
+ "num_tokens": 16793600.0,
+ "step": 1025
+ },
+ {
+ "entropy": 0.7695729732513428,
+ "epoch": 2.0727272727272728,
+ "grad_norm": 2.096400737762451,
+ "learning_rate": 6.195286195286195e-06,
+ "loss": 0.7511329650878906,
+ "mean_token_accuracy": 0.8002564907073975,
+ "num_tokens": 16809984.0,
+ "step": 1026
+ },
+ {
+ "entropy": 0.9394605159759521,
+ "epoch": 2.0747474747474746,
+ "grad_norm": 2.1947765350341797,
+ "learning_rate": 6.181818181818182e-06,
+ "loss": 0.9365648627281189,
+ "mean_token_accuracy": 0.7574499249458313,
+ "num_tokens": 16826368.0,
+ "step": 1027
+ },
+ {
+ "entropy": 0.9941903352737427,
+ "epoch": 2.076767676767677,
+ "grad_norm": 2.017825126647949,
+ "learning_rate": 6.168350168350169e-06,
+ "loss": 0.9688439965248108,
+ "mean_token_accuracy": 0.7499389052391052,
+ "num_tokens": 16842752.0,
+ "step": 1028
+ },
+ {
+ "entropy": 0.6713882088661194,
+ "epoch": 2.0787878787878786,
+ "grad_norm": 2.025696277618408,
+ "learning_rate": 6.154882154882156e-06,
+ "loss": 0.6422317028045654,
+ "mean_token_accuracy": 0.8201636672019958,
+ "num_tokens": 16859136.0,
+ "step": 1029
+ },
+ {
+ "entropy": 1.0641586780548096,
+ "epoch": 2.080808080808081,
+ "grad_norm": 2.1687450408935547,
+ "learning_rate": 6.141414141414141e-06,
+ "loss": 1.055826187133789,
+ "mean_token_accuracy": 0.7419394254684448,
+ "num_tokens": 16875520.0,
+ "step": 1030
+ },
+ {
+ "entropy": 0.9837899804115295,
+ "epoch": 2.0828282828282827,
+ "grad_norm": 2.3160905838012695,
+ "learning_rate": 6.1279461279461286e-06,
+ "loss": 0.9732166528701782,
+ "mean_token_accuracy": 0.7443820238113403,
+ "num_tokens": 16891904.0,
+ "step": 1031
+ },
+ {
+ "entropy": 0.7756745219230652,
+ "epoch": 2.084848484848485,
+ "grad_norm": 2.1464638710021973,
+ "learning_rate": 6.114478114478115e-06,
+ "loss": 0.7793388366699219,
+ "mean_token_accuracy": 0.8052027225494385,
+ "num_tokens": 16908288.0,
+ "step": 1032
+ },
+ {
+ "entropy": 0.9195827841758728,
+ "epoch": 2.0868686868686868,
+ "grad_norm": 2.1330578327178955,
+ "learning_rate": 6.1010101010101015e-06,
+ "loss": 0.9061870574951172,
+ "mean_token_accuracy": 0.7676478028297424,
+ "num_tokens": 16924672.0,
+ "step": 1033
+ },
+ {
+ "entropy": 0.8372635245323181,
+ "epoch": 2.088888888888889,
+ "grad_norm": 1.9992423057556152,
+ "learning_rate": 6.087542087542088e-06,
+ "loss": 0.826106071472168,
+ "mean_token_accuracy": 0.7849902510643005,
+ "num_tokens": 16941056.0,
+ "step": 1034
+ },
+ {
+ "entropy": 0.8915544748306274,
+ "epoch": 2.090909090909091,
+ "grad_norm": 2.100963830947876,
+ "learning_rate": 6.0740740740740745e-06,
+ "loss": 0.8717142343521118,
+ "mean_token_accuracy": 0.7684416174888611,
+ "num_tokens": 16957440.0,
+ "step": 1035
+ },
+ {
+ "entropy": 1.1338574886322021,
+ "epoch": 2.092929292929293,
+ "grad_norm": 2.263211727142334,
+ "learning_rate": 6.060606060606061e-06,
+ "loss": 1.1237502098083496,
+ "mean_token_accuracy": 0.718490481376648,
+ "num_tokens": 16973824.0,
+ "step": 1036
+ },
+ {
+ "entropy": 0.8575322031974792,
+ "epoch": 2.094949494949495,
+ "grad_norm": 2.489158868789673,
+ "learning_rate": 6.047138047138048e-06,
+ "loss": 0.8411874771118164,
+ "mean_token_accuracy": 0.781020998954773,
+ "num_tokens": 16990208.0,
+ "step": 1037
+ },
+ {
+ "entropy": 0.9754064083099365,
+ "epoch": 2.096969696969697,
+ "grad_norm": 2.1259446144104004,
+ "learning_rate": 6.033670033670034e-06,
+ "loss": 0.9752705097198486,
+ "mean_token_accuracy": 0.7573277950286865,
+ "num_tokens": 17006592.0,
+ "step": 1038
+ },
+ {
+ "entropy": 0.830701470375061,
+ "epoch": 2.098989898989899,
+ "grad_norm": 2.162536859512329,
+ "learning_rate": 6.0202020202020204e-06,
+ "loss": 0.8512325882911682,
+ "mean_token_accuracy": 0.7818148732185364,
+ "num_tokens": 17022976.0,
+ "step": 1039
+ },
+ {
+ "entropy": 1.058897614479065,
+ "epoch": 2.101010101010101,
+ "grad_norm": 2.2364299297332764,
+ "learning_rate": 6.006734006734008e-06,
+ "loss": 1.0636553764343262,
+ "mean_token_accuracy": 0.7447484135627747,
+ "num_tokens": 17039360.0,
+ "step": 1040
+ },
+ {
+ "entropy": 0.8365826606750488,
+ "epoch": 2.103030303030303,
+ "grad_norm": 2.1749696731567383,
+ "learning_rate": 5.993265993265994e-06,
+ "loss": 0.8460502028465271,
+ "mean_token_accuracy": 0.7821201682090759,
+ "num_tokens": 17055744.0,
+ "step": 1041
+ },
+ {
+ "entropy": 0.7591535449028015,
+ "epoch": 2.105050505050505,
+ "grad_norm": 2.215092182159424,
+ "learning_rate": 5.97979797979798e-06,
+ "loss": 0.7658643126487732,
+ "mean_token_accuracy": 0.8003175258636475,
+ "num_tokens": 17072128.0,
+ "step": 1042
+ },
+ {
+ "entropy": 1.1817188262939453,
+ "epoch": 2.107070707070707,
+ "grad_norm": 2.094550371170044,
+ "learning_rate": 5.966329966329966e-06,
+ "loss": 1.183272361755371,
+ "mean_token_accuracy": 0.7251465320587158,
+ "num_tokens": 17088512.0,
+ "step": 1043
+ },
+ {
+ "entropy": 0.7743434309959412,
+ "epoch": 2.109090909090909,
+ "grad_norm": 1.9481000900268555,
+ "learning_rate": 5.952861952861954e-06,
+ "loss": 0.7503235936164856,
+ "mean_token_accuracy": 0.8009892702102661,
+ "num_tokens": 17104896.0,
+ "step": 1044
+ },
+ {
+ "entropy": 0.9779418110847473,
+ "epoch": 2.111111111111111,
+ "grad_norm": 10.17557430267334,
+ "learning_rate": 5.93939393939394e-06,
+ "loss": 1.1502952575683594,
+ "mean_token_accuracy": 0.7345505356788635,
+ "num_tokens": 17121280.0,
+ "step": 1045
+ },
+ {
+ "entropy": 0.7385470867156982,
+ "epoch": 2.113131313131313,
+ "grad_norm": 2.2240688800811768,
+ "learning_rate": 5.925925925925926e-06,
+ "loss": 0.7330628633499146,
+ "mean_token_accuracy": 0.7921959161758423,
+ "num_tokens": 17137664.0,
+ "step": 1046
+ },
+ {
+ "entropy": 1.0204321146011353,
+ "epoch": 2.1151515151515152,
+ "grad_norm": 2.159083843231201,
+ "learning_rate": 5.912457912457913e-06,
+ "loss": 1.0087707042694092,
+ "mean_token_accuracy": 0.7565950155258179,
+ "num_tokens": 17154048.0,
+ "step": 1047
+ },
+ {
+ "entropy": 1.3662269115447998,
+ "epoch": 2.117171717171717,
+ "grad_norm": 2.3070926666259766,
+ "learning_rate": 5.8989898989899e-06,
+ "loss": 1.3453154563903809,
+ "mean_token_accuracy": 0.6869198679924011,
+ "num_tokens": 17170432.0,
+ "step": 1048
+ },
+ {
+ "entropy": 1.2418196201324463,
+ "epoch": 2.1191919191919193,
+ "grad_norm": 2.257704734802246,
+ "learning_rate": 5.885521885521886e-06,
+ "loss": 1.2455103397369385,
+ "mean_token_accuracy": 0.709269642829895,
+ "num_tokens": 17186816.0,
+ "step": 1049
+ },
+ {
+ "entropy": 0.9079713821411133,
+ "epoch": 2.121212121212121,
+ "grad_norm": 2.195234537124634,
+ "learning_rate": 5.872053872053873e-06,
+ "loss": 0.8855721950531006,
+ "mean_token_accuracy": 0.7701514363288879,
+ "num_tokens": 17203200.0,
+ "step": 1050
+ },
+ {
+ "entropy": 0.975200891494751,
+ "epoch": 2.1232323232323234,
+ "grad_norm": 2.146843671798706,
+ "learning_rate": 5.858585858585859e-06,
+ "loss": 0.9658151865005493,
+ "mean_token_accuracy": 0.7715559601783752,
+ "num_tokens": 17219584.0,
+ "step": 1051
+ },
+ {
+ "entropy": 0.9608264565467834,
+ "epoch": 2.125252525252525,
+ "grad_norm": 2.330017566680908,
+ "learning_rate": 5.8451178451178456e-06,
+ "loss": 0.9278503656387329,
+ "mean_token_accuracy": 0.7578774094581604,
+ "num_tokens": 17235968.0,
+ "step": 1052
+ },
+ {
+ "entropy": 0.6266511082649231,
+ "epoch": 2.1272727272727274,
+ "grad_norm": 1.9988964796066284,
+ "learning_rate": 5.831649831649833e-06,
+ "loss": 0.6169615983963013,
+ "mean_token_accuracy": 0.8298729658126831,
+ "num_tokens": 17252352.0,
+ "step": 1053
+ },
+ {
+ "entropy": 1.2015810012817383,
+ "epoch": 2.1292929292929292,
+ "grad_norm": 2.2715466022491455,
+ "learning_rate": 5.8181818181818185e-06,
+ "loss": 1.1880292892456055,
+ "mean_token_accuracy": 0.7046898007392883,
+ "num_tokens": 17268736.0,
+ "step": 1054
+ },
+ {
+ "entropy": 1.107398271560669,
+ "epoch": 2.1313131313131315,
+ "grad_norm": 2.1081430912017822,
+ "learning_rate": 5.804713804713805e-06,
+ "loss": 1.1123331785202026,
+ "mean_token_accuracy": 0.7287493944168091,
+ "num_tokens": 17285120.0,
+ "step": 1055
+ },
+ {
+ "entropy": 0.7515788674354553,
+ "epoch": 2.1333333333333333,
+ "grad_norm": 2.2605152130126953,
+ "learning_rate": 5.791245791245792e-06,
+ "loss": 0.7696233987808228,
+ "mean_token_accuracy": 0.7863947153091431,
+ "num_tokens": 17301504.0,
+ "step": 1056
+ },
+ {
+ "entropy": 1.1765409708023071,
+ "epoch": 2.1353535353535356,
+ "grad_norm": 2.3561489582061768,
+ "learning_rate": 5.777777777777778e-06,
+ "loss": 1.1908636093139648,
+ "mean_token_accuracy": 0.7037127614021301,
+ "num_tokens": 17317888.0,
+ "step": 1057
+ },
+ {
+ "entropy": 1.3084092140197754,
+ "epoch": 2.1373737373737374,
+ "grad_norm": 2.1483852863311768,
+ "learning_rate": 5.7643097643097645e-06,
+ "loss": 1.2867857217788696,
+ "mean_token_accuracy": 0.7012090682983398,
+ "num_tokens": 17334272.0,
+ "step": 1058
+ },
+ {
+ "entropy": 1.241392731666565,
+ "epoch": 2.1393939393939396,
+ "grad_norm": 2.342715263366699,
+ "learning_rate": 5.750841750841751e-06,
+ "loss": 1.2093181610107422,
+ "mean_token_accuracy": 0.7300928235054016,
+ "num_tokens": 17350656.0,
+ "step": 1059
+ },
+ {
+ "entropy": 1.0139163732528687,
+ "epoch": 2.1414141414141414,
+ "grad_norm": 2.211137533187866,
+ "learning_rate": 5.737373737373738e-06,
+ "loss": 0.9826461672782898,
+ "mean_token_accuracy": 0.7470688819885254,
+ "num_tokens": 17367040.0,
+ "step": 1060
+ },
+ {
+ "entropy": 1.163819670677185,
+ "epoch": 2.1434343434343432,
+ "grad_norm": 2.1510820388793945,
+ "learning_rate": 5.723905723905724e-06,
+ "loss": 1.141978144645691,
+ "mean_token_accuracy": 0.7236199378967285,
+ "num_tokens": 17383424.0,
+ "step": 1061
+ },
+ {
+ "entropy": 1.2484748363494873,
+ "epoch": 2.1454545454545455,
+ "grad_norm": 2.096975088119507,
+ "learning_rate": 5.71043771043771e-06,
+ "loss": 1.2311407327651978,
+ "mean_token_accuracy": 0.697300910949707,
+ "num_tokens": 17399808.0,
+ "step": 1062
+ },
+ {
+ "entropy": 0.763225793838501,
+ "epoch": 2.1474747474747473,
+ "grad_norm": 2.2745585441589355,
+ "learning_rate": 5.696969696969698e-06,
+ "loss": 0.740476131439209,
+ "mean_token_accuracy": 0.797813892364502,
+ "num_tokens": 17416192.0,
+ "step": 1063
+ },
+ {
+ "entropy": 1.161580204963684,
+ "epoch": 2.1494949494949496,
+ "grad_norm": 2.205702543258667,
+ "learning_rate": 5.683501683501684e-06,
+ "loss": 1.1578772068023682,
+ "mean_token_accuracy": 0.7220322489738464,
+ "num_tokens": 17432576.0,
+ "step": 1064
+ },
+ {
+ "entropy": 1.3429193496704102,
+ "epoch": 2.1515151515151514,
+ "grad_norm": 2.1587917804718018,
+ "learning_rate": 5.67003367003367e-06,
+ "loss": 1.3394683599472046,
+ "mean_token_accuracy": 0.6847215294837952,
+ "num_tokens": 17448960.0,
+ "step": 1065
+ },
+ {
+ "entropy": 0.8381629586219788,
+ "epoch": 2.1535353535353536,
+ "grad_norm": 2.017162322998047,
+ "learning_rate": 5.656565656565657e-06,
+ "loss": 0.8215705156326294,
+ "mean_token_accuracy": 0.7872496247291565,
+ "num_tokens": 17465344.0,
+ "step": 1066
+ },
+ {
+ "entropy": 0.7657965421676636,
+ "epoch": 2.1555555555555554,
+ "grad_norm": 2.1969051361083984,
+ "learning_rate": 5.643097643097644e-06,
+ "loss": 0.778290867805481,
+ "mean_token_accuracy": 0.7841353416442871,
+ "num_tokens": 17481728.0,
+ "step": 1067
+ },
+ {
+ "entropy": 0.8711634874343872,
+ "epoch": 2.1575757575757577,
+ "grad_norm": 2.1608996391296387,
+ "learning_rate": 5.62962962962963e-06,
+ "loss": 0.8894504904747009,
+ "mean_token_accuracy": 0.7703956961631775,
+ "num_tokens": 17498112.0,
+ "step": 1068
+ },
+ {
+ "entropy": 1.127463698387146,
+ "epoch": 2.1595959595959595,
+ "grad_norm": 2.3153634071350098,
+ "learning_rate": 5.616161616161616e-06,
+ "loss": 1.1329922676086426,
+ "mean_token_accuracy": 0.7192232608795166,
+ "num_tokens": 17514496.0,
+ "step": 1069
+ },
+ {
+ "entropy": 0.8976262807846069,
+ "epoch": 2.1616161616161618,
+ "grad_norm": 2.4478628635406494,
+ "learning_rate": 5.602693602693603e-06,
+ "loss": 0.8956177830696106,
+ "mean_token_accuracy": 0.7629457712173462,
+ "num_tokens": 17530880.0,
+ "step": 1070
+ },
+ {
+ "entropy": 1.052526831626892,
+ "epoch": 2.1636363636363636,
+ "grad_norm": 2.4650626182556152,
+ "learning_rate": 5.58922558922559e-06,
+ "loss": 1.0365463495254517,
+ "mean_token_accuracy": 0.7423058152198792,
+ "num_tokens": 17547264.0,
+ "step": 1071
+ },
+ {
+ "entropy": 1.2520077228546143,
+ "epoch": 2.165656565656566,
+ "grad_norm": 2.1537535190582275,
+ "learning_rate": 5.575757575757577e-06,
+ "loss": 1.2596616744995117,
+ "mean_token_accuracy": 0.7003541588783264,
+ "num_tokens": 17563648.0,
+ "step": 1072
+ },
+ {
+ "entropy": 1.2266544103622437,
+ "epoch": 2.1676767676767676,
+ "grad_norm": 2.3401050567626953,
+ "learning_rate": 5.562289562289563e-06,
+ "loss": 1.2335799932479858,
+ "mean_token_accuracy": 0.6985222101211548,
+ "num_tokens": 17580032.0,
+ "step": 1073
+ },
+ {
+ "entropy": 0.9337752461433411,
+ "epoch": 2.16969696969697,
+ "grad_norm": 2.229626178741455,
+ "learning_rate": 5.548821548821549e-06,
+ "loss": 0.9321554899215698,
+ "mean_token_accuracy": 0.7583659291267395,
+ "num_tokens": 17596416.0,
+ "step": 1074
+ },
+ {
+ "entropy": 0.6643204689025879,
+ "epoch": 2.1717171717171717,
+ "grad_norm": 2.038248300552368,
+ "learning_rate": 5.5353535353535355e-06,
+ "loss": 0.6427558064460754,
+ "mean_token_accuracy": 0.8161944150924683,
+ "num_tokens": 17612800.0,
+ "step": 1075
+ },
+ {
+ "entropy": 0.9372988939285278,
+ "epoch": 2.1737373737373735,
+ "grad_norm": 2.398390293121338,
+ "learning_rate": 5.521885521885523e-06,
+ "loss": 0.9131875038146973,
+ "mean_token_accuracy": 0.7569003701210022,
+ "num_tokens": 17629184.0,
+ "step": 1076
+ },
+ {
+ "entropy": 1.3854912519454956,
+ "epoch": 2.175757575757576,
+ "grad_norm": 2.325932264328003,
+ "learning_rate": 5.5084175084175085e-06,
+ "loss": 1.3354313373565674,
+ "mean_token_accuracy": 0.6645700931549072,
+ "num_tokens": 17645568.0,
+ "step": 1077
+ },
+ {
+ "entropy": 0.9905093908309937,
+ "epoch": 2.1777777777777776,
+ "grad_norm": 2.303408622741699,
+ "learning_rate": 5.494949494949495e-06,
+ "loss": 0.9684986472129822,
+ "mean_token_accuracy": 0.7507327795028687,
+ "num_tokens": 17661952.0,
+ "step": 1078
+ },
+ {
+ "entropy": 1.1516029834747314,
+ "epoch": 2.17979797979798,
+ "grad_norm": 2.1588361263275146,
+ "learning_rate": 5.481481481481482e-06,
+ "loss": 1.1108394861221313,
+ "mean_token_accuracy": 0.7329018115997314,
+ "num_tokens": 17678336.0,
+ "step": 1079
+ },
+ {
+ "entropy": 1.1009210348129272,
+ "epoch": 2.1818181818181817,
+ "grad_norm": 2.294980764389038,
+ "learning_rate": 5.468013468013469e-06,
+ "loss": 1.0698065757751465,
+ "mean_token_accuracy": 0.7306423783302307,
+ "num_tokens": 17694720.0,
+ "step": 1080
+ },
+ {
+ "entropy": 0.5653274655342102,
+ "epoch": 2.183838383838384,
+ "grad_norm": 1.7224467992782593,
+ "learning_rate": 5.4545454545454545e-06,
+ "loss": 0.5569928884506226,
+ "mean_token_accuracy": 0.8492305874824524,
+ "num_tokens": 17711104.0,
+ "step": 1081
+ },
+ {
+ "entropy": 0.7761416435241699,
+ "epoch": 2.1858585858585857,
+ "grad_norm": 2.062605142593384,
+ "learning_rate": 5.441077441077442e-06,
+ "loss": 0.7638991475105286,
+ "mean_token_accuracy": 0.8034929037094116,
+ "num_tokens": 17727488.0,
+ "step": 1082
+ },
+ {
+ "entropy": 0.7888216972351074,
+ "epoch": 2.187878787878788,
+ "grad_norm": 1.8905524015426636,
+ "learning_rate": 5.427609427609428e-06,
+ "loss": 0.7825406789779663,
+ "mean_token_accuracy": 0.7937225103378296,
+ "num_tokens": 17743872.0,
+ "step": 1083
+ },
+ {
+ "entropy": 0.9246746897697449,
+ "epoch": 2.18989898989899,
+ "grad_norm": 2.466188669204712,
+ "learning_rate": 5.414141414141415e-06,
+ "loss": 0.9468541145324707,
+ "mean_token_accuracy": 0.7498778700828552,
+ "num_tokens": 17760256.0,
+ "step": 1084
+ },
+ {
+ "entropy": 1.24199640750885,
+ "epoch": 2.191919191919192,
+ "grad_norm": 2.341604232788086,
+ "learning_rate": 5.4006734006734e-06,
+ "loss": 1.218245029449463,
+ "mean_token_accuracy": 0.7104299068450928,
+ "num_tokens": 17776640.0,
+ "step": 1085
+ },
+ {
+ "entropy": 1.120385766029358,
+ "epoch": 2.193939393939394,
+ "grad_norm": 2.2657902240753174,
+ "learning_rate": 5.387205387205388e-06,
+ "loss": 1.10544753074646,
+ "mean_token_accuracy": 0.7314362525939941,
+ "num_tokens": 17793024.0,
+ "step": 1086
+ },
+ {
+ "entropy": 1.1147528886795044,
+ "epoch": 2.195959595959596,
+ "grad_norm": 2.1902315616607666,
+ "learning_rate": 5.373737373737374e-06,
+ "loss": 1.121596097946167,
+ "mean_token_accuracy": 0.7246580123901367,
+ "num_tokens": 17809408.0,
+ "step": 1087
+ },
+ {
+ "entropy": 1.0283604860305786,
+ "epoch": 2.197979797979798,
+ "grad_norm": 2.2043023109436035,
+ "learning_rate": 5.3602693602693615e-06,
+ "loss": 1.0231928825378418,
+ "mean_token_accuracy": 0.7412676811218262,
+ "num_tokens": 17825792.0,
+ "step": 1088
+ },
+ {
+ "entropy": 0.7668783068656921,
+ "epoch": 2.2,
+ "grad_norm": 2.281240701675415,
+ "learning_rate": 5.346801346801347e-06,
+ "loss": 0.7641464471817017,
+ "mean_token_accuracy": 0.7999511361122131,
+ "num_tokens": 17842176.0,
+ "step": 1089
+ },
+ {
+ "entropy": 0.8896923065185547,
+ "epoch": 2.202020202020202,
+ "grad_norm": 2.1806983947753906,
+ "learning_rate": 5.333333333333334e-06,
+ "loss": 0.9093960523605347,
+ "mean_token_accuracy": 0.768991231918335,
+ "num_tokens": 17858560.0,
+ "step": 1090
+ },
+ {
+ "entropy": 0.7874066829681396,
+ "epoch": 2.2040404040404042,
+ "grad_norm": 2.2266900539398193,
+ "learning_rate": 5.31986531986532e-06,
+ "loss": 0.781766414642334,
+ "mean_token_accuracy": 0.784807026386261,
+ "num_tokens": 17874944.0,
+ "step": 1091
+ },
+ {
+ "entropy": 1.0619457960128784,
+ "epoch": 2.206060606060606,
+ "grad_norm": 2.4347457885742188,
+ "learning_rate": 5.3063973063973075e-06,
+ "loss": 1.1253571510314941,
+ "mean_token_accuracy": 0.7411455512046814,
+ "num_tokens": 17891328.0,
+ "step": 1092
+ },
+ {
+ "entropy": 0.8278951048851013,
+ "epoch": 2.2080808080808083,
+ "grad_norm": 2.1931722164154053,
+ "learning_rate": 5.292929292929293e-06,
+ "loss": 0.8328354358673096,
+ "mean_token_accuracy": 0.7835246920585632,
+ "num_tokens": 17907712.0,
+ "step": 1093
+ },
+ {
+ "entropy": 1.0070204734802246,
+ "epoch": 2.21010101010101,
+ "grad_norm": 2.1386632919311523,
+ "learning_rate": 5.27946127946128e-06,
+ "loss": 1.0087940692901611,
+ "mean_token_accuracy": 0.7423668503761292,
+ "num_tokens": 17924096.0,
+ "step": 1094
+ },
+ {
+ "entropy": 1.1913471221923828,
+ "epoch": 2.212121212121212,
+ "grad_norm": 2.0004987716674805,
+ "learning_rate": 5.265993265993267e-06,
+ "loss": 1.1983554363250732,
+ "mean_token_accuracy": 0.7126282453536987,
+ "num_tokens": 17940480.0,
+ "step": 1095
+ },
+ {
+ "entropy": 1.0097228288650513,
+ "epoch": 2.214141414141414,
+ "grad_norm": 2.0011065006256104,
+ "learning_rate": 5.252525252525253e-06,
+ "loss": 1.003847360610962,
+ "mean_token_accuracy": 0.7553126811981201,
+ "num_tokens": 17956864.0,
+ "step": 1096
+ },
+ {
+ "entropy": 0.7539495229721069,
+ "epoch": 2.216161616161616,
+ "grad_norm": 2.109707832336426,
+ "learning_rate": 5.239057239057239e-06,
+ "loss": 0.735792875289917,
+ "mean_token_accuracy": 0.8071568012237549,
+ "num_tokens": 17973248.0,
+ "step": 1097
+ },
+ {
+ "entropy": 1.2859326601028442,
+ "epoch": 2.2181818181818183,
+ "grad_norm": 2.2937707901000977,
+ "learning_rate": 5.225589225589226e-06,
+ "loss": 1.2867331504821777,
+ "mean_token_accuracy": 0.6965070962905884,
+ "num_tokens": 17989632.0,
+ "step": 1098
+ },
+ {
+ "entropy": 0.9745997190475464,
+ "epoch": 2.22020202020202,
+ "grad_norm": 2.4093194007873535,
+ "learning_rate": 5.212121212121213e-06,
+ "loss": 0.9495588541030884,
+ "mean_token_accuracy": 0.7793111801147461,
+ "num_tokens": 18006016.0,
+ "step": 1099
+ },
+ {
+ "entropy": 1.210206389427185,
+ "epoch": 2.2222222222222223,
+ "grad_norm": 2.307809352874756,
+ "learning_rate": 5.198653198653199e-06,
+ "loss": 1.190993070602417,
+ "mean_token_accuracy": 0.7073766589164734,
+ "num_tokens": 18022400.0,
+ "step": 1100
+ },
+ {
+ "epoch": 2.2222222222222223,
+ "eval_entropy": 1.1411552967563752,
+ "eval_loss": 1.6080615520477295,
+ "eval_mean_token_accuracy": 0.6425399270749861,
+ "eval_num_tokens": 18022400.0,
+ "eval_runtime": 43.7895,
+ "eval_samples_per_second": 22.608,
+ "eval_steps_per_second": 2.832,
+ "step": 1100
+ },
+ {
+ "entropy": 0.771600067615509,
+ "epoch": 2.224242424242424,
+ "grad_norm": 2.2073745727539062,
+ "learning_rate": 5.185185185185185e-06,
+ "loss": 0.7411383390426636,
+ "mean_token_accuracy": 0.8007450103759766,
+ "num_tokens": 18038784.0,
+ "step": 1101
+ },
+ {
+ "entropy": 0.8682986497879028,
+ "epoch": 2.2262626262626264,
+ "grad_norm": 2.036909580230713,
+ "learning_rate": 5.171717171717172e-06,
+ "loss": 0.8770009875297546,
+ "mean_token_accuracy": 0.7767464518547058,
+ "num_tokens": 18055168.0,
+ "step": 1102
+ },
+ {
+ "entropy": 1.0334750413894653,
+ "epoch": 2.228282828282828,
+ "grad_norm": 2.139882802963257,
+ "learning_rate": 5.158249158249159e-06,
+ "loss": 1.020939826965332,
+ "mean_token_accuracy": 0.7569614052772522,
+ "num_tokens": 18071552.0,
+ "step": 1103
+ },
+ {
+ "entropy": 1.24951171875,
+ "epoch": 2.2303030303030305,
+ "grad_norm": 2.2494804859161377,
+ "learning_rate": 5.1447811447811444e-06,
+ "loss": 1.2513877153396606,
+ "mean_token_accuracy": 0.7010259032249451,
+ "num_tokens": 18087936.0,
+ "step": 1104
+ },
+ {
+ "entropy": 0.8687318563461304,
+ "epoch": 2.2323232323232323,
+ "grad_norm": 2.346163511276245,
+ "learning_rate": 5.131313131313132e-06,
+ "loss": 0.856663167476654,
+ "mean_token_accuracy": 0.7724108695983887,
+ "num_tokens": 18104320.0,
+ "step": 1105
+ },
+ {
+ "entropy": 0.9489924311637878,
+ "epoch": 2.2343434343434345,
+ "grad_norm": 2.3031630516052246,
+ "learning_rate": 5.117845117845118e-06,
+ "loss": 0.9237314462661743,
+ "mean_token_accuracy": 0.7616023421287537,
+ "num_tokens": 18120704.0,
+ "step": 1106
+ },
+ {
+ "entropy": 0.7660726308822632,
+ "epoch": 2.2363636363636363,
+ "grad_norm": 2.0157992839813232,
+ "learning_rate": 5.104377104377105e-06,
+ "loss": 0.7505912184715271,
+ "mean_token_accuracy": 0.8021494746208191,
+ "num_tokens": 18137088.0,
+ "step": 1107
+ },
+ {
+ "entropy": 0.8646788597106934,
+ "epoch": 2.2383838383838386,
+ "grad_norm": 2.1639404296875,
+ "learning_rate": 5.090909090909091e-06,
+ "loss": 0.8640313148498535,
+ "mean_token_accuracy": 0.7763190269470215,
+ "num_tokens": 18153472.0,
+ "step": 1108
+ },
+ {
+ "entropy": 1.0876270532608032,
+ "epoch": 2.2404040404040404,
+ "grad_norm": 2.2591774463653564,
+ "learning_rate": 5.077441077441078e-06,
+ "loss": 1.0915470123291016,
+ "mean_token_accuracy": 0.7280776500701904,
+ "num_tokens": 18169856.0,
+ "step": 1109
+ },
+ {
+ "entropy": 0.8739895224571228,
+ "epoch": 2.242424242424242,
+ "grad_norm": 2.1637771129608154,
+ "learning_rate": 5.063973063973064e-06,
+ "loss": 0.8555219769477844,
+ "mean_token_accuracy": 0.7708231806755066,
+ "num_tokens": 18186240.0,
+ "step": 1110
+ },
+ {
+ "entropy": 1.0880624055862427,
+ "epoch": 2.2444444444444445,
+ "grad_norm": 2.145496129989624,
+ "learning_rate": 5.0505050505050515e-06,
+ "loss": 1.0848066806793213,
+ "mean_token_accuracy": 0.736993134021759,
+ "num_tokens": 18202624.0,
+ "step": 1111
+ },
+ {
+ "entropy": 1.0834776163101196,
+ "epoch": 2.2464646464646463,
+ "grad_norm": 2.387665033340454,
+ "learning_rate": 5.037037037037037e-06,
+ "loss": 1.0987672805786133,
+ "mean_token_accuracy": 0.7160478830337524,
+ "num_tokens": 18219008.0,
+ "step": 1112
+ },
+ {
+ "entropy": 0.928567111492157,
+ "epoch": 2.2484848484848485,
+ "grad_norm": 2.15563702583313,
+ "learning_rate": 5.023569023569024e-06,
+ "loss": 0.9242000579833984,
+ "mean_token_accuracy": 0.7684416174888611,
+ "num_tokens": 18235392.0,
+ "step": 1113
+ },
+ {
+ "entropy": 0.7360862493515015,
+ "epoch": 2.2505050505050503,
+ "grad_norm": 2.1846494674682617,
+ "learning_rate": 5.010101010101011e-06,
+ "loss": 0.7165976762771606,
+ "mean_token_accuracy": 0.7969589829444885,
+ "num_tokens": 18251776.0,
+ "step": 1114
+ },
+ {
+ "entropy": 0.8648936152458191,
+ "epoch": 2.2525252525252526,
+ "grad_norm": 2.4350926876068115,
+ "learning_rate": 4.996632996632997e-06,
+ "loss": 0.85498046875,
+ "mean_token_accuracy": 0.7721055150032043,
+ "num_tokens": 18268160.0,
+ "step": 1115
+ },
+ {
+ "entropy": 1.095477819442749,
+ "epoch": 2.2545454545454544,
+ "grad_norm": 2.1980817317962646,
+ "learning_rate": 4.983164983164984e-06,
+ "loss": 1.110865592956543,
+ "mean_token_accuracy": 0.7231314182281494,
+ "num_tokens": 18284544.0,
+ "step": 1116
+ },
+ {
+ "entropy": 0.9207680821418762,
+ "epoch": 2.2565656565656567,
+ "grad_norm": 2.0694732666015625,
+ "learning_rate": 4.9696969696969696e-06,
+ "loss": 0.8975710272789001,
+ "mean_token_accuracy": 0.7692965269088745,
+ "num_tokens": 18300928.0,
+ "step": 1117
+ },
+ {
+ "entropy": 1.0263354778289795,
+ "epoch": 2.2585858585858585,
+ "grad_norm": 2.0532844066619873,
+ "learning_rate": 4.956228956228957e-06,
+ "loss": 1.0378568172454834,
+ "mean_token_accuracy": 0.7442598938941956,
+ "num_tokens": 18317312.0,
+ "step": 1118
+ },
+ {
+ "entropy": 0.9276180267333984,
+ "epoch": 2.2606060606060607,
+ "grad_norm": 2.2308754920959473,
+ "learning_rate": 4.942760942760943e-06,
+ "loss": 0.8967354893684387,
+ "mean_token_accuracy": 0.7642892003059387,
+ "num_tokens": 18333696.0,
+ "step": 1119
+ },
+ {
+ "entropy": 1.5558041334152222,
+ "epoch": 2.2626262626262625,
+ "grad_norm": 2.5063514709472656,
+ "learning_rate": 4.92929292929293e-06,
+ "loss": 1.5838005542755127,
+ "mean_token_accuracy": 0.6414875388145447,
+ "num_tokens": 18350080.0,
+ "step": 1120
+ },
+ {
+ "entropy": 0.7984672784805298,
+ "epoch": 2.264646464646465,
+ "grad_norm": 2.019965887069702,
+ "learning_rate": 4.915824915824916e-06,
+ "loss": 0.796333909034729,
+ "mean_token_accuracy": 0.7834025621414185,
+ "num_tokens": 18366464.0,
+ "step": 1121
+ },
+ {
+ "entropy": 1.5188385248184204,
+ "epoch": 2.2666666666666666,
+ "grad_norm": 2.1626179218292236,
+ "learning_rate": 4.902356902356903e-06,
+ "loss": 1.4933388233184814,
+ "mean_token_accuracy": 0.6630434989929199,
+ "num_tokens": 18382848.0,
+ "step": 1122
+ },
+ {
+ "entropy": 0.8135427832603455,
+ "epoch": 2.268686868686869,
+ "grad_norm": 2.1165366172790527,
+ "learning_rate": 4.888888888888889e-06,
+ "loss": 0.815280020236969,
+ "mean_token_accuracy": 0.7854787707328796,
+ "num_tokens": 18399232.0,
+ "step": 1123
+ },
+ {
+ "entropy": 1.0267316102981567,
+ "epoch": 2.2707070707070707,
+ "grad_norm": 2.315552234649658,
+ "learning_rate": 4.875420875420876e-06,
+ "loss": 1.021623969078064,
+ "mean_token_accuracy": 0.7460307478904724,
+ "num_tokens": 18415616.0,
+ "step": 1124
+ },
+ {
+ "entropy": 1.0322307348251343,
+ "epoch": 2.2727272727272725,
+ "grad_norm": 2.4442577362060547,
+ "learning_rate": 4.861952861952862e-06,
+ "loss": 1.0230128765106201,
+ "mean_token_accuracy": 0.7383365631103516,
+ "num_tokens": 18432000.0,
+ "step": 1125
+ },
+ {
+ "entropy": 0.7536509037017822,
+ "epoch": 2.2747474747474747,
+ "grad_norm": 2.087148666381836,
+ "learning_rate": 4.848484848484849e-06,
+ "loss": 0.7206662893295288,
+ "mean_token_accuracy": 0.8106375336647034,
+ "num_tokens": 18448384.0,
+ "step": 1126
+ },
+ {
+ "entropy": 1.132371425628662,
+ "epoch": 2.276767676767677,
+ "grad_norm": 2.144556999206543,
+ "learning_rate": 4.835016835016835e-06,
+ "loss": 1.1253585815429688,
+ "mean_token_accuracy": 0.7419394254684448,
+ "num_tokens": 18464768.0,
+ "step": 1127
+ },
+ {
+ "entropy": 0.6875131130218506,
+ "epoch": 2.278787878787879,
+ "grad_norm": 2.204012632369995,
+ "learning_rate": 4.821548821548822e-06,
+ "loss": 0.6672695875167847,
+ "mean_token_accuracy": 0.8155837655067444,
+ "num_tokens": 18481152.0,
+ "step": 1128
+ },
+ {
+ "entropy": 1.1031876802444458,
+ "epoch": 2.2808080808080806,
+ "grad_norm": 2.0218825340270996,
+ "learning_rate": 4.808080808080808e-06,
+ "loss": 1.106371521949768,
+ "mean_token_accuracy": 0.7319247722625732,
+ "num_tokens": 18497536.0,
+ "step": 1129
+ },
+ {
+ "entropy": 1.3797358274459839,
+ "epoch": 2.282828282828283,
+ "grad_norm": 2.2343342304229736,
+ "learning_rate": 4.7946127946127956e-06,
+ "loss": 1.3944905996322632,
+ "mean_token_accuracy": 0.689728856086731,
+ "num_tokens": 18513920.0,
+ "step": 1130
+ },
+ {
+ "entropy": 1.2295851707458496,
+ "epoch": 2.2848484848484847,
+ "grad_norm": 2.32977294921875,
+ "learning_rate": 4.781144781144781e-06,
+ "loss": 1.2050158977508545,
+ "mean_token_accuracy": 0.7108573317527771,
+ "num_tokens": 18530304.0,
+ "step": 1131
+ },
+ {
+ "entropy": 1.0025029182434082,
+ "epoch": 2.286868686868687,
+ "grad_norm": 2.275966167449951,
+ "learning_rate": 4.7676767676767685e-06,
+ "loss": 1.011232852935791,
+ "mean_token_accuracy": 0.7412066459655762,
+ "num_tokens": 18546688.0,
+ "step": 1132
+ },
+ {
+ "entropy": 1.2037127017974854,
+ "epoch": 2.2888888888888888,
+ "grad_norm": 2.317436933517456,
+ "learning_rate": 4.754208754208754e-06,
+ "loss": 1.2052998542785645,
+ "mean_token_accuracy": 0.7090864777565002,
+ "num_tokens": 18563072.0,
+ "step": 1133
+ },
+ {
+ "entropy": 0.9764331579208374,
+ "epoch": 2.290909090909091,
+ "grad_norm": 2.0774219036102295,
+ "learning_rate": 4.7407407407407415e-06,
+ "loss": 0.9488927125930786,
+ "mean_token_accuracy": 0.7618466019630432,
+ "num_tokens": 18579456.0,
+ "step": 1134
+ },
+ {
+ "entropy": 0.8998712301254272,
+ "epoch": 2.292929292929293,
+ "grad_norm": 1.9016351699829102,
+ "learning_rate": 4.727272727272728e-06,
+ "loss": 0.8935770988464355,
+ "mean_token_accuracy": 0.7713117003440857,
+ "num_tokens": 18595840.0,
+ "step": 1135
+ },
+ {
+ "entropy": 0.9964506030082703,
+ "epoch": 2.294949494949495,
+ "grad_norm": 2.2267792224884033,
+ "learning_rate": 4.7138047138047145e-06,
+ "loss": 0.9771631360054016,
+ "mean_token_accuracy": 0.7502442598342896,
+ "num_tokens": 18612224.0,
+ "step": 1136
+ },
+ {
+ "entropy": 1.2267141342163086,
+ "epoch": 2.296969696969697,
+ "grad_norm": 2.3059990406036377,
+ "learning_rate": 4.700336700336701e-06,
+ "loss": 1.2470276355743408,
+ "mean_token_accuracy": 0.7123228907585144,
+ "num_tokens": 18628608.0,
+ "step": 1137
+ },
+ {
+ "entropy": 0.9602224826812744,
+ "epoch": 2.298989898989899,
+ "grad_norm": 2.163684129714966,
+ "learning_rate": 4.6868686868686874e-06,
+ "loss": 0.9301580190658569,
+ "mean_token_accuracy": 0.7639839053153992,
+ "num_tokens": 18644992.0,
+ "step": 1138
+ },
+ {
+ "entropy": 0.9604593515396118,
+ "epoch": 2.301010101010101,
+ "grad_norm": 2.0252676010131836,
+ "learning_rate": 4.673400673400674e-06,
+ "loss": 0.9598495364189148,
+ "mean_token_accuracy": 0.7568392753601074,
+ "num_tokens": 18661376.0,
+ "step": 1139
+ },
+ {
+ "entropy": 0.8657703995704651,
+ "epoch": 2.303030303030303,
+ "grad_norm": 2.4980340003967285,
+ "learning_rate": 4.65993265993266e-06,
+ "loss": 0.8338019847869873,
+ "mean_token_accuracy": 0.7745481133460999,
+ "num_tokens": 18677760.0,
+ "step": 1140
+ },
+ {
+ "entropy": 1.0187418460845947,
+ "epoch": 2.305050505050505,
+ "grad_norm": 2.1227967739105225,
+ "learning_rate": 4.646464646464647e-06,
+ "loss": 1.0039290189743042,
+ "mean_token_accuracy": 0.7410845160484314,
+ "num_tokens": 18694144.0,
+ "step": 1141
+ },
+ {
+ "entropy": 1.2160016298294067,
+ "epoch": 2.3070707070707073,
+ "grad_norm": 2.737518310546875,
+ "learning_rate": 4.632996632996633e-06,
+ "loss": 1.2041016817092896,
+ "mean_token_accuracy": 0.708170473575592,
+ "num_tokens": 18710528.0,
+ "step": 1142
+ },
+ {
+ "entropy": 0.9344089031219482,
+ "epoch": 2.309090909090909,
+ "grad_norm": 2.058670997619629,
+ "learning_rate": 4.61952861952862e-06,
+ "loss": 0.929522693157196,
+ "mean_token_accuracy": 0.7677088379859924,
+ "num_tokens": 18726912.0,
+ "step": 1143
+ },
+ {
+ "entropy": 0.847671389579773,
+ "epoch": 2.311111111111111,
+ "grad_norm": 2.1489334106445312,
+ "learning_rate": 4.606060606060606e-06,
+ "loss": 0.8554779291152954,
+ "mean_token_accuracy": 0.7794333100318909,
+ "num_tokens": 18743296.0,
+ "step": 1144
+ },
+ {
+ "entropy": 1.2183470726013184,
+ "epoch": 2.313131313131313,
+ "grad_norm": 1.9973106384277344,
+ "learning_rate": 4.592592592592593e-06,
+ "loss": 1.2017948627471924,
+ "mean_token_accuracy": 0.7124450206756592,
+ "num_tokens": 18759680.0,
+ "step": 1145
+ },
+ {
+ "entropy": 1.049198031425476,
+ "epoch": 2.315151515151515,
+ "grad_norm": 2.236171245574951,
+ "learning_rate": 4.57912457912458e-06,
+ "loss": 1.0968973636627197,
+ "mean_token_accuracy": 0.7376648783683777,
+ "num_tokens": 18776064.0,
+ "step": 1146
+ },
+ {
+ "entropy": 0.8611926436424255,
+ "epoch": 2.317171717171717,
+ "grad_norm": 2.1956593990325928,
+ "learning_rate": 4.565656565656566e-06,
+ "loss": 0.8607693910598755,
+ "mean_token_accuracy": 0.7783952355384827,
+ "num_tokens": 18792448.0,
+ "step": 1147
+ },
+ {
+ "entropy": 1.338310956954956,
+ "epoch": 2.319191919191919,
+ "grad_norm": 2.431516647338867,
+ "learning_rate": 4.552188552188553e-06,
+ "loss": 1.3319693803787231,
+ "mean_token_accuracy": 0.6781876087188721,
+ "num_tokens": 18808832.0,
+ "step": 1148
+ },
+ {
+ "entropy": 0.7855240702629089,
+ "epoch": 2.3212121212121213,
+ "grad_norm": 2.083381175994873,
+ "learning_rate": 4.538720538720539e-06,
+ "loss": 0.7579916715621948,
+ "mean_token_accuracy": 0.7961651086807251,
+ "num_tokens": 18825216.0,
+ "step": 1149
+ },
+ {
+ "entropy": 0.9377650022506714,
+ "epoch": 2.323232323232323,
+ "grad_norm": 2.0583441257476807,
+ "learning_rate": 4.525252525252526e-06,
+ "loss": 0.9283360242843628,
+ "mean_token_accuracy": 0.7661822438240051,
+ "num_tokens": 18841600.0,
+ "step": 1150
+ },
+ {
+ "entropy": 0.6888769268989563,
+ "epoch": 2.3252525252525253,
+ "grad_norm": 1.8859190940856934,
+ "learning_rate": 4.5117845117845126e-06,
+ "loss": 0.6679004430770874,
+ "mean_token_accuracy": 0.8202857971191406,
+ "num_tokens": 18857984.0,
+ "step": 1151
+ },
+ {
+ "entropy": 1.2448687553405762,
+ "epoch": 2.327272727272727,
+ "grad_norm": 2.2379355430603027,
+ "learning_rate": 4.498316498316498e-06,
+ "loss": 1.2153626680374146,
+ "mean_token_accuracy": 0.714093804359436,
+ "num_tokens": 18874368.0,
+ "step": 1152
+ },
+ {
+ "entropy": 0.854434609413147,
+ "epoch": 2.3292929292929294,
+ "grad_norm": 2.21307635307312,
+ "learning_rate": 4.4848484848484855e-06,
+ "loss": 0.8640393018722534,
+ "mean_token_accuracy": 0.7816927433013916,
+ "num_tokens": 18890752.0,
+ "step": 1153
+ },
+ {
+ "entropy": 0.897014856338501,
+ "epoch": 2.3313131313131312,
+ "grad_norm": 2.0069496631622314,
+ "learning_rate": 4.471380471380471e-06,
+ "loss": 0.9055691361427307,
+ "mean_token_accuracy": 0.7672814130783081,
+ "num_tokens": 18907136.0,
+ "step": 1154
+ },
+ {
+ "entropy": 0.893920361995697,
+ "epoch": 2.3333333333333335,
+ "grad_norm": 2.1605043411254883,
+ "learning_rate": 4.4579124579124585e-06,
+ "loss": 0.8992359638214111,
+ "mean_token_accuracy": 0.7696629166603088,
+ "num_tokens": 18923520.0,
+ "step": 1155
+ },
+ {
+ "entropy": 0.9284574389457703,
+ "epoch": 2.3353535353535353,
+ "grad_norm": 2.1974165439605713,
+ "learning_rate": 4.444444444444444e-06,
+ "loss": 0.9108461141586304,
+ "mean_token_accuracy": 0.7683194875717163,
+ "num_tokens": 18939904.0,
+ "step": 1156
+ },
+ {
+ "entropy": 1.2936046123504639,
+ "epoch": 2.3373737373737375,
+ "grad_norm": 2.2567598819732666,
+ "learning_rate": 4.4309764309764315e-06,
+ "loss": 1.2691715955734253,
+ "mean_token_accuracy": 0.697300910949707,
+ "num_tokens": 18956288.0,
+ "step": 1157
+ },
+ {
+ "entropy": 0.7441157102584839,
+ "epoch": 2.3393939393939394,
+ "grad_norm": 2.111255168914795,
+ "learning_rate": 4.417508417508418e-06,
+ "loss": 0.725585401058197,
+ "mean_token_accuracy": 0.8006228804588318,
+ "num_tokens": 18972672.0,
+ "step": 1158
+ },
+ {
+ "entropy": 0.917363166809082,
+ "epoch": 2.341414141414141,
+ "grad_norm": 2.9000542163848877,
+ "learning_rate": 4.4040404040404044e-06,
+ "loss": 0.8813672661781311,
+ "mean_token_accuracy": 0.7703956961631775,
+ "num_tokens": 18989056.0,
+ "step": 1159
+ },
+ {
+ "entropy": 0.9396717548370361,
+ "epoch": 2.3434343434343434,
+ "grad_norm": 2.1208033561706543,
+ "learning_rate": 4.390572390572391e-06,
+ "loss": 0.9205979108810425,
+ "mean_token_accuracy": 0.7719223499298096,
+ "num_tokens": 19005440.0,
+ "step": 1160
+ },
+ {
+ "entropy": 0.9639775156974792,
+ "epoch": 2.3454545454545457,
+ "grad_norm": 2.179042339324951,
+ "learning_rate": 4.377104377104377e-06,
+ "loss": 0.9402551054954529,
+ "mean_token_accuracy": 0.7613580822944641,
+ "num_tokens": 19021824.0,
+ "step": 1161
+ },
+ {
+ "entropy": 0.9605804085731506,
+ "epoch": 2.3474747474747475,
+ "grad_norm": 2.201428174972534,
+ "learning_rate": 4.363636363636364e-06,
+ "loss": 0.9531458616256714,
+ "mean_token_accuracy": 0.7554348111152649,
+ "num_tokens": 19038208.0,
+ "step": 1162
+ },
+ {
+ "entropy": 1.2175929546356201,
+ "epoch": 2.3494949494949493,
+ "grad_norm": 2.3427727222442627,
+ "learning_rate": 4.35016835016835e-06,
+ "loss": 1.2165290117263794,
+ "mean_token_accuracy": 0.7007816433906555,
+ "num_tokens": 19054592.0,
+ "step": 1163
+ },
+ {
+ "entropy": 0.8713957667350769,
+ "epoch": 2.3515151515151516,
+ "grad_norm": 2.085918664932251,
+ "learning_rate": 4.336700336700337e-06,
+ "loss": 0.886815071105957,
+ "mean_token_accuracy": 0.7802882194519043,
+ "num_tokens": 19070976.0,
+ "step": 1164
+ },
+ {
+ "entropy": 1.2172973155975342,
+ "epoch": 2.3535353535353534,
+ "grad_norm": 2.2496557235717773,
+ "learning_rate": 4.323232323232323e-06,
+ "loss": 1.2305024862289429,
+ "mean_token_accuracy": 0.6992549896240234,
+ "num_tokens": 19087360.0,
+ "step": 1165
+ },
+ {
+ "entropy": 1.008517861366272,
+ "epoch": 2.3555555555555556,
+ "grad_norm": 2.1454458236694336,
+ "learning_rate": 4.30976430976431e-06,
+ "loss": 1.0096131563186646,
+ "mean_token_accuracy": 0.7481069564819336,
+ "num_tokens": 19103744.0,
+ "step": 1166
+ },
+ {
+ "entropy": 0.8316228985786438,
+ "epoch": 2.3575757575757574,
+ "grad_norm": 2.266805410385132,
+ "learning_rate": 4.296296296296296e-06,
+ "loss": 0.8300824165344238,
+ "mean_token_accuracy": 0.7838299870491028,
+ "num_tokens": 19120128.0,
+ "step": 1167
+ },
+ {
+ "entropy": 0.8297266364097595,
+ "epoch": 2.3595959595959597,
+ "grad_norm": 2.561913251876831,
+ "learning_rate": 4.282828282828283e-06,
+ "loss": 0.8217562437057495,
+ "mean_token_accuracy": 0.7908524870872498,
+ "num_tokens": 19136512.0,
+ "step": 1168
+ },
+ {
+ "entropy": 0.8838104009628296,
+ "epoch": 2.3616161616161615,
+ "grad_norm": 2.295294761657715,
+ "learning_rate": 4.26936026936027e-06,
+ "loss": 0.863752007484436,
+ "mean_token_accuracy": 0.7753419876098633,
+ "num_tokens": 19152896.0,
+ "step": 1169
+ },
+ {
+ "entropy": 0.9900280237197876,
+ "epoch": 2.3636363636363638,
+ "grad_norm": 2.200223684310913,
+ "learning_rate": 4.255892255892256e-06,
+ "loss": 0.9897500872612,
+ "mean_token_accuracy": 0.7403517365455627,
+ "num_tokens": 19169280.0,
+ "step": 1170
+ },
+ {
+ "entropy": 1.2343716621398926,
+ "epoch": 2.3656565656565656,
+ "grad_norm": 2.376347541809082,
+ "learning_rate": 4.242424242424243e-06,
+ "loss": 1.231675148010254,
+ "mean_token_accuracy": 0.7057889699935913,
+ "num_tokens": 19185664.0,
+ "step": 1171
+ },
+ {
+ "entropy": 1.1249511241912842,
+ "epoch": 2.367676767676768,
+ "grad_norm": 2.2998337745666504,
+ "learning_rate": 4.228956228956229e-06,
+ "loss": 1.149594783782959,
+ "mean_token_accuracy": 0.7239863276481628,
+ "num_tokens": 19202048.0,
+ "step": 1172
+ },
+ {
+ "entropy": 1.0526270866394043,
+ "epoch": 2.3696969696969696,
+ "grad_norm": 2.505650043487549,
+ "learning_rate": 4.215488215488216e-06,
+ "loss": 1.0455896854400635,
+ "mean_token_accuracy": 0.7277112603187561,
+ "num_tokens": 19218432.0,
+ "step": 1173
+ },
+ {
+ "entropy": 1.0050756931304932,
+ "epoch": 2.371717171717172,
+ "grad_norm": 2.0753841400146484,
+ "learning_rate": 4.2020202020202026e-06,
+ "loss": 1.0089832544326782,
+ "mean_token_accuracy": 0.7474963068962097,
+ "num_tokens": 19234816.0,
+ "step": 1174
+ },
+ {
+ "entropy": 0.9173234105110168,
+ "epoch": 2.3737373737373737,
+ "grad_norm": 2.2379953861236572,
+ "learning_rate": 4.188552188552189e-06,
+ "loss": 0.9265357255935669,
+ "mean_token_accuracy": 0.7507938742637634,
+ "num_tokens": 19251200.0,
+ "step": 1175
+ },
+ {
+ "entropy": 0.8770121932029724,
+ "epoch": 2.375757575757576,
+ "grad_norm": 2.1281018257141113,
+ "learning_rate": 4.1750841750841755e-06,
+ "loss": 0.8701684474945068,
+ "mean_token_accuracy": 0.7741817235946655,
+ "num_tokens": 19267584.0,
+ "step": 1176
+ },
+ {
+ "entropy": 1.0771297216415405,
+ "epoch": 2.3777777777777778,
+ "grad_norm": 2.2006096839904785,
+ "learning_rate": 4.161616161616162e-06,
+ "loss": 1.0650314092636108,
+ "mean_token_accuracy": 0.7375427484512329,
+ "num_tokens": 19283968.0,
+ "step": 1177
+ },
+ {
+ "entropy": 0.5856462717056274,
+ "epoch": 2.3797979797979796,
+ "grad_norm": 2.043875217437744,
+ "learning_rate": 4.1481481481481485e-06,
+ "loss": 0.5683187246322632,
+ "mean_token_accuracy": 0.8348192572593689,
+ "num_tokens": 19300352.0,
+ "step": 1178
+ },
+ {
+ "entropy": 1.08524751663208,
+ "epoch": 2.381818181818182,
+ "grad_norm": 2.29097318649292,
+ "learning_rate": 4.134680134680135e-06,
+ "loss": 1.0622766017913818,
+ "mean_token_accuracy": 0.7322300672531128,
+ "num_tokens": 19316736.0,
+ "step": 1179
+ },
+ {
+ "entropy": 0.79567551612854,
+ "epoch": 2.3838383838383836,
+ "grad_norm": 2.1101555824279785,
+ "learning_rate": 4.1212121212121215e-06,
+ "loss": 0.7737668752670288,
+ "mean_token_accuracy": 0.7987909317016602,
+ "num_tokens": 19333120.0,
+ "step": 1180
+ },
+ {
+ "entropy": 0.9077378511428833,
+ "epoch": 2.385858585858586,
+ "grad_norm": 2.337980270385742,
+ "learning_rate": 4.107744107744108e-06,
+ "loss": 0.916233479976654,
+ "mean_token_accuracy": 0.7699682712554932,
+ "num_tokens": 19349504.0,
+ "step": 1181
+ },
+ {
+ "entropy": 0.7796720862388611,
+ "epoch": 2.3878787878787877,
+ "grad_norm": 2.0968403816223145,
+ "learning_rate": 4.0942760942760944e-06,
+ "loss": 0.7493729591369629,
+ "mean_token_accuracy": 0.8128969073295593,
+ "num_tokens": 19365888.0,
+ "step": 1182
+ },
+ {
+ "entropy": 0.9127286076545715,
+ "epoch": 2.38989898989899,
+ "grad_norm": 2.166583299636841,
+ "learning_rate": 4.080808080808081e-06,
+ "loss": 0.8826674222946167,
+ "mean_token_accuracy": 0.7838910818099976,
+ "num_tokens": 19382272.0,
+ "step": 1183
+ },
+ {
+ "entropy": 0.794689953327179,
+ "epoch": 2.391919191919192,
+ "grad_norm": 2.130293607711792,
+ "learning_rate": 4.067340067340067e-06,
+ "loss": 0.7883949875831604,
+ "mean_token_accuracy": 0.7930508255958557,
+ "num_tokens": 19398656.0,
+ "step": 1184
+ },
+ {
+ "entropy": 1.2090318202972412,
+ "epoch": 2.393939393939394,
+ "grad_norm": 2.3772876262664795,
+ "learning_rate": 4.053872053872055e-06,
+ "loss": 1.2147963047027588,
+ "mean_token_accuracy": 0.7074987888336182,
+ "num_tokens": 19415040.0,
+ "step": 1185
+ },
+ {
+ "entropy": 0.7899874448776245,
+ "epoch": 2.395959595959596,
+ "grad_norm": 2.068220853805542,
+ "learning_rate": 4.04040404040404e-06,
+ "loss": 0.7576479315757751,
+ "mean_token_accuracy": 0.7963483333587646,
+ "num_tokens": 19431424.0,
+ "step": 1186
+ },
+ {
+ "entropy": 0.5590318441390991,
+ "epoch": 2.397979797979798,
+ "grad_norm": 1.9655200242996216,
+ "learning_rate": 4.026936026936028e-06,
+ "loss": 0.5567639470100403,
+ "mean_token_accuracy": 0.8367733359336853,
+ "num_tokens": 19447808.0,
+ "step": 1187
+ },
+ {
+ "entropy": 0.9547830820083618,
+ "epoch": 2.4,
+ "grad_norm": 2.336129665374756,
+ "learning_rate": 4.013468013468013e-06,
+ "loss": 0.9690505266189575,
+ "mean_token_accuracy": 0.7481069564819336,
+ "num_tokens": 19464192.0,
+ "step": 1188
+ },
+ {
+ "entropy": 0.6599025130271912,
+ "epoch": 2.402020202020202,
+ "grad_norm": 1.9412858486175537,
+ "learning_rate": 4.000000000000001e-06,
+ "loss": 0.6372359991073608,
+ "mean_token_accuracy": 0.830422580242157,
+ "num_tokens": 19480576.0,
+ "step": 1189
+ },
+ {
+ "entropy": 0.9671234488487244,
+ "epoch": 2.404040404040404,
+ "grad_norm": 2.207185983657837,
+ "learning_rate": 3.986531986531987e-06,
+ "loss": 0.9822558164596558,
+ "mean_token_accuracy": 0.7482290863990784,
+ "num_tokens": 19496960.0,
+ "step": 1190
+ },
+ {
+ "entropy": 1.2159186601638794,
+ "epoch": 2.4060606060606062,
+ "grad_norm": 2.3835642337799072,
+ "learning_rate": 3.973063973063974e-06,
+ "loss": 1.2127540111541748,
+ "mean_token_accuracy": 0.7027967572212219,
+ "num_tokens": 19513344.0,
+ "step": 1191
+ },
+ {
+ "entropy": 1.0063631534576416,
+ "epoch": 2.408080808080808,
+ "grad_norm": 2.313455581665039,
+ "learning_rate": 3.95959595959596e-06,
+ "loss": 0.9854291081428528,
+ "mean_token_accuracy": 0.7553126811981201,
+ "num_tokens": 19529728.0,
+ "step": 1192
+ },
+ {
+ "entropy": 1.0717129707336426,
+ "epoch": 2.41010101010101,
+ "grad_norm": 2.2523372173309326,
+ "learning_rate": 3.946127946127947e-06,
+ "loss": 1.0883088111877441,
+ "mean_token_accuracy": 0.723375678062439,
+ "num_tokens": 19546112.0,
+ "step": 1193
+ },
+ {
+ "entropy": 1.3942774534225464,
+ "epoch": 2.412121212121212,
+ "grad_norm": 2.4028871059417725,
+ "learning_rate": 3.932659932659933e-06,
+ "loss": 1.3809077739715576,
+ "mean_token_accuracy": 0.6759281754493713,
+ "num_tokens": 19562496.0,
+ "step": 1194
+ },
+ {
+ "entropy": 0.8493068218231201,
+ "epoch": 2.4141414141414144,
+ "grad_norm": 2.1721816062927246,
+ "learning_rate": 3.9191919191919196e-06,
+ "loss": 0.864505410194397,
+ "mean_token_accuracy": 0.7765021920204163,
+ "num_tokens": 19578880.0,
+ "step": 1195
+ },
+ {
+ "entropy": 1.2580974102020264,
+ "epoch": 2.416161616161616,
+ "grad_norm": 2.396101236343384,
+ "learning_rate": 3.905723905723906e-06,
+ "loss": 1.2584552764892578,
+ "mean_token_accuracy": 0.6950415372848511,
+ "num_tokens": 19595264.0,
+ "step": 1196
+ },
+ {
+ "entropy": 0.8060904741287231,
+ "epoch": 2.418181818181818,
+ "grad_norm": 2.046794891357422,
+ "learning_rate": 3.8922558922558925e-06,
+ "loss": 0.7814371585845947,
+ "mean_token_accuracy": 0.790791392326355,
+ "num_tokens": 19611648.0,
+ "step": 1197
+ },
+ {
+ "entropy": 1.2045812606811523,
+ "epoch": 2.4202020202020202,
+ "grad_norm": 2.247964382171631,
+ "learning_rate": 3.878787878787879e-06,
+ "loss": 1.2171180248260498,
+ "mean_token_accuracy": 0.7079873085021973,
+ "num_tokens": 19628032.0,
+ "step": 1198
+ },
+ {
+ "entropy": 1.1102302074432373,
+ "epoch": 2.422222222222222,
+ "grad_norm": 2.0978305339813232,
+ "learning_rate": 3.8653198653198655e-06,
+ "loss": 1.1046020984649658,
+ "mean_token_accuracy": 0.7316194176673889,
+ "num_tokens": 19644416.0,
+ "step": 1199
+ },
+ {
+ "entropy": 0.840056300163269,
+ "epoch": 2.4242424242424243,
+ "grad_norm": 2.221834659576416,
+ "learning_rate": 3.851851851851852e-06,
+ "loss": 0.8477578163146973,
+ "mean_token_accuracy": 0.7802882194519043,
+ "num_tokens": 19660800.0,
+ "step": 1200
+ },
+ {
+ "epoch": 2.4242424242424243,
+ "eval_entropy": 1.1394333051097008,
+ "eval_loss": 1.6056861877441406,
+ "eval_mean_token_accuracy": 0.6430166306034211,
+ "eval_num_tokens": 19660800.0,
+ "eval_runtime": 43.7916,
+ "eval_samples_per_second": 22.607,
+ "eval_steps_per_second": 2.832,
+ "step": 1200
+ },
+ {
+ "entropy": 1.2870543003082275,
+ "epoch": 2.426262626262626,
+ "grad_norm": 2.17335844039917,
+ "learning_rate": 3.8383838383838385e-06,
+ "loss": 1.3065149784088135,
+ "mean_token_accuracy": 0.6964460015296936,
+ "num_tokens": 19677184.0,
+ "step": 1201
+ },
+ {
+ "entropy": 0.9420839548110962,
+ "epoch": 2.4282828282828284,
+ "grad_norm": 2.2537472248077393,
+ "learning_rate": 3.824915824915825e-06,
+ "loss": 0.9395225644111633,
+ "mean_token_accuracy": 0.7506717443466187,
+ "num_tokens": 19693568.0,
+ "step": 1202
+ },
+ {
+ "entropy": 1.0070335865020752,
+ "epoch": 2.43030303030303,
+ "grad_norm": 2.161827802658081,
+ "learning_rate": 3.8114478114478114e-06,
+ "loss": 1.0133901834487915,
+ "mean_token_accuracy": 0.7372984886169434,
+ "num_tokens": 19709952.0,
+ "step": 1203
+ },
+ {
+ "entropy": 0.8056594729423523,
+ "epoch": 2.4323232323232324,
+ "grad_norm": 1.79896080493927,
+ "learning_rate": 3.7979797979797984e-06,
+ "loss": 0.7943062782287598,
+ "mean_token_accuracy": 0.7943331599235535,
+ "num_tokens": 19726336.0,
+ "step": 1204
+ },
+ {
+ "entropy": 0.9712687134742737,
+ "epoch": 2.4343434343434343,
+ "grad_norm": 1.9938912391662598,
+ "learning_rate": 3.7845117845117844e-06,
+ "loss": 0.9475370645523071,
+ "mean_token_accuracy": 0.7587323188781738,
+ "num_tokens": 19742720.0,
+ "step": 1205
+ },
+ {
+ "entropy": 0.8473865389823914,
+ "epoch": 2.4363636363636365,
+ "grad_norm": 2.0978620052337646,
+ "learning_rate": 3.7710437710437713e-06,
+ "loss": 0.8367564678192139,
+ "mean_token_accuracy": 0.7745481133460999,
+ "num_tokens": 19759104.0,
+ "step": 1206
+ },
+ {
+ "entropy": 0.723155677318573,
+ "epoch": 2.4383838383838383,
+ "grad_norm": 1.8043550252914429,
+ "learning_rate": 3.757575757575758e-06,
+ "loss": 0.7276782989501953,
+ "mean_token_accuracy": 0.8101490139961243,
+ "num_tokens": 19775488.0,
+ "step": 1207
+ },
+ {
+ "entropy": 0.7209433317184448,
+ "epoch": 2.4404040404040406,
+ "grad_norm": 2.128316640853882,
+ "learning_rate": 3.7441077441077443e-06,
+ "loss": 0.70832359790802,
+ "mean_token_accuracy": 0.8017220497131348,
+ "num_tokens": 19791872.0,
+ "step": 1208
+ },
+ {
+ "entropy": 0.7295323610305786,
+ "epoch": 2.4424242424242424,
+ "grad_norm": 2.045367956161499,
+ "learning_rate": 3.7306397306397308e-06,
+ "loss": 0.7077770233154297,
+ "mean_token_accuracy": 0.8126526474952698,
+ "num_tokens": 19808256.0,
+ "step": 1209
+ },
+ {
+ "entropy": 1.0979973077774048,
+ "epoch": 2.4444444444444446,
+ "grad_norm": 2.182023048400879,
+ "learning_rate": 3.7171717171717177e-06,
+ "loss": 1.0852105617523193,
+ "mean_token_accuracy": 0.7303370833396912,
+ "num_tokens": 19824640.0,
+ "step": 1210
+ },
+ {
+ "entropy": 0.9850945472717285,
+ "epoch": 2.4464646464646465,
+ "grad_norm": 2.065443277359009,
+ "learning_rate": 3.7037037037037037e-06,
+ "loss": 0.9809832572937012,
+ "mean_token_accuracy": 0.7525647282600403,
+ "num_tokens": 19841024.0,
+ "step": 1211
+ },
+ {
+ "entropy": 0.8987663388252258,
+ "epoch": 2.4484848484848483,
+ "grad_norm": 2.2341790199279785,
+ "learning_rate": 3.6902356902356906e-06,
+ "loss": 0.8896650671958923,
+ "mean_token_accuracy": 0.7669150233268738,
+ "num_tokens": 19857408.0,
+ "step": 1212
+ },
+ {
+ "entropy": 0.7914950847625732,
+ "epoch": 2.4505050505050505,
+ "grad_norm": 1.9684029817581177,
+ "learning_rate": 3.6767676767676767e-06,
+ "loss": 0.7796467542648315,
+ "mean_token_accuracy": 0.798363447189331,
+ "num_tokens": 19873792.0,
+ "step": 1213
+ },
+ {
+ "entropy": 1.0010449886322021,
+ "epoch": 2.4525252525252528,
+ "grad_norm": 2.1975135803222656,
+ "learning_rate": 3.6632996632996636e-06,
+ "loss": 0.9799243211746216,
+ "mean_token_accuracy": 0.7423668503761292,
+ "num_tokens": 19890176.0,
+ "step": 1214
+ },
+ {
+ "entropy": 1.0015712976455688,
+ "epoch": 2.4545454545454546,
+ "grad_norm": 2.1111021041870117,
+ "learning_rate": 3.64983164983165e-06,
+ "loss": 1.0013333559036255,
+ "mean_token_accuracy": 0.757083535194397,
+ "num_tokens": 19906560.0,
+ "step": 1215
+ },
+ {
+ "entropy": 0.6957159638404846,
+ "epoch": 2.4565656565656564,
+ "grad_norm": 2.203601598739624,
+ "learning_rate": 3.6363636363636366e-06,
+ "loss": 0.6890975832939148,
+ "mean_token_accuracy": 0.8097215294837952,
+ "num_tokens": 19922944.0,
+ "step": 1216
+ },
+ {
+ "entropy": 1.0398633480072021,
+ "epoch": 2.4585858585858587,
+ "grad_norm": 2.1206884384155273,
+ "learning_rate": 3.622895622895623e-06,
+ "loss": 1.020222544670105,
+ "mean_token_accuracy": 0.7437713742256165,
+ "num_tokens": 19939328.0,
+ "step": 1217
+ },
+ {
+ "entropy": 0.948898196220398,
+ "epoch": 2.4606060606060605,
+ "grad_norm": 2.16664981842041,
+ "learning_rate": 3.60942760942761e-06,
+ "loss": 0.9560619592666626,
+ "mean_token_accuracy": 0.7578774094581604,
+ "num_tokens": 19955712.0,
+ "step": 1218
+ },
+ {
+ "entropy": 0.7177059650421143,
+ "epoch": 2.4626262626262627,
+ "grad_norm": 2.0934417247772217,
+ "learning_rate": 3.595959595959596e-06,
+ "loss": 0.721766471862793,
+ "mean_token_accuracy": 0.7979970574378967,
+ "num_tokens": 19972096.0,
+ "step": 1219
+ },
+ {
+ "entropy": 0.8426720499992371,
+ "epoch": 2.4646464646464645,
+ "grad_norm": 2.2284491062164307,
+ "learning_rate": 3.582491582491583e-06,
+ "loss": 0.8461461067199707,
+ "mean_token_accuracy": 0.7724108695983887,
+ "num_tokens": 19988480.0,
+ "step": 1220
+ },
+ {
+ "entropy": 1.0814709663391113,
+ "epoch": 2.466666666666667,
+ "grad_norm": 2.1849255561828613,
+ "learning_rate": 3.569023569023569e-06,
+ "loss": 1.0583312511444092,
+ "mean_token_accuracy": 0.7375427484512329,
+ "num_tokens": 20004864.0,
+ "step": 1221
+ },
+ {
+ "entropy": 0.8736158609390259,
+ "epoch": 2.4686868686868686,
+ "grad_norm": 2.0398738384246826,
+ "learning_rate": 3.555555555555556e-06,
+ "loss": 0.8676871657371521,
+ "mean_token_accuracy": 0.7771128416061401,
+ "num_tokens": 20021248.0,
+ "step": 1222
+ },
+ {
+ "entropy": 0.7232694029808044,
+ "epoch": 2.470707070707071,
+ "grad_norm": 2.105766773223877,
+ "learning_rate": 3.5420875420875424e-06,
+ "loss": 0.7312887907028198,
+ "mean_token_accuracy": 0.8002564907073975,
+ "num_tokens": 20037632.0,
+ "step": 1223
+ },
+ {
+ "entropy": 1.284920334815979,
+ "epoch": 2.4727272727272727,
+ "grad_norm": 2.1596007347106934,
+ "learning_rate": 3.528619528619529e-06,
+ "loss": 1.2728421688079834,
+ "mean_token_accuracy": 0.6981558203697205,
+ "num_tokens": 20054016.0,
+ "step": 1224
+ },
+ {
+ "entropy": 0.7340924739837646,
+ "epoch": 2.474747474747475,
+ "grad_norm": 2.1284775733947754,
+ "learning_rate": 3.5151515151515154e-06,
+ "loss": 0.7263147234916687,
+ "mean_token_accuracy": 0.8036150336265564,
+ "num_tokens": 20070400.0,
+ "step": 1225
+ },
+ {
+ "entropy": 0.801537275314331,
+ "epoch": 2.4767676767676767,
+ "grad_norm": 2.0989158153533936,
+ "learning_rate": 3.5016835016835023e-06,
+ "loss": 0.7869283556938171,
+ "mean_token_accuracy": 0.7972642779350281,
+ "num_tokens": 20086784.0,
+ "step": 1226
+ },
+ {
+ "entropy": 1.1842774152755737,
+ "epoch": 2.4787878787878785,
+ "grad_norm": 2.1426236629486084,
+ "learning_rate": 3.4882154882154883e-06,
+ "loss": 1.1804194450378418,
+ "mean_token_accuracy": 0.7169027924537659,
+ "num_tokens": 20103168.0,
+ "step": 1227
+ },
+ {
+ "entropy": 1.2625954151153564,
+ "epoch": 2.480808080808081,
+ "grad_norm": 2.407891035079956,
+ "learning_rate": 3.4747474747474752e-06,
+ "loss": 1.2765004634857178,
+ "mean_token_accuracy": 0.6922935843467712,
+ "num_tokens": 20119552.0,
+ "step": 1228
+ },
+ {
+ "entropy": 1.5162423849105835,
+ "epoch": 2.482828282828283,
+ "grad_norm": 2.1844899654388428,
+ "learning_rate": 3.4612794612794613e-06,
+ "loss": 1.5104104280471802,
+ "mean_token_accuracy": 0.6639594435691833,
+ "num_tokens": 20135936.0,
+ "step": 1229
+ },
+ {
+ "entropy": 0.9084963798522949,
+ "epoch": 2.484848484848485,
+ "grad_norm": 1.9122270345687866,
+ "learning_rate": 3.447811447811448e-06,
+ "loss": 0.8792629241943359,
+ "mean_token_accuracy": 0.7750977277755737,
+ "num_tokens": 20152320.0,
+ "step": 1230
+ },
+ {
+ "entropy": 1.3305665254592896,
+ "epoch": 2.4868686868686867,
+ "grad_norm": 2.250115394592285,
+ "learning_rate": 3.4343434343434347e-06,
+ "loss": 1.3185248374938965,
+ "mean_token_accuracy": 0.6855154037475586,
+ "num_tokens": 20168704.0,
+ "step": 1231
+ },
+ {
+ "entropy": 0.8350923657417297,
+ "epoch": 2.488888888888889,
+ "grad_norm": 2.0859107971191406,
+ "learning_rate": 3.420875420875421e-06,
+ "loss": 0.8013065457344055,
+ "mean_token_accuracy": 0.7911577820777893,
+ "num_tokens": 20185088.0,
+ "step": 1232
+ },
+ {
+ "entropy": 0.9226766228675842,
+ "epoch": 2.4909090909090907,
+ "grad_norm": 2.030141830444336,
+ "learning_rate": 3.4074074074074077e-06,
+ "loss": 0.9243340492248535,
+ "mean_token_accuracy": 0.7738153338432312,
+ "num_tokens": 20201472.0,
+ "step": 1233
+ },
+ {
+ "entropy": 0.8571307063102722,
+ "epoch": 2.492929292929293,
+ "grad_norm": 2.40063738822937,
+ "learning_rate": 3.3939393939393946e-06,
+ "loss": 0.870561420917511,
+ "mean_token_accuracy": 0.7685027122497559,
+ "num_tokens": 20217856.0,
+ "step": 1234
+ },
+ {
+ "entropy": 0.9109510779380798,
+ "epoch": 2.494949494949495,
+ "grad_norm": 2.1137447357177734,
+ "learning_rate": 3.3804713804713806e-06,
+ "loss": 0.9193125367164612,
+ "mean_token_accuracy": 0.7685637474060059,
+ "num_tokens": 20234240.0,
+ "step": 1235
+ },
+ {
+ "entropy": 0.7725069522857666,
+ "epoch": 2.496969696969697,
+ "grad_norm": 2.1603705883026123,
+ "learning_rate": 3.3670033670033675e-06,
+ "loss": 0.7795518040657043,
+ "mean_token_accuracy": 0.7900586128234863,
+ "num_tokens": 20250624.0,
+ "step": 1236
+ },
+ {
+ "entropy": 1.1810798645019531,
+ "epoch": 2.498989898989899,
+ "grad_norm": 2.321040630340576,
+ "learning_rate": 3.3535353535353536e-06,
+ "loss": 1.197322130203247,
+ "mean_token_accuracy": 0.7245969772338867,
+ "num_tokens": 20267008.0,
+ "step": 1237
+ },
+ {
+ "entropy": 0.9284457564353943,
+ "epoch": 2.501010101010101,
+ "grad_norm": 2.214848518371582,
+ "learning_rate": 3.3400673400673405e-06,
+ "loss": 0.915704071521759,
+ "mean_token_accuracy": 0.7593429684638977,
+ "num_tokens": 20283392.0,
+ "step": 1238
+ },
+ {
+ "entropy": 0.9151167273521423,
+ "epoch": 2.503030303030303,
+ "grad_norm": 2.2259461879730225,
+ "learning_rate": 3.326599326599327e-06,
+ "loss": 0.9136589169502258,
+ "mean_token_accuracy": 0.7659379839897156,
+ "num_tokens": 20299776.0,
+ "step": 1239
+ },
+ {
+ "entropy": 0.937813401222229,
+ "epoch": 2.505050505050505,
+ "grad_norm": 2.3099539279937744,
+ "learning_rate": 3.3131313131313135e-06,
+ "loss": 0.9328974485397339,
+ "mean_token_accuracy": 0.7614191770553589,
+ "num_tokens": 20316160.0,
+ "step": 1240
+ },
+ {
+ "entropy": 1.1104196310043335,
+ "epoch": 2.507070707070707,
+ "grad_norm": 2.3913068771362305,
+ "learning_rate": 3.2996632996633e-06,
+ "loss": 1.0788944959640503,
+ "mean_token_accuracy": 0.7301538586616516,
+ "num_tokens": 20332544.0,
+ "step": 1241
+ },
+ {
+ "entropy": 0.8833880424499512,
+ "epoch": 2.509090909090909,
+ "grad_norm": 2.261202573776245,
+ "learning_rate": 3.286195286195287e-06,
+ "loss": 0.8650795817375183,
+ "mean_token_accuracy": 0.7713727355003357,
+ "num_tokens": 20348928.0,
+ "step": 1242
+ },
+ {
+ "entropy": 1.2574468851089478,
+ "epoch": 2.511111111111111,
+ "grad_norm": 2.414250373840332,
+ "learning_rate": 3.272727272727273e-06,
+ "loss": 1.266160011291504,
+ "mean_token_accuracy": 0.699499249458313,
+ "num_tokens": 20365312.0,
+ "step": 1243
+ },
+ {
+ "entropy": 1.357008695602417,
+ "epoch": 2.5131313131313133,
+ "grad_norm": 2.264491558074951,
+ "learning_rate": 3.25925925925926e-06,
+ "loss": 1.351933240890503,
+ "mean_token_accuracy": 0.689728856086731,
+ "num_tokens": 20381696.0,
+ "step": 1244
+ },
+ {
+ "entropy": 1.2887134552001953,
+ "epoch": 2.515151515151515,
+ "grad_norm": 2.308107614517212,
+ "learning_rate": 3.245791245791246e-06,
+ "loss": 1.289101481437683,
+ "mean_token_accuracy": 0.6892403364181519,
+ "num_tokens": 20398080.0,
+ "step": 1245
+ },
+ {
+ "entropy": 1.3027681112289429,
+ "epoch": 2.517171717171717,
+ "grad_norm": 2.084804058074951,
+ "learning_rate": 3.232323232323233e-06,
+ "loss": 1.296428918838501,
+ "mean_token_accuracy": 0.6989496946334839,
+ "num_tokens": 20414464.0,
+ "step": 1246
+ },
+ {
+ "entropy": 0.9854085445404053,
+ "epoch": 2.519191919191919,
+ "grad_norm": 2.2956128120422363,
+ "learning_rate": 3.2188552188552193e-06,
+ "loss": 0.9593667387962341,
+ "mean_token_accuracy": 0.7523815631866455,
+ "num_tokens": 20430848.0,
+ "step": 1247
+ },
+ {
+ "entropy": 1.1462289094924927,
+ "epoch": 2.5212121212121215,
+ "grad_norm": 2.319286346435547,
+ "learning_rate": 3.2053872053872053e-06,
+ "loss": 1.10858154296875,
+ "mean_token_accuracy": 0.7219711542129517,
+ "num_tokens": 20447232.0,
+ "step": 1248
+ },
+ {
+ "entropy": 1.0656987428665161,
+ "epoch": 2.5232323232323233,
+ "grad_norm": 2.073442220687866,
+ "learning_rate": 3.1919191919191923e-06,
+ "loss": 1.0597574710845947,
+ "mean_token_accuracy": 0.7349780201911926,
+ "num_tokens": 20463616.0,
+ "step": 1249
+ },
+ {
+ "entropy": 0.7793754935264587,
+ "epoch": 2.525252525252525,
+ "grad_norm": 2.2514913082122803,
+ "learning_rate": 3.1784511784511783e-06,
+ "loss": 0.7521334886550903,
+ "mean_token_accuracy": 0.7914631366729736,
+ "num_tokens": 20480000.0,
+ "step": 1250
+ },
+ {
+ "entropy": 1.0033597946166992,
+ "epoch": 2.5272727272727273,
+ "grad_norm": 2.299437999725342,
+ "learning_rate": 3.1649831649831652e-06,
+ "loss": 0.9853575229644775,
+ "mean_token_accuracy": 0.7447484135627747,
+ "num_tokens": 20496384.0,
+ "step": 1251
+ },
+ {
+ "entropy": 0.9790856242179871,
+ "epoch": 2.529292929292929,
+ "grad_norm": 2.2850499153137207,
+ "learning_rate": 3.1515151515151517e-06,
+ "loss": 0.9773675799369812,
+ "mean_token_accuracy": 0.7410845160484314,
+ "num_tokens": 20512768.0,
+ "step": 1252
+ },
+ {
+ "entropy": 1.0001251697540283,
+ "epoch": 2.5313131313131314,
+ "grad_norm": 2.162883758544922,
+ "learning_rate": 3.138047138047138e-06,
+ "loss": 0.9708908796310425,
+ "mean_token_accuracy": 0.7595872282981873,
+ "num_tokens": 20529152.0,
+ "step": 1253
+ },
+ {
+ "entropy": 0.8198298215866089,
+ "epoch": 2.533333333333333,
+ "grad_norm": 2.288684129714966,
+ "learning_rate": 3.1245791245791247e-06,
+ "loss": 0.8112213015556335,
+ "mean_token_accuracy": 0.7818759083747864,
+ "num_tokens": 20545536.0,
+ "step": 1254
+ },
+ {
+ "entropy": 0.9083048701286316,
+ "epoch": 2.5353535353535355,
+ "grad_norm": 2.2135798931121826,
+ "learning_rate": 3.1111111111111116e-06,
+ "loss": 0.908804714679718,
+ "mean_token_accuracy": 0.7622740864753723,
+ "num_tokens": 20561920.0,
+ "step": 1255
+ },
+ {
+ "entropy": 0.795724093914032,
+ "epoch": 2.5373737373737373,
+ "grad_norm": 2.1689705848693848,
+ "learning_rate": 3.0976430976430976e-06,
+ "loss": 0.7923108339309692,
+ "mean_token_accuracy": 0.7879824042320251,
+ "num_tokens": 20578304.0,
+ "step": 1256
+ },
+ {
+ "entropy": 1.4591748714447021,
+ "epoch": 2.5393939393939395,
+ "grad_norm": 2.413419008255005,
+ "learning_rate": 3.0841750841750846e-06,
+ "loss": 1.4487470388412476,
+ "mean_token_accuracy": 0.6948583126068115,
+ "num_tokens": 20594688.0,
+ "step": 1257
+ },
+ {
+ "entropy": 1.0469496250152588,
+ "epoch": 2.5414141414141413,
+ "grad_norm": 2.2782540321350098,
+ "learning_rate": 3.0707070707070706e-06,
+ "loss": 1.0700496435165405,
+ "mean_token_accuracy": 0.7410234212875366,
+ "num_tokens": 20611072.0,
+ "step": 1258
+ },
+ {
+ "entropy": 1.1338191032409668,
+ "epoch": 2.5434343434343436,
+ "grad_norm": 2.370194911956787,
+ "learning_rate": 3.0572390572390575e-06,
+ "loss": 1.1157164573669434,
+ "mean_token_accuracy": 0.729421079158783,
+ "num_tokens": 20627456.0,
+ "step": 1259
+ },
+ {
+ "entropy": 1.2801309823989868,
+ "epoch": 2.5454545454545454,
+ "grad_norm": 2.21651291847229,
+ "learning_rate": 3.043771043771044e-06,
+ "loss": 1.2861847877502441,
+ "mean_token_accuracy": 0.6850268840789795,
+ "num_tokens": 20643840.0,
+ "step": 1260
+ },
+ {
+ "entropy": 1.2465205192565918,
+ "epoch": 2.5474747474747472,
+ "grad_norm": 2.319526433944702,
+ "learning_rate": 3.0303030303030305e-06,
+ "loss": 1.2388584613800049,
+ "mean_token_accuracy": 0.7073766589164734,
+ "num_tokens": 20660224.0,
+ "step": 1261
+ },
+ {
+ "entropy": 1.041487216949463,
+ "epoch": 2.5494949494949495,
+ "grad_norm": 2.3334708213806152,
+ "learning_rate": 3.016835016835017e-06,
+ "loss": 1.0760767459869385,
+ "mean_token_accuracy": 0.7376037836074829,
+ "num_tokens": 20676608.0,
+ "step": 1262
+ },
+ {
+ "entropy": 0.8173713684082031,
+ "epoch": 2.5515151515151517,
+ "grad_norm": 2.1104650497436523,
+ "learning_rate": 3.003367003367004e-06,
+ "loss": 0.8200893998146057,
+ "mean_token_accuracy": 0.7826697826385498,
+ "num_tokens": 20692992.0,
+ "step": 1263
+ },
+ {
+ "entropy": 1.083128809928894,
+ "epoch": 2.5535353535353535,
+ "grad_norm": 2.1319468021392822,
+ "learning_rate": 2.98989898989899e-06,
+ "loss": 1.077414631843567,
+ "mean_token_accuracy": 0.7382755279541016,
+ "num_tokens": 20709376.0,
+ "step": 1264
+ },
+ {
+ "entropy": 0.952279269695282,
+ "epoch": 2.5555555555555554,
+ "grad_norm": 2.261955499649048,
+ "learning_rate": 2.976430976430977e-06,
+ "loss": 0.9421428442001343,
+ "mean_token_accuracy": 0.7682584524154663,
+ "num_tokens": 20725760.0,
+ "step": 1265
+ },
+ {
+ "entropy": 1.4377425909042358,
+ "epoch": 2.5575757575757576,
+ "grad_norm": 2.4162697792053223,
+ "learning_rate": 2.962962962962963e-06,
+ "loss": 1.4387903213500977,
+ "mean_token_accuracy": 0.66841721534729,
+ "num_tokens": 20742144.0,
+ "step": 1266
+ },
+ {
+ "entropy": 1.3983595371246338,
+ "epoch": 2.55959595959596,
+ "grad_norm": 2.3733458518981934,
+ "learning_rate": 2.94949494949495e-06,
+ "loss": 1.407117486000061,
+ "mean_token_accuracy": 0.6690278649330139,
+ "num_tokens": 20758528.0,
+ "step": 1267
+ },
+ {
+ "entropy": 1.1543824672698975,
+ "epoch": 2.5616161616161617,
+ "grad_norm": 2.049203872680664,
+ "learning_rate": 2.9360269360269363e-06,
+ "loss": 1.1425888538360596,
+ "mean_token_accuracy": 0.716292142868042,
+ "num_tokens": 20774912.0,
+ "step": 1268
+ },
+ {
+ "entropy": 1.1245750188827515,
+ "epoch": 2.5636363636363635,
+ "grad_norm": 2.3909695148468018,
+ "learning_rate": 2.9225589225589228e-06,
+ "loss": 1.1087216138839722,
+ "mean_token_accuracy": 0.7255129218101501,
+ "num_tokens": 20791296.0,
+ "step": 1269
+ },
+ {
+ "entropy": 1.0142271518707275,
+ "epoch": 2.5656565656565657,
+ "grad_norm": 2.315854072570801,
+ "learning_rate": 2.9090909090909093e-06,
+ "loss": 1.0216941833496094,
+ "mean_token_accuracy": 0.7418172955513,
+ "num_tokens": 20807680.0,
+ "step": 1270
+ },
+ {
+ "entropy": 0.9911118149757385,
+ "epoch": 2.5676767676767676,
+ "grad_norm": 2.2109222412109375,
+ "learning_rate": 2.895622895622896e-06,
+ "loss": 0.96977698802948,
+ "mean_token_accuracy": 0.7437102794647217,
+ "num_tokens": 20824064.0,
+ "step": 1271
+ },
+ {
+ "entropy": 0.6949873566627502,
+ "epoch": 2.56969696969697,
+ "grad_norm": 2.1211202144622803,
+ "learning_rate": 2.8821548821548822e-06,
+ "loss": 0.6750927567481995,
+ "mean_token_accuracy": 0.8146067261695862,
+ "num_tokens": 20840448.0,
+ "step": 1272
+ },
+ {
+ "entropy": 0.7807356119155884,
+ "epoch": 2.5717171717171716,
+ "grad_norm": 2.1028854846954346,
+ "learning_rate": 2.868686868686869e-06,
+ "loss": 0.7759028673171997,
+ "mean_token_accuracy": 0.7914020419120789,
+ "num_tokens": 20856832.0,
+ "step": 1273
+ },
+ {
+ "entropy": 0.8176921010017395,
+ "epoch": 2.573737373737374,
+ "grad_norm": 1.9491738080978394,
+ "learning_rate": 2.855218855218855e-06,
+ "loss": 0.7880226969718933,
+ "mean_token_accuracy": 0.7891426682472229,
+ "num_tokens": 20873216.0,
+ "step": 1274
+ },
+ {
+ "entropy": 1.1993833780288696,
+ "epoch": 2.5757575757575757,
+ "grad_norm": 2.2878284454345703,
+ "learning_rate": 2.841750841750842e-06,
+ "loss": 1.2149043083190918,
+ "mean_token_accuracy": 0.6962628364562988,
+ "num_tokens": 20889600.0,
+ "step": 1275
+ },
+ {
+ "entropy": 1.2472699880599976,
+ "epoch": 2.5777777777777775,
+ "grad_norm": 2.1730587482452393,
+ "learning_rate": 2.8282828282828286e-06,
+ "loss": 1.2479032278060913,
+ "mean_token_accuracy": 0.6937591433525085,
+ "num_tokens": 20905984.0,
+ "step": 1276
+ },
+ {
+ "entropy": 0.9197280406951904,
+ "epoch": 2.5797979797979798,
+ "grad_norm": 1.955877423286438,
+ "learning_rate": 2.814814814814815e-06,
+ "loss": 0.896048903465271,
+ "mean_token_accuracy": 0.7733879089355469,
+ "num_tokens": 20922368.0,
+ "step": 1277
+ },
+ {
+ "entropy": 1.1877310276031494,
+ "epoch": 2.581818181818182,
+ "grad_norm": 2.2136073112487793,
+ "learning_rate": 2.8013468013468016e-06,
+ "loss": 1.178865909576416,
+ "mean_token_accuracy": 0.7047508358955383,
+ "num_tokens": 20938752.0,
+ "step": 1278
+ },
+ {
+ "entropy": 0.7793066501617432,
+ "epoch": 2.583838383838384,
+ "grad_norm": 2.0132107734680176,
+ "learning_rate": 2.7878787878787885e-06,
+ "loss": 0.7862892150878906,
+ "mean_token_accuracy": 0.7963483333587646,
+ "num_tokens": 20955136.0,
+ "step": 1279
+ },
+ {
+ "entropy": 1.2476853132247925,
+ "epoch": 2.5858585858585856,
+ "grad_norm": 2.3129923343658447,
+ "learning_rate": 2.7744107744107745e-06,
+ "loss": 1.2039775848388672,
+ "mean_token_accuracy": 0.7099413871765137,
+ "num_tokens": 20971520.0,
+ "step": 1280
+ },
+ {
+ "entropy": 1.042412281036377,
+ "epoch": 2.587878787878788,
+ "grad_norm": 2.2215702533721924,
+ "learning_rate": 2.7609427609427614e-06,
+ "loss": 1.051681399345398,
+ "mean_token_accuracy": 0.7425500750541687,
+ "num_tokens": 20987904.0,
+ "step": 1281
+ },
+ {
+ "entropy": 0.7061983346939087,
+ "epoch": 2.58989898989899,
+ "grad_norm": 2.0668582916259766,
+ "learning_rate": 2.7474747474747475e-06,
+ "loss": 0.7098591327667236,
+ "mean_token_accuracy": 0.8134465217590332,
+ "num_tokens": 21004288.0,
+ "step": 1282
+ },
+ {
+ "entropy": 0.8634867668151855,
+ "epoch": 2.591919191919192,
+ "grad_norm": 2.0467185974121094,
+ "learning_rate": 2.7340067340067344e-06,
+ "loss": 0.8574676513671875,
+ "mean_token_accuracy": 0.7678309679031372,
+ "num_tokens": 21020672.0,
+ "step": 1283
+ },
+ {
+ "entropy": 1.071445107460022,
+ "epoch": 2.5939393939393938,
+ "grad_norm": 2.4267988204956055,
+ "learning_rate": 2.720538720538721e-06,
+ "loss": 1.0452513694763184,
+ "mean_token_accuracy": 0.7368099689483643,
+ "num_tokens": 21037056.0,
+ "step": 1284
+ },
+ {
+ "entropy": 1.0232303142547607,
+ "epoch": 2.595959595959596,
+ "grad_norm": 2.244133234024048,
+ "learning_rate": 2.7070707070707074e-06,
+ "loss": 1.0210691690444946,
+ "mean_token_accuracy": 0.7452979683876038,
+ "num_tokens": 21053440.0,
+ "step": 1285
+ },
+ {
+ "entropy": 0.984527587890625,
+ "epoch": 2.597979797979798,
+ "grad_norm": 2.1976277828216553,
+ "learning_rate": 2.693602693602694e-06,
+ "loss": 0.9877492785453796,
+ "mean_token_accuracy": 0.7432217597961426,
+ "num_tokens": 21069824.0,
+ "step": 1286
+ },
+ {
+ "entropy": 1.0401315689086914,
+ "epoch": 2.6,
+ "grad_norm": 2.3494908809661865,
+ "learning_rate": 2.6801346801346808e-06,
+ "loss": 1.0381377935409546,
+ "mean_token_accuracy": 0.7385808229446411,
+ "num_tokens": 21086208.0,
+ "step": 1287
+ },
+ {
+ "entropy": 0.758490800857544,
+ "epoch": 2.602020202020202,
+ "grad_norm": 2.0141663551330566,
+ "learning_rate": 2.666666666666667e-06,
+ "loss": 0.7600662112236023,
+ "mean_token_accuracy": 0.8067293763160706,
+ "num_tokens": 21102592.0,
+ "step": 1288
+ },
+ {
+ "entropy": 1.1525641679763794,
+ "epoch": 2.604040404040404,
+ "grad_norm": 2.1283185482025146,
+ "learning_rate": 2.6531986531986537e-06,
+ "loss": 1.149183750152588,
+ "mean_token_accuracy": 0.7203224301338196,
+ "num_tokens": 21118976.0,
+ "step": 1289
+ },
+ {
+ "entropy": 1.0491526126861572,
+ "epoch": 2.606060606060606,
+ "grad_norm": 2.124562978744507,
+ "learning_rate": 2.63973063973064e-06,
+ "loss": 1.057656168937683,
+ "mean_token_accuracy": 0.7329628467559814,
+ "num_tokens": 21135360.0,
+ "step": 1290
+ },
+ {
+ "entropy": 0.9726903438568115,
+ "epoch": 2.608080808080808,
+ "grad_norm": 2.2004761695861816,
+ "learning_rate": 2.6262626262626267e-06,
+ "loss": 0.963001012802124,
+ "mean_token_accuracy": 0.7552515864372253,
+ "num_tokens": 21151744.0,
+ "step": 1291
+ },
+ {
+ "entropy": 0.7241232395172119,
+ "epoch": 2.61010101010101,
+ "grad_norm": 2.227318525314331,
+ "learning_rate": 2.612794612794613e-06,
+ "loss": 0.7039598226547241,
+ "mean_token_accuracy": 0.8026379942893982,
+ "num_tokens": 21168128.0,
+ "step": 1292
+ },
+ {
+ "entropy": 0.9619441628456116,
+ "epoch": 2.6121212121212123,
+ "grad_norm": 2.043938636779785,
+ "learning_rate": 2.5993265993265997e-06,
+ "loss": 0.9687198996543884,
+ "mean_token_accuracy": 0.7593429684638977,
+ "num_tokens": 21184512.0,
+ "step": 1293
+ },
+ {
+ "entropy": 0.7907335758209229,
+ "epoch": 2.614141414141414,
+ "grad_norm": 2.0104219913482666,
+ "learning_rate": 2.585858585858586e-06,
+ "loss": 0.7938915491104126,
+ "mean_token_accuracy": 0.7873717546463013,
+ "num_tokens": 21200896.0,
+ "step": 1294
+ },
+ {
+ "entropy": 1.2517071962356567,
+ "epoch": 2.616161616161616,
+ "grad_norm": 2.6553330421447754,
+ "learning_rate": 2.5723905723905722e-06,
+ "loss": 1.2589607238769531,
+ "mean_token_accuracy": 0.6866145730018616,
+ "num_tokens": 21217280.0,
+ "step": 1295
+ },
+ {
+ "entropy": 1.0896011590957642,
+ "epoch": 2.618181818181818,
+ "grad_norm": 2.0923056602478027,
+ "learning_rate": 2.558922558922559e-06,
+ "loss": 1.0726165771484375,
+ "mean_token_accuracy": 0.7360160946846008,
+ "num_tokens": 21233664.0,
+ "step": 1296
+ },
+ {
+ "entropy": 0.8060534596443176,
+ "epoch": 2.6202020202020204,
+ "grad_norm": 2.1754214763641357,
+ "learning_rate": 2.5454545454545456e-06,
+ "loss": 0.8015183210372925,
+ "mean_token_accuracy": 0.7854176759719849,
+ "num_tokens": 21250048.0,
+ "step": 1297
+ },
+ {
+ "entropy": 0.8480345606803894,
+ "epoch": 2.6222222222222222,
+ "grad_norm": 2.1952438354492188,
+ "learning_rate": 2.531986531986532e-06,
+ "loss": 0.8521784543991089,
+ "mean_token_accuracy": 0.7794333100318909,
+ "num_tokens": 21266432.0,
+ "step": 1298
+ },
+ {
+ "entropy": 0.9813645482063293,
+ "epoch": 2.624242424242424,
+ "grad_norm": 2.276472330093384,
+ "learning_rate": 2.5185185185185186e-06,
+ "loss": 0.9841573238372803,
+ "mean_token_accuracy": 0.7430996298789978,
+ "num_tokens": 21282816.0,
+ "step": 1299
+ },
+ {
+ "entropy": 1.0559581518173218,
+ "epoch": 2.6262626262626263,
+ "grad_norm": 2.1930594444274902,
+ "learning_rate": 2.5050505050505055e-06,
+ "loss": 1.0522143840789795,
+ "mean_token_accuracy": 0.7405349016189575,
+ "num_tokens": 21299200.0,
+ "step": 1300
+ },
+ {
+ "epoch": 2.6262626262626263,
+ "eval_entropy": 1.136087471919675,
+ "eval_loss": 1.609318733215332,
+ "eval_mean_token_accuracy": 0.6427743410871875,
+ "eval_num_tokens": 21299200.0,
+ "eval_runtime": 43.7802,
+ "eval_samples_per_second": 22.613,
+ "eval_steps_per_second": 2.832,
+ "step": 1300
+ },
+ {
+ "entropy": 1.1001769304275513,
+ "epoch": 2.6282828282828286,
+ "grad_norm": 2.3794357776641846,
+ "learning_rate": 2.491582491582492e-06,
+ "loss": 1.1209263801574707,
+ "mean_token_accuracy": 0.7210552096366882,
+ "num_tokens": 21315584.0,
+ "step": 1301
+ },
+ {
+ "entropy": 0.872175395488739,
+ "epoch": 2.6303030303030304,
+ "grad_norm": 1.983834147453308,
+ "learning_rate": 2.4781144781144785e-06,
+ "loss": 0.8659814596176147,
+ "mean_token_accuracy": 0.7824865579605103,
+ "num_tokens": 21331968.0,
+ "step": 1302
+ },
+ {
+ "entropy": 0.8773177266120911,
+ "epoch": 2.632323232323232,
+ "grad_norm": 2.2786073684692383,
+ "learning_rate": 2.464646464646465e-06,
+ "loss": 0.8917679786682129,
+ "mean_token_accuracy": 0.7721666097640991,
+ "num_tokens": 21348352.0,
+ "step": 1303
+ },
+ {
+ "entropy": 1.4344940185546875,
+ "epoch": 2.6343434343434344,
+ "grad_norm": 2.1941874027252197,
+ "learning_rate": 2.4511784511784514e-06,
+ "loss": 1.3934135437011719,
+ "mean_token_accuracy": 0.6698216795921326,
+ "num_tokens": 21364736.0,
+ "step": 1304
+ },
+ {
+ "entropy": 1.0237466096878052,
+ "epoch": 2.6363636363636362,
+ "grad_norm": 2.210641622543335,
+ "learning_rate": 2.437710437710438e-06,
+ "loss": 1.0018086433410645,
+ "mean_token_accuracy": 0.7518930435180664,
+ "num_tokens": 21381120.0,
+ "step": 1305
+ },
+ {
+ "entropy": 0.8996136784553528,
+ "epoch": 2.6383838383838385,
+ "grad_norm": 2.247093677520752,
+ "learning_rate": 2.4242424242424244e-06,
+ "loss": 0.9008244276046753,
+ "mean_token_accuracy": 0.7721055150032043,
+ "num_tokens": 21397504.0,
+ "step": 1306
+ },
+ {
+ "entropy": 0.6376256346702576,
+ "epoch": 2.6404040404040403,
+ "grad_norm": 2.0698840618133545,
+ "learning_rate": 2.410774410774411e-06,
+ "loss": 0.6202508211135864,
+ "mean_token_accuracy": 0.8296287059783936,
+ "num_tokens": 21413888.0,
+ "step": 1307
+ },
+ {
+ "entropy": 0.7812427282333374,
+ "epoch": 2.6424242424242426,
+ "grad_norm": 1.912300944328308,
+ "learning_rate": 2.3973063973063978e-06,
+ "loss": 0.7610887289047241,
+ "mean_token_accuracy": 0.8078895807266235,
+ "num_tokens": 21430272.0,
+ "step": 1308
+ },
+ {
+ "entropy": 0.8849145174026489,
+ "epoch": 2.6444444444444444,
+ "grad_norm": 2.355889320373535,
+ "learning_rate": 2.3838383838383843e-06,
+ "loss": 0.8553632497787476,
+ "mean_token_accuracy": 0.7741817235946655,
+ "num_tokens": 21446656.0,
+ "step": 1309
+ },
+ {
+ "entropy": 0.7890692353248596,
+ "epoch": 2.6464646464646466,
+ "grad_norm": 1.9500312805175781,
+ "learning_rate": 2.3703703703703707e-06,
+ "loss": 0.7833583950996399,
+ "mean_token_accuracy": 0.7951880693435669,
+ "num_tokens": 21463040.0,
+ "step": 1310
+ },
+ {
+ "entropy": 1.015820860862732,
+ "epoch": 2.6484848484848484,
+ "grad_norm": 2.1424720287323,
+ "learning_rate": 2.3569023569023572e-06,
+ "loss": 1.0118992328643799,
+ "mean_token_accuracy": 0.7511602640151978,
+ "num_tokens": 21479424.0,
+ "step": 1311
+ },
+ {
+ "entropy": 0.9689420461654663,
+ "epoch": 2.6505050505050507,
+ "grad_norm": 2.2687859535217285,
+ "learning_rate": 2.3434343434343437e-06,
+ "loss": 0.9623984694480896,
+ "mean_token_accuracy": 0.7540302872657776,
+ "num_tokens": 21495808.0,
+ "step": 1312
+ },
+ {
+ "entropy": 1.10232412815094,
+ "epoch": 2.6525252525252525,
+ "grad_norm": 2.1755430698394775,
+ "learning_rate": 2.32996632996633e-06,
+ "loss": 1.0716698169708252,
+ "mean_token_accuracy": 0.7326575517654419,
+ "num_tokens": 21512192.0,
+ "step": 1313
+ },
+ {
+ "entropy": 0.8814781904220581,
+ "epoch": 2.6545454545454543,
+ "grad_norm": 2.2704896926879883,
+ "learning_rate": 2.3164983164983167e-06,
+ "loss": 0.8600432872772217,
+ "mean_token_accuracy": 0.7757083773612976,
+ "num_tokens": 21528576.0,
+ "step": 1314
+ },
+ {
+ "entropy": 1.0163686275482178,
+ "epoch": 2.6565656565656566,
+ "grad_norm": 2.1807072162628174,
+ "learning_rate": 2.303030303030303e-06,
+ "loss": 1.015133261680603,
+ "mean_token_accuracy": 0.7492061257362366,
+ "num_tokens": 21544960.0,
+ "step": 1315
+ },
+ {
+ "entropy": 1.06844162940979,
+ "epoch": 2.658585858585859,
+ "grad_norm": 2.3641507625579834,
+ "learning_rate": 2.28956228956229e-06,
+ "loss": 1.04976487159729,
+ "mean_token_accuracy": 0.7343673706054688,
+ "num_tokens": 21561344.0,
+ "step": 1316
+ },
+ {
+ "entropy": 0.8728080987930298,
+ "epoch": 2.6606060606060606,
+ "grad_norm": 2.061201572418213,
+ "learning_rate": 2.2760942760942766e-06,
+ "loss": 0.8450087904930115,
+ "mean_token_accuracy": 0.7807157039642334,
+ "num_tokens": 21577728.0,
+ "step": 1317
+ },
+ {
+ "entropy": 0.8158561587333679,
+ "epoch": 2.6626262626262625,
+ "grad_norm": 2.0881152153015137,
+ "learning_rate": 2.262626262626263e-06,
+ "loss": 0.8148937821388245,
+ "mean_token_accuracy": 0.7823033928871155,
+ "num_tokens": 21594112.0,
+ "step": 1318
+ },
+ {
+ "entropy": 0.9075817465782166,
+ "epoch": 2.6646464646464647,
+ "grad_norm": 2.1599912643432617,
+ "learning_rate": 2.249158249158249e-06,
+ "loss": 0.9027103185653687,
+ "mean_token_accuracy": 0.7571446299552917,
+ "num_tokens": 21610496.0,
+ "step": 1319
+ },
+ {
+ "entropy": 1.037543773651123,
+ "epoch": 2.6666666666666665,
+ "grad_norm": 2.279888391494751,
+ "learning_rate": 2.2356902356902356e-06,
+ "loss": 1.0209299325942993,
+ "mean_token_accuracy": 0.7410845160484314,
+ "num_tokens": 21626880.0,
+ "step": 1320
+ },
+ {
+ "entropy": 0.8195902109146118,
+ "epoch": 2.6686868686868688,
+ "grad_norm": 1.9627693891525269,
+ "learning_rate": 2.222222222222222e-06,
+ "loss": 0.8038985133171082,
+ "mean_token_accuracy": 0.7953101992607117,
+ "num_tokens": 21643264.0,
+ "step": 1321
+ },
+ {
+ "entropy": 0.8495474457740784,
+ "epoch": 2.6707070707070706,
+ "grad_norm": 2.180691957473755,
+ "learning_rate": 2.208754208754209e-06,
+ "loss": 0.8287450075149536,
+ "mean_token_accuracy": 0.7793722748756409,
+ "num_tokens": 21659648.0,
+ "step": 1322
+ },
+ {
+ "entropy": 0.8658807873725891,
+ "epoch": 2.672727272727273,
+ "grad_norm": 2.2783005237579346,
+ "learning_rate": 2.1952861952861955e-06,
+ "loss": 0.8221741914749146,
+ "mean_token_accuracy": 0.7837689518928528,
+ "num_tokens": 21676032.0,
+ "step": 1323
+ },
+ {
+ "entropy": 1.3661859035491943,
+ "epoch": 2.6747474747474747,
+ "grad_norm": 2.3197426795959473,
+ "learning_rate": 2.181818181818182e-06,
+ "loss": 1.3951282501220703,
+ "mean_token_accuracy": 0.6698216795921326,
+ "num_tokens": 21692416.0,
+ "step": 1324
+ },
+ {
+ "entropy": 0.8162605166435242,
+ "epoch": 2.676767676767677,
+ "grad_norm": 2.2228052616119385,
+ "learning_rate": 2.1683501683501684e-06,
+ "loss": 0.8109195232391357,
+ "mean_token_accuracy": 0.7856619358062744,
+ "num_tokens": 21708800.0,
+ "step": 1325
+ },
+ {
+ "entropy": 0.686102032661438,
+ "epoch": 2.6787878787878787,
+ "grad_norm": 2.090847969055176,
+ "learning_rate": 2.154882154882155e-06,
+ "loss": 0.685897171497345,
+ "mean_token_accuracy": 0.8203468322753906,
+ "num_tokens": 21725184.0,
+ "step": 1326
+ },
+ {
+ "entropy": 0.7880700826644897,
+ "epoch": 2.680808080808081,
+ "grad_norm": 2.205867052078247,
+ "learning_rate": 2.1414141414141414e-06,
+ "loss": 0.7863162755966187,
+ "mean_token_accuracy": 0.7872496247291565,
+ "num_tokens": 21741568.0,
+ "step": 1327
+ },
+ {
+ "entropy": 1.1857227087020874,
+ "epoch": 2.682828282828283,
+ "grad_norm": 2.4101028442382812,
+ "learning_rate": 2.127946127946128e-06,
+ "loss": 1.186779499053955,
+ "mean_token_accuracy": 0.7098192572593689,
+ "num_tokens": 21757952.0,
+ "step": 1328
+ },
+ {
+ "entropy": 0.6934559345245361,
+ "epoch": 2.6848484848484846,
+ "grad_norm": 2.0098626613616943,
+ "learning_rate": 2.1144781144781144e-06,
+ "loss": 0.6930664777755737,
+ "mean_token_accuracy": 0.8147899508476257,
+ "num_tokens": 21774336.0,
+ "step": 1329
+ },
+ {
+ "entropy": 0.9050189256668091,
+ "epoch": 2.686868686868687,
+ "grad_norm": 2.298691987991333,
+ "learning_rate": 2.1010101010101013e-06,
+ "loss": 0.9015864133834839,
+ "mean_token_accuracy": 0.7663043737411499,
+ "num_tokens": 21790720.0,
+ "step": 1330
+ },
+ {
+ "entropy": 0.8668895959854126,
+ "epoch": 2.688888888888889,
+ "grad_norm": 2.20314359664917,
+ "learning_rate": 2.0875420875420878e-06,
+ "loss": 0.8361119627952576,
+ "mean_token_accuracy": 0.7816316485404968,
+ "num_tokens": 21807104.0,
+ "step": 1331
+ },
+ {
+ "entropy": 0.6271505951881409,
+ "epoch": 2.690909090909091,
+ "grad_norm": 2.095332622528076,
+ "learning_rate": 2.0740740740740742e-06,
+ "loss": 0.6370486617088318,
+ "mean_token_accuracy": 0.827186107635498,
+ "num_tokens": 21823488.0,
+ "step": 1332
+ },
+ {
+ "entropy": 0.9386110901832581,
+ "epoch": 2.6929292929292927,
+ "grad_norm": 2.3780136108398438,
+ "learning_rate": 2.0606060606060607e-06,
+ "loss": 0.9280458688735962,
+ "mean_token_accuracy": 0.7595872282981873,
+ "num_tokens": 21839872.0,
+ "step": 1333
+ },
+ {
+ "entropy": 0.6763703227043152,
+ "epoch": 2.694949494949495,
+ "grad_norm": 2.014997720718384,
+ "learning_rate": 2.0471380471380472e-06,
+ "loss": 0.6596795320510864,
+ "mean_token_accuracy": 0.8192476630210876,
+ "num_tokens": 21856256.0,
+ "step": 1334
+ },
+ {
+ "entropy": 0.9701617360115051,
+ "epoch": 2.6969696969696972,
+ "grad_norm": 2.1106598377227783,
+ "learning_rate": 2.0336700336700337e-06,
+ "loss": 0.9544191360473633,
+ "mean_token_accuracy": 0.7517098188400269,
+ "num_tokens": 21872640.0,
+ "step": 1335
+ },
+ {
+ "entropy": 1.0414639711380005,
+ "epoch": 2.698989898989899,
+ "grad_norm": 2.058302640914917,
+ "learning_rate": 2.02020202020202e-06,
+ "loss": 1.0318336486816406,
+ "mean_token_accuracy": 0.7472520470619202,
+ "num_tokens": 21889024.0,
+ "step": 1336
+ },
+ {
+ "entropy": 1.1898949146270752,
+ "epoch": 2.701010101010101,
+ "grad_norm": 2.3403096199035645,
+ "learning_rate": 2.0067340067340067e-06,
+ "loss": 1.1814141273498535,
+ "mean_token_accuracy": 0.7099413871765137,
+ "num_tokens": 21905408.0,
+ "step": 1337
+ },
+ {
+ "entropy": 0.7575987577438354,
+ "epoch": 2.703030303030303,
+ "grad_norm": 2.305110454559326,
+ "learning_rate": 1.9932659932659936e-06,
+ "loss": 0.7617212533950806,
+ "mean_token_accuracy": 0.79347825050354,
+ "num_tokens": 21921792.0,
+ "step": 1338
+ },
+ {
+ "entropy": 1.2738142013549805,
+ "epoch": 2.705050505050505,
+ "grad_norm": 2.1600685119628906,
+ "learning_rate": 1.97979797979798e-06,
+ "loss": 1.26939058303833,
+ "mean_token_accuracy": 0.7040180563926697,
+ "num_tokens": 21938176.0,
+ "step": 1339
+ },
+ {
+ "entropy": 1.20439612865448,
+ "epoch": 2.707070707070707,
+ "grad_norm": 2.13100528717041,
+ "learning_rate": 1.9663299663299665e-06,
+ "loss": 1.19621741771698,
+ "mean_token_accuracy": 0.7227039337158203,
+ "num_tokens": 21954560.0,
+ "step": 1340
+ },
+ {
+ "entropy": 0.8565170764923096,
+ "epoch": 2.709090909090909,
+ "grad_norm": 2.153420925140381,
+ "learning_rate": 1.952861952861953e-06,
+ "loss": 0.8627563714981079,
+ "mean_token_accuracy": 0.7792501449584961,
+ "num_tokens": 21970944.0,
+ "step": 1341
+ },
+ {
+ "entropy": 1.0674846172332764,
+ "epoch": 2.7111111111111112,
+ "grad_norm": 2.2488465309143066,
+ "learning_rate": 1.9393939393939395e-06,
+ "loss": 1.1025105714797974,
+ "mean_token_accuracy": 0.725024402141571,
+ "num_tokens": 21987328.0,
+ "step": 1342
+ },
+ {
+ "entropy": 1.098164677619934,
+ "epoch": 2.713131313131313,
+ "grad_norm": 1.880246639251709,
+ "learning_rate": 1.925925925925926e-06,
+ "loss": 1.093496322631836,
+ "mean_token_accuracy": 0.7441987991333008,
+ "num_tokens": 22003712.0,
+ "step": 1343
+ },
+ {
+ "entropy": 1.007258415222168,
+ "epoch": 2.7151515151515153,
+ "grad_norm": 2.064815044403076,
+ "learning_rate": 1.9124579124579125e-06,
+ "loss": 0.9783555865287781,
+ "mean_token_accuracy": 0.7542135119438171,
+ "num_tokens": 22020096.0,
+ "step": 1344
+ },
+ {
+ "entropy": 0.56569904088974,
+ "epoch": 2.717171717171717,
+ "grad_norm": 2.032102346420288,
+ "learning_rate": 1.8989898989898992e-06,
+ "loss": 0.5754306316375732,
+ "mean_token_accuracy": 0.8329873085021973,
+ "num_tokens": 22036480.0,
+ "step": 1345
+ },
+ {
+ "entropy": 1.2911049127578735,
+ "epoch": 2.7191919191919194,
+ "grad_norm": 2.148770570755005,
+ "learning_rate": 1.8855218855218857e-06,
+ "loss": 1.295262098312378,
+ "mean_token_accuracy": 0.6909501552581787,
+ "num_tokens": 22052864.0,
+ "step": 1346
+ },
+ {
+ "entropy": 1.0127439498901367,
+ "epoch": 2.721212121212121,
+ "grad_norm": 2.073178291320801,
+ "learning_rate": 1.8720538720538721e-06,
+ "loss": 1.026899814605713,
+ "mean_token_accuracy": 0.7647166848182678,
+ "num_tokens": 22069248.0,
+ "step": 1347
+ },
+ {
+ "entropy": 1.3102049827575684,
+ "epoch": 2.723232323232323,
+ "grad_norm": 3.1892120838165283,
+ "learning_rate": 1.8585858585858588e-06,
+ "loss": 1.3353972434997559,
+ "mean_token_accuracy": 0.6885075569152832,
+ "num_tokens": 22085632.0,
+ "step": 1348
+ },
+ {
+ "entropy": 1.234026551246643,
+ "epoch": 2.7252525252525253,
+ "grad_norm": 2.2144527435302734,
+ "learning_rate": 1.8451178451178453e-06,
+ "loss": 1.263453722000122,
+ "mean_token_accuracy": 0.7058500051498413,
+ "num_tokens": 22102016.0,
+ "step": 1349
+ },
+ {
+ "entropy": 0.8032963871955872,
+ "epoch": 2.7272727272727275,
+ "grad_norm": 2.0798792839050293,
+ "learning_rate": 1.8316498316498318e-06,
+ "loss": 0.7893553972244263,
+ "mean_token_accuracy": 0.7901807427406311,
+ "num_tokens": 22118400.0,
+ "step": 1350
+ },
+ {
+ "entropy": 0.8658150434494019,
+ "epoch": 2.7292929292929293,
+ "grad_norm": 2.3371081352233887,
+ "learning_rate": 1.8181818181818183e-06,
+ "loss": 0.8607960939407349,
+ "mean_token_accuracy": 0.765693724155426,
+ "num_tokens": 22134784.0,
+ "step": 1351
+ },
+ {
+ "entropy": 0.7587262988090515,
+ "epoch": 2.731313131313131,
+ "grad_norm": 2.0964932441711426,
+ "learning_rate": 1.804713804713805e-06,
+ "loss": 0.7469046711921692,
+ "mean_token_accuracy": 0.7841353416442871,
+ "num_tokens": 22151168.0,
+ "step": 1352
+ },
+ {
+ "entropy": 1.0758448839187622,
+ "epoch": 2.7333333333333334,
+ "grad_norm": 2.216097593307495,
+ "learning_rate": 1.7912457912457915e-06,
+ "loss": 1.0564570426940918,
+ "mean_token_accuracy": 0.7307645082473755,
+ "num_tokens": 22167552.0,
+ "step": 1353
+ },
+ {
+ "entropy": 0.6904771327972412,
+ "epoch": 2.735353535353535,
+ "grad_norm": 2.1575329303741455,
+ "learning_rate": 1.777777777777778e-06,
+ "loss": 0.694258451461792,
+ "mean_token_accuracy": 0.8085002303123474,
+ "num_tokens": 22183936.0,
+ "step": 1354
+ },
+ {
+ "entropy": 0.9386518001556396,
+ "epoch": 2.7373737373737375,
+ "grad_norm": 2.3370704650878906,
+ "learning_rate": 1.7643097643097644e-06,
+ "loss": 0.9242473840713501,
+ "mean_token_accuracy": 0.758671224117279,
+ "num_tokens": 22200320.0,
+ "step": 1355
+ },
+ {
+ "entropy": 1.350234866142273,
+ "epoch": 2.7393939393939393,
+ "grad_norm": 2.273244619369507,
+ "learning_rate": 1.7508417508417511e-06,
+ "loss": 1.3637861013412476,
+ "mean_token_accuracy": 0.7142769694328308,
+ "num_tokens": 22216704.0,
+ "step": 1356
+ },
+ {
+ "entropy": 0.9347048401832581,
+ "epoch": 2.7414141414141415,
+ "grad_norm": 2.2230279445648193,
+ "learning_rate": 1.7373737373737376e-06,
+ "loss": 0.9385714530944824,
+ "mean_token_accuracy": 0.7549462914466858,
+ "num_tokens": 22233088.0,
+ "step": 1357
+ },
+ {
+ "entropy": 0.8205846548080444,
+ "epoch": 2.7434343434343433,
+ "grad_norm": 2.119422435760498,
+ "learning_rate": 1.723905723905724e-06,
+ "loss": 0.8094472885131836,
+ "mean_token_accuracy": 0.7930508255958557,
+ "num_tokens": 22249472.0,
+ "step": 1358
+ },
+ {
+ "entropy": 1.3506226539611816,
+ "epoch": 2.7454545454545456,
+ "grad_norm": 2.1955182552337646,
+ "learning_rate": 1.7104377104377106e-06,
+ "loss": 1.34646475315094,
+ "mean_token_accuracy": 0.6719589829444885,
+ "num_tokens": 22265856.0,
+ "step": 1359
+ },
+ {
+ "entropy": 0.9343435168266296,
+ "epoch": 2.7474747474747474,
+ "grad_norm": 2.149738073348999,
+ "learning_rate": 1.6969696969696973e-06,
+ "loss": 0.9155716300010681,
+ "mean_token_accuracy": 0.7647777199745178,
+ "num_tokens": 22282240.0,
+ "step": 1360
+ },
+ {
+ "entropy": 1.245186448097229,
+ "epoch": 2.7494949494949497,
+ "grad_norm": 2.2064425945281982,
+ "learning_rate": 1.6835016835016838e-06,
+ "loss": 1.2376378774642944,
+ "mean_token_accuracy": 0.702125072479248,
+ "num_tokens": 22298624.0,
+ "step": 1361
+ },
+ {
+ "entropy": 0.794501781463623,
+ "epoch": 2.7515151515151515,
+ "grad_norm": 2.13567852973938,
+ "learning_rate": 1.6700336700336703e-06,
+ "loss": 0.7857789993286133,
+ "mean_token_accuracy": 0.7925012111663818,
+ "num_tokens": 22315008.0,
+ "step": 1362
+ },
+ {
+ "entropy": 0.7703561186790466,
+ "epoch": 2.7535353535353533,
+ "grad_norm": 2.251387596130371,
+ "learning_rate": 1.6565656565656567e-06,
+ "loss": 0.7742305397987366,
+ "mean_token_accuracy": 0.7852345108985901,
+ "num_tokens": 22331392.0,
+ "step": 1363
+ },
+ {
+ "entropy": 1.0444586277008057,
+ "epoch": 2.7555555555555555,
+ "grad_norm": 2.2355873584747314,
+ "learning_rate": 1.6430976430976434e-06,
+ "loss": 1.0516932010650635,
+ "mean_token_accuracy": 0.7522594332695007,
+ "num_tokens": 22347776.0,
+ "step": 1364
+ },
+ {
+ "entropy": 1.1597771644592285,
+ "epoch": 2.757575757575758,
+ "grad_norm": 2.19389009475708,
+ "learning_rate": 1.62962962962963e-06,
+ "loss": 1.1565136909484863,
+ "mean_token_accuracy": 0.7207498550415039,
+ "num_tokens": 22364160.0,
+ "step": 1365
+ },
+ {
+ "entropy": 1.2256853580474854,
+ "epoch": 2.7595959595959596,
+ "grad_norm": 2.273327350616455,
+ "learning_rate": 1.6161616161616164e-06,
+ "loss": 1.2160989046096802,
+ "mean_token_accuracy": 0.7077430486679077,
+ "num_tokens": 22380544.0,
+ "step": 1366
+ },
+ {
+ "entropy": 1.1024588346481323,
+ "epoch": 2.7616161616161614,
+ "grad_norm": 2.3130714893341064,
+ "learning_rate": 1.6026936026936027e-06,
+ "loss": 1.0891623497009277,
+ "mean_token_accuracy": 0.7457864880561829,
+ "num_tokens": 22396928.0,
+ "step": 1367
+ },
+ {
+ "entropy": 1.1749509572982788,
+ "epoch": 2.7636363636363637,
+ "grad_norm": 2.1840877532958984,
+ "learning_rate": 1.5892255892255892e-06,
+ "loss": 1.1486189365386963,
+ "mean_token_accuracy": 0.7224596738815308,
+ "num_tokens": 22413312.0,
+ "step": 1368
+ },
+ {
+ "entropy": 0.755344569683075,
+ "epoch": 2.765656565656566,
+ "grad_norm": 2.225905179977417,
+ "learning_rate": 1.5757575757575759e-06,
+ "loss": 0.7560299634933472,
+ "mean_token_accuracy": 0.8009281754493713,
+ "num_tokens": 22429696.0,
+ "step": 1369
+ },
+ {
+ "entropy": 1.11491858959198,
+ "epoch": 2.7676767676767677,
+ "grad_norm": 2.181978702545166,
+ "learning_rate": 1.5622895622895623e-06,
+ "loss": 1.082735300064087,
+ "mean_token_accuracy": 0.7393136024475098,
+ "num_tokens": 22446080.0,
+ "step": 1370
+ },
+ {
+ "entropy": 1.203733205795288,
+ "epoch": 2.7696969696969695,
+ "grad_norm": 2.1899125576019287,
+ "learning_rate": 1.5488215488215488e-06,
+ "loss": 1.2145953178405762,
+ "mean_token_accuracy": 0.7018197178840637,
+ "num_tokens": 22462464.0,
+ "step": 1371
+ },
+ {
+ "entropy": 1.042996883392334,
+ "epoch": 2.771717171717172,
+ "grad_norm": 2.1655640602111816,
+ "learning_rate": 1.5353535353535353e-06,
+ "loss": 1.0236766338348389,
+ "mean_token_accuracy": 0.7488397359848022,
+ "num_tokens": 22478848.0,
+ "step": 1372
+ },
+ {
+ "entropy": 0.9719638228416443,
+ "epoch": 2.7737373737373736,
+ "grad_norm": 2.123048782348633,
+ "learning_rate": 1.521885521885522e-06,
+ "loss": 0.9863395690917969,
+ "mean_token_accuracy": 0.748961865901947,
+ "num_tokens": 22495232.0,
+ "step": 1373
+ },
+ {
+ "entropy": 0.9668793082237244,
+ "epoch": 2.775757575757576,
+ "grad_norm": 2.0465495586395264,
+ "learning_rate": 1.5084175084175085e-06,
+ "loss": 0.9484689235687256,
+ "mean_token_accuracy": 0.761907696723938,
+ "num_tokens": 22511616.0,
+ "step": 1374
+ },
+ {
+ "entropy": 1.125752329826355,
+ "epoch": 2.7777777777777777,
+ "grad_norm": 2.090346336364746,
+ "learning_rate": 1.494949494949495e-06,
+ "loss": 1.1148427724838257,
+ "mean_token_accuracy": 0.7297874689102173,
+ "num_tokens": 22528000.0,
+ "step": 1375
+ },
+ {
+ "entropy": 0.9548814296722412,
+ "epoch": 2.77979797979798,
+ "grad_norm": 2.199824333190918,
+ "learning_rate": 1.4814814814814815e-06,
+ "loss": 0.9301331043243408,
+ "mean_token_accuracy": 0.7628236413002014,
+ "num_tokens": 22544384.0,
+ "step": 1376
+ },
+ {
+ "entropy": 1.0585272312164307,
+ "epoch": 2.7818181818181817,
+ "grad_norm": 2.337399482727051,
+ "learning_rate": 1.4680134680134681e-06,
+ "loss": 1.0728713274002075,
+ "mean_token_accuracy": 0.7237420678138733,
+ "num_tokens": 22560768.0,
+ "step": 1377
+ },
+ {
+ "entropy": 1.1996891498565674,
+ "epoch": 2.783838383838384,
+ "grad_norm": 2.465836763381958,
+ "learning_rate": 1.4545454545454546e-06,
+ "loss": 1.2366418838500977,
+ "mean_token_accuracy": 0.6919882893562317,
+ "num_tokens": 22577152.0,
+ "step": 1378
+ },
+ {
+ "entropy": 0.8297094702720642,
+ "epoch": 2.785858585858586,
+ "grad_norm": 2.303840398788452,
+ "learning_rate": 1.4410774410774411e-06,
+ "loss": 0.8407220840454102,
+ "mean_token_accuracy": 0.7727161645889282,
+ "num_tokens": 22593536.0,
+ "step": 1379
+ },
+ {
+ "entropy": 1.0284315347671509,
+ "epoch": 2.787878787878788,
+ "grad_norm": 2.5417709350585938,
+ "learning_rate": 1.4276094276094276e-06,
+ "loss": 1.0437777042388916,
+ "mean_token_accuracy": 0.738092303276062,
+ "num_tokens": 22609920.0,
+ "step": 1380
+ },
+ {
+ "entropy": 0.9829826951026917,
+ "epoch": 2.78989898989899,
+ "grad_norm": 2.1615004539489746,
+ "learning_rate": 1.4141414141414143e-06,
+ "loss": 0.9619671106338501,
+ "mean_token_accuracy": 0.7565339803695679,
+ "num_tokens": 22626304.0,
+ "step": 1381
+ },
+ {
+ "entropy": 0.6986419558525085,
+ "epoch": 2.7919191919191917,
+ "grad_norm": 2.059281349182129,
+ "learning_rate": 1.4006734006734008e-06,
+ "loss": 0.6810094714164734,
+ "mean_token_accuracy": 0.8127137422561646,
+ "num_tokens": 22642688.0,
+ "step": 1382
+ },
+ {
+ "entropy": 1.1219873428344727,
+ "epoch": 2.793939393939394,
+ "grad_norm": 4.745248794555664,
+ "learning_rate": 1.3872053872053873e-06,
+ "loss": 1.17073655128479,
+ "mean_token_accuracy": 0.7132999300956726,
+ "num_tokens": 22659072.0,
+ "step": 1383
+ },
+ {
+ "entropy": 1.0922656059265137,
+ "epoch": 2.795959595959596,
+ "grad_norm": 2.024942398071289,
+ "learning_rate": 1.3737373737373738e-06,
+ "loss": 1.088742971420288,
+ "mean_token_accuracy": 0.7467024922370911,
+ "num_tokens": 22675456.0,
+ "step": 1384
+ },
+ {
+ "entropy": 1.038475513458252,
+ "epoch": 2.797979797979798,
+ "grad_norm": 2.274622917175293,
+ "learning_rate": 1.3602693602693604e-06,
+ "loss": 1.0326749086380005,
+ "mean_token_accuracy": 0.7448094487190247,
+ "num_tokens": 22691840.0,
+ "step": 1385
+ },
+ {
+ "entropy": 0.9840977787971497,
+ "epoch": 2.8,
+ "grad_norm": 2.2080237865448,
+ "learning_rate": 1.346801346801347e-06,
+ "loss": 0.9834811687469482,
+ "mean_token_accuracy": 0.7443820238113403,
+ "num_tokens": 22708224.0,
+ "step": 1386
+ },
+ {
+ "entropy": 0.8895727396011353,
+ "epoch": 2.802020202020202,
+ "grad_norm": 1.9656678438186646,
+ "learning_rate": 1.3333333333333334e-06,
+ "loss": 0.868664562702179,
+ "mean_token_accuracy": 0.7685027122497559,
+ "num_tokens": 22724608.0,
+ "step": 1387
+ },
+ {
+ "entropy": 1.1341460943222046,
+ "epoch": 2.804040404040404,
+ "grad_norm": 2.34385347366333,
+ "learning_rate": 1.31986531986532e-06,
+ "loss": 1.1425552368164062,
+ "mean_token_accuracy": 0.718551516532898,
+ "num_tokens": 22740992.0,
+ "step": 1388
+ },
+ {
+ "entropy": 0.8935446739196777,
+ "epoch": 2.806060606060606,
+ "grad_norm": 2.0570411682128906,
+ "learning_rate": 1.3063973063973066e-06,
+ "loss": 0.8920376300811768,
+ "mean_token_accuracy": 0.7730825543403625,
+ "num_tokens": 22757376.0,
+ "step": 1389
+ },
+ {
+ "entropy": 0.8422355055809021,
+ "epoch": 2.808080808080808,
+ "grad_norm": 2.287170886993408,
+ "learning_rate": 1.292929292929293e-06,
+ "loss": 0.8516101837158203,
+ "mean_token_accuracy": 0.788532018661499,
+ "num_tokens": 22773760.0,
+ "step": 1390
+ },
+ {
+ "entropy": 0.9526816606521606,
+ "epoch": 2.81010101010101,
+ "grad_norm": 2.069354772567749,
+ "learning_rate": 1.2794612794612796e-06,
+ "loss": 0.9390783309936523,
+ "mean_token_accuracy": 0.7687469720840454,
+ "num_tokens": 22790144.0,
+ "step": 1391
+ },
+ {
+ "entropy": 1.3697867393493652,
+ "epoch": 2.812121212121212,
+ "grad_norm": 2.668720006942749,
+ "learning_rate": 1.265993265993266e-06,
+ "loss": 1.3849083185195923,
+ "mean_token_accuracy": 0.6784929037094116,
+ "num_tokens": 22806528.0,
+ "step": 1392
+ },
+ {
+ "entropy": 0.9275345802307129,
+ "epoch": 2.8141414141414143,
+ "grad_norm": 2.1365058422088623,
+ "learning_rate": 1.2525252525252527e-06,
+ "loss": 0.9283688068389893,
+ "mean_token_accuracy": 0.7622129917144775,
+ "num_tokens": 22822912.0,
+ "step": 1393
+ },
+ {
+ "entropy": 0.8800023198127747,
+ "epoch": 2.816161616161616,
+ "grad_norm": 2.3574256896972656,
+ "learning_rate": 1.2390572390572392e-06,
+ "loss": 0.8732290267944336,
+ "mean_token_accuracy": 0.7710674405097961,
+ "num_tokens": 22839296.0,
+ "step": 1394
+ },
+ {
+ "entropy": 1.4244937896728516,
+ "epoch": 2.8181818181818183,
+ "grad_norm": 2.1275830268859863,
+ "learning_rate": 1.2255892255892257e-06,
+ "loss": 1.3897889852523804,
+ "mean_token_accuracy": 0.6743404865264893,
+ "num_tokens": 22855680.0,
+ "step": 1395
+ },
+ {
+ "entropy": 1.0230107307434082,
+ "epoch": 2.82020202020202,
+ "grad_norm": 2.1384198665618896,
+ "learning_rate": 1.2121212121212122e-06,
+ "loss": 1.034029483795166,
+ "mean_token_accuracy": 0.7398632168769836,
+ "num_tokens": 22872064.0,
+ "step": 1396
+ },
+ {
+ "entropy": 0.7272607088088989,
+ "epoch": 2.822222222222222,
+ "grad_norm": 2.208526611328125,
+ "learning_rate": 1.1986531986531989e-06,
+ "loss": 0.71860671043396,
+ "mean_token_accuracy": 0.8063018918037415,
+ "num_tokens": 22888448.0,
+ "step": 1397
+ },
+ {
+ "entropy": 1.2365843057632446,
+ "epoch": 2.824242424242424,
+ "grad_norm": 2.191222667694092,
+ "learning_rate": 1.1851851851851854e-06,
+ "loss": 1.2230392694473267,
+ "mean_token_accuracy": 0.7057278752326965,
+ "num_tokens": 22904832.0,
+ "step": 1398
+ },
+ {
+ "entropy": 0.8831301331520081,
+ "epoch": 2.8262626262626265,
+ "grad_norm": 2.2478153705596924,
+ "learning_rate": 1.1717171717171719e-06,
+ "loss": 0.8597757816314697,
+ "mean_token_accuracy": 0.7766854166984558,
+ "num_tokens": 22921216.0,
+ "step": 1399
+ },
+ {
+ "entropy": 0.8236691951751709,
+ "epoch": 2.8282828282828283,
+ "grad_norm": 2.3858163356781006,
+ "learning_rate": 1.1582491582491583e-06,
+ "loss": 0.8266822099685669,
+ "mean_token_accuracy": 0.7748534679412842,
+ "num_tokens": 22937600.0,
+ "step": 1400
+ },
+ {
+ "epoch": 2.8282828282828283,
+ "eval_entropy": 1.1389604489649496,
+ "eval_loss": 1.6066142320632935,
+ "eval_mean_token_accuracy": 0.6432013021361443,
+ "eval_num_tokens": 22937600.0,
+ "eval_runtime": 43.792,
+ "eval_samples_per_second": 22.607,
+ "eval_steps_per_second": 2.832,
+ "step": 1400
+ },
+ {
+ "entropy": 0.8862301111221313,
+ "epoch": 2.83030303030303,
+ "grad_norm": 2.362920045852661,
+ "learning_rate": 1.144781144781145e-06,
+ "loss": 0.8503944277763367,
+ "mean_token_accuracy": 0.7711284756660461,
+ "num_tokens": 22953984.0,
+ "step": 1401
+ },
+ {
+ "entropy": 1.0803418159484863,
+ "epoch": 2.8323232323232324,
+ "grad_norm": 2.292283058166504,
+ "learning_rate": 1.1313131313131315e-06,
+ "loss": 1.0977113246917725,
+ "mean_token_accuracy": 0.7307645082473755,
+ "num_tokens": 22970368.0,
+ "step": 1402
+ },
+ {
+ "entropy": 1.593738079071045,
+ "epoch": 2.8343434343434346,
+ "grad_norm": 2.6801810264587402,
+ "learning_rate": 1.1178451178451178e-06,
+ "loss": 1.5267648696899414,
+ "mean_token_accuracy": 0.6606009006500244,
+ "num_tokens": 22986752.0,
+ "step": 1403
+ },
+ {
+ "entropy": 0.9804041981697083,
+ "epoch": 2.8363636363636364,
+ "grad_norm": 2.235811471939087,
+ "learning_rate": 1.1043771043771045e-06,
+ "loss": 0.9784083962440491,
+ "mean_token_accuracy": 0.752198338508606,
+ "num_tokens": 23003136.0,
+ "step": 1404
+ },
+ {
+ "entropy": 0.7044815421104431,
+ "epoch": 2.8383838383838382,
+ "grad_norm": 2.034780263900757,
+ "learning_rate": 1.090909090909091e-06,
+ "loss": 0.6678165197372437,
+ "mean_token_accuracy": 0.8119198679924011,
+ "num_tokens": 23019520.0,
+ "step": 1405
+ },
+ {
+ "entropy": 0.9826809167861938,
+ "epoch": 2.8404040404040405,
+ "grad_norm": 2.5305731296539307,
+ "learning_rate": 1.0774410774410775e-06,
+ "loss": 0.9591178894042969,
+ "mean_token_accuracy": 0.7456033229827881,
+ "num_tokens": 23035904.0,
+ "step": 1406
+ },
+ {
+ "entropy": 1.1601169109344482,
+ "epoch": 2.8424242424242423,
+ "grad_norm": 2.191697597503662,
+ "learning_rate": 1.063973063973064e-06,
+ "loss": 1.181044578552246,
+ "mean_token_accuracy": 0.7303370833396912,
+ "num_tokens": 23052288.0,
+ "step": 1407
+ },
+ {
+ "entropy": 0.6280582547187805,
+ "epoch": 2.8444444444444446,
+ "grad_norm": 2.06825590133667,
+ "learning_rate": 1.0505050505050506e-06,
+ "loss": 0.622545599937439,
+ "mean_token_accuracy": 0.8232169151306152,
+ "num_tokens": 23068672.0,
+ "step": 1408
+ },
+ {
+ "entropy": 0.9755560755729675,
+ "epoch": 2.8464646464646464,
+ "grad_norm": 2.2459638118743896,
+ "learning_rate": 1.0370370370370371e-06,
+ "loss": 0.9764548540115356,
+ "mean_token_accuracy": 0.7656326293945312,
+ "num_tokens": 23085056.0,
+ "step": 1409
+ },
+ {
+ "entropy": 1.4480527639389038,
+ "epoch": 2.8484848484848486,
+ "grad_norm": 2.12251615524292,
+ "learning_rate": 1.0235690235690236e-06,
+ "loss": 1.4220662117004395,
+ "mean_token_accuracy": 0.677760124206543,
+ "num_tokens": 23101440.0,
+ "step": 1410
+ },
+ {
+ "entropy": 0.7127546072006226,
+ "epoch": 2.8505050505050504,
+ "grad_norm": 2.2641944885253906,
+ "learning_rate": 1.01010101010101e-06,
+ "loss": 0.7147482633590698,
+ "mean_token_accuracy": 0.7989130616188049,
+ "num_tokens": 23117824.0,
+ "step": 1411
+ },
+ {
+ "entropy": 0.9932766556739807,
+ "epoch": 2.8525252525252527,
+ "grad_norm": 2.1933536529541016,
+ "learning_rate": 9.966329966329968e-07,
+ "loss": 0.977810263633728,
+ "mean_token_accuracy": 0.7449315786361694,
+ "num_tokens": 23134208.0,
+ "step": 1412
+ },
+ {
+ "entropy": 0.9120132327079773,
+ "epoch": 2.8545454545454545,
+ "grad_norm": 2.004495859146118,
+ "learning_rate": 9.831649831649833e-07,
+ "loss": 0.9141850471496582,
+ "mean_token_accuracy": 0.7762579321861267,
+ "num_tokens": 23150592.0,
+ "step": 1413
+ },
+ {
+ "entropy": 1.0467151403427124,
+ "epoch": 2.8565656565656568,
+ "grad_norm": 2.4027388095855713,
+ "learning_rate": 9.696969696969698e-07,
+ "loss": 1.0727280378341675,
+ "mean_token_accuracy": 0.7319247722625732,
+ "num_tokens": 23166976.0,
+ "step": 1414
+ },
+ {
+ "entropy": 1.058577299118042,
+ "epoch": 2.8585858585858586,
+ "grad_norm": 2.2488481998443604,
+ "learning_rate": 9.562289562289562e-07,
+ "loss": 1.0526621341705322,
+ "mean_token_accuracy": 0.7397410869598389,
+ "num_tokens": 23183360.0,
+ "step": 1415
+ },
+ {
+ "entropy": 1.078447937965393,
+ "epoch": 2.8606060606060604,
+ "grad_norm": 2.3252272605895996,
+ "learning_rate": 9.427609427609428e-07,
+ "loss": 1.0718753337860107,
+ "mean_token_accuracy": 0.7324743270874023,
+ "num_tokens": 23199744.0,
+ "step": 1416
+ },
+ {
+ "entropy": 1.008683681488037,
+ "epoch": 2.8626262626262626,
+ "grad_norm": 2.1366565227508545,
+ "learning_rate": 9.292929292929294e-07,
+ "loss": 1.0080740451812744,
+ "mean_token_accuracy": 0.7556790709495544,
+ "num_tokens": 23216128.0,
+ "step": 1417
+ },
+ {
+ "entropy": 1.3291339874267578,
+ "epoch": 2.864646464646465,
+ "grad_norm": 2.276232957839966,
+ "learning_rate": 9.158249158249159e-07,
+ "loss": 1.304205060005188,
+ "mean_token_accuracy": 0.6845993995666504,
+ "num_tokens": 23232512.0,
+ "step": 1418
+ },
+ {
+ "entropy": 1.0262223482131958,
+ "epoch": 2.8666666666666667,
+ "grad_norm": 2.078489303588867,
+ "learning_rate": 9.023569023569025e-07,
+ "loss": 1.007215142250061,
+ "mean_token_accuracy": 0.7423668503761292,
+ "num_tokens": 23248896.0,
+ "step": 1419
+ },
+ {
+ "entropy": 1.0474964380264282,
+ "epoch": 2.8686868686868685,
+ "grad_norm": 2.276054620742798,
+ "learning_rate": 8.88888888888889e-07,
+ "loss": 1.047877550125122,
+ "mean_token_accuracy": 0.7332682013511658,
+ "num_tokens": 23265280.0,
+ "step": 1420
+ },
+ {
+ "entropy": 0.8027759194374084,
+ "epoch": 2.8707070707070708,
+ "grad_norm": 2.192600727081299,
+ "learning_rate": 8.754208754208756e-07,
+ "loss": 0.7764748334884644,
+ "mean_token_accuracy": 0.7879213690757751,
+ "num_tokens": 23281664.0,
+ "step": 1421
+ },
+ {
+ "entropy": 0.661788284778595,
+ "epoch": 2.8727272727272726,
+ "grad_norm": 2.0137529373168945,
+ "learning_rate": 8.61952861952862e-07,
+ "loss": 0.6363742351531982,
+ "mean_token_accuracy": 0.8241939544677734,
+ "num_tokens": 23298048.0,
+ "step": 1422
+ },
+ {
+ "entropy": 1.087633728981018,
+ "epoch": 2.874747474747475,
+ "grad_norm": 2.175304651260376,
+ "learning_rate": 8.484848484848486e-07,
+ "loss": 1.0498201847076416,
+ "mean_token_accuracy": 0.7443209290504456,
+ "num_tokens": 23314432.0,
+ "step": 1423
+ },
+ {
+ "entropy": 0.9076523184776306,
+ "epoch": 2.8767676767676766,
+ "grad_norm": 2.026049852371216,
+ "learning_rate": 8.350168350168351e-07,
+ "loss": 0.8883987665176392,
+ "mean_token_accuracy": 0.7653883695602417,
+ "num_tokens": 23330816.0,
+ "step": 1424
+ },
+ {
+ "entropy": 0.8226358294487,
+ "epoch": 2.878787878787879,
+ "grad_norm": 2.2744534015655518,
+ "learning_rate": 8.215488215488217e-07,
+ "loss": 0.812698245048523,
+ "mean_token_accuracy": 0.7820591330528259,
+ "num_tokens": 23347200.0,
+ "step": 1425
+ },
+ {
+ "entropy": 1.0673551559448242,
+ "epoch": 2.8808080808080807,
+ "grad_norm": 2.0791378021240234,
+ "learning_rate": 8.080808080808082e-07,
+ "loss": 1.0502647161483765,
+ "mean_token_accuracy": 0.7533586025238037,
+ "num_tokens": 23363584.0,
+ "step": 1426
+ },
+ {
+ "entropy": 0.9955654144287109,
+ "epoch": 2.882828282828283,
+ "grad_norm": 2.1457395553588867,
+ "learning_rate": 7.946127946127946e-07,
+ "loss": 0.9881689548492432,
+ "mean_token_accuracy": 0.7527479529380798,
+ "num_tokens": 23379968.0,
+ "step": 1427
+ },
+ {
+ "entropy": 1.1520777940750122,
+ "epoch": 2.8848484848484848,
+ "grad_norm": 2.173024892807007,
+ "learning_rate": 7.811447811447812e-07,
+ "loss": 1.156259536743164,
+ "mean_token_accuracy": 0.7158646583557129,
+ "num_tokens": 23396352.0,
+ "step": 1428
+ },
+ {
+ "entropy": 1.0403298139572144,
+ "epoch": 2.886868686868687,
+ "grad_norm": 2.422877311706543,
+ "learning_rate": 7.676767676767677e-07,
+ "loss": 1.0122939348220825,
+ "mean_token_accuracy": 0.7454200983047485,
+ "num_tokens": 23412736.0,
+ "step": 1429
+ },
+ {
+ "entropy": 1.0343279838562012,
+ "epoch": 2.888888888888889,
+ "grad_norm": 2.0776565074920654,
+ "learning_rate": 7.542087542087542e-07,
+ "loss": 1.0315957069396973,
+ "mean_token_accuracy": 0.7367488741874695,
+ "num_tokens": 23429120.0,
+ "step": 1430
+ },
+ {
+ "entropy": 0.8493256568908691,
+ "epoch": 2.8909090909090907,
+ "grad_norm": 2.327897787094116,
+ "learning_rate": 7.407407407407407e-07,
+ "loss": 0.8583534955978394,
+ "mean_token_accuracy": 0.7767464518547058,
+ "num_tokens": 23445504.0,
+ "step": 1431
+ },
+ {
+ "entropy": 1.148512363433838,
+ "epoch": 2.892929292929293,
+ "grad_norm": 2.1868507862091064,
+ "learning_rate": 7.272727272727273e-07,
+ "loss": 1.1311705112457275,
+ "mean_token_accuracy": 0.7248412370681763,
+ "num_tokens": 23461888.0,
+ "step": 1432
+ },
+ {
+ "entropy": 1.1566276550292969,
+ "epoch": 2.894949494949495,
+ "grad_norm": 2.2865636348724365,
+ "learning_rate": 7.138047138047138e-07,
+ "loss": 1.1686909198760986,
+ "mean_token_accuracy": 0.7239863276481628,
+ "num_tokens": 23478272.0,
+ "step": 1433
+ },
+ {
+ "entropy": 1.0386736392974854,
+ "epoch": 2.896969696969697,
+ "grad_norm": 2.052959680557251,
+ "learning_rate": 7.003367003367004e-07,
+ "loss": 1.01320481300354,
+ "mean_token_accuracy": 0.7514655590057373,
+ "num_tokens": 23494656.0,
+ "step": 1434
+ },
+ {
+ "entropy": 1.1572645902633667,
+ "epoch": 2.898989898989899,
+ "grad_norm": 2.215139627456665,
+ "learning_rate": 6.868686868686869e-07,
+ "loss": 1.1418699026107788,
+ "mean_token_accuracy": 0.7198949456214905,
+ "num_tokens": 23511040.0,
+ "step": 1435
+ },
+ {
+ "entropy": 1.1387436389923096,
+ "epoch": 2.901010101010101,
+ "grad_norm": 2.5032405853271484,
+ "learning_rate": 6.734006734006735e-07,
+ "loss": 1.1193574666976929,
+ "mean_token_accuracy": 0.7191011309623718,
+ "num_tokens": 23527424.0,
+ "step": 1436
+ },
+ {
+ "entropy": 0.7306453585624695,
+ "epoch": 2.9030303030303033,
+ "grad_norm": 2.1751644611358643,
+ "learning_rate": 6.5993265993266e-07,
+ "loss": 0.710832417011261,
+ "mean_token_accuracy": 0.810332179069519,
+ "num_tokens": 23543808.0,
+ "step": 1437
+ },
+ {
+ "entropy": 0.751767635345459,
+ "epoch": 2.905050505050505,
+ "grad_norm": 2.215428590774536,
+ "learning_rate": 6.464646464646465e-07,
+ "loss": 0.7304487824440002,
+ "mean_token_accuracy": 0.8015388250350952,
+ "num_tokens": 23560192.0,
+ "step": 1438
+ },
+ {
+ "entropy": 1.0094949007034302,
+ "epoch": 2.907070707070707,
+ "grad_norm": 2.2883613109588623,
+ "learning_rate": 6.32996632996633e-07,
+ "loss": 1.0086544752120972,
+ "mean_token_accuracy": 0.7419394254684448,
+ "num_tokens": 23576576.0,
+ "step": 1439
+ },
+ {
+ "entropy": 0.5964215397834778,
+ "epoch": 2.909090909090909,
+ "grad_norm": 1.9728953838348389,
+ "learning_rate": 6.195286195286196e-07,
+ "loss": 0.5952176451683044,
+ "mean_token_accuracy": 0.8348802924156189,
+ "num_tokens": 23592960.0,
+ "step": 1440
+ },
+ {
+ "entropy": 0.6571753621101379,
+ "epoch": 2.911111111111111,
+ "grad_norm": 1.9908696413040161,
+ "learning_rate": 6.060606060606061e-07,
+ "loss": 0.6432311534881592,
+ "mean_token_accuracy": 0.8191866278648376,
+ "num_tokens": 23609344.0,
+ "step": 1441
+ },
+ {
+ "entropy": 0.6548683047294617,
+ "epoch": 2.9131313131313132,
+ "grad_norm": 2.1132285594940186,
+ "learning_rate": 5.925925925925927e-07,
+ "loss": 0.6500411033630371,
+ "mean_token_accuracy": 0.8267586827278137,
+ "num_tokens": 23625728.0,
+ "step": 1442
+ },
+ {
+ "entropy": 0.6230350732803345,
+ "epoch": 2.915151515151515,
+ "grad_norm": 1.970469355583191,
+ "learning_rate": 5.791245791245792e-07,
+ "loss": 0.6239666938781738,
+ "mean_token_accuracy": 0.8229726552963257,
+ "num_tokens": 23642112.0,
+ "step": 1443
+ },
+ {
+ "entropy": 0.7765372395515442,
+ "epoch": 2.9171717171717173,
+ "grad_norm": 2.221811532974243,
+ "learning_rate": 5.656565656565658e-07,
+ "loss": 0.7877421975135803,
+ "mean_token_accuracy": 0.7888984084129333,
+ "num_tokens": 23658496.0,
+ "step": 1444
+ },
+ {
+ "entropy": 0.7895376682281494,
+ "epoch": 2.919191919191919,
+ "grad_norm": 2.3417742252349854,
+ "learning_rate": 5.521885521885522e-07,
+ "loss": 0.7840774059295654,
+ "mean_token_accuracy": 0.7839521169662476,
+ "num_tokens": 23674880.0,
+ "step": 1445
+ },
+ {
+ "entropy": 0.9695510268211365,
+ "epoch": 2.9212121212121214,
+ "grad_norm": 2.0251240730285645,
+ "learning_rate": 5.387205387205387e-07,
+ "loss": 0.9858237504959106,
+ "mean_token_accuracy": 0.7494503855705261,
+ "num_tokens": 23691264.0,
+ "step": 1446
+ },
+ {
+ "entropy": 1.0403461456298828,
+ "epoch": 2.923232323232323,
+ "grad_norm": 2.1194682121276855,
+ "learning_rate": 5.252525252525253e-07,
+ "loss": 1.0414793491363525,
+ "mean_token_accuracy": 0.7365046143531799,
+ "num_tokens": 23707648.0,
+ "step": 1447
+ },
+ {
+ "entropy": 0.8436279296875,
+ "epoch": 2.9252525252525254,
+ "grad_norm": 2.288971424102783,
+ "learning_rate": 5.117845117845118e-07,
+ "loss": 0.8372863531112671,
+ "mean_token_accuracy": 0.7795554399490356,
+ "num_tokens": 23724032.0,
+ "step": 1448
+ },
+ {
+ "entropy": 1.2964613437652588,
+ "epoch": 2.9272727272727272,
+ "grad_norm": 2.3658535480499268,
+ "learning_rate": 4.983164983164984e-07,
+ "loss": 1.3105628490447998,
+ "mean_token_accuracy": 0.6841108798980713,
+ "num_tokens": 23740416.0,
+ "step": 1449
+ },
+ {
+ "entropy": 0.9370822310447693,
+ "epoch": 2.929292929292929,
+ "grad_norm": 2.259519100189209,
+ "learning_rate": 4.848484848484849e-07,
+ "loss": 0.913622260093689,
+ "mean_token_accuracy": 0.7578774094581604,
+ "num_tokens": 23756800.0,
+ "step": 1450
+ },
+ {
+ "entropy": 0.8995789885520935,
+ "epoch": 2.9313131313131313,
+ "grad_norm": 2.1428349018096924,
+ "learning_rate": 4.713804713804714e-07,
+ "loss": 0.894747257232666,
+ "mean_token_accuracy": 0.7653883695602417,
+ "num_tokens": 23773184.0,
+ "step": 1451
+ },
+ {
+ "entropy": 1.014516830444336,
+ "epoch": 2.9333333333333336,
+ "grad_norm": 2.1029326915740967,
+ "learning_rate": 4.5791245791245795e-07,
+ "loss": 0.9938762187957764,
+ "mean_token_accuracy": 0.751038134098053,
+ "num_tokens": 23789568.0,
+ "step": 1452
+ },
+ {
+ "entropy": 1.4249095916748047,
+ "epoch": 2.9353535353535354,
+ "grad_norm": 2.294334888458252,
+ "learning_rate": 4.444444444444445e-07,
+ "loss": 1.392219066619873,
+ "mean_token_accuracy": 0.663593053817749,
+ "num_tokens": 23805952.0,
+ "step": 1453
+ },
+ {
+ "entropy": 0.9197583794593811,
+ "epoch": 2.937373737373737,
+ "grad_norm": 2.3255887031555176,
+ "learning_rate": 4.30976430976431e-07,
+ "loss": 0.9103667140007019,
+ "mean_token_accuracy": 0.7593429684638977,
+ "num_tokens": 23822336.0,
+ "step": 1454
+ },
+ {
+ "entropy": 1.0247950553894043,
+ "epoch": 2.9393939393939394,
+ "grad_norm": 2.212773323059082,
+ "learning_rate": 4.1750841750841756e-07,
+ "loss": 1.0136611461639404,
+ "mean_token_accuracy": 0.7379091382026672,
+ "num_tokens": 23838720.0,
+ "step": 1455
+ },
+ {
+ "entropy": 0.8738380670547485,
+ "epoch": 2.9414141414141413,
+ "grad_norm": 2.332756280899048,
+ "learning_rate": 4.040404040404041e-07,
+ "loss": 0.8523008823394775,
+ "mean_token_accuracy": 0.7714948654174805,
+ "num_tokens": 23855104.0,
+ "step": 1456
+ },
+ {
+ "entropy": 0.9150345325469971,
+ "epoch": 2.9434343434343435,
+ "grad_norm": 2.202171564102173,
+ "learning_rate": 3.905723905723906e-07,
+ "loss": 0.9201558828353882,
+ "mean_token_accuracy": 0.7680752277374268,
+ "num_tokens": 23871488.0,
+ "step": 1457
+ },
+ {
+ "entropy": 0.9043408036231995,
+ "epoch": 2.9454545454545453,
+ "grad_norm": 2.40272855758667,
+ "learning_rate": 3.771043771043771e-07,
+ "loss": 0.868109941482544,
+ "mean_token_accuracy": 0.77167809009552,
+ "num_tokens": 23887872.0,
+ "step": 1458
+ },
+ {
+ "entropy": 0.8504953980445862,
+ "epoch": 2.9474747474747476,
+ "grad_norm": 2.1198537349700928,
+ "learning_rate": 3.6363636363636366e-07,
+ "loss": 0.8543665409088135,
+ "mean_token_accuracy": 0.7743649482727051,
+ "num_tokens": 23904256.0,
+ "step": 1459
+ },
+ {
+ "entropy": 1.1891556978225708,
+ "epoch": 2.9494949494949494,
+ "grad_norm": 2.2647321224212646,
+ "learning_rate": 3.501683501683502e-07,
+ "loss": 1.2097854614257812,
+ "mean_token_accuracy": 0.7138495445251465,
+ "num_tokens": 23920640.0,
+ "step": 1460
+ },
+ {
+ "entropy": 0.9867375493049622,
+ "epoch": 2.9515151515151516,
+ "grad_norm": 2.232421636581421,
+ "learning_rate": 3.3670033670033673e-07,
+ "loss": 0.979575514793396,
+ "mean_token_accuracy": 0.7459086179733276,
+ "num_tokens": 23937024.0,
+ "step": 1461
+ },
+ {
+ "entropy": 0.9686830639839172,
+ "epoch": 2.9535353535353535,
+ "grad_norm": 2.1404521465301514,
+ "learning_rate": 3.2323232323232327e-07,
+ "loss": 0.967602550983429,
+ "mean_token_accuracy": 0.7539692521095276,
+ "num_tokens": 23953408.0,
+ "step": 1462
+ },
+ {
+ "entropy": 0.7671990990638733,
+ "epoch": 2.9555555555555557,
+ "grad_norm": 2.133831262588501,
+ "learning_rate": 3.097643097643098e-07,
+ "loss": 0.7617117166519165,
+ "mean_token_accuracy": 0.8006228804588318,
+ "num_tokens": 23969792.0,
+ "step": 1463
+ },
+ {
+ "entropy": 1.2517712116241455,
+ "epoch": 2.9575757575757575,
+ "grad_norm": 8.452461242675781,
+ "learning_rate": 2.9629629629629634e-07,
+ "loss": 1.4012014865875244,
+ "mean_token_accuracy": 0.6907058954238892,
+ "num_tokens": 23986176.0,
+ "step": 1464
+ },
+ {
+ "entropy": 0.9953964948654175,
+ "epoch": 2.9595959595959593,
+ "grad_norm": 2.425787925720215,
+ "learning_rate": 2.828282828282829e-07,
+ "loss": 0.9884676933288574,
+ "mean_token_accuracy": 0.7476184368133545,
+ "num_tokens": 24002560.0,
+ "step": 1465
+ },
+ {
+ "entropy": 1.3182833194732666,
+ "epoch": 2.9616161616161616,
+ "grad_norm": 2.174762725830078,
+ "learning_rate": 2.6936026936026936e-07,
+ "loss": 1.3243706226348877,
+ "mean_token_accuracy": 0.6910722851753235,
+ "num_tokens": 24018944.0,
+ "step": 1466
+ },
+ {
+ "entropy": 0.8355759978294373,
+ "epoch": 2.963636363636364,
+ "grad_norm": 2.1497325897216797,
+ "learning_rate": 2.558922558922559e-07,
+ "loss": 0.8269426822662354,
+ "mean_token_accuracy": 0.7804714441299438,
+ "num_tokens": 24035328.0,
+ "step": 1467
+ },
+ {
+ "entropy": 1.119064450263977,
+ "epoch": 2.9656565656565657,
+ "grad_norm": 2.4803388118743896,
+ "learning_rate": 2.4242424242424244e-07,
+ "loss": 1.1198925971984863,
+ "mean_token_accuracy": 0.7159257531166077,
+ "num_tokens": 24051712.0,
+ "step": 1468
+ },
+ {
+ "entropy": 0.8541463017463684,
+ "epoch": 2.9676767676767675,
+ "grad_norm": 2.0393052101135254,
+ "learning_rate": 2.2895622895622898e-07,
+ "loss": 0.8403464555740356,
+ "mean_token_accuracy": 0.7884098887443542,
+ "num_tokens": 24068096.0,
+ "step": 1469
+ },
+ {
+ "entropy": 0.49429821968078613,
+ "epoch": 2.9696969696969697,
+ "grad_norm": 1.9169429540634155,
+ "learning_rate": 2.154882154882155e-07,
+ "loss": 0.4926488995552063,
+ "mean_token_accuracy": 0.8597337603569031,
+ "num_tokens": 24084480.0,
+ "step": 1470
+ },
+ {
+ "entropy": 1.157039999961853,
+ "epoch": 2.971717171717172,
+ "grad_norm": 2.274075508117676,
+ "learning_rate": 2.0202020202020205e-07,
+ "loss": 1.1713600158691406,
+ "mean_token_accuracy": 0.7217879891395569,
+ "num_tokens": 24100864.0,
+ "step": 1471
+ },
+ {
+ "entropy": 1.3693535327911377,
+ "epoch": 2.973737373737374,
+ "grad_norm": 2.233780860900879,
+ "learning_rate": 1.8855218855218856e-07,
+ "loss": 1.3571951389312744,
+ "mean_token_accuracy": 0.6933317184448242,
+ "num_tokens": 24117248.0,
+ "step": 1472
+ },
+ {
+ "entropy": 1.060738444328308,
+ "epoch": 2.9757575757575756,
+ "grad_norm": 2.376091480255127,
+ "learning_rate": 1.750841750841751e-07,
+ "loss": 1.0911407470703125,
+ "mean_token_accuracy": 0.7194064259529114,
+ "num_tokens": 24133632.0,
+ "step": 1473
+ },
+ {
+ "entropy": 1.0141303539276123,
+ "epoch": 2.977777777777778,
+ "grad_norm": 2.206777811050415,
+ "learning_rate": 1.6161616161616163e-07,
+ "loss": 1.0147366523742676,
+ "mean_token_accuracy": 0.7421225905418396,
+ "num_tokens": 24150016.0,
+ "step": 1474
+ },
+ {
+ "entropy": 1.094582438468933,
+ "epoch": 2.9797979797979797,
+ "grad_norm": 2.13834810256958,
+ "learning_rate": 1.4814814814814817e-07,
+ "loss": 1.0652586221694946,
+ "mean_token_accuracy": 0.7409012913703918,
+ "num_tokens": 24166400.0,
+ "step": 1475
+ },
+ {
+ "entropy": 0.9652472138404846,
+ "epoch": 2.981818181818182,
+ "grad_norm": 2.1181185245513916,
+ "learning_rate": 1.3468013468013468e-07,
+ "loss": 0.9842501282691956,
+ "mean_token_accuracy": 0.7498167753219604,
+ "num_tokens": 24182784.0,
+ "step": 1476
+ },
+ {
+ "entropy": 0.9012734293937683,
+ "epoch": 2.9838383838383837,
+ "grad_norm": 2.388531446456909,
+ "learning_rate": 1.2121212121212122e-07,
+ "loss": 0.9103577136993408,
+ "mean_token_accuracy": 0.7572667598724365,
+ "num_tokens": 24199168.0,
+ "step": 1477
+ },
+ {
+ "entropy": 1.0186536312103271,
+ "epoch": 2.985858585858586,
+ "grad_norm": 2.272202730178833,
+ "learning_rate": 1.0774410774410776e-07,
+ "loss": 0.9883908629417419,
+ "mean_token_accuracy": 0.7520151734352112,
+ "num_tokens": 24215552.0,
+ "step": 1478
+ },
+ {
+ "entropy": 1.0810787677764893,
+ "epoch": 2.987878787878788,
+ "grad_norm": 2.195042848587036,
+ "learning_rate": 9.427609427609428e-08,
+ "loss": 1.0988293886184692,
+ "mean_token_accuracy": 0.7214215993881226,
+ "num_tokens": 24231936.0,
+ "step": 1479
+ },
+ {
+ "entropy": 0.8350635170936584,
+ "epoch": 2.98989898989899,
+ "grad_norm": 1.9644523859024048,
+ "learning_rate": 8.080808080808082e-08,
+ "loss": 0.8452527523040771,
+ "mean_token_accuracy": 0.7926233410835266,
+ "num_tokens": 24248320.0,
+ "step": 1480
+ },
+ {
+ "entropy": 0.8076480627059937,
+ "epoch": 2.991919191919192,
+ "grad_norm": 2.1587746143341064,
+ "learning_rate": 6.734006734006734e-08,
+ "loss": 0.8024610280990601,
+ "mean_token_accuracy": 0.7873107194900513,
+ "num_tokens": 24264704.0,
+ "step": 1481
+ },
+ {
+ "entropy": 0.8234610557556152,
+ "epoch": 2.993939393939394,
+ "grad_norm": 2.1396384239196777,
+ "learning_rate": 5.387205387205388e-08,
+ "loss": 0.8239790201187134,
+ "mean_token_accuracy": 0.7902418375015259,
+ "num_tokens": 24281088.0,
+ "step": 1482
+ },
+ {
+ "entropy": 1.0003139972686768,
+ "epoch": 2.995959595959596,
+ "grad_norm": 2.406714677810669,
+ "learning_rate": 4.040404040404041e-08,
+ "loss": 0.9654958844184875,
+ "mean_token_accuracy": 0.7531143426895142,
+ "num_tokens": 24297472.0,
+ "step": 1483
+ },
+ {
+ "entropy": 1.1883150339126587,
+ "epoch": 2.9979797979797977,
+ "grad_norm": 2.3837242126464844,
+ "learning_rate": 2.693602693602694e-08,
+ "loss": 1.2012369632720947,
+ "mean_token_accuracy": 0.7122007608413696,
+ "num_tokens": 24313856.0,
+ "step": 1484
+ },
+ {
+ "entropy": 1.122470498085022,
+ "epoch": 3.0,
+ "grad_norm": 2.012483596801758,
+ "learning_rate": 1.346801346801347e-08,
+ "loss": 1.1389365196228027,
+ "mean_token_accuracy": 0.7195896506309509,
+ "num_tokens": 24330240.0,
+ "step": 1485
+ },
+ {
+ "epoch": 3.0,
+ "eval_entropy": 1.132785740879274,
+ "eval_loss": 1.6079809665679932,
+ "eval_mean_token_accuracy": 0.6431884972318527,
+ "eval_num_tokens": 24330240.0,
+ "eval_runtime": 43.8459,
+ "eval_samples_per_second": 22.579,
+ "eval_steps_per_second": 2.828,
+ "step": 1485
+ }
+ ],
+ "logging_steps": 1,
+ "max_steps": 1485,
+ "num_input_tokens_seen": 0,
+ "num_train_epochs": 3,
+ "save_steps": 500,
+ "stateful_callbacks": {
+ "TrainerControl": {
+ "args": {
+ "should_epoch_stop": false,
+ "should_evaluate": false,
+ "should_log": false,
+ "should_save": true,
+ "should_training_stop": true
+ },
+ "attributes": {}
+ }
+ },
+ "total_flos": 2.0574770835750912e+17,
+ "train_batch_size": 4,
+ "trial_name": null,
+ "trial_params": null
+}
diff --git a/qwen3-1.7b-teutonic-2e5/checkpoint-1485/training_args.bin b/qwen3-1.7b-teutonic-2e5/checkpoint-1485/training_args.bin
new file mode 100644
index 0000000000000000000000000000000000000000..98c558883023185245bfac817f2ca585caa9d94f
--- /dev/null
+++ b/qwen3-1.7b-teutonic-2e5/checkpoint-1485/training_args.bin
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:2c573bdac95c796f8f7368ab9af6d35c5687f53373737ea965b8102d518df7cb
+size 7121
diff --git a/qwen3-1.7b-teutonic-2e5/checkpoint-1485/zero_to_fp32.py b/qwen3-1.7b-teutonic-2e5/checkpoint-1485/zero_to_fp32.py
new file mode 100644
index 0000000000000000000000000000000000000000..3970548c400fd4361bd923b763db90e963ddaf8c
--- /dev/null
+++ b/qwen3-1.7b-teutonic-2e5/checkpoint-1485/zero_to_fp32.py
@@ -0,0 +1,854 @@
+#!/usr/bin/env python
+
+# Copyright (c) Microsoft Corporation.
+# SPDX-License-Identifier: Apache-2.0
+
+# DeepSpeed Team
+
+# This script extracts fp32 consolidated weights from a zero 1, 2 and 3 DeepSpeed checkpoints. It gets
+# copied into the top level checkpoint dir, so the user can easily do the conversion at any point in
+# the future. Once extracted, the weights don't require DeepSpeed and can be used in any
+# application.
+#
+# example:
+# python zero_to_fp32.py . output_dir/
+# or
+# python zero_to_fp32.py . output_dir/ --safe_serialization
+
+import argparse
+import torch
+import glob
+import math
+import os
+import re
+import gc
+import json
+import numpy as np
+from tqdm import tqdm
+from collections import OrderedDict
+from dataclasses import dataclass
+
+# while this script doesn't use deepspeed to recover data, since the checkpoints are pickled with
+# DeepSpeed data structures it has to be available in the current python environment.
+from deepspeed.utils import logger
+from deepspeed.checkpoint.constants import (DS_VERSION, OPTIMIZER_STATE_DICT, SINGLE_PARTITION_OF_FP32_GROUPS,
+ FP32_FLAT_GROUPS, ZERO_STAGE, PARTITION_COUNT, PARAM_SHAPES, BUFFER_NAMES,
+ FROZEN_PARAM_SHAPES, FROZEN_PARAM_FRAGMENTS, AUTOEP_LAYERS_KEY,
+ AUTOEP_LAYERS_KEY_LEGACY, AUTOEP_ZERO3_EXPERT_STATE_FORMAT_KEY,
+ AUTOEP_ZERO3_PARTITIONED_EXPERT_STATE_FORMAT, PARAM_ALIGNMENT_PADDINGS)
+
+
+@dataclass
+class zero_model_state:
+ buffers: dict()
+ param_shapes: dict()
+ shared_params: list
+ ds_version: int
+ frozen_param_shapes: dict()
+ frozen_param_fragments: dict()
+
+
+debug = 0
+
+# load to cpu
+device = torch.device('cpu')
+
+OUTPUT_DTYPE_NAMES = {
+ 'float32': torch.float32,
+ 'fp32': torch.float32,
+ 'float16': torch.float16,
+ 'fp16': torch.float16,
+ 'bfloat16': torch.bfloat16,
+ 'bf16': torch.bfloat16,
+}
+
+
+def _resolve_output_dtype(dtype):
+ requested_dtype = dtype
+ if isinstance(dtype, str):
+ dtype_name = dtype[6:] if dtype.startswith('torch.') else dtype
+ dtype = OUTPUT_DTYPE_NAMES.get(dtype_name.lower())
+ if dtype not in set(OUTPUT_DTYPE_NAMES.values()):
+ supported = ', '.join(sorted(OUTPUT_DTYPE_NAMES))
+ raise ValueError(f"Unsupported output dtype {requested_dtype!r}. Choose one of: {supported}")
+ return dtype
+
+
+def atoi(text):
+ return int(text) if text.isdigit() else text
+
+
+def natural_keys(text):
+ '''
+ alist.sort(key=natural_keys) sorts in human order
+ http://nedbatchelder.com/blog/200712/human_sorting.html
+ (See Toothy's implementation in the comments)
+ '''
+ return [atoi(c) for c in re.split(r'(\d+)', text)]
+
+
+def get_model_state_file(checkpoint_dir, zero_stage):
+ if not os.path.isdir(checkpoint_dir):
+ raise FileNotFoundError(f"Directory '{checkpoint_dir}' doesn't exist")
+
+ # there should be only one file
+ if zero_stage <= 2:
+ file = os.path.join(checkpoint_dir, "mp_rank_00_model_states.pt")
+ elif zero_stage == 3:
+ file = os.path.join(checkpoint_dir, "zero_pp_rank_0_mp_rank_00_model_states.pt")
+
+ if not os.path.exists(file):
+ raise FileNotFoundError(f"can't find model states file at '{file}'")
+
+ return file
+
+
+def get_checkpoint_files(checkpoint_dir, glob_pattern):
+ # XXX: need to test that this simple glob rule works for multi-node setup too
+ ckpt_files = sorted(glob.glob(os.path.join(checkpoint_dir, glob_pattern)), key=natural_keys)
+
+ if len(ckpt_files) == 0:
+ raise FileNotFoundError(f"can't find {glob_pattern} files in directory '{checkpoint_dir}'")
+
+ return ckpt_files
+
+
+def get_optim_files(checkpoint_dir):
+ return get_checkpoint_files(checkpoint_dir, "*_optim_states.pt")
+
+
+def get_model_state_files(checkpoint_dir):
+ return get_checkpoint_files(checkpoint_dir, "*_model_states.pt")
+
+
+def _has_autoep_zero3_partitioned_metadata(state_dict):
+ autoep_layers = state_dict.get(AUTOEP_LAYERS_KEY)
+ if autoep_layers is None:
+ autoep_layers = state_dict.get(AUTOEP_LAYERS_KEY_LEGACY)
+ if not isinstance(autoep_layers, list):
+ return False
+ return any(
+ isinstance(entry, dict)
+ and entry.get(AUTOEP_ZERO3_EXPERT_STATE_FORMAT_KEY) == AUTOEP_ZERO3_PARTITIONED_EXPERT_STATE_FORMAT
+ for entry in autoep_layers)
+
+
+def _raise_if_autoep_zero3_partitioned_state(state_dict):
+ if _has_autoep_zero3_partitioned_metadata(state_dict):
+ raise NotImplementedError("zero_to_fp32 does not support AutoEP ZeRO-3 partition-native checkpoints. "
+ "AutoEP expert parameters are partitioned over expert replica groups, so "
+ "global data-parallel consolidation would produce incomplete expert tensors. "
+ "Use ds_to_universal.py for expert-aware conversion.")
+
+
+def _raise_if_autoep_zero3_partitioned_checkpoint(model_files):
+ for file in model_files:
+ state_dict = torch.load(file, map_location=device, weights_only=False)
+ _raise_if_autoep_zero3_partitioned_state(state_dict)
+
+
+def parse_model_states(files):
+ zero_model_states = []
+ for file in files:
+ state_dict = torch.load(file, map_location=device, weights_only=False)
+ _raise_if_autoep_zero3_partitioned_state(state_dict)
+
+ if BUFFER_NAMES not in state_dict:
+ raise ValueError(f"{file} is not a model state checkpoint")
+ buffer_names = state_dict[BUFFER_NAMES]
+ if debug:
+ print("Found buffers:", buffer_names)
+
+ # recover just the buffers while restoring them to fp32 if they were saved in fp16
+ buffers = {k: v.float() for k, v in state_dict["module"].items() if k in buffer_names}
+ param_shapes = state_dict[PARAM_SHAPES]
+
+ # collect parameters that are included in param_shapes
+ param_names = []
+ for s in param_shapes:
+ for name in s.keys():
+ param_names.append(name)
+
+ # update with frozen parameters
+ frozen_param_shapes = state_dict.get(FROZEN_PARAM_SHAPES, None)
+ if frozen_param_shapes is not None:
+ if debug:
+ print(f"Found frozen_param_shapes: {frozen_param_shapes}")
+ param_names += list(frozen_param_shapes.keys())
+
+ # handle shared params
+ shared_params = [[k, v] for k, v in state_dict["shared_params"].items()]
+
+ ds_version = state_dict.get(DS_VERSION, None)
+
+ frozen_param_fragments = state_dict.get(FROZEN_PARAM_FRAGMENTS, None)
+
+ z_model_state = zero_model_state(buffers=buffers,
+ param_shapes=param_shapes,
+ shared_params=shared_params,
+ ds_version=ds_version,
+ frozen_param_shapes=frozen_param_shapes,
+ frozen_param_fragments=frozen_param_fragments)
+ zero_model_states.append(z_model_state)
+
+ return zero_model_states
+
+
+def parse_optim_states(files, ds_checkpoint_dir):
+ total_files = len(files)
+ state_dicts = []
+ for f in tqdm(files, desc='Loading checkpoint shards'):
+ state_dict = torch.load(f, map_location=device, mmap=True, weights_only=False)
+ # immediately discard the potentially huge 2 optimizer states as we only care for fp32 master weights
+ # and also handle the case where it was already removed by another helper script
+ state_dict["optimizer_state_dict"].pop("optimizer_state_dict", None)
+ state_dicts.append(state_dict)
+
+ if ZERO_STAGE not in state_dicts[0][OPTIMIZER_STATE_DICT]:
+ raise ValueError(f"{files[0]} is not a zero checkpoint")
+ zero_stage = state_dicts[0][OPTIMIZER_STATE_DICT][ZERO_STAGE]
+ world_size = state_dicts[0][OPTIMIZER_STATE_DICT][PARTITION_COUNT]
+
+ # For ZeRO-2 each param group can have different partition_count as data parallelism for expert
+ # parameters can be different from data parallelism for non-expert parameters. So we can just
+ # use the max of the partition_count to get the dp world_size.
+
+ if type(world_size) is list:
+ world_size = max(world_size)
+
+ if world_size != total_files:
+ raise ValueError(
+ f"Expected {world_size} of '*_optim_states.pt' under '{ds_checkpoint_dir}' but found {total_files} files. "
+ "Possibly due to an overwrite of an old checkpoint, or a checkpoint didn't get saved by one or more processes."
+ )
+
+ # the groups are named differently in each stage
+ if zero_stage <= 2:
+ fp32_groups_key = SINGLE_PARTITION_OF_FP32_GROUPS
+ elif zero_stage == 3:
+ fp32_groups_key = FP32_FLAT_GROUPS
+ else:
+ raise ValueError(f"unknown zero stage {zero_stage}")
+
+ fp32_flat_groups = [state_dicts[i][OPTIMIZER_STATE_DICT][fp32_groups_key] for i in range(len(state_dicts))]
+ param_alignment_paddings = state_dicts[0][OPTIMIZER_STATE_DICT].get(PARAM_ALIGNMENT_PADDINGS)
+ return zero_stage, world_size, fp32_flat_groups, param_alignment_paddings
+
+
+def _get_fp32_state_dict_from_zero_checkpoint(ds_checkpoint_dir, exclude_frozen_parameters):
+ """
+ Returns fp32 state_dict reconstructed from ds checkpoint
+
+ Args:
+ - ``ds_checkpoint_dir``: path to the deepspeed checkpoint folder (where the optimizer files are)
+
+ """
+ print(f"Processing zero checkpoint '{ds_checkpoint_dir}'")
+
+ # parse_model_states rejects AutoEP ZeRO-3 partition-native checkpoints
+ # before the expensive optimizer-shard load below.
+ model_files = get_model_state_files(ds_checkpoint_dir)
+ zero_model_states = parse_model_states(model_files)
+ print(f'Parsing checkpoint created by deepspeed=={zero_model_states[0].ds_version}')
+
+ optim_files = get_optim_files(ds_checkpoint_dir)
+ zero_stage, world_size, fp32_flat_groups, param_alignment_paddings = parse_optim_states(
+ optim_files, ds_checkpoint_dir)
+ print(f"Detected checkpoint of type zero stage {zero_stage}, world_size: {world_size}")
+
+ if zero_stage <= 2:
+ return _get_fp32_state_dict_from_zero2_checkpoint(world_size, fp32_flat_groups, zero_model_states,
+ exclude_frozen_parameters, param_alignment_paddings)
+ elif zero_stage == 3:
+ return _get_fp32_state_dict_from_zero3_checkpoint(world_size, fp32_flat_groups, zero_model_states,
+ exclude_frozen_parameters)
+
+
+def _zero2_merge_frozen_params(state_dict, zero_model_states):
+ if zero_model_states[0].frozen_param_shapes is None or len(zero_model_states[0].frozen_param_shapes) == 0:
+ return
+
+ frozen_param_shapes = zero_model_states[0].frozen_param_shapes
+ frozen_param_fragments = zero_model_states[0].frozen_param_fragments
+
+ if debug:
+ num_elem = sum(s.numel() for s in frozen_param_shapes.values())
+ print(f'rank 0: {FROZEN_PARAM_SHAPES}.numel = {num_elem}')
+
+ wanted_params = len(frozen_param_shapes)
+ wanted_numel = sum(s.numel() for s in frozen_param_shapes.values())
+ avail_numel = sum([p.numel() for p in frozen_param_fragments.values()])
+ print(f'Frozen params: Have {avail_numel} numels to process.')
+ print(f'Frozen params: Need {wanted_numel} numels in {wanted_params} params')
+
+ total_params = 0
+ total_numel = 0
+ for name, shape in frozen_param_shapes.items():
+ total_params += 1
+ unpartitioned_numel = shape.numel()
+ total_numel += unpartitioned_numel
+
+ state_dict[name] = frozen_param_fragments[name]
+
+ if debug:
+ print(f"{name} full shape: {shape} unpartitioned numel {unpartitioned_numel} ")
+
+ print(f"Reconstructed Frozen fp32 state dict with {total_params} params {total_numel} elements")
+
+
+def _has_callable(obj, fn):
+ attr = getattr(obj, fn, None)
+ return callable(attr)
+
+
+def _zero2_merge_trainable_params(state_dict,
+ world_size,
+ fp32_flat_groups,
+ zero_model_states,
+ param_alignment_paddings=None):
+ param_shapes = zero_model_states[0].param_shapes
+
+ # Reconstruction protocol:
+ #
+ # XXX: document this
+
+ if debug:
+ for i in range(world_size):
+ for j in range(len(fp32_flat_groups[0])):
+ print(f"{FP32_FLAT_GROUPS}[{i}][{j}].shape={fp32_flat_groups[i][j].shape}")
+
+ # XXX: memory usage doubles here (zero2)
+ num_param_groups = len(fp32_flat_groups[0])
+ merged_single_partition_of_fp32_groups = []
+ for i in range(num_param_groups):
+ merged_partitions = [sd[i] for sd in fp32_flat_groups]
+ full_single_fp32_vector = torch.cat(merged_partitions, 0)
+ merged_single_partition_of_fp32_groups.append(full_single_fp32_vector)
+ avail_numel = sum(
+ [full_single_fp32_vector.numel() for full_single_fp32_vector in merged_single_partition_of_fp32_groups])
+
+ if debug:
+ wanted_params = sum([len(shapes) for shapes in param_shapes])
+ wanted_numel = sum([sum(shape.numel() for shape in shapes.values()) for shapes in param_shapes])
+ # not asserting if there is a mismatch due to possible padding
+ print(f"Have {avail_numel} numels to process.")
+ print(f"Need {wanted_numel} numels in {wanted_params} params.")
+
+ # params
+ # XXX: for huge models that can't fit into the host's RAM we will have to recode this to support
+ # out-of-core computing solution
+ total_numel = 0
+ total_params = 0
+ for group_idx, (shapes,
+ full_single_fp32_vector) in enumerate(zip(param_shapes, merged_single_partition_of_fp32_groups)):
+ offset = 0
+ avail_numel = full_single_fp32_vector.numel()
+ group_alignment_paddings = None
+ if param_alignment_paddings is not None:
+ group_alignment_paddings = param_alignment_paddings[group_idx]
+ if len(group_alignment_paddings) != len(shapes):
+ raise ValueError(f"Expected {len(shapes)} parameter alignment paddings for group {group_idx}, "
+ f"but found {len(group_alignment_paddings)}")
+
+ for param_idx, (name, shape) in enumerate(shapes.items()):
+
+ unpartitioned_numel = shape.numel() if _has_callable(shape, 'numel') else math.prod(shape)
+ total_numel += unpartitioned_numel
+ total_params += 1
+
+ if debug:
+ print(f"{name} full shape: {shape} unpartitioned numel {unpartitioned_numel} ")
+ state_dict[name] = full_single_fp32_vector.narrow(0, offset, unpartitioned_numel).view(shape)
+ offset += unpartitioned_numel
+ if group_alignment_paddings is not None:
+ offset += group_alignment_paddings[param_idx]
+
+ # Z2 started to align to 2*world_size to improve nccl performance. Therefore both offset and
+ # avail_numel can differ by anywhere between 0..2*world_size. Due to two unrelated complex
+ # paddings performed in the code it's almost impossible to predict the exact numbers w/o the
+ # live optimizer object, so we are checking that the numbers are within the right range
+ align_to = 2 * world_size
+
+ def zero2_align(x):
+ return align_to * math.ceil(x / align_to)
+
+ if debug:
+ print(f"original offset={offset}, avail_numel={avail_numel}")
+
+ offset = zero2_align(offset)
+ avail_numel = zero2_align(avail_numel)
+
+ if debug:
+ print(f"aligned offset={offset}, avail_numel={avail_numel}")
+
+ # Sanity check
+ if offset != avail_numel:
+ raise ValueError(f"consumed {offset} numels out of {avail_numel} - something is wrong")
+
+ print(f"Reconstructed fp32 state dict with {total_params} params {total_numel} elements")
+
+
+def _get_fp32_state_dict_from_zero2_checkpoint(world_size,
+ fp32_flat_groups,
+ zero_model_states,
+ exclude_frozen_parameters,
+ param_alignment_paddings=None):
+ state_dict = OrderedDict()
+
+ # buffers
+ buffers = zero_model_states[0].buffers
+ state_dict.update(buffers)
+ if debug:
+ print(f"added {len(buffers)} buffers")
+
+ if not exclude_frozen_parameters:
+ _zero2_merge_frozen_params(state_dict, zero_model_states)
+
+ _zero2_merge_trainable_params(state_dict, world_size, fp32_flat_groups, zero_model_states,
+ param_alignment_paddings)
+
+ # recover shared parameters
+ for pair in zero_model_states[0].shared_params:
+ if pair[1] in state_dict:
+ state_dict[pair[0]] = state_dict[pair[1]]
+
+ return state_dict
+
+
+def zero3_partitioned_param_info(unpartitioned_numel, world_size):
+ remainder = unpartitioned_numel % world_size
+ padding_numel = (world_size - remainder) if remainder else 0
+ partitioned_numel = math.ceil(unpartitioned_numel / world_size)
+ return partitioned_numel, padding_numel
+
+
+def _zero3_merge_frozen_params(state_dict, world_size, zero_model_states):
+ if zero_model_states[0].frozen_param_shapes is None or len(zero_model_states[0].frozen_param_shapes) == 0:
+ return
+
+ if debug:
+ for i in range(world_size):
+ num_elem = sum(s.numel() for s in zero_model_states[i].frozen_param_fragments.values())
+ print(f'rank {i}: {FROZEN_PARAM_SHAPES}.numel = {num_elem}')
+
+ frozen_param_shapes = zero_model_states[0].frozen_param_shapes
+ wanted_params = len(frozen_param_shapes)
+ wanted_numel = sum(s.numel() for s in frozen_param_shapes.values())
+ avail_numel = sum([p.numel() for p in zero_model_states[0].frozen_param_fragments.values()]) * world_size
+ print(f'Frozen params: Have {avail_numel} numels to process.')
+ print(f'Frozen params: Need {wanted_numel} numels in {wanted_params} params')
+
+ total_params = 0
+ total_numel = 0
+ for name, shape in zero_model_states[0].frozen_param_shapes.items():
+ total_params += 1
+ unpartitioned_numel = shape.numel()
+ total_numel += unpartitioned_numel
+
+ param_frags = tuple(model_state.frozen_param_fragments[name] for model_state in zero_model_states)
+ state_dict[name] = torch.cat(param_frags, 0).narrow(0, 0, unpartitioned_numel).view(shape)
+
+ partitioned_numel, partitioned_padding_numel = zero3_partitioned_param_info(unpartitioned_numel, world_size)
+
+ if debug:
+ print(
+ f"Frozen params: {total_params} {name} full shape: {shape} partition0 numel={partitioned_numel} partitioned_padding_numel={partitioned_padding_numel}"
+ )
+
+ print(f"Reconstructed Frozen fp32 state dict with {total_params} params {total_numel} elements")
+
+
+class GatheredTensor:
+ """
+ A pseudo tensor that collects partitioned weights.
+ It is more memory efficient when there are multiple groups.
+ """
+
+ def __init__(self, flat_groups, flat_groups_offset, offset, partitioned_numel, shape):
+ self.flat_groups = flat_groups
+ self.flat_groups_offset = flat_groups_offset
+ self.offset = offset
+ self.partitioned_numel = partitioned_numel
+ self.shape = shape
+ self.dtype = self.flat_groups[0][0].dtype
+
+ def contiguous(self):
+ """
+ Merge partitioned weights from flat_groups into a single tensor.
+ """
+ end_idx = self.offset + self.partitioned_numel
+ world_size = len(self.flat_groups)
+ pad_flat_param_chunks = []
+
+ for rank_i in range(world_size):
+ # for each rank, we need to collect weights from related group/groups
+ flat_groups_at_rank_i = self.flat_groups[rank_i]
+ start_group_id = None
+ end_group_id = None
+ for group_id in range(len(self.flat_groups_offset)):
+ if self.flat_groups_offset[group_id] <= self.offset < self.flat_groups_offset[group_id + 1]:
+ start_group_id = group_id
+ if self.flat_groups_offset[group_id] < end_idx <= self.flat_groups_offset[group_id + 1]:
+ end_group_id = group_id
+ break
+ # collect weights from related group/groups
+ for group_id in range(start_group_id, end_group_id + 1):
+ flat_tensor = flat_groups_at_rank_i[group_id]
+ start_offset = self.offset - self.flat_groups_offset[group_id]
+ end_offset = min(end_idx, self.flat_groups_offset[group_id + 1]) - self.flat_groups_offset[group_id]
+ pad_flat_param_chunks.append(flat_tensor[start_offset:end_offset])
+
+ # collect weights from all ranks
+ pad_flat_param = torch.cat(pad_flat_param_chunks, dim=0)
+ param = pad_flat_param[:self.shape.numel()].view(self.shape).contiguous()
+ return param
+
+
+def _zero3_merge_trainable_params(state_dict, world_size, fp32_flat_groups, zero_model_states):
+ param_shapes = zero_model_states[0].param_shapes
+ avail_numel = sum([flat_group.numel() for flat_group in fp32_flat_groups[0]]) * world_size
+
+ # Reconstruction protocol: For zero3 we need to zip the partitions together at boundary of each
+ # param, re-consolidating each param, while dealing with padding if any
+
+ # merge list of dicts, preserving order
+ param_shapes = {k: v for d in param_shapes for k, v in d.items()}
+
+ if debug:
+ for i in range(world_size):
+ print(f"{FP32_FLAT_GROUPS}[{i}].shape={fp32_flat_groups[i].shape}")
+
+ wanted_params = len(param_shapes)
+ wanted_numel = sum(shape.numel() for shape in param_shapes.values())
+ # not asserting if there is a mismatch due to possible padding
+ avail_numel = fp32_flat_groups[0].numel() * world_size
+ print(f"Trainable params: Have {avail_numel} numels to process.")
+ print(f"Trainable params: Need {wanted_numel} numels in {wanted_params} params.")
+
+ # params
+ # XXX: for huge models that can't fit into the host's RAM we will have to recode this to support
+ # out-of-core computing solution
+ offset = 0
+ total_numel = 0
+ total_params = 0
+ flat_groups_offset = [0] + list(np.cumsum([flat_tensor.numel() for flat_tensor in fp32_flat_groups[0]]))
+ for name, shape in tqdm(param_shapes.items(), desc='Gathering sharded weights'):
+ unpartitioned_numel = shape.numel()
+ total_numel += unpartitioned_numel
+ total_params += 1
+ partitioned_numel, partitioned_padding_numel = zero3_partitioned_param_info(unpartitioned_numel, world_size)
+
+ if debug:
+ print(
+ f"Trainable params: {total_params} {name} full shape: {shape} partition0 numel={partitioned_numel} partitioned_padding_numel={partitioned_padding_numel}"
+ )
+
+ # memory efficient tensor
+ tensor = GatheredTensor(fp32_flat_groups, flat_groups_offset, offset, partitioned_numel, shape)
+ state_dict[name] = tensor
+ offset += partitioned_numel
+
+ offset *= world_size
+
+ # Sanity check
+ if offset != avail_numel:
+ raise ValueError(f"consumed {offset} numels out of {avail_numel} - something is wrong")
+
+ print(f"Reconstructed Trainable fp32 state dict with {total_params} params {total_numel} elements")
+
+
+def _get_fp32_state_dict_from_zero3_checkpoint(world_size, fp32_flat_groups, zero_model_states,
+ exclude_frozen_parameters):
+ state_dict = OrderedDict()
+
+ # buffers
+ buffers = zero_model_states[0].buffers
+ state_dict.update(buffers)
+ if debug:
+ print(f"added {len(buffers)} buffers")
+
+ if not exclude_frozen_parameters:
+ _zero3_merge_frozen_params(state_dict, world_size, zero_model_states)
+
+ _zero3_merge_trainable_params(state_dict, world_size, fp32_flat_groups, zero_model_states)
+
+ # recover shared parameters
+ for pair in zero_model_states[0].shared_params:
+ if pair[1] in state_dict:
+ state_dict[pair[0]] = state_dict[pair[1]]
+
+ return state_dict
+
+
+def to_torch_tensor(state_dict, return_empty_tensor=False, dtype=None):
+ """
+ Convert state_dict of GatheredTensor to torch tensor
+ """
+ if dtype is not None:
+ dtype = _resolve_output_dtype(dtype)
+
+ torch_state_dict = {}
+ converted_tensors = {}
+ for name, tensor in state_dict.items():
+ tensor_id = id(tensor)
+ if tensor_id in converted_tensors: # shared tensors
+ shared_tensor = torch_state_dict[converted_tensors[tensor_id]]
+ torch_state_dict[name] = shared_tensor
+ else:
+ converted_tensors[tensor_id] = name
+ if return_empty_tensor:
+ torch_state_dict[name] = torch.empty(tensor.shape, dtype=dtype or tensor.dtype)
+ else:
+ contiguous_tensor = tensor.contiguous()
+ torch_state_dict[name] = contiguous_tensor.to(dtype=dtype) if dtype else contiguous_tensor
+ return torch_state_dict
+
+
+def get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir,
+ tag=None,
+ exclude_frozen_parameters=False,
+ lazy_mode=False):
+ """
+ Convert ZeRO 2 or 3 checkpoint into a single fp32 consolidated state_dict that can be loaded with
+ ``load_state_dict()`` and used for training without DeepSpeed or shared with others, for example
+ via a model hub.
+
+ Args:
+ - ``checkpoint_dir``: path to the desired checkpoint folder
+ - ``tag``: checkpoint tag used as a unique identifier for checkpoint. If not provided will attempt to load tag in 'latest' file. e.g., ``global_step14``
+ - ``exclude_frozen_parameters``: exclude frozen parameters
+ - ``lazy_mode``: get state_dict in lazy mode. It returns a dict of pesduo tensor instead of torch tensor, which is more memory efficient.
+ Convert the pesduo tensor to torch tensor by ``.contiguous()``
+
+ Returns:
+ - pytorch ``state_dict``
+
+ A typical usage might be ::
+
+ from deepspeed.utils.zero_to_fp32 import get_fp32_state_dict_from_zero_checkpoint
+ # do the training and checkpoint saving
+ state_dict = get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir) # already on cpu
+ model = model.cpu() # move to cpu
+ model.load_state_dict(state_dict)
+ # submit to model hub or save the model to share with others
+
+ In this example the ``model`` will no longer be usable in the deepspeed context of the same
+ application. i.e. you will need to re-initialize the deepspeed engine, since
+ ``model.load_state_dict(state_dict)`` will remove all the deepspeed magic from it.
+
+ If you want it all done for you, use ``load_state_dict_from_zero_checkpoint`` instead.
+
+ Note: the above usage may not work if your application doesn't have sufficient free CPU memory.
+ You may need to use the offline approach using the ``zero_to_fp32.py`` script that is saved with
+ the checkpoint. Or you can load state_dict in lazy mode ::
+
+ from deepspeed.utils.zero_to_fp32 import get_fp32_state_dict_from_zero_checkpoint
+ state_dict = get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir, lazy_mode=True) # not on cpu
+ for name, lazy_tensor in state_dict.item():
+ tensor = lazy_tensor.contiguous() # to cpu
+ print(name, tensor)
+ # del tensor to release memory if it no longer in use
+ """
+ if tag is None:
+ latest_path = os.path.join(checkpoint_dir, 'latest')
+ if os.path.isfile(latest_path):
+ with open(latest_path, 'r') as fd:
+ tag = fd.read().strip()
+ else:
+ raise ValueError(f"Unable to find 'latest' file at {latest_path}")
+
+ ds_checkpoint_dir = os.path.join(checkpoint_dir, tag)
+
+ if not os.path.isdir(ds_checkpoint_dir):
+ raise FileNotFoundError(f"Directory '{ds_checkpoint_dir}' doesn't exist")
+
+ state_dict = _get_fp32_state_dict_from_zero_checkpoint(ds_checkpoint_dir, exclude_frozen_parameters)
+ if lazy_mode:
+ return state_dict
+ else:
+ return to_torch_tensor(state_dict)
+
+
+def convert_zero_checkpoint_to_state_dict(checkpoint_dir,
+ output_dir,
+ dtype=torch.float32,
+ max_shard_size="5GB",
+ safe_serialization=False,
+ tag=None,
+ exclude_frozen_parameters=False):
+ """
+ Convert ZeRO 2 or 3 checkpoint into a consolidated ``state_dict`` file that can be
+ loaded with ``torch.load(file)`` + ``load_state_dict()`` and used for training without DeepSpeed.
+
+ Args:
+ - ``checkpoint_dir``: path to the desired checkpoint folder. (one that contains the tag-folder, like ``global_step14``)
+ - ``output_dir``: directory for the PyTorch state_dict output files
+ - ``dtype``: output tensor dtype. Supports float32, float16, and bfloat16 as strings or torch dtypes.
+ - ``max_shard_size``: the maximum size for a checkpoint before being sharded, default value is 5GB
+ - ``safe_serialization``: whether to save the model using `safetensors` or the traditional PyTorch way (that uses `pickle`).
+ - ``tag``: checkpoint tag used as a unique identifier for checkpoint. If not provided will attempt to load tag in the file named ``latest`` in the checkpoint folder, e.g., ``global_step14``
+ - ``exclude_frozen_parameters``: exclude frozen parameters
+ """
+
+ dtype = _resolve_output_dtype(dtype)
+
+ # Dependency pre-check
+ if safe_serialization:
+ try:
+ from safetensors.torch import save_file
+ except ImportError:
+ print('If you want to use `safe_serialization`, please `pip install safetensors`')
+ raise
+ if max_shard_size is not None:
+ try:
+ from huggingface_hub import split_torch_state_dict_into_shards
+ except ImportError:
+ print('If you want to use `max_shard_size`, please `pip install huggingface_hub`')
+ raise
+
+ # Convert zero checkpoint to state_dict
+ state_dict = get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir,
+ tag,
+ exclude_frozen_parameters,
+ lazy_mode=True)
+
+ # Shard the model if it is too big.
+ weights_name = "model.safetensors" if safe_serialization else "pytorch_model.bin"
+ if max_shard_size is not None:
+ filename_pattern = weights_name.replace(".bin", "{suffix}.bin").replace(".safetensors", "{suffix}.safetensors")
+ # an memory-efficient approach for sharding
+ empty_state_dict = to_torch_tensor(state_dict, return_empty_tensor=True, dtype=dtype)
+ state_dict_split = split_torch_state_dict_into_shards(empty_state_dict,
+ filename_pattern=filename_pattern,
+ max_shard_size=max_shard_size)
+ else:
+ from collections import namedtuple
+ StateDictSplit = namedtuple("StateDictSplit", ["is_sharded", "filename_to_tensors"])
+ state_dict_split = StateDictSplit(is_sharded=False,
+ filename_to_tensors={weights_name: list(state_dict.keys())})
+
+ # Save the model by shard
+ os.makedirs(output_dir, exist_ok=True)
+ filename_to_tensors = state_dict_split.filename_to_tensors.items()
+ for shard_file, tensors in tqdm(filename_to_tensors, desc="Saving checkpoint shards"):
+ shard_state_dict = {tensor_name: state_dict[tensor_name] for tensor_name in tensors}
+ shard_state_dict = to_torch_tensor(shard_state_dict, dtype=dtype)
+ output_path = os.path.join(output_dir, shard_file)
+ if safe_serialization:
+ save_file(shard_state_dict, output_path, metadata={"format": "pt"})
+ else:
+ torch.save(shard_state_dict, output_path)
+ # release the memory of current shard
+ for tensor_name in list(shard_state_dict.keys()):
+ del state_dict[tensor_name]
+ del shard_state_dict[tensor_name]
+ del shard_state_dict
+ gc.collect()
+
+ # Save index if sharded
+ if state_dict_split.is_sharded:
+ index = {
+ "metadata": state_dict_split.metadata,
+ "weight_map": state_dict_split.tensor_to_filename,
+ }
+ save_index_file = "model.safetensors.index.json" if safe_serialization else "pytorch_model.bin.index.json"
+ save_index_file = os.path.join(output_dir, save_index_file)
+ with open(save_index_file, "w", encoding="utf-8") as f:
+ content = json.dumps(index, indent=2, sort_keys=True) + "\n"
+ f.write(content)
+
+
+def convert_zero_checkpoint_to_fp32_state_dict(checkpoint_dir,
+ output_dir,
+ max_shard_size="5GB",
+ safe_serialization=False,
+ tag=None,
+ exclude_frozen_parameters=False):
+ """Backward-compatible fp32 checkpoint conversion."""
+ return convert_zero_checkpoint_to_state_dict(checkpoint_dir,
+ output_dir,
+ dtype=torch.float32,
+ max_shard_size=max_shard_size,
+ safe_serialization=safe_serialization,
+ tag=tag,
+ exclude_frozen_parameters=exclude_frozen_parameters)
+
+
+def load_state_dict_from_zero_checkpoint(model, checkpoint_dir, tag=None):
+ """
+ 1. Put the provided model to cpu
+ 2. Convert ZeRO 2 or 3 checkpoint into a single fp32 consolidated ``state_dict``
+ 3. Load it into the provided model
+
+ Args:
+ - ``model``: the model object to update
+ - ``checkpoint_dir``: path to the desired checkpoint folder. (one that contains the tag-folder, like ``global_step14``)
+ - ``tag``: checkpoint tag used as a unique identifier for checkpoint. If not provided will attempt to load tag in the file named ``latest`` in the checkpoint folder, e.g., ``global_step14``
+
+ Returns:
+ - ``model`: modified model
+
+ Make sure you have plenty of CPU memory available before you call this function. If you don't
+ have enough use the ``zero_to_fp32.py`` utility to do the conversion. You will find it
+ conveniently placed for you in the checkpoint folder.
+
+ A typical usage might be ::
+
+ from deepspeed.utils.zero_to_fp32 import load_state_dict_from_zero_checkpoint
+ model = load_state_dict_from_zero_checkpoint(trainer.model, checkpoint_dir)
+ # submit to model hub or save the model to share with others
+
+ Note, that once this was run, the ``model`` will no longer be usable in the deepspeed context
+ of the same application. i.e. you will need to re-initialize the deepspeed engine, since
+ ``model.load_state_dict(state_dict)`` will remove all the deepspeed magic from it.
+
+ """
+ logger.info("Extracting fp32 weights")
+ state_dict = get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir, tag)
+
+ logger.info("Overwriting model with fp32 weights")
+ model = model.cpu()
+ model.load_state_dict(state_dict, strict=False)
+
+ return model
+
+
+if __name__ == "__main__":
+ parser = argparse.ArgumentParser()
+ parser.add_argument("checkpoint_dir",
+ type=str,
+ help="path to the desired checkpoint folder, e.g., path/checkpoint-12")
+ parser.add_argument("output_dir",
+ type=str,
+ help="directory to the pytorch fp32 state_dict output files"
+ "(e.g. path/checkpoint-12-output/)")
+ parser.add_argument(
+ "--max_shard_size",
+ type=str,
+ default="5GB",
+ help="The maximum size for a checkpoint before being sharded. Checkpoints shard will then be each of size"
+ "lower than this size. If expressed as a string, needs to be digits followed by a unit (like `5MB`"
+ "We default it to 5GB in order for models to be able to run easily on free-tier google colab instances"
+ "without CPU OOM issues.")
+ parser.add_argument(
+ "--safe_serialization",
+ default=False,
+ action='store_true',
+ help="Whether to save the model using `safetensors` or the traditional PyTorch way (that uses `pickle`).")
+ parser.add_argument("-t",
+ "--tag",
+ type=str,
+ default=None,
+ help="checkpoint tag used as a unique identifier for checkpoint. e.g., global_step1")
+ parser.add_argument("--exclude_frozen_parameters", action='store_true', help="exclude frozen parameters")
+ parser.add_argument("-d", "--debug", action='store_true', help="enable debug")
+ args = parser.parse_args()
+
+ debug = args.debug
+
+ convert_zero_checkpoint_to_fp32_state_dict(args.checkpoint_dir,
+ args.output_dir,
+ max_shard_size=args.max_shard_size,
+ safe_serialization=args.safe_serialization,
+ tag=args.tag,
+ exclude_frozen_parameters=args.exclude_frozen_parameters)
diff --git a/qwen3-1.7b-teutonic-2e5/checkpoint-1485/zero_to_torch.py b/qwen3-1.7b-teutonic-2e5/checkpoint-1485/zero_to_torch.py
new file mode 100644
index 0000000000000000000000000000000000000000..81312e252400d77f03cc1a88522f8f18ebe70ee7
--- /dev/null
+++ b/qwen3-1.7b-teutonic-2e5/checkpoint-1485/zero_to_torch.py
@@ -0,0 +1,54 @@
+#!/usr/bin/env python
+
+# SPDX-License-Identifier: Apache-2.0
+# DeepSpeed Team
+
+import argparse
+
+if __package__:
+ from . import zero_to_fp32
+else:
+ import zero_to_fp32
+
+
+def main(args=None):
+ parser = argparse.ArgumentParser(
+ description="Convert a DeepSpeed ZeRO checkpoint to float32, float16, or bfloat16 PyTorch weights.")
+ parser.add_argument("checkpoint_dir",
+ type=str,
+ help="path to the desired checkpoint folder, e.g., path/checkpoint-12")
+ parser.add_argument("output_dir", type=str, help="directory for the converted PyTorch state_dict files")
+ parser.add_argument("--dtype",
+ type=str,
+ choices=sorted(zero_to_fp32.OUTPUT_DTYPE_NAMES),
+ required=True,
+ help="output tensor dtype")
+ parser.add_argument("--max_shard_size",
+ type=str,
+ default="5GB",
+ help="maximum size of each checkpoint shard, such as 5GB or 500MB")
+ parser.add_argument("--safe_serialization",
+ default=False,
+ action='store_true',
+ help="save with safetensors instead of PyTorch pickle serialization")
+ parser.add_argument("-t",
+ "--tag",
+ type=str,
+ default=None,
+ help="checkpoint tag used as a unique identifier, e.g., global_step1")
+ parser.add_argument("--exclude_frozen_parameters", action='store_true', help="exclude frozen parameters")
+ parser.add_argument("-d", "--debug", action='store_true', help="enable debug output")
+ parsed_args = parser.parse_args(args)
+
+ zero_to_fp32.debug = parsed_args.debug
+ zero_to_fp32.convert_zero_checkpoint_to_state_dict(parsed_args.checkpoint_dir,
+ parsed_args.output_dir,
+ dtype=parsed_args.dtype,
+ max_shard_size=parsed_args.max_shard_size,
+ safe_serialization=parsed_args.safe_serialization,
+ tag=parsed_args.tag,
+ exclude_frozen_parameters=parsed_args.exclude_frozen_parameters)
+
+
+if __name__ == "__main__":
+ main()
diff --git a/qwen3-1.7b-teutonic-2e5/checkpoint-500/chat_template.jinja b/qwen3-1.7b-teutonic-2e5/checkpoint-500/chat_template.jinja
new file mode 100644
index 0000000000000000000000000000000000000000..01be9b307daa2d425f7c168c9fb145a286e0afb4
--- /dev/null
+++ b/qwen3-1.7b-teutonic-2e5/checkpoint-500/chat_template.jinja
@@ -0,0 +1,89 @@
+{%- if tools %}
+ {{- '<|im_start|>system\n' }}
+ {%- if messages[0].role == 'system' %}
+ {{- messages[0].content + '\n\n' }}
+ {%- endif %}
+ {{- "# Tools\n\nYou may call one or more functions to assist with the user query.\n\nYou are provided with function signatures within XML tags:\n" }}
+ {%- for tool in tools %}
+ {{- "\n" }}
+ {{- tool | tojson }}
+ {%- endfor %}
+ {{- "\n\n\nFor each function call, return a json object with function name and arguments within XML tags:\n\n{\"name\": , \"arguments\": }\n<|im_end|>\n" }}
+{%- else %}
+ {%- if messages[0].role == 'system' %}
+ {{- '<|im_start|>system\n' + messages[0].content + '<|im_end|>\n' }}
+ {%- endif %}
+{%- endif %}
+{%- set ns = namespace(multi_step_tool=true, last_query_index=messages|length - 1) %}
+{%- for message in messages[::-1] %}
+ {%- set index = (messages|length - 1) - loop.index0 %}
+ {%- if ns.multi_step_tool and message.role == "user" and message.content is string and not(message.content.startswith('') and message.content.endswith('')) %}
+ {%- set ns.multi_step_tool = false %}
+ {%- set ns.last_query_index = index %}
+ {%- endif %}
+{%- endfor %}
+{%- for message in messages %}
+ {%- if message.content is string %}
+ {%- set content = message.content %}
+ {%- else %}
+ {%- set content = '' %}
+ {%- endif %}
+ {%- if (message.role == "user") or (message.role == "system" and not loop.first) %}
+ {{- '<|im_start|>' + message.role + '\n' + content + '<|im_end|>' + '\n' }}
+ {%- elif message.role == "assistant" %}
+ {%- set reasoning_content = '' %}
+ {%- if message.reasoning_content is string %}
+ {%- set reasoning_content = message.reasoning_content %}
+ {%- else %}
+ {%- if '' in content %}
+ {%- set reasoning_content = content.split('')[0].rstrip('\n').split('')[-1].lstrip('\n') %}
+ {%- set content = content.split('')[-1].lstrip('\n') %}
+ {%- endif %}
+ {%- endif %}
+ {%- if loop.index0 > ns.last_query_index %}
+ {%- if loop.last or (not loop.last and reasoning_content) %}
+ {{- '<|im_start|>' + message.role + '\n\n' + reasoning_content.strip('\n') + '\n\n\n' + content.lstrip('\n') }}
+ {%- else %}
+ {{- '<|im_start|>' + message.role + '\n' + content }}
+ {%- endif %}
+ {%- else %}
+ {{- '<|im_start|>' + message.role + '\n' + content }}
+ {%- endif %}
+ {%- if message.tool_calls %}
+ {%- for tool_call in message.tool_calls %}
+ {%- if (loop.first and content) or (not loop.first) %}
+ {{- '\n' }}
+ {%- endif %}
+ {%- if tool_call.function %}
+ {%- set tool_call = tool_call.function %}
+ {%- endif %}
+ {{- '\n{"name": "' }}
+ {{- tool_call.name }}
+ {{- '", "arguments": ' }}
+ {%- if tool_call.arguments is string %}
+ {{- tool_call.arguments }}
+ {%- else %}
+ {{- tool_call.arguments | tojson }}
+ {%- endif %}
+ {{- '}\n' }}
+ {%- endfor %}
+ {%- endif %}
+ {{- '<|im_end|>\n' }}
+ {%- elif message.role == "tool" %}
+ {%- if loop.first or (messages[loop.index0 - 1].role != "tool") %}
+ {{- '<|im_start|>user' }}
+ {%- endif %}
+ {{- '\n\n' }}
+ {{- content }}
+ {{- '\n' }}
+ {%- if loop.last or (messages[loop.index0 + 1].role != "tool") %}
+ {{- '<|im_end|>\n' }}
+ {%- endif %}
+ {%- endif %}
+{%- endfor %}
+{%- if add_generation_prompt %}
+ {{- '<|im_start|>assistant\n' }}
+ {%- if enable_thinking is defined and enable_thinking is false %}
+ {{- '\n\n\n\n' }}
+ {%- endif %}
+{%- endif %}
\ No newline at end of file
diff --git a/qwen3-1.7b-teutonic-2e5/checkpoint-500/config.json b/qwen3-1.7b-teutonic-2e5/checkpoint-500/config.json
new file mode 100644
index 0000000000000000000000000000000000000000..5d9cef07396baadff5390e4508eb33025967a029
--- /dev/null
+++ b/qwen3-1.7b-teutonic-2e5/checkpoint-500/config.json
@@ -0,0 +1,63 @@
+{
+ "architectures": [
+ "Qwen3ForCausalLM"
+ ],
+ "attention_bias": false,
+ "attention_dropout": 0.0,
+ "bos_token_id": null,
+ "dtype": "bfloat16",
+ "eos_token_id": 151645,
+ "head_dim": 128,
+ "hidden_act": "silu",
+ "hidden_size": 2048,
+ "initializer_range": 0.02,
+ "intermediate_size": 6144,
+ "layer_types": [
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention"
+ ],
+ "max_position_embeddings": 40960,
+ "max_window_layers": 28,
+ "model_type": "qwen3",
+ "num_attention_heads": 16,
+ "num_hidden_layers": 28,
+ "num_key_value_heads": 8,
+ "pad_token_id": 151643,
+ "rms_norm_eps": 1e-06,
+ "rope_parameters": {
+ "rope_theta": 1000000,
+ "rope_type": "default"
+ },
+ "sliding_window": null,
+ "tie_word_embeddings": true,
+ "transformers_version": "5.17.0",
+ "use_cache": false,
+ "use_sliding_window": false,
+ "vocab_size": 151936
+}
diff --git a/qwen3-1.7b-teutonic-2e5/checkpoint-500/generation_config.json b/qwen3-1.7b-teutonic-2e5/checkpoint-500/generation_config.json
new file mode 100644
index 0000000000000000000000000000000000000000..e6941fe4b04f26113d6eef6255d005c4d7090bda
--- /dev/null
+++ b/qwen3-1.7b-teutonic-2e5/checkpoint-500/generation_config.json
@@ -0,0 +1,12 @@
+{
+ "do_sample": true,
+ "eos_token_id": [
+ 151645,
+ 151643
+ ],
+ "pad_token_id": 151643,
+ "temperature": 0.6,
+ "top_k": 20,
+ "top_p": 0.95,
+ "transformers_version": "5.17.0"
+}
diff --git a/qwen3-1.7b-teutonic-2e5/checkpoint-500/global_step500/bf16_zero_pp_rank_0_mp_rank_00_optim_states.pt b/qwen3-1.7b-teutonic-2e5/checkpoint-500/global_step500/bf16_zero_pp_rank_0_mp_rank_00_optim_states.pt
new file mode 100644
index 0000000000000000000000000000000000000000..81580ef63b233cd2e8c6659d7354b75fa7253257
--- /dev/null
+++ b/qwen3-1.7b-teutonic-2e5/checkpoint-500/global_step500/bf16_zero_pp_rank_0_mp_rank_00_optim_states.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:729d5481732775300f3678e55abd25181ac14249b2102bb960f3d12e0de8c1dd
+size 10323466465
diff --git a/qwen3-1.7b-teutonic-2e5/checkpoint-500/global_step500/bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt b/qwen3-1.7b-teutonic-2e5/checkpoint-500/global_step500/bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt
new file mode 100644
index 0000000000000000000000000000000000000000..48bdbe4cb43bdeb1dba841f0790f1c6e38b7c6e9
--- /dev/null
+++ b/qwen3-1.7b-teutonic-2e5/checkpoint-500/global_step500/bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:42810e16642c209d4e06c04f15001786db4ee9f99a890dfd7da8da2ad0a68cc1
+size 10323469601
diff --git a/qwen3-1.7b-teutonic-2e5/checkpoint-500/global_step500/mp_rank_00_model_states.pt b/qwen3-1.7b-teutonic-2e5/checkpoint-500/global_step500/mp_rank_00_model_states.pt
new file mode 100644
index 0000000000000000000000000000000000000000..3b1e5d5215246770a372fb9a56e2246f26125c47
--- /dev/null
+++ b/qwen3-1.7b-teutonic-2e5/checkpoint-500/global_step500/mp_rank_00_model_states.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:8e3a2e69f57fd2c64c50cd6430722e31c6c0aa5e4bd3bae2ee2afd9aaefba424
+size 3441239653
diff --git a/qwen3-1.7b-teutonic-2e5/checkpoint-500/latest b/qwen3-1.7b-teutonic-2e5/checkpoint-500/latest
new file mode 100644
index 0000000000000000000000000000000000000000..f0b47ce15fff9a01b2a416a473b2148085048a50
--- /dev/null
+++ b/qwen3-1.7b-teutonic-2e5/checkpoint-500/latest
@@ -0,0 +1 @@
+global_step500
\ No newline at end of file
diff --git a/qwen3-1.7b-teutonic-2e5/checkpoint-500/model.safetensors b/qwen3-1.7b-teutonic-2e5/checkpoint-500/model.safetensors
new file mode 100644
index 0000000000000000000000000000000000000000..74eed8deee7f42bfea72bf4166ecaa1f93311588
--- /dev/null
+++ b/qwen3-1.7b-teutonic-2e5/checkpoint-500/model.safetensors
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:05699b742b2507f4eff84eafc1be8993c2ee2e467a5eaf97c9f506698cda6244
+size 4063515640
diff --git a/qwen3-1.7b-teutonic-2e5/checkpoint-500/rng_state_0.pth b/qwen3-1.7b-teutonic-2e5/checkpoint-500/rng_state_0.pth
new file mode 100644
index 0000000000000000000000000000000000000000..4216f98cda667c0eec56821cfba089005aed1288
--- /dev/null
+++ b/qwen3-1.7b-teutonic-2e5/checkpoint-500/rng_state_0.pth
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:958a6e27892fb703ed10a62b2ead762f94b60ec3bb4677c08b5468a57d8e339b
+size 14917
diff --git a/qwen3-1.7b-teutonic-2e5/checkpoint-500/rng_state_1.pth b/qwen3-1.7b-teutonic-2e5/checkpoint-500/rng_state_1.pth
new file mode 100644
index 0000000000000000000000000000000000000000..1a1c44d1b25f44ae0e4d1fb6be15190c4defe223
--- /dev/null
+++ b/qwen3-1.7b-teutonic-2e5/checkpoint-500/rng_state_1.pth
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:378c8813b6f7f592a17b751f070068187707a01baf668eb0520492c2223e28e8
+size 14917
diff --git a/qwen3-1.7b-teutonic-2e5/checkpoint-500/scheduler.pt b/qwen3-1.7b-teutonic-2e5/checkpoint-500/scheduler.pt
new file mode 100644
index 0000000000000000000000000000000000000000..882f340b4d52889e5a393164c8fafd815dad9612
--- /dev/null
+++ b/qwen3-1.7b-teutonic-2e5/checkpoint-500/scheduler.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:de1d54e3a77abffcb08b973a58f94dc815bdaee26ce117745a3a339bb5263473
+size 1465
diff --git a/qwen3-1.7b-teutonic-2e5/checkpoint-500/tokenizer.json b/qwen3-1.7b-teutonic-2e5/checkpoint-500/tokenizer.json
new file mode 100644
index 0000000000000000000000000000000000000000..c7afbed2efcdf019f88ab0572ec29d3bf595dfe2
--- /dev/null
+++ b/qwen3-1.7b-teutonic-2e5/checkpoint-500/tokenizer.json
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:be75606093db2094d7cd20f3c2f385c212750648bd6ea4fb2bf507a6a4c55506
+size 11422650
diff --git a/qwen3-1.7b-teutonic-2e5/checkpoint-500/tokenizer_config.json b/qwen3-1.7b-teutonic-2e5/checkpoint-500/tokenizer_config.json
new file mode 100644
index 0000000000000000000000000000000000000000..770e41d6c92519d525eede4cbcf3ba27f6425311
--- /dev/null
+++ b/qwen3-1.7b-teutonic-2e5/checkpoint-500/tokenizer_config.json
@@ -0,0 +1,30 @@
+{
+ "add_prefix_space": false,
+ "backend": "tokenizers",
+ "bos_token": null,
+ "clean_up_tokenization_spaces": false,
+ "eos_token": "<|im_end|>",
+ "errors": "replace",
+ "extra_special_tokens": [
+ "<|im_start|>",
+ "<|im_end|>",
+ "<|object_ref_start|>",
+ "<|object_ref_end|>",
+ "<|box_start|>",
+ "<|box_end|>",
+ "<|quad_start|>",
+ "<|quad_end|>",
+ "<|vision_start|>",
+ "<|vision_end|>",
+ "<|vision_pad|>",
+ "<|image_pad|>",
+ "<|video_pad|>"
+ ],
+ "is_local": false,
+ "local_files_only": false,
+ "model_max_length": 131072,
+ "pad_token": "<|endoftext|>",
+ "split_special_tokens": false,
+ "tokenizer_class": "Qwen2Tokenizer",
+ "unk_token": null
+}
diff --git a/qwen3-1.7b-teutonic-2e5/checkpoint-500/trainer_state.json b/qwen3-1.7b-teutonic-2e5/checkpoint-500/trainer_state.json
new file mode 100644
index 0000000000000000000000000000000000000000..e510d11d3f65389d124f92af1e87715f8bb1ef96
--- /dev/null
+++ b/qwen3-1.7b-teutonic-2e5/checkpoint-500/trainer_state.json
@@ -0,0 +1,5089 @@
+{
+ "best_global_step": 400,
+ "best_metric": 1.5199862718582153,
+ "best_model_checkpoint": null,
+ "epoch": 1.0101010101010102,
+ "eval_steps": 100,
+ "global_step": 500,
+ "is_hyper_param_search": false,
+ "is_local_process_zero": true,
+ "is_world_process_zero": true,
+ "log_history": [
+ {
+ "entropy": 0.8295683860778809,
+ "epoch": 0.00202020202020202,
+ "grad_norm": 10.885663032531738,
+ "learning_rate": 2e-05,
+ "loss": 1.5185801982879639,
+ "mean_token_accuracy": 0.672874927520752,
+ "num_tokens": 16384.0,
+ "step": 1
+ },
+ {
+ "entropy": 1.3341505527496338,
+ "epoch": 0.00404040404040404,
+ "grad_norm": 9.276060104370117,
+ "learning_rate": 1.9986531986531986e-05,
+ "loss": 2.0802578926086426,
+ "mean_token_accuracy": 0.5727283954620361,
+ "num_tokens": 32768.0,
+ "step": 2
+ },
+ {
+ "entropy": 1.159377932548523,
+ "epoch": 0.006060606060606061,
+ "grad_norm": 3.999565839767456,
+ "learning_rate": 1.9973063973063975e-05,
+ "loss": 1.4128143787384033,
+ "mean_token_accuracy": 0.6849657893180847,
+ "num_tokens": 49152.0,
+ "step": 3
+ },
+ {
+ "entropy": 1.504081130027771,
+ "epoch": 0.00808080808080808,
+ "grad_norm": 3.018310546875,
+ "learning_rate": 1.995959595959596e-05,
+ "loss": 1.5479036569595337,
+ "mean_token_accuracy": 0.647777259349823,
+ "num_tokens": 65536.0,
+ "step": 4
+ },
+ {
+ "entropy": 2.1211423873901367,
+ "epoch": 0.010101010101010102,
+ "grad_norm": 4.352237701416016,
+ "learning_rate": 1.9946127946127948e-05,
+ "loss": 1.7965757846832275,
+ "mean_token_accuracy": 0.5992916226387024,
+ "num_tokens": 81920.0,
+ "step": 5
+ },
+ {
+ "entropy": 1.6345494985580444,
+ "epoch": 0.012121212121212121,
+ "grad_norm": 3.2360711097717285,
+ "learning_rate": 1.9932659932659936e-05,
+ "loss": 1.4024577140808105,
+ "mean_token_accuracy": 0.6747679710388184,
+ "num_tokens": 98304.0,
+ "step": 6
+ },
+ {
+ "entropy": 2.105585813522339,
+ "epoch": 0.014141414141414142,
+ "grad_norm": 2.84971284866333,
+ "learning_rate": 1.991919191919192e-05,
+ "loss": 1.9915739297866821,
+ "mean_token_accuracy": 0.5882999300956726,
+ "num_tokens": 114688.0,
+ "step": 7
+ },
+ {
+ "entropy": 1.7230671644210815,
+ "epoch": 0.01616161616161616,
+ "grad_norm": 2.686546802520752,
+ "learning_rate": 1.990572390572391e-05,
+ "loss": 1.630468726158142,
+ "mean_token_accuracy": 0.6269540786743164,
+ "num_tokens": 131072.0,
+ "step": 8
+ },
+ {
+ "entropy": 2.048288106918335,
+ "epoch": 0.01818181818181818,
+ "grad_norm": 2.6931567192077637,
+ "learning_rate": 1.9892255892255894e-05,
+ "loss": 2.212648391723633,
+ "mean_token_accuracy": 0.5294333100318909,
+ "num_tokens": 147456.0,
+ "step": 9
+ },
+ {
+ "entropy": 1.3021576404571533,
+ "epoch": 0.020202020202020204,
+ "grad_norm": 2.5286052227020264,
+ "learning_rate": 1.9878787878787878e-05,
+ "loss": 1.4357099533081055,
+ "mean_token_accuracy": 0.6633487939834595,
+ "num_tokens": 163840.0,
+ "step": 10
+ },
+ {
+ "entropy": 1.6998594999313354,
+ "epoch": 0.022222222222222223,
+ "grad_norm": 2.673692226409912,
+ "learning_rate": 1.9865319865319866e-05,
+ "loss": 1.8616552352905273,
+ "mean_token_accuracy": 0.6129091382026672,
+ "num_tokens": 180224.0,
+ "step": 11
+ },
+ {
+ "entropy": 1.5349891185760498,
+ "epoch": 0.024242424242424242,
+ "grad_norm": 2.882964611053467,
+ "learning_rate": 1.9851851851851855e-05,
+ "loss": 1.7540476322174072,
+ "mean_token_accuracy": 0.6111993193626404,
+ "num_tokens": 196608.0,
+ "step": 12
+ },
+ {
+ "entropy": 1.4202098846435547,
+ "epoch": 0.026262626262626262,
+ "grad_norm": 2.3892722129821777,
+ "learning_rate": 1.983838383838384e-05,
+ "loss": 1.5638885498046875,
+ "mean_token_accuracy": 0.642892062664032,
+ "num_tokens": 212992.0,
+ "step": 13
+ },
+ {
+ "entropy": 1.462074875831604,
+ "epoch": 0.028282828282828285,
+ "grad_norm": 2.357353925704956,
+ "learning_rate": 1.9824915824915828e-05,
+ "loss": 1.5339899063110352,
+ "mean_token_accuracy": 0.643991231918335,
+ "num_tokens": 229376.0,
+ "step": 14
+ },
+ {
+ "entropy": 1.6386674642562866,
+ "epoch": 0.030303030303030304,
+ "grad_norm": 2.1600515842437744,
+ "learning_rate": 1.9811447811447812e-05,
+ "loss": 1.6242434978485107,
+ "mean_token_accuracy": 0.6472887396812439,
+ "num_tokens": 245760.0,
+ "step": 15
+ },
+ {
+ "entropy": 1.882306456565857,
+ "epoch": 0.03232323232323232,
+ "grad_norm": 2.478746175765991,
+ "learning_rate": 1.97979797979798e-05,
+ "loss": 1.83274245262146,
+ "mean_token_accuracy": 0.5993527173995972,
+ "num_tokens": 262144.0,
+ "step": 16
+ },
+ {
+ "entropy": 1.5836528539657593,
+ "epoch": 0.03434343434343434,
+ "grad_norm": 2.5016613006591797,
+ "learning_rate": 1.9784511784511785e-05,
+ "loss": 1.5364878177642822,
+ "mean_token_accuracy": 0.6441743969917297,
+ "num_tokens": 278528.0,
+ "step": 17
+ },
+ {
+ "entropy": 1.5695534944534302,
+ "epoch": 0.03636363636363636,
+ "grad_norm": 2.2971248626708984,
+ "learning_rate": 1.9771043771043774e-05,
+ "loss": 1.5179738998413086,
+ "mean_token_accuracy": 0.6437469720840454,
+ "num_tokens": 294912.0,
+ "step": 18
+ },
+ {
+ "entropy": 1.3835400342941284,
+ "epoch": 0.03838383838383838,
+ "grad_norm": 2.1497576236724854,
+ "learning_rate": 1.975757575757576e-05,
+ "loss": 1.3505406379699707,
+ "mean_token_accuracy": 0.6729970574378967,
+ "num_tokens": 311296.0,
+ "step": 19
+ },
+ {
+ "entropy": 1.4660464525222778,
+ "epoch": 0.04040404040404041,
+ "grad_norm": 2.048318862915039,
+ "learning_rate": 1.9744107744107747e-05,
+ "loss": 1.3898100852966309,
+ "mean_token_accuracy": 0.6654250025749207,
+ "num_tokens": 327680.0,
+ "step": 20
+ },
+ {
+ "entropy": 1.959633469581604,
+ "epoch": 0.04242424242424243,
+ "grad_norm": 2.2084245681762695,
+ "learning_rate": 1.973063973063973e-05,
+ "loss": 1.9230711460113525,
+ "mean_token_accuracy": 0.5726062655448914,
+ "num_tokens": 344064.0,
+ "step": 21
+ },
+ {
+ "entropy": 1.4872093200683594,
+ "epoch": 0.044444444444444446,
+ "grad_norm": 2.116982936859131,
+ "learning_rate": 1.971717171717172e-05,
+ "loss": 1.514385461807251,
+ "mean_token_accuracy": 0.6615168452262878,
+ "num_tokens": 360448.0,
+ "step": 22
+ },
+ {
+ "entropy": 1.845888614654541,
+ "epoch": 0.046464646464646465,
+ "grad_norm": 2.4377331733703613,
+ "learning_rate": 1.9703703703703704e-05,
+ "loss": 1.8750195503234863,
+ "mean_token_accuracy": 0.5962994694709778,
+ "num_tokens": 376832.0,
+ "step": 23
+ },
+ {
+ "entropy": 1.5549575090408325,
+ "epoch": 0.048484848484848485,
+ "grad_norm": 2.44534969329834,
+ "learning_rate": 1.9690235690235692e-05,
+ "loss": 1.627150058746338,
+ "mean_token_accuracy": 0.6215803623199463,
+ "num_tokens": 393216.0,
+ "step": 24
+ },
+ {
+ "entropy": 1.4516701698303223,
+ "epoch": 0.050505050505050504,
+ "grad_norm": 2.2948875427246094,
+ "learning_rate": 1.9676767676767677e-05,
+ "loss": 1.4912647008895874,
+ "mean_token_accuracy": 0.6431363224983215,
+ "num_tokens": 409600.0,
+ "step": 25
+ },
+ {
+ "entropy": 1.3493973016738892,
+ "epoch": 0.052525252525252523,
+ "grad_norm": 2.1061222553253174,
+ "learning_rate": 1.9663299663299665e-05,
+ "loss": 1.44138765335083,
+ "mean_token_accuracy": 0.6621885895729065,
+ "num_tokens": 425984.0,
+ "step": 26
+ },
+ {
+ "entropy": 1.7119438648223877,
+ "epoch": 0.05454545454545454,
+ "grad_norm": 2.5149970054626465,
+ "learning_rate": 1.9649831649831654e-05,
+ "loss": 1.776426076889038,
+ "mean_token_accuracy": 0.5944675207138062,
+ "num_tokens": 442368.0,
+ "step": 27
+ },
+ {
+ "entropy": 1.361685872077942,
+ "epoch": 0.05656565656565657,
+ "grad_norm": 2.343843460083008,
+ "learning_rate": 1.963636363636364e-05,
+ "loss": 1.404232144355774,
+ "mean_token_accuracy": 0.6639594435691833,
+ "num_tokens": 458752.0,
+ "step": 28
+ },
+ {
+ "entropy": 1.2213996648788452,
+ "epoch": 0.05858585858585859,
+ "grad_norm": 1.9002994298934937,
+ "learning_rate": 1.9622895622895623e-05,
+ "loss": 1.249969244003296,
+ "mean_token_accuracy": 0.6933927536010742,
+ "num_tokens": 475136.0,
+ "step": 29
+ },
+ {
+ "entropy": 1.7157055139541626,
+ "epoch": 0.06060606060606061,
+ "grad_norm": 2.21567702293396,
+ "learning_rate": 1.960942760942761e-05,
+ "loss": 1.7103240489959717,
+ "mean_token_accuracy": 0.6218856573104858,
+ "num_tokens": 491520.0,
+ "step": 30
+ },
+ {
+ "entropy": 1.8233915567398071,
+ "epoch": 0.06262626262626263,
+ "grad_norm": 2.0189297199249268,
+ "learning_rate": 1.9595959595959596e-05,
+ "loss": 1.820570707321167,
+ "mean_token_accuracy": 0.6126648783683777,
+ "num_tokens": 507904.0,
+ "step": 31
+ },
+ {
+ "entropy": 1.82878839969635,
+ "epoch": 0.06464646464646465,
+ "grad_norm": 2.556058883666992,
+ "learning_rate": 1.9582491582491584e-05,
+ "loss": 1.773336410522461,
+ "mean_token_accuracy": 0.5889716744422913,
+ "num_tokens": 524288.0,
+ "step": 32
+ },
+ {
+ "entropy": 1.4608416557312012,
+ "epoch": 0.06666666666666667,
+ "grad_norm": 2.0254712104797363,
+ "learning_rate": 1.956902356902357e-05,
+ "loss": 1.4261068105697632,
+ "mean_token_accuracy": 0.6561431288719177,
+ "num_tokens": 540672.0,
+ "step": 33
+ },
+ {
+ "entropy": 1.4706993103027344,
+ "epoch": 0.06868686868686869,
+ "grad_norm": 2.1344153881073,
+ "learning_rate": 1.9555555555555557e-05,
+ "loss": 1.4610451459884644,
+ "mean_token_accuracy": 0.6472276449203491,
+ "num_tokens": 557056.0,
+ "step": 34
+ },
+ {
+ "entropy": 1.9629759788513184,
+ "epoch": 0.0707070707070707,
+ "grad_norm": 2.209073066711426,
+ "learning_rate": 1.9542087542087545e-05,
+ "loss": 1.9688678979873657,
+ "mean_token_accuracy": 0.5702857971191406,
+ "num_tokens": 573440.0,
+ "step": 35
+ },
+ {
+ "entropy": 1.5821412801742554,
+ "epoch": 0.07272727272727272,
+ "grad_norm": 1.9711852073669434,
+ "learning_rate": 1.952861952861953e-05,
+ "loss": 1.5614874362945557,
+ "mean_token_accuracy": 0.639167070388794,
+ "num_tokens": 589824.0,
+ "step": 36
+ },
+ {
+ "entropy": 1.4696686267852783,
+ "epoch": 0.07474747474747474,
+ "grad_norm": 2.07849383354187,
+ "learning_rate": 1.9515151515151515e-05,
+ "loss": 1.497229814529419,
+ "mean_token_accuracy": 0.6555324792861938,
+ "num_tokens": 606208.0,
+ "step": 37
+ },
+ {
+ "entropy": 1.481307864189148,
+ "epoch": 0.07676767676767676,
+ "grad_norm": 2.028839588165283,
+ "learning_rate": 1.9501683501683503e-05,
+ "loss": 1.495100975036621,
+ "mean_token_accuracy": 0.6472276449203491,
+ "num_tokens": 622592.0,
+ "step": 38
+ },
+ {
+ "entropy": 1.515073537826538,
+ "epoch": 0.07878787878787878,
+ "grad_norm": 2.2417402267456055,
+ "learning_rate": 1.9488215488215488e-05,
+ "loss": 1.6139262914657593,
+ "mean_token_accuracy": 0.6312286257743835,
+ "num_tokens": 638976.0,
+ "step": 39
+ },
+ {
+ "entropy": 1.9048941135406494,
+ "epoch": 0.08080808080808081,
+ "grad_norm": 2.232630968093872,
+ "learning_rate": 1.9474747474747476e-05,
+ "loss": 1.9573429822921753,
+ "mean_token_accuracy": 0.5796287059783936,
+ "num_tokens": 655360.0,
+ "step": 40
+ },
+ {
+ "entropy": 1.7393985986709595,
+ "epoch": 0.08282828282828283,
+ "grad_norm": 2.1077799797058105,
+ "learning_rate": 1.9461279461279464e-05,
+ "loss": 1.825897216796875,
+ "mean_token_accuracy": 0.5946506857872009,
+ "num_tokens": 671744.0,
+ "step": 41
+ },
+ {
+ "entropy": 1.6294668912887573,
+ "epoch": 0.08484848484848485,
+ "grad_norm": 2.299520492553711,
+ "learning_rate": 1.944781144781145e-05,
+ "loss": 1.6667912006378174,
+ "mean_token_accuracy": 0.6225574016571045,
+ "num_tokens": 688128.0,
+ "step": 42
+ },
+ {
+ "entropy": 1.4176290035247803,
+ "epoch": 0.08686868686868687,
+ "grad_norm": 1.9385138750076294,
+ "learning_rate": 1.9434343434343437e-05,
+ "loss": 1.422924280166626,
+ "mean_token_accuracy": 0.6719589829444885,
+ "num_tokens": 704512.0,
+ "step": 43
+ },
+ {
+ "entropy": 1.6570991277694702,
+ "epoch": 0.08888888888888889,
+ "grad_norm": 2.1650922298431396,
+ "learning_rate": 1.9420875420875422e-05,
+ "loss": 1.6351218223571777,
+ "mean_token_accuracy": 0.6342818737030029,
+ "num_tokens": 720896.0,
+ "step": 44
+ },
+ {
+ "entropy": 1.520472764968872,
+ "epoch": 0.09090909090909091,
+ "grad_norm": 2.116955280303955,
+ "learning_rate": 1.9407407407407407e-05,
+ "loss": 1.4788711071014404,
+ "mean_token_accuracy": 0.6557767391204834,
+ "num_tokens": 737280.0,
+ "step": 45
+ },
+ {
+ "entropy": 1.3090498447418213,
+ "epoch": 0.09292929292929293,
+ "grad_norm": 2.0493314266204834,
+ "learning_rate": 1.9393939393939395e-05,
+ "loss": 1.3372726440429688,
+ "mean_token_accuracy": 0.6762945652008057,
+ "num_tokens": 753664.0,
+ "step": 46
+ },
+ {
+ "entropy": 1.6449471712112427,
+ "epoch": 0.09494949494949495,
+ "grad_norm": 2.2450437545776367,
+ "learning_rate": 1.9380471380471383e-05,
+ "loss": 1.6683790683746338,
+ "mean_token_accuracy": 0.6266487836837769,
+ "num_tokens": 770048.0,
+ "step": 47
+ },
+ {
+ "entropy": 1.2516838312149048,
+ "epoch": 0.09696969696969697,
+ "grad_norm": 2.168203592300415,
+ "learning_rate": 1.9367003367003368e-05,
+ "loss": 1.2885866165161133,
+ "mean_token_accuracy": 0.689667820930481,
+ "num_tokens": 786432.0,
+ "step": 48
+ },
+ {
+ "entropy": 1.7858967781066895,
+ "epoch": 0.09898989898989899,
+ "grad_norm": 2.0490918159484863,
+ "learning_rate": 1.9353535353535356e-05,
+ "loss": 1.8125935792922974,
+ "mean_token_accuracy": 0.5957498550415039,
+ "num_tokens": 802816.0,
+ "step": 49
+ },
+ {
+ "entropy": 1.6833406686782837,
+ "epoch": 0.10101010101010101,
+ "grad_norm": 1.9622541666030884,
+ "learning_rate": 1.934006734006734e-05,
+ "loss": 1.715425729751587,
+ "mean_token_accuracy": 0.606008768081665,
+ "num_tokens": 819200.0,
+ "step": 50
+ },
+ {
+ "entropy": 1.4103842973709106,
+ "epoch": 0.10303030303030303,
+ "grad_norm": 2.015503406524658,
+ "learning_rate": 1.932659932659933e-05,
+ "loss": 1.4374600648880005,
+ "mean_token_accuracy": 0.6615168452262878,
+ "num_tokens": 835584.0,
+ "step": 51
+ },
+ {
+ "entropy": 1.6805938482284546,
+ "epoch": 0.10505050505050505,
+ "grad_norm": 2.139333963394165,
+ "learning_rate": 1.9313131313131314e-05,
+ "loss": 1.684693455696106,
+ "mean_token_accuracy": 0.6270151734352112,
+ "num_tokens": 851968.0,
+ "step": 52
+ },
+ {
+ "entropy": 1.770232915878296,
+ "epoch": 0.10707070707070707,
+ "grad_norm": 2.3331124782562256,
+ "learning_rate": 1.92996632996633e-05,
+ "loss": 1.781606674194336,
+ "mean_token_accuracy": 0.6006350517272949,
+ "num_tokens": 868352.0,
+ "step": 53
+ },
+ {
+ "entropy": 1.676267147064209,
+ "epoch": 0.10909090909090909,
+ "grad_norm": 2.1863765716552734,
+ "learning_rate": 1.9286195286195287e-05,
+ "loss": 1.6873016357421875,
+ "mean_token_accuracy": 0.6182828545570374,
+ "num_tokens": 884736.0,
+ "step": 54
+ },
+ {
+ "entropy": 1.3133333921432495,
+ "epoch": 0.1111111111111111,
+ "grad_norm": 2.455390691757202,
+ "learning_rate": 1.9272727272727275e-05,
+ "loss": 1.336955189704895,
+ "mean_token_accuracy": 0.682584285736084,
+ "num_tokens": 901120.0,
+ "step": 55
+ },
+ {
+ "entropy": 1.8064881563186646,
+ "epoch": 0.11313131313131314,
+ "grad_norm": 2.218869686126709,
+ "learning_rate": 1.925925925925926e-05,
+ "loss": 1.8163663148880005,
+ "mean_token_accuracy": 0.5947728157043457,
+ "num_tokens": 917504.0,
+ "step": 56
+ },
+ {
+ "entropy": 1.495352864265442,
+ "epoch": 0.11515151515151516,
+ "grad_norm": 2.040064573287964,
+ "learning_rate": 1.9245791245791248e-05,
+ "loss": 1.4638073444366455,
+ "mean_token_accuracy": 0.6512579321861267,
+ "num_tokens": 933888.0,
+ "step": 57
+ },
+ {
+ "entropy": 1.5757899284362793,
+ "epoch": 0.11717171717171718,
+ "grad_norm": 1.8449796438217163,
+ "learning_rate": 1.9232323232323233e-05,
+ "loss": 1.5526244640350342,
+ "mean_token_accuracy": 0.6346482634544373,
+ "num_tokens": 950272.0,
+ "step": 58
+ },
+ {
+ "entropy": 1.6834555864334106,
+ "epoch": 0.1191919191919192,
+ "grad_norm": 2.0705320835113525,
+ "learning_rate": 1.921885521885522e-05,
+ "loss": 1.753828763961792,
+ "mean_token_accuracy": 0.6237176060676575,
+ "num_tokens": 966656.0,
+ "step": 59
+ },
+ {
+ "entropy": 1.5231870412826538,
+ "epoch": 0.12121212121212122,
+ "grad_norm": 2.1398122310638428,
+ "learning_rate": 1.9205387205387206e-05,
+ "loss": 1.515265941619873,
+ "mean_token_accuracy": 0.6452125310897827,
+ "num_tokens": 983040.0,
+ "step": 60
+ },
+ {
+ "entropy": 1.6915754079818726,
+ "epoch": 0.12323232323232323,
+ "grad_norm": 2.179158926010132,
+ "learning_rate": 1.9191919191919194e-05,
+ "loss": 1.6949875354766846,
+ "mean_token_accuracy": 0.6109550595283508,
+ "num_tokens": 999424.0,
+ "step": 61
+ },
+ {
+ "entropy": 1.580087661743164,
+ "epoch": 0.12525252525252525,
+ "grad_norm": 2.8643219470977783,
+ "learning_rate": 1.9178451178451182e-05,
+ "loss": 1.6217050552368164,
+ "mean_token_accuracy": 0.6498534679412842,
+ "num_tokens": 1015808.0,
+ "step": 62
+ },
+ {
+ "entropy": 1.7331494092941284,
+ "epoch": 0.12727272727272726,
+ "grad_norm": 2.0613038539886475,
+ "learning_rate": 1.9164983164983167e-05,
+ "loss": 1.7527930736541748,
+ "mean_token_accuracy": 0.59513920545578,
+ "num_tokens": 1032192.0,
+ "step": 63
+ },
+ {
+ "entropy": 1.3311079740524292,
+ "epoch": 0.1292929292929293,
+ "grad_norm": 1.996009111404419,
+ "learning_rate": 1.9151515151515152e-05,
+ "loss": 1.3299833536148071,
+ "mean_token_accuracy": 0.6760503053665161,
+ "num_tokens": 1048576.0,
+ "step": 64
+ },
+ {
+ "entropy": 1.1875317096710205,
+ "epoch": 0.13131313131313133,
+ "grad_norm": 1.9458681344985962,
+ "learning_rate": 1.913804713804714e-05,
+ "loss": 1.1885290145874023,
+ "mean_token_accuracy": 0.6996213793754578,
+ "num_tokens": 1064960.0,
+ "step": 65
+ },
+ {
+ "entropy": 1.0825527906417847,
+ "epoch": 0.13333333333333333,
+ "grad_norm": 1.8164746761322021,
+ "learning_rate": 1.9124579124579125e-05,
+ "loss": 1.0918856859207153,
+ "mean_token_accuracy": 0.7477405667304993,
+ "num_tokens": 1081344.0,
+ "step": 66
+ },
+ {
+ "entropy": 1.5940357446670532,
+ "epoch": 0.13535353535353536,
+ "grad_norm": 1.9322525262832642,
+ "learning_rate": 1.9111111111111113e-05,
+ "loss": 1.6157622337341309,
+ "mean_token_accuracy": 0.6340376138687134,
+ "num_tokens": 1097728.0,
+ "step": 67
+ },
+ {
+ "entropy": 1.683976173400879,
+ "epoch": 0.13737373737373737,
+ "grad_norm": 2.1792430877685547,
+ "learning_rate": 1.9097643097643098e-05,
+ "loss": 1.7643585205078125,
+ "mean_token_accuracy": 0.5998412370681763,
+ "num_tokens": 1114112.0,
+ "step": 68
+ },
+ {
+ "entropy": 1.5753607749938965,
+ "epoch": 0.1393939393939394,
+ "grad_norm": 2.033538818359375,
+ "learning_rate": 1.9084175084175086e-05,
+ "loss": 1.628584861755371,
+ "mean_token_accuracy": 0.6401441097259521,
+ "num_tokens": 1130496.0,
+ "step": 69
+ },
+ {
+ "entropy": 1.4061329364776611,
+ "epoch": 0.1414141414141414,
+ "grad_norm": 1.9239107370376587,
+ "learning_rate": 1.9070707070707074e-05,
+ "loss": 1.4355204105377197,
+ "mean_token_accuracy": 0.6727527976036072,
+ "num_tokens": 1146880.0,
+ "step": 70
+ },
+ {
+ "entropy": 1.1722302436828613,
+ "epoch": 0.14343434343434344,
+ "grad_norm": 2.0911052227020264,
+ "learning_rate": 1.905723905723906e-05,
+ "loss": 1.2008472681045532,
+ "mean_token_accuracy": 0.7064606547355652,
+ "num_tokens": 1163264.0,
+ "step": 71
+ },
+ {
+ "entropy": 1.5291107892990112,
+ "epoch": 0.14545454545454545,
+ "grad_norm": 2.1054158210754395,
+ "learning_rate": 1.9043771043771044e-05,
+ "loss": 1.574873924255371,
+ "mean_token_accuracy": 0.6356253027915955,
+ "num_tokens": 1179648.0,
+ "step": 72
+ },
+ {
+ "entropy": 1.3679379224777222,
+ "epoch": 0.14747474747474748,
+ "grad_norm": 1.928666353225708,
+ "learning_rate": 1.9030303030303032e-05,
+ "loss": 1.3473236560821533,
+ "mean_token_accuracy": 0.679347813129425,
+ "num_tokens": 1196032.0,
+ "step": 73
+ },
+ {
+ "entropy": 1.7593064308166504,
+ "epoch": 0.1494949494949495,
+ "grad_norm": 2.3172972202301025,
+ "learning_rate": 1.9016835016835017e-05,
+ "loss": 1.7953016757965088,
+ "mean_token_accuracy": 0.6107107996940613,
+ "num_tokens": 1212416.0,
+ "step": 74
+ },
+ {
+ "entropy": 1.3841301202774048,
+ "epoch": 0.15151515151515152,
+ "grad_norm": 1.9938782453536987,
+ "learning_rate": 1.9003367003367005e-05,
+ "loss": 1.3923728466033936,
+ "mean_token_accuracy": 0.6686614751815796,
+ "num_tokens": 1228800.0,
+ "step": 75
+ },
+ {
+ "entropy": 1.6030631065368652,
+ "epoch": 0.15353535353535352,
+ "grad_norm": 2.0514845848083496,
+ "learning_rate": 1.8989898989898993e-05,
+ "loss": 1.5902166366577148,
+ "mean_token_accuracy": 0.6376404762268066,
+ "num_tokens": 1245184.0,
+ "step": 76
+ },
+ {
+ "entropy": 1.5902619361877441,
+ "epoch": 0.15555555555555556,
+ "grad_norm": 2.115469217300415,
+ "learning_rate": 1.8976430976430978e-05,
+ "loss": 1.5841395854949951,
+ "mean_token_accuracy": 0.6361138224601746,
+ "num_tokens": 1261568.0,
+ "step": 77
+ },
+ {
+ "entropy": 1.7821043729782104,
+ "epoch": 0.15757575757575756,
+ "grad_norm": 2.008962869644165,
+ "learning_rate": 1.8962962962962966e-05,
+ "loss": 1.7826645374298096,
+ "mean_token_accuracy": 0.6053370833396912,
+ "num_tokens": 1277952.0,
+ "step": 78
+ },
+ {
+ "entropy": 1.833274006843567,
+ "epoch": 0.1595959595959596,
+ "grad_norm": 2.0258517265319824,
+ "learning_rate": 1.894949494949495e-05,
+ "loss": 1.8566474914550781,
+ "mean_token_accuracy": 0.5800561904907227,
+ "num_tokens": 1294336.0,
+ "step": 79
+ },
+ {
+ "entropy": 1.785520315170288,
+ "epoch": 0.16161616161616163,
+ "grad_norm": 2.0918498039245605,
+ "learning_rate": 1.8936026936026936e-05,
+ "loss": 1.775907278060913,
+ "mean_token_accuracy": 0.6063141226768494,
+ "num_tokens": 1310720.0,
+ "step": 80
+ },
+ {
+ "entropy": 1.9341778755187988,
+ "epoch": 0.16363636363636364,
+ "grad_norm": 2.0399975776672363,
+ "learning_rate": 1.8922558922558924e-05,
+ "loss": 1.9579840898513794,
+ "mean_token_accuracy": 0.5619809627532959,
+ "num_tokens": 1327104.0,
+ "step": 81
+ },
+ {
+ "entropy": 1.4945461750030518,
+ "epoch": 0.16565656565656567,
+ "grad_norm": 1.9975999593734741,
+ "learning_rate": 1.8909090909090912e-05,
+ "loss": 1.5436582565307617,
+ "mean_token_accuracy": 0.6423424482345581,
+ "num_tokens": 1343488.0,
+ "step": 82
+ },
+ {
+ "entropy": 1.6639432907104492,
+ "epoch": 0.16767676767676767,
+ "grad_norm": 2.0375208854675293,
+ "learning_rate": 1.8895622895622897e-05,
+ "loss": 1.679243564605713,
+ "mean_token_accuracy": 0.6226184368133545,
+ "num_tokens": 1359872.0,
+ "step": 83
+ },
+ {
+ "entropy": 1.4505668878555298,
+ "epoch": 0.1696969696969697,
+ "grad_norm": 1.8894681930541992,
+ "learning_rate": 1.8882154882154885e-05,
+ "loss": 1.4778342247009277,
+ "mean_token_accuracy": 0.6637151837348938,
+ "num_tokens": 1376256.0,
+ "step": 84
+ },
+ {
+ "entropy": 1.3845125436782837,
+ "epoch": 0.1717171717171717,
+ "grad_norm": 2.1110754013061523,
+ "learning_rate": 1.886868686868687e-05,
+ "loss": 1.4153435230255127,
+ "mean_token_accuracy": 0.6737298369407654,
+ "num_tokens": 1392640.0,
+ "step": 85
+ },
+ {
+ "entropy": 1.8785539865493774,
+ "epoch": 0.17373737373737375,
+ "grad_norm": 2.0894815921783447,
+ "learning_rate": 1.8855218855218858e-05,
+ "loss": 1.8658334016799927,
+ "mean_token_accuracy": 0.5846360325813293,
+ "num_tokens": 1409024.0,
+ "step": 86
+ },
+ {
+ "entropy": 1.6006972789764404,
+ "epoch": 0.17575757575757575,
+ "grad_norm": 2.088477849960327,
+ "learning_rate": 1.8841750841750843e-05,
+ "loss": 1.6257164478302002,
+ "mean_token_accuracy": 0.623900830745697,
+ "num_tokens": 1425408.0,
+ "step": 87
+ },
+ {
+ "entropy": 1.1098109483718872,
+ "epoch": 0.17777777777777778,
+ "grad_norm": 1.934228539466858,
+ "learning_rate": 1.8828282828282827e-05,
+ "loss": 1.0948461294174194,
+ "mean_token_accuracy": 0.7248412370681763,
+ "num_tokens": 1441792.0,
+ "step": 88
+ },
+ {
+ "entropy": 1.4807215929031372,
+ "epoch": 0.1797979797979798,
+ "grad_norm": 2.06062388420105,
+ "learning_rate": 1.8814814814814816e-05,
+ "loss": 1.5115268230438232,
+ "mean_token_accuracy": 0.6463117003440857,
+ "num_tokens": 1458176.0,
+ "step": 89
+ },
+ {
+ "entropy": 1.07956862449646,
+ "epoch": 0.18181818181818182,
+ "grad_norm": 1.8913685083389282,
+ "learning_rate": 1.8801346801346804e-05,
+ "loss": 1.0902754068374634,
+ "mean_token_accuracy": 0.7308256030082703,
+ "num_tokens": 1474560.0,
+ "step": 90
+ },
+ {
+ "entropy": 1.4571733474731445,
+ "epoch": 0.18383838383838383,
+ "grad_norm": 2.2199525833129883,
+ "learning_rate": 1.8787878787878792e-05,
+ "loss": 1.4686330556869507,
+ "mean_token_accuracy": 0.6566927433013916,
+ "num_tokens": 1490944.0,
+ "step": 91
+ },
+ {
+ "entropy": 1.4484740495681763,
+ "epoch": 0.18585858585858586,
+ "grad_norm": 2.0670580863952637,
+ "learning_rate": 1.8774410774410777e-05,
+ "loss": 1.4791338443756104,
+ "mean_token_accuracy": 0.6493649482727051,
+ "num_tokens": 1507328.0,
+ "step": 92
+ },
+ {
+ "entropy": 1.498313546180725,
+ "epoch": 0.18787878787878787,
+ "grad_norm": 2.024329423904419,
+ "learning_rate": 1.876094276094276e-05,
+ "loss": 1.4964686632156372,
+ "mean_token_accuracy": 0.6457620859146118,
+ "num_tokens": 1523712.0,
+ "step": 93
+ },
+ {
+ "entropy": 1.4806159734725952,
+ "epoch": 0.1898989898989899,
+ "grad_norm": 2.1369152069091797,
+ "learning_rate": 1.874747474747475e-05,
+ "loss": 1.4587206840515137,
+ "mean_token_accuracy": 0.6521739363670349,
+ "num_tokens": 1540096.0,
+ "step": 94
+ },
+ {
+ "entropy": 1.3746109008789062,
+ "epoch": 0.1919191919191919,
+ "grad_norm": 2.216958522796631,
+ "learning_rate": 1.8734006734006735e-05,
+ "loss": 1.3600832223892212,
+ "mean_token_accuracy": 0.671775758266449,
+ "num_tokens": 1556480.0,
+ "step": 95
+ },
+ {
+ "entropy": 1.3492207527160645,
+ "epoch": 0.19393939393939394,
+ "grad_norm": 2.093956708908081,
+ "learning_rate": 1.8720538720538723e-05,
+ "loss": 1.3867243528366089,
+ "mean_token_accuracy": 0.6625549793243408,
+ "num_tokens": 1572864.0,
+ "step": 96
+ },
+ {
+ "entropy": 1.560738205909729,
+ "epoch": 0.19595959595959597,
+ "grad_norm": 2.0691707134246826,
+ "learning_rate": 1.8707070707070707e-05,
+ "loss": 1.5875897407531738,
+ "mean_token_accuracy": 0.6345872282981873,
+ "num_tokens": 1589248.0,
+ "step": 97
+ },
+ {
+ "entropy": 1.46450674533844,
+ "epoch": 0.19797979797979798,
+ "grad_norm": 2.064547538757324,
+ "learning_rate": 1.8693602693602696e-05,
+ "loss": 1.4920742511749268,
+ "mean_token_accuracy": 0.6411822438240051,
+ "num_tokens": 1605632.0,
+ "step": 98
+ },
+ {
+ "entropy": 1.5433454513549805,
+ "epoch": 0.2,
+ "grad_norm": 2.014409303665161,
+ "learning_rate": 1.868013468013468e-05,
+ "loss": 1.5737671852111816,
+ "mean_token_accuracy": 0.6349536180496216,
+ "num_tokens": 1622016.0,
+ "step": 99
+ },
+ {
+ "entropy": 1.358772873878479,
+ "epoch": 0.20202020202020202,
+ "grad_norm": 1.9765456914901733,
+ "learning_rate": 1.866666666666667e-05,
+ "loss": 1.3583091497421265,
+ "mean_token_accuracy": 0.680446982383728,
+ "num_tokens": 1638400.0,
+ "step": 100
+ },
+ {
+ "epoch": 0.20202020202020202,
+ "eval_entropy": 1.5664977690865916,
+ "eval_loss": 1.5804312229156494,
+ "eval_mean_token_accuracy": 0.63647972239602,
+ "eval_num_tokens": 1638400.0,
+ "eval_runtime": 44.1541,
+ "eval_samples_per_second": 22.421,
+ "eval_steps_per_second": 2.808,
+ "step": 100
+ },
+ {
+ "entropy": 1.5879693031311035,
+ "epoch": 0.20404040404040405,
+ "grad_norm": 1.8491015434265137,
+ "learning_rate": 1.8653198653198653e-05,
+ "loss": 1.5892077684402466,
+ "mean_token_accuracy": 0.6665852665901184,
+ "num_tokens": 1654784.0,
+ "step": 101
+ },
+ {
+ "entropy": 1.5789942741394043,
+ "epoch": 0.20606060606060606,
+ "grad_norm": 1.9096908569335938,
+ "learning_rate": 1.863973063973064e-05,
+ "loss": 1.594193458557129,
+ "mean_token_accuracy": 0.6293966770172119,
+ "num_tokens": 1671168.0,
+ "step": 102
+ },
+ {
+ "entropy": 1.5245429277420044,
+ "epoch": 0.2080808080808081,
+ "grad_norm": 1.986904501914978,
+ "learning_rate": 1.8626262626262626e-05,
+ "loss": 1.5463616847991943,
+ "mean_token_accuracy": 0.6354421377182007,
+ "num_tokens": 1687552.0,
+ "step": 103
+ },
+ {
+ "entropy": 2.015364408493042,
+ "epoch": 0.2101010101010101,
+ "grad_norm": 2.3297250270843506,
+ "learning_rate": 1.8612794612794615e-05,
+ "loss": 1.987194538116455,
+ "mean_token_accuracy": 0.5604543089866638,
+ "num_tokens": 1703936.0,
+ "step": 104
+ },
+ {
+ "entropy": 1.770432472229004,
+ "epoch": 0.21212121212121213,
+ "grad_norm": 2.3580551147460938,
+ "learning_rate": 1.8599326599326603e-05,
+ "loss": 1.7715529203414917,
+ "mean_token_accuracy": 0.5786516666412354,
+ "num_tokens": 1720320.0,
+ "step": 105
+ },
+ {
+ "entropy": 1.3456813097000122,
+ "epoch": 0.21414141414141413,
+ "grad_norm": 1.9782105684280396,
+ "learning_rate": 1.8585858585858588e-05,
+ "loss": 1.342590570449829,
+ "mean_token_accuracy": 0.6779433488845825,
+ "num_tokens": 1736704.0,
+ "step": 106
+ },
+ {
+ "entropy": 1.4614694118499756,
+ "epoch": 0.21616161616161617,
+ "grad_norm": 1.9486815929412842,
+ "learning_rate": 1.8572390572390572e-05,
+ "loss": 1.4493391513824463,
+ "mean_token_accuracy": 0.6618832349777222,
+ "num_tokens": 1753088.0,
+ "step": 107
+ },
+ {
+ "entropy": 2.007932186126709,
+ "epoch": 0.21818181818181817,
+ "grad_norm": 2.0571465492248535,
+ "learning_rate": 1.855892255892256e-05,
+ "loss": 2.033851385116577,
+ "mean_token_accuracy": 0.5746213793754578,
+ "num_tokens": 1769472.0,
+ "step": 108
+ },
+ {
+ "entropy": 1.506022572517395,
+ "epoch": 0.2202020202020202,
+ "grad_norm": 1.9980531930923462,
+ "learning_rate": 1.8545454545454545e-05,
+ "loss": 1.5305016040802002,
+ "mean_token_accuracy": 0.6416096687316895,
+ "num_tokens": 1785856.0,
+ "step": 109
+ },
+ {
+ "entropy": 1.479506492614746,
+ "epoch": 0.2222222222222222,
+ "grad_norm": 2.411241292953491,
+ "learning_rate": 1.8531986531986533e-05,
+ "loss": 1.5501817464828491,
+ "mean_token_accuracy": 0.6364802122116089,
+ "num_tokens": 1802240.0,
+ "step": 110
+ },
+ {
+ "entropy": 1.7731081247329712,
+ "epoch": 0.22424242424242424,
+ "grad_norm": 2.1088449954986572,
+ "learning_rate": 1.851851851851852e-05,
+ "loss": 1.7582051753997803,
+ "mean_token_accuracy": 0.5985588431358337,
+ "num_tokens": 1818624.0,
+ "step": 111
+ },
+ {
+ "entropy": 1.7963190078735352,
+ "epoch": 0.22626262626262628,
+ "grad_norm": 2.180325508117676,
+ "learning_rate": 1.8505050505050506e-05,
+ "loss": 1.8436882495880127,
+ "mean_token_accuracy": 0.5850635170936584,
+ "num_tokens": 1835008.0,
+ "step": 112
+ },
+ {
+ "entropy": 1.513211727142334,
+ "epoch": 0.22828282828282828,
+ "grad_norm": 2.1563429832458496,
+ "learning_rate": 1.8491582491582495e-05,
+ "loss": 1.547263741493225,
+ "mean_token_accuracy": 0.6399609446525574,
+ "num_tokens": 1851392.0,
+ "step": 113
+ },
+ {
+ "entropy": 1.8904774188995361,
+ "epoch": 0.23030303030303031,
+ "grad_norm": 2.2191243171691895,
+ "learning_rate": 1.847811447811448e-05,
+ "loss": 1.9237418174743652,
+ "mean_token_accuracy": 0.5852466821670532,
+ "num_tokens": 1867776.0,
+ "step": 114
+ },
+ {
+ "entropy": 1.706113338470459,
+ "epoch": 0.23232323232323232,
+ "grad_norm": 2.1411335468292236,
+ "learning_rate": 1.8464646464646464e-05,
+ "loss": 1.7145740985870361,
+ "mean_token_accuracy": 0.6138861775398254,
+ "num_tokens": 1884160.0,
+ "step": 115
+ },
+ {
+ "entropy": 1.726441502571106,
+ "epoch": 0.23434343434343435,
+ "grad_norm": 2.212477922439575,
+ "learning_rate": 1.8451178451178452e-05,
+ "loss": 1.7427434921264648,
+ "mean_token_accuracy": 0.6087567210197449,
+ "num_tokens": 1900544.0,
+ "step": 116
+ },
+ {
+ "entropy": 1.49800443649292,
+ "epoch": 0.23636363636363636,
+ "grad_norm": 2.003248929977417,
+ "learning_rate": 1.8437710437710437e-05,
+ "loss": 1.4677059650421143,
+ "mean_token_accuracy": 0.6617000699043274,
+ "num_tokens": 1916928.0,
+ "step": 117
+ },
+ {
+ "entropy": 1.5079678297042847,
+ "epoch": 0.2383838383838384,
+ "grad_norm": 1.9702556133270264,
+ "learning_rate": 1.8424242424242425e-05,
+ "loss": 1.5194823741912842,
+ "mean_token_accuracy": 0.6593796014785767,
+ "num_tokens": 1933312.0,
+ "step": 118
+ },
+ {
+ "entropy": 1.760438323020935,
+ "epoch": 0.2404040404040404,
+ "grad_norm": 1.9484418630599976,
+ "learning_rate": 1.8410774410774414e-05,
+ "loss": 1.756869912147522,
+ "mean_token_accuracy": 0.6193209290504456,
+ "num_tokens": 1949696.0,
+ "step": 119
+ },
+ {
+ "entropy": 1.3706170320510864,
+ "epoch": 0.24242424242424243,
+ "grad_norm": 2.078357458114624,
+ "learning_rate": 1.83973063973064e-05,
+ "loss": 1.3532414436340332,
+ "mean_token_accuracy": 0.6680508255958557,
+ "num_tokens": 1966080.0,
+ "step": 120
+ },
+ {
+ "entropy": 1.2994399070739746,
+ "epoch": 0.24444444444444444,
+ "grad_norm": 1.9178324937820435,
+ "learning_rate": 1.8383838383838387e-05,
+ "loss": 1.298889398574829,
+ "mean_token_accuracy": 0.6831949353218079,
+ "num_tokens": 1982464.0,
+ "step": 121
+ },
+ {
+ "entropy": 1.6806142330169678,
+ "epoch": 0.24646464646464647,
+ "grad_norm": 2.287158966064453,
+ "learning_rate": 1.837037037037037e-05,
+ "loss": 1.728592038154602,
+ "mean_token_accuracy": 0.6161455512046814,
+ "num_tokens": 1998848.0,
+ "step": 122
+ },
+ {
+ "entropy": 1.2894387245178223,
+ "epoch": 0.24848484848484848,
+ "grad_norm": 1.9998525381088257,
+ "learning_rate": 1.8356902356902356e-05,
+ "loss": 1.3326170444488525,
+ "mean_token_accuracy": 0.6693331599235535,
+ "num_tokens": 2015232.0,
+ "step": 123
+ },
+ {
+ "entropy": 1.5466209650039673,
+ "epoch": 0.2505050505050505,
+ "grad_norm": 1.8439521789550781,
+ "learning_rate": 1.8343434343434344e-05,
+ "loss": 1.5606887340545654,
+ "mean_token_accuracy": 0.6571201682090759,
+ "num_tokens": 2031616.0,
+ "step": 124
+ },
+ {
+ "entropy": 1.2692567110061646,
+ "epoch": 0.25252525252525254,
+ "grad_norm": 1.9568793773651123,
+ "learning_rate": 1.8329966329966332e-05,
+ "loss": 1.308868408203125,
+ "mean_token_accuracy": 0.6954689621925354,
+ "num_tokens": 2048000.0,
+ "step": 125
+ },
+ {
+ "entropy": 1.5780351161956787,
+ "epoch": 0.2545454545454545,
+ "grad_norm": 2.151250123977661,
+ "learning_rate": 1.831649831649832e-05,
+ "loss": 1.60304856300354,
+ "mean_token_accuracy": 0.6258549094200134,
+ "num_tokens": 2064384.0,
+ "step": 126
+ },
+ {
+ "entropy": 1.487567663192749,
+ "epoch": 0.25656565656565655,
+ "grad_norm": 1.9134294986724854,
+ "learning_rate": 1.8303030303030305e-05,
+ "loss": 1.514064073562622,
+ "mean_token_accuracy": 0.6348925232887268,
+ "num_tokens": 2080768.0,
+ "step": 127
+ },
+ {
+ "entropy": 1.6072919368743896,
+ "epoch": 0.2585858585858586,
+ "grad_norm": 2.119290351867676,
+ "learning_rate": 1.828956228956229e-05,
+ "loss": 1.6062684059143066,
+ "mean_token_accuracy": 0.6332437992095947,
+ "num_tokens": 2097152.0,
+ "step": 128
+ },
+ {
+ "entropy": 1.1168174743652344,
+ "epoch": 0.2606060606060606,
+ "grad_norm": 1.8520931005477905,
+ "learning_rate": 1.827609427609428e-05,
+ "loss": 1.1421526670455933,
+ "mean_token_accuracy": 0.7078651785850525,
+ "num_tokens": 2113536.0,
+ "step": 129
+ },
+ {
+ "entropy": 1.349792718887329,
+ "epoch": 0.26262626262626265,
+ "grad_norm": 1.8463630676269531,
+ "learning_rate": 1.8262626262626263e-05,
+ "loss": 1.3712577819824219,
+ "mean_token_accuracy": 0.6736077070236206,
+ "num_tokens": 2129920.0,
+ "step": 130
+ },
+ {
+ "entropy": 1.8531655073165894,
+ "epoch": 0.26464646464646463,
+ "grad_norm": 2.0503969192504883,
+ "learning_rate": 1.824915824915825e-05,
+ "loss": 1.8971941471099854,
+ "mean_token_accuracy": 0.5845749974250793,
+ "num_tokens": 2146304.0,
+ "step": 131
+ },
+ {
+ "entropy": 1.436646819114685,
+ "epoch": 0.26666666666666666,
+ "grad_norm": 1.8517117500305176,
+ "learning_rate": 1.8235690235690236e-05,
+ "loss": 1.4255880117416382,
+ "mean_token_accuracy": 0.6653639674186707,
+ "num_tokens": 2162688.0,
+ "step": 132
+ },
+ {
+ "entropy": 1.9219673871994019,
+ "epoch": 0.2686868686868687,
+ "grad_norm": 2.1424307823181152,
+ "learning_rate": 1.8222222222222224e-05,
+ "loss": 1.9088413715362549,
+ "mean_token_accuracy": 0.5868954658508301,
+ "num_tokens": 2179072.0,
+ "step": 133
+ },
+ {
+ "entropy": 1.4034194946289062,
+ "epoch": 0.27070707070707073,
+ "grad_norm": 1.9987515211105347,
+ "learning_rate": 1.8208754208754212e-05,
+ "loss": 1.3612333536148071,
+ "mean_token_accuracy": 0.6721421480178833,
+ "num_tokens": 2195456.0,
+ "step": 134
+ },
+ {
+ "entropy": 2.064425230026245,
+ "epoch": 0.2727272727272727,
+ "grad_norm": 2.0800976753234863,
+ "learning_rate": 1.8195286195286197e-05,
+ "loss": 2.0595853328704834,
+ "mean_token_accuracy": 0.5694919228553772,
+ "num_tokens": 2211840.0,
+ "step": 135
+ },
+ {
+ "entropy": 1.2883177995681763,
+ "epoch": 0.27474747474747474,
+ "grad_norm": 1.9042892456054688,
+ "learning_rate": 1.8181818181818182e-05,
+ "loss": 1.3220078945159912,
+ "mean_token_accuracy": 0.6805080771446228,
+ "num_tokens": 2228224.0,
+ "step": 136
+ },
+ {
+ "entropy": 1.101396918296814,
+ "epoch": 0.2767676767676768,
+ "grad_norm": 1.857073187828064,
+ "learning_rate": 1.816835016835017e-05,
+ "loss": 1.1156296730041504,
+ "mean_token_accuracy": 0.7214826345443726,
+ "num_tokens": 2244608.0,
+ "step": 137
+ },
+ {
+ "entropy": 1.376597285270691,
+ "epoch": 0.2787878787878788,
+ "grad_norm": 1.9103689193725586,
+ "learning_rate": 1.8154882154882155e-05,
+ "loss": 1.3915824890136719,
+ "mean_token_accuracy": 0.6824010610580444,
+ "num_tokens": 2260992.0,
+ "step": 138
+ },
+ {
+ "entropy": 1.4654942750930786,
+ "epoch": 0.2808080808080808,
+ "grad_norm": 1.9513856172561646,
+ "learning_rate": 1.8141414141414143e-05,
+ "loss": 1.5000076293945312,
+ "mean_token_accuracy": 0.6457620859146118,
+ "num_tokens": 2277376.0,
+ "step": 139
+ },
+ {
+ "entropy": 1.1999047994613647,
+ "epoch": 0.2828282828282828,
+ "grad_norm": 1.824987530708313,
+ "learning_rate": 1.812794612794613e-05,
+ "loss": 1.2217535972595215,
+ "mean_token_accuracy": 0.7034685015678406,
+ "num_tokens": 2293760.0,
+ "step": 140
+ },
+ {
+ "entropy": 1.2822942733764648,
+ "epoch": 0.28484848484848485,
+ "grad_norm": 2.0979108810424805,
+ "learning_rate": 1.8114478114478116e-05,
+ "loss": 1.309725046157837,
+ "mean_token_accuracy": 0.6855764389038086,
+ "num_tokens": 2310144.0,
+ "step": 141
+ },
+ {
+ "entropy": 1.5589981079101562,
+ "epoch": 0.2868686868686869,
+ "grad_norm": 2.030648708343506,
+ "learning_rate": 1.8101010101010104e-05,
+ "loss": 1.5918951034545898,
+ "mean_token_accuracy": 0.6286638975143433,
+ "num_tokens": 2326528.0,
+ "step": 142
+ },
+ {
+ "entropy": 1.5600453615188599,
+ "epoch": 0.28888888888888886,
+ "grad_norm": 2.023409605026245,
+ "learning_rate": 1.808754208754209e-05,
+ "loss": 1.5725011825561523,
+ "mean_token_accuracy": 0.6384342908859253,
+ "num_tokens": 2342912.0,
+ "step": 143
+ },
+ {
+ "entropy": 1.5066077709197998,
+ "epoch": 0.2909090909090909,
+ "grad_norm": 1.9963997602462769,
+ "learning_rate": 1.8074074074074074e-05,
+ "loss": 1.5066845417022705,
+ "mean_token_accuracy": 0.6474108695983887,
+ "num_tokens": 2359296.0,
+ "step": 144
+ },
+ {
+ "entropy": 1.8313122987747192,
+ "epoch": 0.29292929292929293,
+ "grad_norm": 2.0791213512420654,
+ "learning_rate": 1.8060606060606062e-05,
+ "loss": 1.8148126602172852,
+ "mean_token_accuracy": 0.5820102691650391,
+ "num_tokens": 2375680.0,
+ "step": 145
+ },
+ {
+ "entropy": 1.8828215599060059,
+ "epoch": 0.29494949494949496,
+ "grad_norm": 2.1346335411071777,
+ "learning_rate": 1.804713804713805e-05,
+ "loss": 1.905554175376892,
+ "mean_token_accuracy": 0.6041768193244934,
+ "num_tokens": 2392064.0,
+ "step": 146
+ },
+ {
+ "entropy": 1.1984127759933472,
+ "epoch": 0.296969696969697,
+ "grad_norm": 1.6585179567337036,
+ "learning_rate": 1.8033670033670035e-05,
+ "loss": 1.2255632877349854,
+ "mean_token_accuracy": 0.712506115436554,
+ "num_tokens": 2408448.0,
+ "step": 147
+ },
+ {
+ "entropy": 1.4248595237731934,
+ "epoch": 0.298989898989899,
+ "grad_norm": 2.468965530395508,
+ "learning_rate": 1.8020202020202023e-05,
+ "loss": 1.4362621307373047,
+ "mean_token_accuracy": 0.6464948654174805,
+ "num_tokens": 2424832.0,
+ "step": 148
+ },
+ {
+ "entropy": 1.7295368909835815,
+ "epoch": 0.301010101010101,
+ "grad_norm": 1.819359302520752,
+ "learning_rate": 1.8006734006734008e-05,
+ "loss": 1.7290053367614746,
+ "mean_token_accuracy": 0.6312897205352783,
+ "num_tokens": 2441216.0,
+ "step": 149
+ },
+ {
+ "entropy": 1.7334001064300537,
+ "epoch": 0.30303030303030304,
+ "grad_norm": 2.149543285369873,
+ "learning_rate": 1.7993265993265993e-05,
+ "loss": 1.7014907598495483,
+ "mean_token_accuracy": 0.5981314182281494,
+ "num_tokens": 2457600.0,
+ "step": 150
+ },
+ {
+ "entropy": 2.0550050735473633,
+ "epoch": 0.30505050505050507,
+ "grad_norm": 1.9880189895629883,
+ "learning_rate": 1.797979797979798e-05,
+ "loss": 2.001784324645996,
+ "mean_token_accuracy": 0.5552638173103333,
+ "num_tokens": 2473984.0,
+ "step": 151
+ },
+ {
+ "entropy": 1.5352036952972412,
+ "epoch": 0.30707070707070705,
+ "grad_norm": 2.031998634338379,
+ "learning_rate": 1.7966329966329966e-05,
+ "loss": 1.5262815952301025,
+ "mean_token_accuracy": 0.6328163146972656,
+ "num_tokens": 2490368.0,
+ "step": 152
+ },
+ {
+ "entropy": 1.4278677701950073,
+ "epoch": 0.3090909090909091,
+ "grad_norm": 2.101398229598999,
+ "learning_rate": 1.7952861952861954e-05,
+ "loss": 1.4288041591644287,
+ "mean_token_accuracy": 0.6647533178329468,
+ "num_tokens": 2506752.0,
+ "step": 153
+ },
+ {
+ "entropy": 1.7469983100891113,
+ "epoch": 0.3111111111111111,
+ "grad_norm": 2.0824313163757324,
+ "learning_rate": 1.7939393939393942e-05,
+ "loss": 1.770402193069458,
+ "mean_token_accuracy": 0.5997191071510315,
+ "num_tokens": 2523136.0,
+ "step": 154
+ },
+ {
+ "entropy": 1.6565595865249634,
+ "epoch": 0.31313131313131315,
+ "grad_norm": 2.0815961360931396,
+ "learning_rate": 1.7925925925925927e-05,
+ "loss": 1.7024259567260742,
+ "mean_token_accuracy": 0.6109550595283508,
+ "num_tokens": 2539520.0,
+ "step": 155
+ },
+ {
+ "entropy": 1.4104127883911133,
+ "epoch": 0.3151515151515151,
+ "grad_norm": 2.129694700241089,
+ "learning_rate": 1.7912457912457915e-05,
+ "loss": 1.4378812313079834,
+ "mean_token_accuracy": 0.6489985585212708,
+ "num_tokens": 2555904.0,
+ "step": 156
+ },
+ {
+ "entropy": 1.5129127502441406,
+ "epoch": 0.31717171717171716,
+ "grad_norm": 2.180267095565796,
+ "learning_rate": 1.78989898989899e-05,
+ "loss": 1.5564348697662354,
+ "mean_token_accuracy": 0.6327552795410156,
+ "num_tokens": 2572288.0,
+ "step": 157
+ },
+ {
+ "entropy": 1.5240321159362793,
+ "epoch": 0.3191919191919192,
+ "grad_norm": 1.968405842781067,
+ "learning_rate": 1.7885521885521885e-05,
+ "loss": 1.585756778717041,
+ "mean_token_accuracy": 0.629885196685791,
+ "num_tokens": 2588672.0,
+ "step": 158
+ },
+ {
+ "entropy": 1.3169562816619873,
+ "epoch": 0.3212121212121212,
+ "grad_norm": 2.0452873706817627,
+ "learning_rate": 1.7872053872053873e-05,
+ "loss": 1.3569687604904175,
+ "mean_token_accuracy": 0.671775758266449,
+ "num_tokens": 2605056.0,
+ "step": 159
+ },
+ {
+ "entropy": 1.3630796670913696,
+ "epoch": 0.32323232323232326,
+ "grad_norm": 1.9721522331237793,
+ "learning_rate": 1.785858585858586e-05,
+ "loss": 1.3965554237365723,
+ "mean_token_accuracy": 0.6626160144805908,
+ "num_tokens": 2621440.0,
+ "step": 160
+ },
+ {
+ "entropy": 1.178250789642334,
+ "epoch": 0.32525252525252524,
+ "grad_norm": 2.0231528282165527,
+ "learning_rate": 1.7845117845117846e-05,
+ "loss": 1.2126985788345337,
+ "mean_token_accuracy": 0.6977894306182861,
+ "num_tokens": 2637824.0,
+ "step": 161
+ },
+ {
+ "entropy": 1.8399475812911987,
+ "epoch": 0.32727272727272727,
+ "grad_norm": 2.0615484714508057,
+ "learning_rate": 1.7831649831649834e-05,
+ "loss": 1.868000864982605,
+ "mean_token_accuracy": 0.5822545289993286,
+ "num_tokens": 2654208.0,
+ "step": 162
+ },
+ {
+ "entropy": 1.6604139804840088,
+ "epoch": 0.3292929292929293,
+ "grad_norm": 1.9838379621505737,
+ "learning_rate": 1.781818181818182e-05,
+ "loss": 1.6677067279815674,
+ "mean_token_accuracy": 0.621275007724762,
+ "num_tokens": 2670592.0,
+ "step": 163
+ },
+ {
+ "entropy": 1.3761085271835327,
+ "epoch": 0.33131313131313134,
+ "grad_norm": 2.0093271732330322,
+ "learning_rate": 1.7804713804713807e-05,
+ "loss": 1.3829190731048584,
+ "mean_token_accuracy": 0.6655471324920654,
+ "num_tokens": 2686976.0,
+ "step": 164
+ },
+ {
+ "entropy": 1.578093409538269,
+ "epoch": 0.3333333333333333,
+ "grad_norm": 1.9710773229599,
+ "learning_rate": 1.7791245791245792e-05,
+ "loss": 1.560583472251892,
+ "mean_token_accuracy": 0.6344650983810425,
+ "num_tokens": 2703360.0,
+ "step": 165
+ },
+ {
+ "entropy": 1.0447157621383667,
+ "epoch": 0.33535353535353535,
+ "grad_norm": 1.68338942527771,
+ "learning_rate": 1.7777777777777777e-05,
+ "loss": 1.0116058588027954,
+ "mean_token_accuracy": 0.7449315786361694,
+ "num_tokens": 2719744.0,
+ "step": 166
+ },
+ {
+ "entropy": 1.3687469959259033,
+ "epoch": 0.3373737373737374,
+ "grad_norm": 2.062744617462158,
+ "learning_rate": 1.7764309764309765e-05,
+ "loss": 1.3179712295532227,
+ "mean_token_accuracy": 0.6761724352836609,
+ "num_tokens": 2736128.0,
+ "step": 167
+ },
+ {
+ "entropy": 1.8639556169509888,
+ "epoch": 0.3393939393939394,
+ "grad_norm": 2.041898250579834,
+ "learning_rate": 1.7750841750841753e-05,
+ "loss": 1.8531380891799927,
+ "mean_token_accuracy": 0.5920859575271606,
+ "num_tokens": 2752512.0,
+ "step": 168
+ },
+ {
+ "entropy": 1.6900824308395386,
+ "epoch": 0.3414141414141414,
+ "grad_norm": 1.9403504133224487,
+ "learning_rate": 1.773737373737374e-05,
+ "loss": 1.701782464981079,
+ "mean_token_accuracy": 0.6102222800254822,
+ "num_tokens": 2768896.0,
+ "step": 169
+ },
+ {
+ "entropy": 1.6202301979064941,
+ "epoch": 0.3434343434343434,
+ "grad_norm": 2.027329683303833,
+ "learning_rate": 1.7723905723905726e-05,
+ "loss": 1.6471121311187744,
+ "mean_token_accuracy": 0.6310454607009888,
+ "num_tokens": 2785280.0,
+ "step": 170
+ },
+ {
+ "entropy": 1.2701244354248047,
+ "epoch": 0.34545454545454546,
+ "grad_norm": 1.8625534772872925,
+ "learning_rate": 1.771043771043771e-05,
+ "loss": 1.291702151298523,
+ "mean_token_accuracy": 0.67659991979599,
+ "num_tokens": 2801664.0,
+ "step": 171
+ },
+ {
+ "entropy": 1.5759357213974,
+ "epoch": 0.3474747474747475,
+ "grad_norm": 1.945141077041626,
+ "learning_rate": 1.76969696969697e-05,
+ "loss": 1.6189839839935303,
+ "mean_token_accuracy": 0.6225574016571045,
+ "num_tokens": 2818048.0,
+ "step": 172
+ },
+ {
+ "entropy": 1.3319392204284668,
+ "epoch": 0.34949494949494947,
+ "grad_norm": 1.8485770225524902,
+ "learning_rate": 1.7683501683501684e-05,
+ "loss": 1.3553425073623657,
+ "mean_token_accuracy": 0.6759281754493713,
+ "num_tokens": 2834432.0,
+ "step": 173
+ },
+ {
+ "entropy": 1.3251745700836182,
+ "epoch": 0.3515151515151515,
+ "grad_norm": 2.0564401149749756,
+ "learning_rate": 1.7670033670033672e-05,
+ "loss": 1.359522819519043,
+ "mean_token_accuracy": 0.6678065657615662,
+ "num_tokens": 2850816.0,
+ "step": 174
+ },
+ {
+ "entropy": 1.2764136791229248,
+ "epoch": 0.35353535353535354,
+ "grad_norm": 2.1749041080474854,
+ "learning_rate": 1.765656565656566e-05,
+ "loss": 1.3260173797607422,
+ "mean_token_accuracy": 0.6710429787635803,
+ "num_tokens": 2867200.0,
+ "step": 175
+ },
+ {
+ "entropy": 1.6382335424423218,
+ "epoch": 0.35555555555555557,
+ "grad_norm": 1.923366904258728,
+ "learning_rate": 1.7643097643097645e-05,
+ "loss": 1.6868517398834229,
+ "mean_token_accuracy": 0.6268930435180664,
+ "num_tokens": 2883584.0,
+ "step": 176
+ },
+ {
+ "entropy": 1.2399169206619263,
+ "epoch": 0.3575757575757576,
+ "grad_norm": 1.795649766921997,
+ "learning_rate": 1.7629629629629633e-05,
+ "loss": 1.2266355752944946,
+ "mean_token_accuracy": 0.689667820930481,
+ "num_tokens": 2899968.0,
+ "step": 177
+ },
+ {
+ "entropy": 1.5761929750442505,
+ "epoch": 0.3595959595959596,
+ "grad_norm": 2.0536038875579834,
+ "learning_rate": 1.7616161616161618e-05,
+ "loss": 1.5698940753936768,
+ "mean_token_accuracy": 0.6263434290885925,
+ "num_tokens": 2916352.0,
+ "step": 178
+ },
+ {
+ "entropy": 1.3612396717071533,
+ "epoch": 0.3616161616161616,
+ "grad_norm": 2.003840684890747,
+ "learning_rate": 1.7602693602693603e-05,
+ "loss": 1.359879732131958,
+ "mean_token_accuracy": 0.6798974275588989,
+ "num_tokens": 2932736.0,
+ "step": 179
+ },
+ {
+ "entropy": 1.0548239946365356,
+ "epoch": 0.36363636363636365,
+ "grad_norm": 1.6967436075210571,
+ "learning_rate": 1.758922558922559e-05,
+ "loss": 1.040539264678955,
+ "mean_token_accuracy": 0.7433438897132874,
+ "num_tokens": 2949120.0,
+ "step": 180
+ },
+ {
+ "entropy": 1.16105055809021,
+ "epoch": 0.3656565656565657,
+ "grad_norm": 1.9793111085891724,
+ "learning_rate": 1.7575757575757576e-05,
+ "loss": 1.1532450914382935,
+ "mean_token_accuracy": 0.7135441899299622,
+ "num_tokens": 2965504.0,
+ "step": 181
+ },
+ {
+ "entropy": 1.4421862363815308,
+ "epoch": 0.36767676767676766,
+ "grad_norm": 1.8337509632110596,
+ "learning_rate": 1.7562289562289564e-05,
+ "loss": 1.419133186340332,
+ "mean_token_accuracy": 0.659807026386261,
+ "num_tokens": 2981888.0,
+ "step": 182
+ },
+ {
+ "entropy": 1.6186654567718506,
+ "epoch": 0.3696969696969697,
+ "grad_norm": 2.0585362911224365,
+ "learning_rate": 1.7548821548821552e-05,
+ "loss": 1.5880110263824463,
+ "mean_token_accuracy": 0.6322056651115417,
+ "num_tokens": 2998272.0,
+ "step": 183
+ },
+ {
+ "entropy": 1.76968252658844,
+ "epoch": 0.3717171717171717,
+ "grad_norm": 2.139742374420166,
+ "learning_rate": 1.7535353535353537e-05,
+ "loss": 1.797312617301941,
+ "mean_token_accuracy": 0.6096116304397583,
+ "num_tokens": 3014656.0,
+ "step": 184
+ },
+ {
+ "entropy": 1.0575193166732788,
+ "epoch": 0.37373737373737376,
+ "grad_norm": 1.8547794818878174,
+ "learning_rate": 1.7521885521885525e-05,
+ "loss": 1.0575863122940063,
+ "mean_token_accuracy": 0.7325354218482971,
+ "num_tokens": 3031040.0,
+ "step": 185
+ },
+ {
+ "entropy": 1.2937911748886108,
+ "epoch": 0.37575757575757573,
+ "grad_norm": 2.3985140323638916,
+ "learning_rate": 1.750841750841751e-05,
+ "loss": 1.3277016878128052,
+ "mean_token_accuracy": 0.6654860973358154,
+ "num_tokens": 3047424.0,
+ "step": 186
+ },
+ {
+ "entropy": 1.1829862594604492,
+ "epoch": 0.37777777777777777,
+ "grad_norm": 1.948241114616394,
+ "learning_rate": 1.7494949494949494e-05,
+ "loss": 1.2146751880645752,
+ "mean_token_accuracy": 0.6964460015296936,
+ "num_tokens": 3063808.0,
+ "step": 187
+ },
+ {
+ "entropy": 1.4858520030975342,
+ "epoch": 0.3797979797979798,
+ "grad_norm": 2.2712836265563965,
+ "learning_rate": 1.7481481481481483e-05,
+ "loss": 1.5523631572723389,
+ "mean_token_accuracy": 0.6392281651496887,
+ "num_tokens": 3080192.0,
+ "step": 188
+ },
+ {
+ "entropy": 1.7905362844467163,
+ "epoch": 0.38181818181818183,
+ "grad_norm": 1.890699863433838,
+ "learning_rate": 1.746801346801347e-05,
+ "loss": 1.8139519691467285,
+ "mean_token_accuracy": 0.6015510559082031,
+ "num_tokens": 3096576.0,
+ "step": 189
+ },
+ {
+ "entropy": 1.3264763355255127,
+ "epoch": 0.3838383838383838,
+ "grad_norm": 2.1567318439483643,
+ "learning_rate": 1.7454545454545456e-05,
+ "loss": 1.3780615329742432,
+ "mean_token_accuracy": 0.6636541485786438,
+ "num_tokens": 3112960.0,
+ "step": 190
+ },
+ {
+ "entropy": 1.9898782968521118,
+ "epoch": 0.38585858585858585,
+ "grad_norm": 2.008547306060791,
+ "learning_rate": 1.7441077441077444e-05,
+ "loss": 2.0208630561828613,
+ "mean_token_accuracy": 0.5498290061950684,
+ "num_tokens": 3129344.0,
+ "step": 191
+ },
+ {
+ "entropy": 1.2579785585403442,
+ "epoch": 0.3878787878787879,
+ "grad_norm": 1.921314001083374,
+ "learning_rate": 1.742760942760943e-05,
+ "loss": 1.271723985671997,
+ "mean_token_accuracy": 0.6882632970809937,
+ "num_tokens": 3145728.0,
+ "step": 192
+ },
+ {
+ "entropy": 1.3366814851760864,
+ "epoch": 0.3898989898989899,
+ "grad_norm": 1.9525961875915527,
+ "learning_rate": 1.7414141414141413e-05,
+ "loss": 1.3561700582504272,
+ "mean_token_accuracy": 0.6822789311408997,
+ "num_tokens": 3162112.0,
+ "step": 193
+ },
+ {
+ "entropy": 1.5505253076553345,
+ "epoch": 0.39191919191919194,
+ "grad_norm": 2.044635057449341,
+ "learning_rate": 1.74006734006734e-05,
+ "loss": 1.5198928117752075,
+ "mean_token_accuracy": 0.6326331496238708,
+ "num_tokens": 3178496.0,
+ "step": 194
+ },
+ {
+ "entropy": 1.4198534488677979,
+ "epoch": 0.3939393939393939,
+ "grad_norm": 1.9955503940582275,
+ "learning_rate": 1.738720538720539e-05,
+ "loss": 1.4072260856628418,
+ "mean_token_accuracy": 0.6631045341491699,
+ "num_tokens": 3194880.0,
+ "step": 195
+ },
+ {
+ "entropy": 2.0099895000457764,
+ "epoch": 0.39595959595959596,
+ "grad_norm": 2.110247850418091,
+ "learning_rate": 1.7373737373737375e-05,
+ "loss": 2.0012335777282715,
+ "mean_token_accuracy": 0.5668661594390869,
+ "num_tokens": 3211264.0,
+ "step": 196
+ },
+ {
+ "entropy": 2.1619439125061035,
+ "epoch": 0.397979797979798,
+ "grad_norm": 2.0869383811950684,
+ "learning_rate": 1.7360269360269363e-05,
+ "loss": 2.1793670654296875,
+ "mean_token_accuracy": 0.5569125413894653,
+ "num_tokens": 3227648.0,
+ "step": 197
+ },
+ {
+ "entropy": 1.6383775472640991,
+ "epoch": 0.4,
+ "grad_norm": 1.7467572689056396,
+ "learning_rate": 1.7346801346801347e-05,
+ "loss": 1.656137228012085,
+ "mean_token_accuracy": 0.6395334601402283,
+ "num_tokens": 3244032.0,
+ "step": 198
+ },
+ {
+ "entropy": 1.5911450386047363,
+ "epoch": 0.402020202020202,
+ "grad_norm": 1.9362678527832031,
+ "learning_rate": 1.7333333333333336e-05,
+ "loss": 1.5865838527679443,
+ "mean_token_accuracy": 0.621275007724762,
+ "num_tokens": 3260416.0,
+ "step": 199
+ },
+ {
+ "entropy": 1.3983819484710693,
+ "epoch": 0.40404040404040403,
+ "grad_norm": 1.925863265991211,
+ "learning_rate": 1.731986531986532e-05,
+ "loss": 1.3835841417312622,
+ "mean_token_accuracy": 0.6618221998214722,
+ "num_tokens": 3276800.0,
+ "step": 200
+ },
+ {
+ "epoch": 0.40404040404040403,
+ "eval_entropy": 1.5366105552642577,
+ "eval_loss": 1.5502140522003174,
+ "eval_mean_token_accuracy": 0.6411546406246,
+ "eval_num_tokens": 3276800.0,
+ "eval_runtime": 43.8218,
+ "eval_samples_per_second": 22.592,
+ "eval_steps_per_second": 2.83,
+ "step": 200
+ },
+ {
+ "entropy": 1.291393756866455,
+ "epoch": 0.40606060606060607,
+ "grad_norm": 1.9057531356811523,
+ "learning_rate": 1.7306397306397305e-05,
+ "loss": 1.2847933769226074,
+ "mean_token_accuracy": 0.6901563405990601,
+ "num_tokens": 3293184.0,
+ "step": 201
+ },
+ {
+ "entropy": 1.1511558294296265,
+ "epoch": 0.4080808080808081,
+ "grad_norm": 1.849436640739441,
+ "learning_rate": 1.7292929292929293e-05,
+ "loss": 1.1661030054092407,
+ "mean_token_accuracy": 0.7095139026641846,
+ "num_tokens": 3309568.0,
+ "step": 202
+ },
+ {
+ "entropy": 1.5591504573822021,
+ "epoch": 0.4101010101010101,
+ "grad_norm": 1.9354233741760254,
+ "learning_rate": 1.727946127946128e-05,
+ "loss": 1.6045410633087158,
+ "mean_token_accuracy": 0.6226795315742493,
+ "num_tokens": 3325952.0,
+ "step": 203
+ },
+ {
+ "entropy": 1.7634649276733398,
+ "epoch": 0.4121212121212121,
+ "grad_norm": 2.414632797241211,
+ "learning_rate": 1.726599326599327e-05,
+ "loss": 1.8021953105926514,
+ "mean_token_accuracy": 0.6099780201911926,
+ "num_tokens": 3342336.0,
+ "step": 204
+ },
+ {
+ "entropy": 1.6080671548843384,
+ "epoch": 0.41414141414141414,
+ "grad_norm": 1.9936028718948364,
+ "learning_rate": 1.7252525252525255e-05,
+ "loss": 1.630000352859497,
+ "mean_token_accuracy": 0.6256106495857239,
+ "num_tokens": 3358720.0,
+ "step": 205
+ },
+ {
+ "entropy": 1.6845343112945557,
+ "epoch": 0.4161616161616162,
+ "grad_norm": 1.9131947755813599,
+ "learning_rate": 1.723905723905724e-05,
+ "loss": 1.6912329196929932,
+ "mean_token_accuracy": 0.6263434290885925,
+ "num_tokens": 3375104.0,
+ "step": 206
+ },
+ {
+ "entropy": 1.5946446657180786,
+ "epoch": 0.41818181818181815,
+ "grad_norm": 2.0423905849456787,
+ "learning_rate": 1.7225589225589228e-05,
+ "loss": 1.647271752357483,
+ "mean_token_accuracy": 0.6296409368515015,
+ "num_tokens": 3391488.0,
+ "step": 207
+ },
+ {
+ "entropy": 1.3124741315841675,
+ "epoch": 0.4202020202020202,
+ "grad_norm": 1.8896982669830322,
+ "learning_rate": 1.7212121212121212e-05,
+ "loss": 1.3216124773025513,
+ "mean_token_accuracy": 0.687469482421875,
+ "num_tokens": 3407872.0,
+ "step": 208
+ },
+ {
+ "entropy": 1.3931281566619873,
+ "epoch": 0.4222222222222222,
+ "grad_norm": 1.7883074283599854,
+ "learning_rate": 1.71986531986532e-05,
+ "loss": 1.4264600276947021,
+ "mean_token_accuracy": 0.6568148732185364,
+ "num_tokens": 3424256.0,
+ "step": 209
+ },
+ {
+ "entropy": 1.1427687406539917,
+ "epoch": 0.42424242424242425,
+ "grad_norm": 1.9367003440856934,
+ "learning_rate": 1.7185185185185185e-05,
+ "loss": 1.1611723899841309,
+ "mean_token_accuracy": 0.7058500051498413,
+ "num_tokens": 3440640.0,
+ "step": 210
+ },
+ {
+ "entropy": 1.7226167917251587,
+ "epoch": 0.4262626262626263,
+ "grad_norm": 2.175487518310547,
+ "learning_rate": 1.7171717171717173e-05,
+ "loss": 1.7383192777633667,
+ "mean_token_accuracy": 0.6075964570045471,
+ "num_tokens": 3457024.0,
+ "step": 211
+ },
+ {
+ "entropy": 1.3780062198638916,
+ "epoch": 0.42828282828282827,
+ "grad_norm": 1.9817614555358887,
+ "learning_rate": 1.715824915824916e-05,
+ "loss": 1.3755927085876465,
+ "mean_token_accuracy": 0.6750732660293579,
+ "num_tokens": 3473408.0,
+ "step": 212
+ },
+ {
+ "entropy": 1.637842059135437,
+ "epoch": 0.4303030303030303,
+ "grad_norm": 2.2326242923736572,
+ "learning_rate": 1.7144781144781146e-05,
+ "loss": 1.635596752166748,
+ "mean_token_accuracy": 0.6110771894454956,
+ "num_tokens": 3489792.0,
+ "step": 213
+ },
+ {
+ "entropy": 1.9970118999481201,
+ "epoch": 0.43232323232323233,
+ "grad_norm": 2.1303606033325195,
+ "learning_rate": 1.713131313131313e-05,
+ "loss": 2.0099592208862305,
+ "mean_token_accuracy": 0.5781631469726562,
+ "num_tokens": 3506176.0,
+ "step": 214
+ },
+ {
+ "entropy": 1.4759869575500488,
+ "epoch": 0.43434343434343436,
+ "grad_norm": 2.043727397918701,
+ "learning_rate": 1.711784511784512e-05,
+ "loss": 1.4756600856781006,
+ "mean_token_accuracy": 0.6483879089355469,
+ "num_tokens": 3522560.0,
+ "step": 215
+ },
+ {
+ "entropy": 1.2833726406097412,
+ "epoch": 0.43636363636363634,
+ "grad_norm": 1.9260340929031372,
+ "learning_rate": 1.7104377104377104e-05,
+ "loss": 1.2887630462646484,
+ "mean_token_accuracy": 0.6900342106819153,
+ "num_tokens": 3538944.0,
+ "step": 216
+ },
+ {
+ "entropy": 1.3936302661895752,
+ "epoch": 0.4383838383838384,
+ "grad_norm": 1.8590774536132812,
+ "learning_rate": 1.7090909090909092e-05,
+ "loss": 1.3784823417663574,
+ "mean_token_accuracy": 0.6736077070236206,
+ "num_tokens": 3555328.0,
+ "step": 217
+ },
+ {
+ "entropy": 1.2413936853408813,
+ "epoch": 0.4404040404040404,
+ "grad_norm": 2.1111643314361572,
+ "learning_rate": 1.707744107744108e-05,
+ "loss": 1.2740635871887207,
+ "mean_token_accuracy": 0.6883854269981384,
+ "num_tokens": 3571712.0,
+ "step": 218
+ },
+ {
+ "entropy": 1.4879416227340698,
+ "epoch": 0.44242424242424244,
+ "grad_norm": 2.127614736557007,
+ "learning_rate": 1.7063973063973065e-05,
+ "loss": 1.48406183719635,
+ "mean_token_accuracy": 0.649975597858429,
+ "num_tokens": 3588096.0,
+ "step": 219
+ },
+ {
+ "entropy": 1.4226471185684204,
+ "epoch": 0.4444444444444444,
+ "grad_norm": 2.164325475692749,
+ "learning_rate": 1.7050505050505054e-05,
+ "loss": 1.466269850730896,
+ "mean_token_accuracy": 0.6502808928489685,
+ "num_tokens": 3604480.0,
+ "step": 220
+ },
+ {
+ "entropy": 1.6183370351791382,
+ "epoch": 0.44646464646464645,
+ "grad_norm": 2.018846273422241,
+ "learning_rate": 1.7037037037037038e-05,
+ "loss": 1.6348307132720947,
+ "mean_token_accuracy": 0.627198338508606,
+ "num_tokens": 3620864.0,
+ "step": 221
+ },
+ {
+ "entropy": 1.6979204416275024,
+ "epoch": 0.4484848484848485,
+ "grad_norm": 2.1574854850769043,
+ "learning_rate": 1.7023569023569023e-05,
+ "loss": 1.7019602060317993,
+ "mean_token_accuracy": 0.6157181262969971,
+ "num_tokens": 3637248.0,
+ "step": 222
+ },
+ {
+ "entropy": 1.753490686416626,
+ "epoch": 0.4505050505050505,
+ "grad_norm": 2.0253705978393555,
+ "learning_rate": 1.701010101010101e-05,
+ "loss": 1.7466539144515991,
+ "mean_token_accuracy": 0.6008793115615845,
+ "num_tokens": 3653632.0,
+ "step": 223
+ },
+ {
+ "entropy": 1.717929720878601,
+ "epoch": 0.45252525252525255,
+ "grad_norm": 1.9840974807739258,
+ "learning_rate": 1.6996632996633e-05,
+ "loss": 1.703988790512085,
+ "mean_token_accuracy": 0.6099780201911926,
+ "num_tokens": 3670016.0,
+ "step": 224
+ },
+ {
+ "entropy": 1.377287745475769,
+ "epoch": 0.45454545454545453,
+ "grad_norm": 2.1027839183807373,
+ "learning_rate": 1.6983164983164984e-05,
+ "loss": 1.3902864456176758,
+ "mean_token_accuracy": 0.660906195640564,
+ "num_tokens": 3686400.0,
+ "step": 225
+ },
+ {
+ "entropy": 1.237000584602356,
+ "epoch": 0.45656565656565656,
+ "grad_norm": 1.9116249084472656,
+ "learning_rate": 1.6969696969696972e-05,
+ "loss": 1.221006155014038,
+ "mean_token_accuracy": 0.7045676708221436,
+ "num_tokens": 3702784.0,
+ "step": 226
+ },
+ {
+ "entropy": 1.4198882579803467,
+ "epoch": 0.4585858585858586,
+ "grad_norm": 2.099358558654785,
+ "learning_rate": 1.6956228956228957e-05,
+ "loss": 1.4211962223052979,
+ "mean_token_accuracy": 0.6549829244613647,
+ "num_tokens": 3719168.0,
+ "step": 227
+ },
+ {
+ "entropy": 1.4647245407104492,
+ "epoch": 0.46060606060606063,
+ "grad_norm": 1.8873655796051025,
+ "learning_rate": 1.6942760942760945e-05,
+ "loss": 1.4848006963729858,
+ "mean_token_accuracy": 0.6534562706947327,
+ "num_tokens": 3735552.0,
+ "step": 228
+ },
+ {
+ "entropy": 1.348371982574463,
+ "epoch": 0.4626262626262626,
+ "grad_norm": 2.0487565994262695,
+ "learning_rate": 1.692929292929293e-05,
+ "loss": 1.355329155921936,
+ "mean_token_accuracy": 0.6709208488464355,
+ "num_tokens": 3751936.0,
+ "step": 229
+ },
+ {
+ "entropy": 1.6201715469360352,
+ "epoch": 0.46464646464646464,
+ "grad_norm": 1.9640510082244873,
+ "learning_rate": 1.6915824915824915e-05,
+ "loss": 1.6176421642303467,
+ "mean_token_accuracy": 0.6221910119056702,
+ "num_tokens": 3768320.0,
+ "step": 230
+ },
+ {
+ "entropy": 1.461037039756775,
+ "epoch": 0.4666666666666667,
+ "grad_norm": 1.9843394756317139,
+ "learning_rate": 1.6902356902356903e-05,
+ "loss": 1.4502949714660645,
+ "mean_token_accuracy": 0.6463727355003357,
+ "num_tokens": 3784704.0,
+ "step": 231
+ },
+ {
+ "entropy": 1.4964231252670288,
+ "epoch": 0.4686868686868687,
+ "grad_norm": 2.0087051391601562,
+ "learning_rate": 1.688888888888889e-05,
+ "loss": 1.5107879638671875,
+ "mean_token_accuracy": 0.6536394953727722,
+ "num_tokens": 3801088.0,
+ "step": 232
+ },
+ {
+ "entropy": 1.3013520240783691,
+ "epoch": 0.4707070707070707,
+ "grad_norm": 2.1042368412017822,
+ "learning_rate": 1.6875420875420876e-05,
+ "loss": 1.312164306640625,
+ "mean_token_accuracy": 0.673363447189331,
+ "num_tokens": 3817472.0,
+ "step": 233
+ },
+ {
+ "entropy": 1.6242352724075317,
+ "epoch": 0.4727272727272727,
+ "grad_norm": 1.8541103601455688,
+ "learning_rate": 1.6861952861952864e-05,
+ "loss": 1.6327502727508545,
+ "mean_token_accuracy": 0.6376404762268066,
+ "num_tokens": 3833856.0,
+ "step": 234
+ },
+ {
+ "entropy": 1.4227592945098877,
+ "epoch": 0.47474747474747475,
+ "grad_norm": 1.8749339580535889,
+ "learning_rate": 1.684848484848485e-05,
+ "loss": 1.436142921447754,
+ "mean_token_accuracy": 0.6705544590950012,
+ "num_tokens": 3850240.0,
+ "step": 235
+ },
+ {
+ "entropy": 1.4807487726211548,
+ "epoch": 0.4767676767676768,
+ "grad_norm": 2.1475114822387695,
+ "learning_rate": 1.6835016835016837e-05,
+ "loss": 1.4670445919036865,
+ "mean_token_accuracy": 0.6385564208030701,
+ "num_tokens": 3866624.0,
+ "step": 236
+ },
+ {
+ "entropy": 2.0623066425323486,
+ "epoch": 0.47878787878787876,
+ "grad_norm": 2.1698546409606934,
+ "learning_rate": 1.6821548821548822e-05,
+ "loss": 2.08683180809021,
+ "mean_token_accuracy": 0.5531876087188721,
+ "num_tokens": 3883008.0,
+ "step": 237
+ },
+ {
+ "entropy": 1.6894460916519165,
+ "epoch": 0.4808080808080808,
+ "grad_norm": 2.158062219619751,
+ "learning_rate": 1.680808080808081e-05,
+ "loss": 1.7244186401367188,
+ "mean_token_accuracy": 0.6055813431739807,
+ "num_tokens": 3899392.0,
+ "step": 238
+ },
+ {
+ "entropy": 1.2741345167160034,
+ "epoch": 0.48282828282828283,
+ "grad_norm": 1.918306827545166,
+ "learning_rate": 1.67946127946128e-05,
+ "loss": 1.2937202453613281,
+ "mean_token_accuracy": 0.6878969073295593,
+ "num_tokens": 3915776.0,
+ "step": 239
+ },
+ {
+ "entropy": 1.556649088859558,
+ "epoch": 0.48484848484848486,
+ "grad_norm": 2.0560667514801025,
+ "learning_rate": 1.6781144781144783e-05,
+ "loss": 1.5751030445098877,
+ "mean_token_accuracy": 0.6324499249458313,
+ "num_tokens": 3932160.0,
+ "step": 240
+ },
+ {
+ "entropy": 1.7632875442504883,
+ "epoch": 0.4868686868686869,
+ "grad_norm": 1.8408738374710083,
+ "learning_rate": 1.6767676767676768e-05,
+ "loss": 1.7745842933654785,
+ "mean_token_accuracy": 0.6091231107711792,
+ "num_tokens": 3948544.0,
+ "step": 241
+ },
+ {
+ "entropy": 1.4793967008590698,
+ "epoch": 0.4888888888888889,
+ "grad_norm": 2.0134575366973877,
+ "learning_rate": 1.6754208754208756e-05,
+ "loss": 1.514477014541626,
+ "mean_token_accuracy": 0.6414875388145447,
+ "num_tokens": 3964928.0,
+ "step": 242
+ },
+ {
+ "entropy": 1.4762561321258545,
+ "epoch": 0.4909090909090909,
+ "grad_norm": 2.087501287460327,
+ "learning_rate": 1.674074074074074e-05,
+ "loss": 1.5195722579956055,
+ "mean_token_accuracy": 0.6289081573486328,
+ "num_tokens": 3981312.0,
+ "step": 243
+ },
+ {
+ "entropy": 1.6629496812820435,
+ "epoch": 0.49292929292929294,
+ "grad_norm": 1.7386047840118408,
+ "learning_rate": 1.672727272727273e-05,
+ "loss": 1.6932936906814575,
+ "mean_token_accuracy": 0.6289692521095276,
+ "num_tokens": 3997696.0,
+ "step": 244
+ },
+ {
+ "entropy": 1.4506888389587402,
+ "epoch": 0.494949494949495,
+ "grad_norm": 1.8268823623657227,
+ "learning_rate": 1.6713804713804714e-05,
+ "loss": 1.4257543087005615,
+ "mean_token_accuracy": 0.6678065657615662,
+ "num_tokens": 4014080.0,
+ "step": 245
+ },
+ {
+ "entropy": 1.6132714748382568,
+ "epoch": 0.49696969696969695,
+ "grad_norm": 2.168206214904785,
+ "learning_rate": 1.6700336700336702e-05,
+ "loss": 1.602414608001709,
+ "mean_token_accuracy": 0.6202979683876038,
+ "num_tokens": 4030464.0,
+ "step": 246
+ },
+ {
+ "entropy": 1.2860945463180542,
+ "epoch": 0.498989898989899,
+ "grad_norm": 1.830817699432373,
+ "learning_rate": 1.668686868686869e-05,
+ "loss": 1.284934163093567,
+ "mean_token_accuracy": 0.6854543089866638,
+ "num_tokens": 4046848.0,
+ "step": 247
+ },
+ {
+ "entropy": 1.4948641061782837,
+ "epoch": 0.501010101010101,
+ "grad_norm": 1.9775508642196655,
+ "learning_rate": 1.6673400673400675e-05,
+ "loss": 1.497206449508667,
+ "mean_token_accuracy": 0.6471055150032043,
+ "num_tokens": 4063232.0,
+ "step": 248
+ },
+ {
+ "entropy": 1.490235447883606,
+ "epoch": 0.503030303030303,
+ "grad_norm": 2.207184076309204,
+ "learning_rate": 1.665993265993266e-05,
+ "loss": 1.5091016292572021,
+ "mean_token_accuracy": 0.6482657790184021,
+ "num_tokens": 4079616.0,
+ "step": 249
+ },
+ {
+ "entropy": 1.5752832889556885,
+ "epoch": 0.5050505050505051,
+ "grad_norm": 2.0718111991882324,
+ "learning_rate": 1.6646464646464648e-05,
+ "loss": 1.576171875,
+ "mean_token_accuracy": 0.6375793814659119,
+ "num_tokens": 4096000.0,
+ "step": 250
+ },
+ {
+ "entropy": 1.3935530185699463,
+ "epoch": 0.5070707070707071,
+ "grad_norm": 1.9379569292068481,
+ "learning_rate": 1.6632996632996633e-05,
+ "loss": 1.4019020795822144,
+ "mean_token_accuracy": 0.6742794513702393,
+ "num_tokens": 4112384.0,
+ "step": 251
+ },
+ {
+ "entropy": 1.3622194528579712,
+ "epoch": 0.509090909090909,
+ "grad_norm": 1.9684133529663086,
+ "learning_rate": 1.661952861952862e-05,
+ "loss": 1.3581968545913696,
+ "mean_token_accuracy": 0.6631045341491699,
+ "num_tokens": 4128768.0,
+ "step": 252
+ },
+ {
+ "entropy": 2.1161346435546875,
+ "epoch": 0.5111111111111111,
+ "grad_norm": 1.9295378923416138,
+ "learning_rate": 1.660606060606061e-05,
+ "loss": 2.1316990852355957,
+ "mean_token_accuracy": 0.5643624663352966,
+ "num_tokens": 4145152.0,
+ "step": 253
+ },
+ {
+ "entropy": 1.4322566986083984,
+ "epoch": 0.5131313131313131,
+ "grad_norm": 2.0198557376861572,
+ "learning_rate": 1.6592592592592594e-05,
+ "loss": 1.4390883445739746,
+ "mean_token_accuracy": 0.6648143529891968,
+ "num_tokens": 4161536.0,
+ "step": 254
+ },
+ {
+ "entropy": 1.5892291069030762,
+ "epoch": 0.5151515151515151,
+ "grad_norm": 2.0127851963043213,
+ "learning_rate": 1.6579124579124582e-05,
+ "loss": 1.6437480449676514,
+ "mean_token_accuracy": 0.6223131418228149,
+ "num_tokens": 4177920.0,
+ "step": 255
+ },
+ {
+ "entropy": 1.445371150970459,
+ "epoch": 0.5171717171717172,
+ "grad_norm": 1.9758318662643433,
+ "learning_rate": 1.6565656565656567e-05,
+ "loss": 1.4749128818511963,
+ "mean_token_accuracy": 0.6457010507583618,
+ "num_tokens": 4194304.0,
+ "step": 256
+ },
+ {
+ "entropy": 1.5587478876113892,
+ "epoch": 0.5191919191919192,
+ "grad_norm": 1.918809413909912,
+ "learning_rate": 1.6552188552188552e-05,
+ "loss": 1.5739175081253052,
+ "mean_token_accuracy": 0.6334269642829895,
+ "num_tokens": 4210688.0,
+ "step": 257
+ },
+ {
+ "entropy": 1.5112392902374268,
+ "epoch": 0.5212121212121212,
+ "grad_norm": 2.0176963806152344,
+ "learning_rate": 1.653872053872054e-05,
+ "loss": 1.5279680490493774,
+ "mean_token_accuracy": 0.640754759311676,
+ "num_tokens": 4227072.0,
+ "step": 258
+ },
+ {
+ "entropy": 1.4898114204406738,
+ "epoch": 0.5232323232323233,
+ "grad_norm": 2.0637850761413574,
+ "learning_rate": 1.6525252525252528e-05,
+ "loss": 1.5081419944763184,
+ "mean_token_accuracy": 0.6375183463096619,
+ "num_tokens": 4243456.0,
+ "step": 259
+ },
+ {
+ "entropy": 1.4768201112747192,
+ "epoch": 0.5252525252525253,
+ "grad_norm": 1.835053563117981,
+ "learning_rate": 1.6511784511784513e-05,
+ "loss": 1.494248390197754,
+ "mean_token_accuracy": 0.6496092081069946,
+ "num_tokens": 4259840.0,
+ "step": 260
+ },
+ {
+ "entropy": 1.655643105506897,
+ "epoch": 0.5272727272727272,
+ "grad_norm": 1.9655805826187134,
+ "learning_rate": 1.64983164983165e-05,
+ "loss": 1.6360158920288086,
+ "mean_token_accuracy": 0.6209086179733276,
+ "num_tokens": 4276224.0,
+ "step": 261
+ },
+ {
+ "entropy": 1.5621604919433594,
+ "epoch": 0.5292929292929293,
+ "grad_norm": 1.9304734468460083,
+ "learning_rate": 1.6484848484848486e-05,
+ "loss": 1.5850739479064941,
+ "mean_token_accuracy": 0.6422203183174133,
+ "num_tokens": 4292608.0,
+ "step": 262
+ },
+ {
+ "entropy": 1.592617392539978,
+ "epoch": 0.5313131313131313,
+ "grad_norm": 1.9590504169464111,
+ "learning_rate": 1.6471380471380474e-05,
+ "loss": 1.583874225616455,
+ "mean_token_accuracy": 0.6230459213256836,
+ "num_tokens": 4308992.0,
+ "step": 263
+ },
+ {
+ "entropy": 1.5268265008926392,
+ "epoch": 0.5333333333333333,
+ "grad_norm": 2.0704433917999268,
+ "learning_rate": 1.645791245791246e-05,
+ "loss": 1.5327314138412476,
+ "mean_token_accuracy": 0.6398388147354126,
+ "num_tokens": 4325376.0,
+ "step": 264
+ },
+ {
+ "entropy": 1.1762311458587646,
+ "epoch": 0.5353535353535354,
+ "grad_norm": 2.0408453941345215,
+ "learning_rate": 1.6444444444444444e-05,
+ "loss": 1.1851191520690918,
+ "mean_token_accuracy": 0.7027967572212219,
+ "num_tokens": 4341760.0,
+ "step": 265
+ },
+ {
+ "entropy": 1.567710518836975,
+ "epoch": 0.5373737373737374,
+ "grad_norm": 1.8334625959396362,
+ "learning_rate": 1.6430976430976432e-05,
+ "loss": 1.564115047454834,
+ "mean_token_accuracy": 0.6340376138687134,
+ "num_tokens": 4358144.0,
+ "step": 266
+ },
+ {
+ "entropy": 1.2294297218322754,
+ "epoch": 0.5393939393939394,
+ "grad_norm": 3.5099127292633057,
+ "learning_rate": 1.641750841750842e-05,
+ "loss": 1.2887773513793945,
+ "mean_token_accuracy": 0.688629686832428,
+ "num_tokens": 4374528.0,
+ "step": 267
+ },
+ {
+ "entropy": 1.5130258798599243,
+ "epoch": 0.5414141414141415,
+ "grad_norm": 1.8295154571533203,
+ "learning_rate": 1.6404040404040405e-05,
+ "loss": 1.581455945968628,
+ "mean_token_accuracy": 0.639655590057373,
+ "num_tokens": 4390912.0,
+ "step": 268
+ },
+ {
+ "entropy": 1.6612389087677002,
+ "epoch": 0.5434343434343434,
+ "grad_norm": 2.1676483154296875,
+ "learning_rate": 1.6390572390572393e-05,
+ "loss": 1.6873081922531128,
+ "mean_token_accuracy": 0.6049095988273621,
+ "num_tokens": 4407296.0,
+ "step": 269
+ },
+ {
+ "entropy": 1.416680932044983,
+ "epoch": 0.5454545454545454,
+ "grad_norm": 2.044541358947754,
+ "learning_rate": 1.6377104377104378e-05,
+ "loss": 1.4432693719863892,
+ "mean_token_accuracy": 0.6580972075462341,
+ "num_tokens": 4423680.0,
+ "step": 270
+ },
+ {
+ "entropy": 1.6253215074539185,
+ "epoch": 0.5474747474747474,
+ "grad_norm": 2.1488161087036133,
+ "learning_rate": 1.6363636363636366e-05,
+ "loss": 1.6667883396148682,
+ "mean_token_accuracy": 0.6122373938560486,
+ "num_tokens": 4440064.0,
+ "step": 271
+ },
+ {
+ "entropy": 1.492803931236267,
+ "epoch": 0.5494949494949495,
+ "grad_norm": 2.0591108798980713,
+ "learning_rate": 1.635016835016835e-05,
+ "loss": 1.5023317337036133,
+ "mean_token_accuracy": 0.6420371532440186,
+ "num_tokens": 4456448.0,
+ "step": 272
+ },
+ {
+ "entropy": 1.5818493366241455,
+ "epoch": 0.5515151515151515,
+ "grad_norm": 1.9004875421524048,
+ "learning_rate": 1.633670033670034e-05,
+ "loss": 1.5762417316436768,
+ "mean_token_accuracy": 0.6375793814659119,
+ "num_tokens": 4472832.0,
+ "step": 273
+ },
+ {
+ "entropy": 1.2263695001602173,
+ "epoch": 0.5535353535353535,
+ "grad_norm": 1.8976677656173706,
+ "learning_rate": 1.6323232323232324e-05,
+ "loss": 1.2397428750991821,
+ "mean_token_accuracy": 0.70658278465271,
+ "num_tokens": 4489216.0,
+ "step": 274
+ },
+ {
+ "entropy": 1.370070219039917,
+ "epoch": 0.5555555555555556,
+ "grad_norm": 1.875389575958252,
+ "learning_rate": 1.6309764309764312e-05,
+ "loss": 1.3603096008300781,
+ "mean_token_accuracy": 0.6843551397323608,
+ "num_tokens": 4505600.0,
+ "step": 275
+ },
+ {
+ "entropy": 1.9219107627868652,
+ "epoch": 0.5575757575757576,
+ "grad_norm": 2.057508945465088,
+ "learning_rate": 1.6296296296296297e-05,
+ "loss": 1.9082988500595093,
+ "mean_token_accuracy": 0.5834758281707764,
+ "num_tokens": 4521984.0,
+ "step": 276
+ },
+ {
+ "entropy": 1.6965564489364624,
+ "epoch": 0.5595959595959596,
+ "grad_norm": 1.9217735528945923,
+ "learning_rate": 1.6282828282828285e-05,
+ "loss": 1.7024450302124023,
+ "mean_token_accuracy": 0.6050928235054016,
+ "num_tokens": 4538368.0,
+ "step": 277
+ },
+ {
+ "entropy": 1.7517896890640259,
+ "epoch": 0.5616161616161616,
+ "grad_norm": 2.029672145843506,
+ "learning_rate": 1.626936026936027e-05,
+ "loss": 1.7533857822418213,
+ "mean_token_accuracy": 0.5986199378967285,
+ "num_tokens": 4554752.0,
+ "step": 278
+ },
+ {
+ "entropy": 1.6086634397506714,
+ "epoch": 0.5636363636363636,
+ "grad_norm": 1.980076551437378,
+ "learning_rate": 1.6255892255892258e-05,
+ "loss": 1.5998293161392212,
+ "mean_token_accuracy": 0.6469223499298096,
+ "num_tokens": 4571136.0,
+ "step": 279
+ },
+ {
+ "entropy": 1.5670934915542603,
+ "epoch": 0.5656565656565656,
+ "grad_norm": 1.9693994522094727,
+ "learning_rate": 1.6242424242424243e-05,
+ "loss": 1.583737850189209,
+ "mean_token_accuracy": 0.6189545392990112,
+ "num_tokens": 4587520.0,
+ "step": 280
+ },
+ {
+ "entropy": 1.5290220975875854,
+ "epoch": 0.5676767676767677,
+ "grad_norm": 1.9464402198791504,
+ "learning_rate": 1.622895622895623e-05,
+ "loss": 1.523187518119812,
+ "mean_token_accuracy": 0.6289692521095276,
+ "num_tokens": 4603904.0,
+ "step": 281
+ },
+ {
+ "entropy": 1.2070591449737549,
+ "epoch": 0.5696969696969697,
+ "grad_norm": 2.1152102947235107,
+ "learning_rate": 1.621548821548822e-05,
+ "loss": 1.2107012271881104,
+ "mean_token_accuracy": 0.695652186870575,
+ "num_tokens": 4620288.0,
+ "step": 282
+ },
+ {
+ "entropy": 1.4039427042007446,
+ "epoch": 0.5717171717171717,
+ "grad_norm": 1.936285376548767,
+ "learning_rate": 1.6202020202020204e-05,
+ "loss": 1.425846815109253,
+ "mean_token_accuracy": 0.663593053817749,
+ "num_tokens": 4636672.0,
+ "step": 283
+ },
+ {
+ "entropy": 1.2910168170928955,
+ "epoch": 0.5737373737373738,
+ "grad_norm": 1.8882776498794556,
+ "learning_rate": 1.618855218855219e-05,
+ "loss": 1.3109780550003052,
+ "mean_token_accuracy": 0.6896067261695862,
+ "num_tokens": 4653056.0,
+ "step": 284
+ },
+ {
+ "entropy": 1.7089474201202393,
+ "epoch": 0.5757575757575758,
+ "grad_norm": 2.0113184452056885,
+ "learning_rate": 1.6175084175084177e-05,
+ "loss": 1.7272329330444336,
+ "mean_token_accuracy": 0.600024402141571,
+ "num_tokens": 4669440.0,
+ "step": 285
+ },
+ {
+ "entropy": 1.12257719039917,
+ "epoch": 0.5777777777777777,
+ "grad_norm": 2.0191409587860107,
+ "learning_rate": 1.616161616161616e-05,
+ "loss": 1.1193959712982178,
+ "mean_token_accuracy": 0.7199560403823853,
+ "num_tokens": 4685824.0,
+ "step": 286
+ },
+ {
+ "entropy": 1.2052935361862183,
+ "epoch": 0.5797979797979798,
+ "grad_norm": 1.9647032022476196,
+ "learning_rate": 1.614814814814815e-05,
+ "loss": 1.2192071676254272,
+ "mean_token_accuracy": 0.6977894306182861,
+ "num_tokens": 4702208.0,
+ "step": 287
+ },
+ {
+ "entropy": 1.5017658472061157,
+ "epoch": 0.5818181818181818,
+ "grad_norm": 2.0078749656677246,
+ "learning_rate": 1.6134680134680138e-05,
+ "loss": 1.5069187879562378,
+ "mean_token_accuracy": 0.6404494643211365,
+ "num_tokens": 4718592.0,
+ "step": 288
+ },
+ {
+ "entropy": 1.4429490566253662,
+ "epoch": 0.5838383838383838,
+ "grad_norm": 1.8666913509368896,
+ "learning_rate": 1.6121212121212123e-05,
+ "loss": 1.4498789310455322,
+ "mean_token_accuracy": 0.6577919125556946,
+ "num_tokens": 4734976.0,
+ "step": 289
+ },
+ {
+ "entropy": 1.216312050819397,
+ "epoch": 0.5858585858585859,
+ "grad_norm": 1.8042014837265015,
+ "learning_rate": 1.610774410774411e-05,
+ "loss": 1.2346259355545044,
+ "mean_token_accuracy": 0.7015144228935242,
+ "num_tokens": 4751360.0,
+ "step": 290
+ },
+ {
+ "entropy": 1.1411411762237549,
+ "epoch": 0.5878787878787879,
+ "grad_norm": 1.7743948698043823,
+ "learning_rate": 1.6094276094276096e-05,
+ "loss": 1.158776044845581,
+ "mean_token_accuracy": 0.7111626863479614,
+ "num_tokens": 4767744.0,
+ "step": 291
+ },
+ {
+ "entropy": 1.5691558122634888,
+ "epoch": 0.5898989898989899,
+ "grad_norm": 2.0790789127349854,
+ "learning_rate": 1.608080808080808e-05,
+ "loss": 1.5900537967681885,
+ "mean_token_accuracy": 0.6259770393371582,
+ "num_tokens": 4784128.0,
+ "step": 292
+ },
+ {
+ "entropy": 1.5600252151489258,
+ "epoch": 0.591919191919192,
+ "grad_norm": 1.8288682699203491,
+ "learning_rate": 1.606734006734007e-05,
+ "loss": 1.5642855167388916,
+ "mean_token_accuracy": 0.6497313380241394,
+ "num_tokens": 4800512.0,
+ "step": 293
+ },
+ {
+ "entropy": 1.316757082939148,
+ "epoch": 0.593939393939394,
+ "grad_norm": 2.051280975341797,
+ "learning_rate": 1.6053872053872053e-05,
+ "loss": 1.3361237049102783,
+ "mean_token_accuracy": 0.6642647981643677,
+ "num_tokens": 4816896.0,
+ "step": 294
+ },
+ {
+ "entropy": 1.4421989917755127,
+ "epoch": 0.5959595959595959,
+ "grad_norm": 2.257932662963867,
+ "learning_rate": 1.604040404040404e-05,
+ "loss": 1.4325928688049316,
+ "mean_token_accuracy": 0.6571201682090759,
+ "num_tokens": 4833280.0,
+ "step": 295
+ },
+ {
+ "entropy": 1.4986436367034912,
+ "epoch": 0.597979797979798,
+ "grad_norm": 2.0747647285461426,
+ "learning_rate": 1.602693602693603e-05,
+ "loss": 1.508068323135376,
+ "mean_token_accuracy": 0.6551050543785095,
+ "num_tokens": 4849664.0,
+ "step": 296
+ },
+ {
+ "entropy": 1.4758206605911255,
+ "epoch": 0.6,
+ "grad_norm": 1.874723196029663,
+ "learning_rate": 1.6013468013468014e-05,
+ "loss": 1.4831781387329102,
+ "mean_token_accuracy": 0.6479604244232178,
+ "num_tokens": 4866048.0,
+ "step": 297
+ },
+ {
+ "entropy": 1.8049858808517456,
+ "epoch": 0.602020202020202,
+ "grad_norm": 2.089683771133423,
+ "learning_rate": 1.6000000000000003e-05,
+ "loss": 1.8325893878936768,
+ "mean_token_accuracy": 0.5932462215423584,
+ "num_tokens": 4882432.0,
+ "step": 298
+ },
+ {
+ "entropy": 1.6346150636672974,
+ "epoch": 0.604040404040404,
+ "grad_norm": 1.9526349306106567,
+ "learning_rate": 1.5986531986531987e-05,
+ "loss": 1.6568968296051025,
+ "mean_token_accuracy": 0.6246336102485657,
+ "num_tokens": 4898816.0,
+ "step": 299
+ },
+ {
+ "entropy": 1.431725025177002,
+ "epoch": 0.6060606060606061,
+ "grad_norm": 2.109987497329712,
+ "learning_rate": 1.5973063973063972e-05,
+ "loss": 1.4448115825653076,
+ "mean_token_accuracy": 0.6481436491012573,
+ "num_tokens": 4915200.0,
+ "step": 300
+ },
+ {
+ "epoch": 0.6060606060606061,
+ "eval_entropy": 1.524387352889584,
+ "eval_loss": 1.5324952602386475,
+ "eval_mean_token_accuracy": 0.6443250953189789,
+ "eval_num_tokens": 4915200.0,
+ "eval_runtime": 43.7714,
+ "eval_samples_per_second": 22.618,
+ "eval_steps_per_second": 2.833,
+ "step": 300
+ },
+ {
+ "entropy": 1.4816409349441528,
+ "epoch": 0.6080808080808081,
+ "grad_norm": 3.3732845783233643,
+ "learning_rate": 1.595959595959596e-05,
+ "loss": 1.4913065433502197,
+ "mean_token_accuracy": 0.658707857131958,
+ "num_tokens": 4931584.0,
+ "step": 301
+ },
+ {
+ "entropy": 1.4020944833755493,
+ "epoch": 0.6101010101010101,
+ "grad_norm": 2.1152141094207764,
+ "learning_rate": 1.594612794612795e-05,
+ "loss": 1.4000526666641235,
+ "mean_token_accuracy": 0.656509518623352,
+ "num_tokens": 4947968.0,
+ "step": 302
+ },
+ {
+ "entropy": 1.4965202808380127,
+ "epoch": 0.6121212121212121,
+ "grad_norm": 2.1385467052459717,
+ "learning_rate": 1.5932659932659933e-05,
+ "loss": 1.5221803188323975,
+ "mean_token_accuracy": 0.6567537784576416,
+ "num_tokens": 4964352.0,
+ "step": 303
+ },
+ {
+ "entropy": 1.4024577140808105,
+ "epoch": 0.6141414141414141,
+ "grad_norm": 1.9162694215774536,
+ "learning_rate": 1.591919191919192e-05,
+ "loss": 1.4050084352493286,
+ "mean_token_accuracy": 0.6670737862586975,
+ "num_tokens": 4980736.0,
+ "step": 304
+ },
+ {
+ "entropy": 1.430273413658142,
+ "epoch": 0.6161616161616161,
+ "grad_norm": 2.0312552452087402,
+ "learning_rate": 1.5905723905723906e-05,
+ "loss": 1.418628454208374,
+ "mean_token_accuracy": 0.6585857272148132,
+ "num_tokens": 4997120.0,
+ "step": 305
+ },
+ {
+ "entropy": 1.700549602508545,
+ "epoch": 0.6181818181818182,
+ "grad_norm": 2.080810546875,
+ "learning_rate": 1.5892255892255895e-05,
+ "loss": 1.740492820739746,
+ "mean_token_accuracy": 0.6042989492416382,
+ "num_tokens": 5013504.0,
+ "step": 306
+ },
+ {
+ "entropy": 1.6465320587158203,
+ "epoch": 0.6202020202020202,
+ "grad_norm": 1.8810964822769165,
+ "learning_rate": 1.587878787878788e-05,
+ "loss": 1.6414886713027954,
+ "mean_token_accuracy": 0.629824161529541,
+ "num_tokens": 5029888.0,
+ "step": 307
+ },
+ {
+ "entropy": 1.3717931509017944,
+ "epoch": 0.6222222222222222,
+ "grad_norm": 1.921536922454834,
+ "learning_rate": 1.5865319865319868e-05,
+ "loss": 1.3700361251831055,
+ "mean_token_accuracy": 0.6630434989929199,
+ "num_tokens": 5046272.0,
+ "step": 308
+ },
+ {
+ "entropy": 1.4906446933746338,
+ "epoch": 0.6242424242424243,
+ "grad_norm": 2.1185555458068848,
+ "learning_rate": 1.5851851851851852e-05,
+ "loss": 1.4850339889526367,
+ "mean_token_accuracy": 0.6402662396430969,
+ "num_tokens": 5062656.0,
+ "step": 309
+ },
+ {
+ "entropy": 1.4417473077774048,
+ "epoch": 0.6262626262626263,
+ "grad_norm": 2.0196895599365234,
+ "learning_rate": 1.583838383838384e-05,
+ "loss": 1.4567816257476807,
+ "mean_token_accuracy": 0.6760503053665161,
+ "num_tokens": 5079040.0,
+ "step": 310
+ },
+ {
+ "entropy": 1.4662094116210938,
+ "epoch": 0.6282828282828283,
+ "grad_norm": 1.8859882354736328,
+ "learning_rate": 1.5824915824915825e-05,
+ "loss": 1.4870874881744385,
+ "mean_token_accuracy": 0.6618221998214722,
+ "num_tokens": 5095424.0,
+ "step": 311
+ },
+ {
+ "entropy": 1.8497077226638794,
+ "epoch": 0.6303030303030303,
+ "grad_norm": 1.974537968635559,
+ "learning_rate": 1.5811447811447813e-05,
+ "loss": 1.840538501739502,
+ "mean_token_accuracy": 0.589093804359436,
+ "num_tokens": 5111808.0,
+ "step": 312
+ },
+ {
+ "entropy": 1.4185130596160889,
+ "epoch": 0.6323232323232323,
+ "grad_norm": 1.8612359762191772,
+ "learning_rate": 1.5797979797979798e-05,
+ "loss": 1.4404246807098389,
+ "mean_token_accuracy": 0.664631187915802,
+ "num_tokens": 5128192.0,
+ "step": 313
+ },
+ {
+ "entropy": 1.8270463943481445,
+ "epoch": 0.6343434343434343,
+ "grad_norm": 2.1050970554351807,
+ "learning_rate": 1.5784511784511786e-05,
+ "loss": 1.8682940006256104,
+ "mean_token_accuracy": 0.5893380641937256,
+ "num_tokens": 5144576.0,
+ "step": 314
+ },
+ {
+ "entropy": 1.5919326543807983,
+ "epoch": 0.6363636363636364,
+ "grad_norm": 1.8464620113372803,
+ "learning_rate": 1.577104377104377e-05,
+ "loss": 1.6087713241577148,
+ "mean_token_accuracy": 0.6350146532058716,
+ "num_tokens": 5160960.0,
+ "step": 315
+ },
+ {
+ "entropy": 1.4776946306228638,
+ "epoch": 0.6383838383838384,
+ "grad_norm": 2.031834602355957,
+ "learning_rate": 1.575757575757576e-05,
+ "loss": 1.5114507675170898,
+ "mean_token_accuracy": 0.6498534679412842,
+ "num_tokens": 5177344.0,
+ "step": 316
+ },
+ {
+ "entropy": 1.3789927959442139,
+ "epoch": 0.6404040404040404,
+ "grad_norm": 1.8604289293289185,
+ "learning_rate": 1.5744107744107748e-05,
+ "loss": 1.3792052268981934,
+ "mean_token_accuracy": 0.6712262034416199,
+ "num_tokens": 5193728.0,
+ "step": 317
+ },
+ {
+ "entropy": 1.5859184265136719,
+ "epoch": 0.6424242424242425,
+ "grad_norm": 2.1134088039398193,
+ "learning_rate": 1.5730639730639732e-05,
+ "loss": 1.595099925994873,
+ "mean_token_accuracy": 0.6351367831230164,
+ "num_tokens": 5210112.0,
+ "step": 318
+ },
+ {
+ "entropy": 1.3681901693344116,
+ "epoch": 0.6444444444444445,
+ "grad_norm": 1.9200235605239868,
+ "learning_rate": 1.5717171717171717e-05,
+ "loss": 1.3700852394104004,
+ "mean_token_accuracy": 0.6671959161758423,
+ "num_tokens": 5226496.0,
+ "step": 319
+ },
+ {
+ "entropy": 1.594885230064392,
+ "epoch": 0.6464646464646465,
+ "grad_norm": 1.7683591842651367,
+ "learning_rate": 1.5703703703703705e-05,
+ "loss": 1.5906047821044922,
+ "mean_token_accuracy": 0.6350757479667664,
+ "num_tokens": 5242880.0,
+ "step": 320
+ },
+ {
+ "entropy": 1.240937352180481,
+ "epoch": 0.6484848484848484,
+ "grad_norm": 2.117145299911499,
+ "learning_rate": 1.569023569023569e-05,
+ "loss": 1.2188963890075684,
+ "mean_token_accuracy": 0.6958353519439697,
+ "num_tokens": 5259264.0,
+ "step": 321
+ },
+ {
+ "entropy": 1.244801640510559,
+ "epoch": 0.6505050505050505,
+ "grad_norm": 1.983111023902893,
+ "learning_rate": 1.5676767676767678e-05,
+ "loss": 1.246530294418335,
+ "mean_token_accuracy": 0.6966902613639832,
+ "num_tokens": 5275648.0,
+ "step": 322
+ },
+ {
+ "entropy": 1.5355960130691528,
+ "epoch": 0.6525252525252525,
+ "grad_norm": 2.126024007797241,
+ "learning_rate": 1.5663299663299666e-05,
+ "loss": 1.576147198677063,
+ "mean_token_accuracy": 0.6414265036582947,
+ "num_tokens": 5292032.0,
+ "step": 323
+ },
+ {
+ "entropy": 1.13125741481781,
+ "epoch": 0.6545454545454545,
+ "grad_norm": 1.8933371305465698,
+ "learning_rate": 1.564983164983165e-05,
+ "loss": 1.1167645454406738,
+ "mean_token_accuracy": 0.7080483436584473,
+ "num_tokens": 5308416.0,
+ "step": 324
+ },
+ {
+ "entropy": 1.6553086042404175,
+ "epoch": 0.6565656565656566,
+ "grad_norm": 1.9253427982330322,
+ "learning_rate": 1.563636363636364e-05,
+ "loss": 1.6746933460235596,
+ "mean_token_accuracy": 0.6138250827789307,
+ "num_tokens": 5324800.0,
+ "step": 325
+ },
+ {
+ "entropy": 1.7187142372131348,
+ "epoch": 0.6585858585858586,
+ "grad_norm": 1.8655303716659546,
+ "learning_rate": 1.5622895622895624e-05,
+ "loss": 1.7368483543395996,
+ "mean_token_accuracy": 0.6073521971702576,
+ "num_tokens": 5341184.0,
+ "step": 326
+ },
+ {
+ "entropy": 1.4150934219360352,
+ "epoch": 0.6606060606060606,
+ "grad_norm": 1.8396921157836914,
+ "learning_rate": 1.560942760942761e-05,
+ "loss": 1.450613260269165,
+ "mean_token_accuracy": 0.6628602743148804,
+ "num_tokens": 5357568.0,
+ "step": 327
+ },
+ {
+ "entropy": 1.6463909149169922,
+ "epoch": 0.6626262626262627,
+ "grad_norm": 2.0179622173309326,
+ "learning_rate": 1.5595959595959597e-05,
+ "loss": 1.6767137050628662,
+ "mean_token_accuracy": 0.6147410869598389,
+ "num_tokens": 5373952.0,
+ "step": 328
+ },
+ {
+ "entropy": 1.2962068319320679,
+ "epoch": 0.6646464646464646,
+ "grad_norm": 1.9322903156280518,
+ "learning_rate": 1.5582491582491582e-05,
+ "loss": 1.288381814956665,
+ "mean_token_accuracy": 0.6877137422561646,
+ "num_tokens": 5390336.0,
+ "step": 329
+ },
+ {
+ "entropy": 1.4675830602645874,
+ "epoch": 0.6666666666666666,
+ "grad_norm": 1.7735666036605835,
+ "learning_rate": 1.556902356902357e-05,
+ "loss": 1.4585455656051636,
+ "mean_token_accuracy": 0.671775758266449,
+ "num_tokens": 5406720.0,
+ "step": 330
+ },
+ {
+ "entropy": 1.55418062210083,
+ "epoch": 0.6686868686868687,
+ "grad_norm": 1.9255602359771729,
+ "learning_rate": 1.555555555555556e-05,
+ "loss": 1.6033880710601807,
+ "mean_token_accuracy": 0.6486321687698364,
+ "num_tokens": 5423104.0,
+ "step": 331
+ },
+ {
+ "entropy": 1.4288218021392822,
+ "epoch": 0.6707070707070707,
+ "grad_norm": 1.8075612783432007,
+ "learning_rate": 1.5542087542087543e-05,
+ "loss": 1.457895278930664,
+ "mean_token_accuracy": 0.6725085377693176,
+ "num_tokens": 5439488.0,
+ "step": 332
+ },
+ {
+ "entropy": 1.3419297933578491,
+ "epoch": 0.6727272727272727,
+ "grad_norm": 2.1730353832244873,
+ "learning_rate": 1.552861952861953e-05,
+ "loss": 1.3681403398513794,
+ "mean_token_accuracy": 0.6664020419120789,
+ "num_tokens": 5455872.0,
+ "step": 333
+ },
+ {
+ "entropy": 1.706923007965088,
+ "epoch": 0.6747474747474748,
+ "grad_norm": 1.9297505617141724,
+ "learning_rate": 1.5515151515151516e-05,
+ "loss": 1.7304211854934692,
+ "mean_token_accuracy": 0.6020395755767822,
+ "num_tokens": 5472256.0,
+ "step": 334
+ },
+ {
+ "entropy": 1.117404818534851,
+ "epoch": 0.6767676767676768,
+ "grad_norm": 1.7363555431365967,
+ "learning_rate": 1.55016835016835e-05,
+ "loss": 1.1002353429794312,
+ "mean_token_accuracy": 0.7285661697387695,
+ "num_tokens": 5488640.0,
+ "step": 335
+ },
+ {
+ "entropy": 1.409253478050232,
+ "epoch": 0.6787878787878788,
+ "grad_norm": 2.029304265975952,
+ "learning_rate": 1.548821548821549e-05,
+ "loss": 1.4001437425613403,
+ "mean_token_accuracy": 0.6604176759719849,
+ "num_tokens": 5505024.0,
+ "step": 336
+ },
+ {
+ "entropy": 1.0427494049072266,
+ "epoch": 0.6808080808080809,
+ "grad_norm": 2.0266847610473633,
+ "learning_rate": 1.5474747474747477e-05,
+ "loss": 1.0636063814163208,
+ "mean_token_accuracy": 0.7298485636711121,
+ "num_tokens": 5521408.0,
+ "step": 337
+ },
+ {
+ "entropy": 1.479506254196167,
+ "epoch": 0.6828282828282828,
+ "grad_norm": 1.937677264213562,
+ "learning_rate": 1.5461279461279462e-05,
+ "loss": 1.4711894989013672,
+ "mean_token_accuracy": 0.6436248421669006,
+ "num_tokens": 5537792.0,
+ "step": 338
+ },
+ {
+ "entropy": 1.636492371559143,
+ "epoch": 0.6848484848484848,
+ "grad_norm": 1.9746829271316528,
+ "learning_rate": 1.544781144781145e-05,
+ "loss": 1.638418436050415,
+ "mean_token_accuracy": 0.6204200983047485,
+ "num_tokens": 5554176.0,
+ "step": 339
+ },
+ {
+ "entropy": 1.6094971895217896,
+ "epoch": 0.6868686868686869,
+ "grad_norm": 1.998382806777954,
+ "learning_rate": 1.5434343434343435e-05,
+ "loss": 1.6025879383087158,
+ "mean_token_accuracy": 0.6193209290504456,
+ "num_tokens": 5570560.0,
+ "step": 340
+ },
+ {
+ "entropy": 1.3113542795181274,
+ "epoch": 0.6888888888888889,
+ "grad_norm": 2.059739589691162,
+ "learning_rate": 1.5420875420875423e-05,
+ "loss": 1.32335364818573,
+ "mean_token_accuracy": 0.6776379942893982,
+ "num_tokens": 5586944.0,
+ "step": 341
+ },
+ {
+ "entropy": 1.3761622905731201,
+ "epoch": 0.6909090909090909,
+ "grad_norm": 1.9774043560028076,
+ "learning_rate": 1.5407407407407408e-05,
+ "loss": 1.3731458187103271,
+ "mean_token_accuracy": 0.681546151638031,
+ "num_tokens": 5603328.0,
+ "step": 342
+ },
+ {
+ "entropy": 1.3822734355926514,
+ "epoch": 0.692929292929293,
+ "grad_norm": 5.41459321975708,
+ "learning_rate": 1.5393939393939393e-05,
+ "loss": 1.4077341556549072,
+ "mean_token_accuracy": 0.6688446402549744,
+ "num_tokens": 5619712.0,
+ "step": 343
+ },
+ {
+ "entropy": 1.4885843992233276,
+ "epoch": 0.694949494949495,
+ "grad_norm": 1.9256173372268677,
+ "learning_rate": 1.538047138047138e-05,
+ "loss": 1.5017898082733154,
+ "mean_token_accuracy": 0.6411211490631104,
+ "num_tokens": 5636096.0,
+ "step": 344
+ },
+ {
+ "entropy": 1.4558689594268799,
+ "epoch": 0.696969696969697,
+ "grad_norm": 1.923056721687317,
+ "learning_rate": 1.536700336700337e-05,
+ "loss": 1.4409637451171875,
+ "mean_token_accuracy": 0.6607840657234192,
+ "num_tokens": 5652480.0,
+ "step": 345
+ },
+ {
+ "entropy": 1.3396257162094116,
+ "epoch": 0.6989898989898989,
+ "grad_norm": 1.9321191310882568,
+ "learning_rate": 1.5353535353535354e-05,
+ "loss": 1.3465441465377808,
+ "mean_token_accuracy": 0.6682339906692505,
+ "num_tokens": 5668864.0,
+ "step": 346
+ },
+ {
+ "entropy": 1.931525468826294,
+ "epoch": 0.701010101010101,
+ "grad_norm": 2.11252498626709,
+ "learning_rate": 1.5340067340067342e-05,
+ "loss": 1.947523832321167,
+ "mean_token_accuracy": 0.5751709938049316,
+ "num_tokens": 5685248.0,
+ "step": 347
+ },
+ {
+ "entropy": 1.4246045351028442,
+ "epoch": 0.703030303030303,
+ "grad_norm": 2.017690658569336,
+ "learning_rate": 1.5326599326599327e-05,
+ "loss": 1.431095004081726,
+ "mean_token_accuracy": 0.6621274948120117,
+ "num_tokens": 5701632.0,
+ "step": 348
+ },
+ {
+ "entropy": 1.2840481996536255,
+ "epoch": 0.705050505050505,
+ "grad_norm": 1.897916555404663,
+ "learning_rate": 1.5313131313131315e-05,
+ "loss": 1.2944300174713135,
+ "mean_token_accuracy": 0.7013311982154846,
+ "num_tokens": 5718016.0,
+ "step": 349
+ },
+ {
+ "entropy": 1.1308226585388184,
+ "epoch": 0.7070707070707071,
+ "grad_norm": 1.8326802253723145,
+ "learning_rate": 1.52996632996633e-05,
+ "loss": 1.1564277410507202,
+ "mean_token_accuracy": 0.714154839515686,
+ "num_tokens": 5734400.0,
+ "step": 350
+ },
+ {
+ "entropy": 1.4157475233078003,
+ "epoch": 0.7090909090909091,
+ "grad_norm": 1.9305709600448608,
+ "learning_rate": 1.5286195286195288e-05,
+ "loss": 1.421119213104248,
+ "mean_token_accuracy": 0.6651197075843811,
+ "num_tokens": 5750784.0,
+ "step": 351
+ },
+ {
+ "entropy": 1.353966474533081,
+ "epoch": 0.7111111111111111,
+ "grad_norm": 2.058542251586914,
+ "learning_rate": 1.5272727272727276e-05,
+ "loss": 1.3575413227081299,
+ "mean_token_accuracy": 0.6724475026130676,
+ "num_tokens": 5767168.0,
+ "step": 352
+ },
+ {
+ "entropy": 1.7946882247924805,
+ "epoch": 0.7131313131313132,
+ "grad_norm": 1.9397060871124268,
+ "learning_rate": 1.525925925925926e-05,
+ "loss": 1.7882269620895386,
+ "mean_token_accuracy": 0.6064972877502441,
+ "num_tokens": 5783552.0,
+ "step": 353
+ },
+ {
+ "entropy": 1.4986019134521484,
+ "epoch": 0.7151515151515152,
+ "grad_norm": 1.9084810018539429,
+ "learning_rate": 1.5245791245791246e-05,
+ "loss": 1.5188498497009277,
+ "mean_token_accuracy": 0.6610283255577087,
+ "num_tokens": 5799936.0,
+ "step": 354
+ },
+ {
+ "entropy": 1.9081071615219116,
+ "epoch": 0.7171717171717171,
+ "grad_norm": 2.1528689861297607,
+ "learning_rate": 1.5232323232323234e-05,
+ "loss": 1.8967041969299316,
+ "mean_token_accuracy": 0.5691255331039429,
+ "num_tokens": 5816320.0,
+ "step": 355
+ },
+ {
+ "entropy": 1.641337513923645,
+ "epoch": 0.7191919191919192,
+ "grad_norm": 1.9197942018508911,
+ "learning_rate": 1.521885521885522e-05,
+ "loss": 1.6193110942840576,
+ "mean_token_accuracy": 0.6286638975143433,
+ "num_tokens": 5832704.0,
+ "step": 356
+ },
+ {
+ "entropy": 1.348304271697998,
+ "epoch": 0.7212121212121212,
+ "grad_norm": 1.9729533195495605,
+ "learning_rate": 1.5205387205387207e-05,
+ "loss": 1.3781111240386963,
+ "mean_token_accuracy": 0.6654250025749207,
+ "num_tokens": 5849088.0,
+ "step": 357
+ },
+ {
+ "entropy": 1.5427544116973877,
+ "epoch": 0.7232323232323232,
+ "grad_norm": 2.0730769634246826,
+ "learning_rate": 1.5191919191919193e-05,
+ "loss": 1.5770654678344727,
+ "mean_token_accuracy": 0.6273815631866455,
+ "num_tokens": 5865472.0,
+ "step": 358
+ },
+ {
+ "entropy": 1.5560660362243652,
+ "epoch": 0.7252525252525253,
+ "grad_norm": 2.0436644554138184,
+ "learning_rate": 1.5178451178451178e-05,
+ "loss": 1.5465143918991089,
+ "mean_token_accuracy": 0.6335490942001343,
+ "num_tokens": 5881856.0,
+ "step": 359
+ },
+ {
+ "entropy": 1.5396223068237305,
+ "epoch": 0.7272727272727273,
+ "grad_norm": 1.9955254793167114,
+ "learning_rate": 1.5164983164983166e-05,
+ "loss": 1.5742223262786865,
+ "mean_token_accuracy": 0.6344650983810425,
+ "num_tokens": 5898240.0,
+ "step": 360
+ },
+ {
+ "entropy": 1.1664644479751587,
+ "epoch": 0.7292929292929293,
+ "grad_norm": 1.6942992210388184,
+ "learning_rate": 1.5151515151515153e-05,
+ "loss": 1.159040927886963,
+ "mean_token_accuracy": 0.7144601941108704,
+ "num_tokens": 5914624.0,
+ "step": 361
+ },
+ {
+ "entropy": 1.356544017791748,
+ "epoch": 0.7313131313131314,
+ "grad_norm": 1.890787124633789,
+ "learning_rate": 1.5138047138047138e-05,
+ "loss": 1.347895622253418,
+ "mean_token_accuracy": 0.6813629865646362,
+ "num_tokens": 5931008.0,
+ "step": 362
+ },
+ {
+ "entropy": 1.6837173700332642,
+ "epoch": 0.7333333333333333,
+ "grad_norm": 2.1780381202697754,
+ "learning_rate": 1.5124579124579126e-05,
+ "loss": 1.6997990608215332,
+ "mean_token_accuracy": 0.6098558902740479,
+ "num_tokens": 5947392.0,
+ "step": 363
+ },
+ {
+ "entropy": 1.1412957906723022,
+ "epoch": 0.7353535353535353,
+ "grad_norm": 1.801152229309082,
+ "learning_rate": 1.5111111111111112e-05,
+ "loss": 1.161649227142334,
+ "mean_token_accuracy": 0.705483615398407,
+ "num_tokens": 5963776.0,
+ "step": 364
+ },
+ {
+ "entropy": 1.2495859861373901,
+ "epoch": 0.7373737373737373,
+ "grad_norm": 1.8535690307617188,
+ "learning_rate": 1.50976430976431e-05,
+ "loss": 1.2595994472503662,
+ "mean_token_accuracy": 0.6926599740982056,
+ "num_tokens": 5980160.0,
+ "step": 365
+ },
+ {
+ "entropy": 1.8037965297698975,
+ "epoch": 0.7393939393939394,
+ "grad_norm": 2.0506303310394287,
+ "learning_rate": 1.5084175084175085e-05,
+ "loss": 1.8499953746795654,
+ "mean_token_accuracy": 0.5861626863479614,
+ "num_tokens": 5996544.0,
+ "step": 366
+ },
+ {
+ "entropy": 1.2877405881881714,
+ "epoch": 0.7414141414141414,
+ "grad_norm": 1.8159914016723633,
+ "learning_rate": 1.5070707070707072e-05,
+ "loss": 1.3165576457977295,
+ "mean_token_accuracy": 0.685271143913269,
+ "num_tokens": 6012928.0,
+ "step": 367
+ },
+ {
+ "entropy": 1.345953106880188,
+ "epoch": 0.7434343434343434,
+ "grad_norm": 2.003962516784668,
+ "learning_rate": 1.505723905723906e-05,
+ "loss": 1.3270621299743652,
+ "mean_token_accuracy": 0.6671348214149475,
+ "num_tokens": 6029312.0,
+ "step": 368
+ },
+ {
+ "entropy": 1.408361792564392,
+ "epoch": 0.7454545454545455,
+ "grad_norm": 1.9815948009490967,
+ "learning_rate": 1.5043771043771045e-05,
+ "loss": 1.4097518920898438,
+ "mean_token_accuracy": 0.652662456035614,
+ "num_tokens": 6045696.0,
+ "step": 369
+ },
+ {
+ "entropy": 1.4007829427719116,
+ "epoch": 0.7474747474747475,
+ "grad_norm": 1.928751826286316,
+ "learning_rate": 1.5030303030303031e-05,
+ "loss": 1.4142816066741943,
+ "mean_token_accuracy": 0.671775758266449,
+ "num_tokens": 6062080.0,
+ "step": 370
+ },
+ {
+ "entropy": 1.7548134326934814,
+ "epoch": 0.7494949494949495,
+ "grad_norm": 2.081939458847046,
+ "learning_rate": 1.5016835016835018e-05,
+ "loss": 1.792219877243042,
+ "mean_token_accuracy": 0.6044821739196777,
+ "num_tokens": 6078464.0,
+ "step": 371
+ },
+ {
+ "entropy": 1.6162793636322021,
+ "epoch": 0.7515151515151515,
+ "grad_norm": 1.8544763326644897,
+ "learning_rate": 1.5003367003367004e-05,
+ "loss": 1.650557041168213,
+ "mean_token_accuracy": 0.6289692521095276,
+ "num_tokens": 6094848.0,
+ "step": 372
+ },
+ {
+ "entropy": 1.6329439878463745,
+ "epoch": 0.7535353535353535,
+ "grad_norm": 1.8025282621383667,
+ "learning_rate": 1.4989898989898992e-05,
+ "loss": 1.6812629699707031,
+ "mean_token_accuracy": 0.6405104994773865,
+ "num_tokens": 6111232.0,
+ "step": 373
+ },
+ {
+ "entropy": 1.2201604843139648,
+ "epoch": 0.7555555555555555,
+ "grad_norm": 1.9814586639404297,
+ "learning_rate": 1.4976430976430977e-05,
+ "loss": 1.2086223363876343,
+ "mean_token_accuracy": 0.7020029425621033,
+ "num_tokens": 6127616.0,
+ "step": 374
+ },
+ {
+ "entropy": 1.8105695247650146,
+ "epoch": 0.7575757575757576,
+ "grad_norm": 2.0217790603637695,
+ "learning_rate": 1.4962962962962964e-05,
+ "loss": 1.8443559408187866,
+ "mean_token_accuracy": 0.5897654891014099,
+ "num_tokens": 6144000.0,
+ "step": 375
+ },
+ {
+ "entropy": 1.197646141052246,
+ "epoch": 0.7595959595959596,
+ "grad_norm": 1.9286304712295532,
+ "learning_rate": 1.4949494949494952e-05,
+ "loss": 1.2183899879455566,
+ "mean_token_accuracy": 0.6925989389419556,
+ "num_tokens": 6160384.0,
+ "step": 376
+ },
+ {
+ "entropy": 1.4825936555862427,
+ "epoch": 0.7616161616161616,
+ "grad_norm": 1.6694327592849731,
+ "learning_rate": 1.4936026936026937e-05,
+ "loss": 1.4815235137939453,
+ "mean_token_accuracy": 0.6572422981262207,
+ "num_tokens": 6176768.0,
+ "step": 377
+ },
+ {
+ "entropy": 1.3497473001480103,
+ "epoch": 0.7636363636363637,
+ "grad_norm": 2.0366406440734863,
+ "learning_rate": 1.4922558922558923e-05,
+ "loss": 1.3637127876281738,
+ "mean_token_accuracy": 0.6690889000892639,
+ "num_tokens": 6193152.0,
+ "step": 378
+ },
+ {
+ "entropy": 1.314116358757019,
+ "epoch": 0.7656565656565657,
+ "grad_norm": 1.742874026298523,
+ "learning_rate": 1.4909090909090911e-05,
+ "loss": 1.300222396850586,
+ "mean_token_accuracy": 0.6977283954620361,
+ "num_tokens": 6209536.0,
+ "step": 379
+ },
+ {
+ "entropy": 1.4347912073135376,
+ "epoch": 0.7676767676767676,
+ "grad_norm": 1.9317418336868286,
+ "learning_rate": 1.4895622895622896e-05,
+ "loss": 1.429826021194458,
+ "mean_token_accuracy": 0.6650586128234863,
+ "num_tokens": 6225920.0,
+ "step": 380
+ },
+ {
+ "entropy": 1.448641061782837,
+ "epoch": 0.7696969696969697,
+ "grad_norm": 1.8417843580245972,
+ "learning_rate": 1.4882154882154884e-05,
+ "loss": 1.4439187049865723,
+ "mean_token_accuracy": 0.6615168452262878,
+ "num_tokens": 6242304.0,
+ "step": 381
+ },
+ {
+ "entropy": 1.2918578386306763,
+ "epoch": 0.7717171717171717,
+ "grad_norm": 1.8720030784606934,
+ "learning_rate": 1.486868686868687e-05,
+ "loss": 1.3158597946166992,
+ "mean_token_accuracy": 0.6865534782409668,
+ "num_tokens": 6258688.0,
+ "step": 382
+ },
+ {
+ "entropy": 1.5822116136550903,
+ "epoch": 0.7737373737373737,
+ "grad_norm": 1.9301713705062866,
+ "learning_rate": 1.4855218855218856e-05,
+ "loss": 1.6046046018600464,
+ "mean_token_accuracy": 0.6361138224601746,
+ "num_tokens": 6275072.0,
+ "step": 383
+ },
+ {
+ "entropy": 1.6837408542633057,
+ "epoch": 0.7757575757575758,
+ "grad_norm": 2.270636796951294,
+ "learning_rate": 1.4841750841750844e-05,
+ "loss": 1.7323150634765625,
+ "mean_token_accuracy": 0.6006350517272949,
+ "num_tokens": 6291456.0,
+ "step": 384
+ },
+ {
+ "entropy": 1.4837889671325684,
+ "epoch": 0.7777777777777778,
+ "grad_norm": 1.908146619796753,
+ "learning_rate": 1.482828282828283e-05,
+ "loss": 1.4938652515411377,
+ "mean_token_accuracy": 0.64667809009552,
+ "num_tokens": 6307840.0,
+ "step": 385
+ },
+ {
+ "entropy": 1.680192232131958,
+ "epoch": 0.7797979797979798,
+ "grad_norm": 2.2369024753570557,
+ "learning_rate": 1.4814814814814815e-05,
+ "loss": 1.692443609237671,
+ "mean_token_accuracy": 0.6279922127723694,
+ "num_tokens": 6324224.0,
+ "step": 386
+ },
+ {
+ "entropy": 1.686415195465088,
+ "epoch": 0.7818181818181819,
+ "grad_norm": 1.8252373933792114,
+ "learning_rate": 1.4801346801346803e-05,
+ "loss": 1.7022712230682373,
+ "mean_token_accuracy": 0.6302515864372253,
+ "num_tokens": 6340608.0,
+ "step": 387
+ },
+ {
+ "entropy": 1.6967185735702515,
+ "epoch": 0.7838383838383839,
+ "grad_norm": 2.102458953857422,
+ "learning_rate": 1.478787878787879e-05,
+ "loss": 1.6931922435760498,
+ "mean_token_accuracy": 0.6149853467941284,
+ "num_tokens": 6356992.0,
+ "step": 388
+ },
+ {
+ "entropy": 1.201438307762146,
+ "epoch": 0.7858585858585858,
+ "grad_norm": 1.8396239280700684,
+ "learning_rate": 1.4774410774410774e-05,
+ "loss": 1.2081820964813232,
+ "mean_token_accuracy": 0.6987664699554443,
+ "num_tokens": 6373376.0,
+ "step": 389
+ },
+ {
+ "entropy": 1.5559548139572144,
+ "epoch": 0.7878787878787878,
+ "grad_norm": 2.063498020172119,
+ "learning_rate": 1.4760942760942763e-05,
+ "loss": 1.545433759689331,
+ "mean_token_accuracy": 0.6267709136009216,
+ "num_tokens": 6389760.0,
+ "step": 390
+ },
+ {
+ "entropy": 1.7186503410339355,
+ "epoch": 0.7898989898989899,
+ "grad_norm": 2.0129787921905518,
+ "learning_rate": 1.4747474747474747e-05,
+ "loss": 1.717472791671753,
+ "mean_token_accuracy": 0.6540058851242065,
+ "num_tokens": 6406144.0,
+ "step": 391
+ },
+ {
+ "entropy": 1.643048882484436,
+ "epoch": 0.7919191919191919,
+ "grad_norm": 1.9443073272705078,
+ "learning_rate": 1.4734006734006736e-05,
+ "loss": 1.6441841125488281,
+ "mean_token_accuracy": 0.6458231806755066,
+ "num_tokens": 6422528.0,
+ "step": 392
+ },
+ {
+ "entropy": 1.3627581596374512,
+ "epoch": 0.793939393939394,
+ "grad_norm": 1.9852598905563354,
+ "learning_rate": 1.4720538720538722e-05,
+ "loss": 1.3558071851730347,
+ "mean_token_accuracy": 0.6720200181007385,
+ "num_tokens": 6438912.0,
+ "step": 393
+ },
+ {
+ "entropy": 0.8946540951728821,
+ "epoch": 0.795959595959596,
+ "grad_norm": 1.54441237449646,
+ "learning_rate": 1.4707070707070707e-05,
+ "loss": 0.8874151706695557,
+ "mean_token_accuracy": 0.7685027122497559,
+ "num_tokens": 6455296.0,
+ "step": 394
+ },
+ {
+ "entropy": 1.35861074924469,
+ "epoch": 0.797979797979798,
+ "grad_norm": 2.158553123474121,
+ "learning_rate": 1.4693602693602695e-05,
+ "loss": 1.4291828870773315,
+ "mean_token_accuracy": 0.6647533178329468,
+ "num_tokens": 6471680.0,
+ "step": 395
+ },
+ {
+ "entropy": 1.7047442197799683,
+ "epoch": 0.8,
+ "grad_norm": 2.1242175102233887,
+ "learning_rate": 1.4680134680134681e-05,
+ "loss": 1.7065966129302979,
+ "mean_token_accuracy": 0.6058256030082703,
+ "num_tokens": 6488064.0,
+ "step": 396
+ },
+ {
+ "entropy": 1.7153913974761963,
+ "epoch": 0.802020202020202,
+ "grad_norm": 1.9085419178009033,
+ "learning_rate": 1.4666666666666666e-05,
+ "loss": 1.7373592853546143,
+ "mean_token_accuracy": 0.6020395755767822,
+ "num_tokens": 6504448.0,
+ "step": 397
+ },
+ {
+ "entropy": 1.1062475442886353,
+ "epoch": 0.804040404040404,
+ "grad_norm": 1.9173986911773682,
+ "learning_rate": 1.4653198653198654e-05,
+ "loss": 1.0991405248641968,
+ "mean_token_accuracy": 0.7195896506309509,
+ "num_tokens": 6520832.0,
+ "step": 398
+ },
+ {
+ "entropy": 1.5717406272888184,
+ "epoch": 0.806060606060606,
+ "grad_norm": 2.0077219009399414,
+ "learning_rate": 1.4639730639730641e-05,
+ "loss": 1.595954418182373,
+ "mean_token_accuracy": 0.6270151734352112,
+ "num_tokens": 6537216.0,
+ "step": 399
+ },
+ {
+ "entropy": 1.3960795402526855,
+ "epoch": 0.8080808080808081,
+ "grad_norm": 2.2864415645599365,
+ "learning_rate": 1.4626262626262629e-05,
+ "loss": 1.416130781173706,
+ "mean_token_accuracy": 0.6631656289100647,
+ "num_tokens": 6553600.0,
+ "step": 400
+ },
+ {
+ "epoch": 0.8080808080808081,
+ "eval_entropy": 1.510355769626556,
+ "eval_loss": 1.5199862718582153,
+ "eval_mean_token_accuracy": 0.6461201098657423,
+ "eval_num_tokens": 6553600.0,
+ "eval_runtime": 43.8634,
+ "eval_samples_per_second": 22.57,
+ "eval_steps_per_second": 2.827,
+ "step": 400
+ },
+ {
+ "entropy": 1.5662645101547241,
+ "epoch": 0.8101010101010101,
+ "grad_norm": 2.1806085109710693,
+ "learning_rate": 1.4612794612794614e-05,
+ "loss": 1.5763804912567139,
+ "mean_token_accuracy": 0.6330605745315552,
+ "num_tokens": 6569984.0,
+ "step": 401
+ },
+ {
+ "entropy": 1.4046087265014648,
+ "epoch": 0.8121212121212121,
+ "grad_norm": 1.9650650024414062,
+ "learning_rate": 1.45993265993266e-05,
+ "loss": 1.396510124206543,
+ "mean_token_accuracy": 0.6568148732185364,
+ "num_tokens": 6586368.0,
+ "step": 402
+ },
+ {
+ "entropy": 1.084455966949463,
+ "epoch": 0.8141414141414142,
+ "grad_norm": 1.926537275314331,
+ "learning_rate": 1.4585858585858587e-05,
+ "loss": 1.096333622932434,
+ "mean_token_accuracy": 0.7233145833015442,
+ "num_tokens": 6602752.0,
+ "step": 403
+ },
+ {
+ "entropy": 1.379700779914856,
+ "epoch": 0.8161616161616162,
+ "grad_norm": 1.957202672958374,
+ "learning_rate": 1.4572390572390573e-05,
+ "loss": 1.4035027027130127,
+ "mean_token_accuracy": 0.674462616443634,
+ "num_tokens": 6619136.0,
+ "step": 404
+ },
+ {
+ "entropy": 1.3775221109390259,
+ "epoch": 0.8181818181818182,
+ "grad_norm": 1.9095491170883179,
+ "learning_rate": 1.455892255892256e-05,
+ "loss": 1.3915553092956543,
+ "mean_token_accuracy": 0.6811187267303467,
+ "num_tokens": 6635520.0,
+ "step": 405
+ },
+ {
+ "entropy": 1.4745222330093384,
+ "epoch": 0.8202020202020202,
+ "grad_norm": 2.0017123222351074,
+ "learning_rate": 1.4545454545454546e-05,
+ "loss": 1.4845868349075317,
+ "mean_token_accuracy": 0.6557767391204834,
+ "num_tokens": 6651904.0,
+ "step": 406
+ },
+ {
+ "entropy": 1.5129855871200562,
+ "epoch": 0.8222222222222222,
+ "grad_norm": 2.127979040145874,
+ "learning_rate": 1.4531986531986533e-05,
+ "loss": 1.5081605911254883,
+ "mean_token_accuracy": 0.6425256729125977,
+ "num_tokens": 6668288.0,
+ "step": 407
+ },
+ {
+ "entropy": 1.1438099145889282,
+ "epoch": 0.8242424242424242,
+ "grad_norm": 1.7389986515045166,
+ "learning_rate": 1.4518518518518521e-05,
+ "loss": 1.1446681022644043,
+ "mean_token_accuracy": 0.7180629968643188,
+ "num_tokens": 6684672.0,
+ "step": 408
+ },
+ {
+ "entropy": 1.448157787322998,
+ "epoch": 0.8262626262626263,
+ "grad_norm": 1.8483424186706543,
+ "learning_rate": 1.4505050505050506e-05,
+ "loss": 1.4317030906677246,
+ "mean_token_accuracy": 0.6659746170043945,
+ "num_tokens": 6701056.0,
+ "step": 409
+ },
+ {
+ "entropy": 1.419940710067749,
+ "epoch": 0.8282828282828283,
+ "grad_norm": 1.961125135421753,
+ "learning_rate": 1.4491582491582492e-05,
+ "loss": 1.42207670211792,
+ "mean_token_accuracy": 0.672874927520752,
+ "num_tokens": 6717440.0,
+ "step": 410
+ },
+ {
+ "entropy": 1.5707522630691528,
+ "epoch": 0.8303030303030303,
+ "grad_norm": 2.0862255096435547,
+ "learning_rate": 1.447811447811448e-05,
+ "loss": 1.5721523761749268,
+ "mean_token_accuracy": 0.6305569410324097,
+ "num_tokens": 6733824.0,
+ "step": 411
+ },
+ {
+ "entropy": 1.4308481216430664,
+ "epoch": 0.8323232323232324,
+ "grad_norm": 2.1505820751190186,
+ "learning_rate": 1.4464646464646465e-05,
+ "loss": 1.4522666931152344,
+ "mean_token_accuracy": 0.6502198576927185,
+ "num_tokens": 6750208.0,
+ "step": 412
+ },
+ {
+ "entropy": 1.7424806356430054,
+ "epoch": 0.8343434343434344,
+ "grad_norm": 2.1787710189819336,
+ "learning_rate": 1.4451178451178452e-05,
+ "loss": 1.7719662189483643,
+ "mean_token_accuracy": 0.5897654891014099,
+ "num_tokens": 6766592.0,
+ "step": 413
+ },
+ {
+ "entropy": 1.9777010679244995,
+ "epoch": 0.8363636363636363,
+ "grad_norm": 2.1178369522094727,
+ "learning_rate": 1.443771043771044e-05,
+ "loss": 1.9885730743408203,
+ "mean_token_accuracy": 0.5628358721733093,
+ "num_tokens": 6782976.0,
+ "step": 414
+ },
+ {
+ "entropy": 1.701235055923462,
+ "epoch": 0.8383838383838383,
+ "grad_norm": 1.9038937091827393,
+ "learning_rate": 1.4424242424242425e-05,
+ "loss": 1.6970818042755127,
+ "mean_token_accuracy": 0.6209086179733276,
+ "num_tokens": 6799360.0,
+ "step": 415
+ },
+ {
+ "entropy": 1.5118876695632935,
+ "epoch": 0.8404040404040404,
+ "grad_norm": 1.8924365043640137,
+ "learning_rate": 1.4410774410774413e-05,
+ "loss": 1.5288946628570557,
+ "mean_token_accuracy": 0.6480215191841125,
+ "num_tokens": 6815744.0,
+ "step": 416
+ },
+ {
+ "entropy": 1.5526877641677856,
+ "epoch": 0.8424242424242424,
+ "grad_norm": 2.109499216079712,
+ "learning_rate": 1.43973063973064e-05,
+ "loss": 1.5581820011138916,
+ "mean_token_accuracy": 0.6351978778839111,
+ "num_tokens": 6832128.0,
+ "step": 417
+ },
+ {
+ "entropy": 1.8132506608963013,
+ "epoch": 0.8444444444444444,
+ "grad_norm": 2.0389890670776367,
+ "learning_rate": 1.4383838383838384e-05,
+ "loss": 1.8378231525421143,
+ "mean_token_accuracy": 0.592391312122345,
+ "num_tokens": 6848512.0,
+ "step": 418
+ },
+ {
+ "entropy": 1.3766752481460571,
+ "epoch": 0.8464646464646465,
+ "grad_norm": 1.8586320877075195,
+ "learning_rate": 1.4370370370370372e-05,
+ "loss": 1.3741214275360107,
+ "mean_token_accuracy": 0.6796531677246094,
+ "num_tokens": 6864896.0,
+ "step": 419
+ },
+ {
+ "entropy": 1.1854407787322998,
+ "epoch": 0.8484848484848485,
+ "grad_norm": 1.875899314880371,
+ "learning_rate": 1.4356902356902359e-05,
+ "loss": 1.2393877506256104,
+ "mean_token_accuracy": 0.704323410987854,
+ "num_tokens": 6881280.0,
+ "step": 420
+ },
+ {
+ "entropy": 1.498558759689331,
+ "epoch": 0.8505050505050505,
+ "grad_norm": 1.898848295211792,
+ "learning_rate": 1.4343434343434344e-05,
+ "loss": 1.4963322877883911,
+ "mean_token_accuracy": 0.6542501449584961,
+ "num_tokens": 6897664.0,
+ "step": 421
+ },
+ {
+ "entropy": 1.5070384740829468,
+ "epoch": 0.8525252525252526,
+ "grad_norm": 2.095853090286255,
+ "learning_rate": 1.4329966329966332e-05,
+ "loss": 1.5026702880859375,
+ "mean_token_accuracy": 0.6561431288719177,
+ "num_tokens": 6914048.0,
+ "step": 422
+ },
+ {
+ "entropy": 1.7848025560379028,
+ "epoch": 0.8545454545454545,
+ "grad_norm": 1.9265769720077515,
+ "learning_rate": 1.4316498316498317e-05,
+ "loss": 1.8066134452819824,
+ "mean_token_accuracy": 0.5936736464500427,
+ "num_tokens": 6930432.0,
+ "step": 423
+ },
+ {
+ "entropy": 1.4278018474578857,
+ "epoch": 0.8565656565656565,
+ "grad_norm": 1.900985836982727,
+ "learning_rate": 1.4303030303030305e-05,
+ "loss": 1.4045631885528564,
+ "mean_token_accuracy": 0.6630434989929199,
+ "num_tokens": 6946816.0,
+ "step": 424
+ },
+ {
+ "entropy": 1.728216290473938,
+ "epoch": 0.8585858585858586,
+ "grad_norm": 1.952841877937317,
+ "learning_rate": 1.4289562289562291e-05,
+ "loss": 1.728804349899292,
+ "mean_token_accuracy": 0.613031268119812,
+ "num_tokens": 6963200.0,
+ "step": 425
+ },
+ {
+ "entropy": 1.6632241010665894,
+ "epoch": 0.8606060606060606,
+ "grad_norm": 2.187973737716675,
+ "learning_rate": 1.4276094276094276e-05,
+ "loss": 1.6703932285308838,
+ "mean_token_accuracy": 0.6150463819503784,
+ "num_tokens": 6979584.0,
+ "step": 426
+ },
+ {
+ "entropy": 1.761826992034912,
+ "epoch": 0.8626262626262626,
+ "grad_norm": 2.203636646270752,
+ "learning_rate": 1.4262626262626264e-05,
+ "loss": 1.727257490158081,
+ "mean_token_accuracy": 0.5861626863479614,
+ "num_tokens": 6995968.0,
+ "step": 427
+ },
+ {
+ "entropy": 1.4528838396072388,
+ "epoch": 0.8646464646464647,
+ "grad_norm": 1.7967053651809692,
+ "learning_rate": 1.424915824915825e-05,
+ "loss": 1.4786958694458008,
+ "mean_token_accuracy": 0.6589521169662476,
+ "num_tokens": 7012352.0,
+ "step": 428
+ },
+ {
+ "entropy": 1.2221835851669312,
+ "epoch": 0.8666666666666667,
+ "grad_norm": 1.8118345737457275,
+ "learning_rate": 1.4235690235690235e-05,
+ "loss": 1.247727632522583,
+ "mean_token_accuracy": 0.7024303674697876,
+ "num_tokens": 7028736.0,
+ "step": 429
+ },
+ {
+ "entropy": 1.5066863298416138,
+ "epoch": 0.8686868686868687,
+ "grad_norm": 1.8390358686447144,
+ "learning_rate": 1.4222222222222224e-05,
+ "loss": 1.5283207893371582,
+ "mean_token_accuracy": 0.6553493142127991,
+ "num_tokens": 7045120.0,
+ "step": 430
+ },
+ {
+ "entropy": 1.346379041671753,
+ "epoch": 0.8707070707070707,
+ "grad_norm": 2.070962905883789,
+ "learning_rate": 1.420875420875421e-05,
+ "loss": 1.386117935180664,
+ "mean_token_accuracy": 0.6690889000892639,
+ "num_tokens": 7061504.0,
+ "step": 431
+ },
+ {
+ "entropy": 1.8691747188568115,
+ "epoch": 0.8727272727272727,
+ "grad_norm": 1.7868962287902832,
+ "learning_rate": 1.4195286195286198e-05,
+ "loss": 1.9285637140274048,
+ "mean_token_accuracy": 0.5680874586105347,
+ "num_tokens": 7077888.0,
+ "step": 432
+ },
+ {
+ "entropy": 1.8500115871429443,
+ "epoch": 0.8747474747474747,
+ "grad_norm": 1.8853468894958496,
+ "learning_rate": 1.4181818181818183e-05,
+ "loss": 1.8733386993408203,
+ "mean_token_accuracy": 0.5956888198852539,
+ "num_tokens": 7094272.0,
+ "step": 433
+ },
+ {
+ "entropy": 1.3737305402755737,
+ "epoch": 0.8767676767676768,
+ "grad_norm": 1.8363661766052246,
+ "learning_rate": 1.416835016835017e-05,
+ "loss": 1.4020421504974365,
+ "mean_token_accuracy": 0.6723253726959229,
+ "num_tokens": 7110656.0,
+ "step": 434
+ },
+ {
+ "entropy": 1.5069265365600586,
+ "epoch": 0.8787878787878788,
+ "grad_norm": 2.1154301166534424,
+ "learning_rate": 1.4154882154882156e-05,
+ "loss": 1.5137630701065063,
+ "mean_token_accuracy": 0.628724992275238,
+ "num_tokens": 7127040.0,
+ "step": 435
+ },
+ {
+ "entropy": 1.6280715465545654,
+ "epoch": 0.8808080808080808,
+ "grad_norm": 1.9827117919921875,
+ "learning_rate": 1.4141414141414143e-05,
+ "loss": 1.6629055738449097,
+ "mean_token_accuracy": 0.6300684213638306,
+ "num_tokens": 7143424.0,
+ "step": 436
+ },
+ {
+ "entropy": 1.0767987966537476,
+ "epoch": 0.8828282828282829,
+ "grad_norm": 1.8503392934799194,
+ "learning_rate": 1.4127946127946129e-05,
+ "loss": 1.0964105129241943,
+ "mean_token_accuracy": 0.7118954658508301,
+ "num_tokens": 7159808.0,
+ "step": 437
+ },
+ {
+ "entropy": 1.6979929208755493,
+ "epoch": 0.8848484848484849,
+ "grad_norm": 2.114170551300049,
+ "learning_rate": 1.4114478114478116e-05,
+ "loss": 1.7339307069778442,
+ "mean_token_accuracy": 0.6071690320968628,
+ "num_tokens": 7176192.0,
+ "step": 438
+ },
+ {
+ "entropy": 1.840633511543274,
+ "epoch": 0.8868686868686869,
+ "grad_norm": 2.1661946773529053,
+ "learning_rate": 1.4101010101010102e-05,
+ "loss": 1.866344690322876,
+ "mean_token_accuracy": 0.5988031029701233,
+ "num_tokens": 7192576.0,
+ "step": 439
+ },
+ {
+ "entropy": 1.8278590440750122,
+ "epoch": 0.8888888888888888,
+ "grad_norm": 1.9278390407562256,
+ "learning_rate": 1.4087542087542087e-05,
+ "loss": 1.81584632396698,
+ "mean_token_accuracy": 0.5878114104270935,
+ "num_tokens": 7208960.0,
+ "step": 440
+ },
+ {
+ "entropy": 1.405017375946045,
+ "epoch": 0.8909090909090909,
+ "grad_norm": 1.9446384906768799,
+ "learning_rate": 1.4074074074074075e-05,
+ "loss": 1.4040675163269043,
+ "mean_token_accuracy": 0.6647533178329468,
+ "num_tokens": 7225344.0,
+ "step": 441
+ },
+ {
+ "entropy": 1.0967023372650146,
+ "epoch": 0.8929292929292929,
+ "grad_norm": 1.8880785703659058,
+ "learning_rate": 1.4060606060606061e-05,
+ "loss": 1.0798790454864502,
+ "mean_token_accuracy": 0.7167806625366211,
+ "num_tokens": 7241728.0,
+ "step": 442
+ },
+ {
+ "entropy": 1.6126327514648438,
+ "epoch": 0.8949494949494949,
+ "grad_norm": 2.2159016132354736,
+ "learning_rate": 1.404713804713805e-05,
+ "loss": 1.620417833328247,
+ "mean_token_accuracy": 0.6215192675590515,
+ "num_tokens": 7258112.0,
+ "step": 443
+ },
+ {
+ "entropy": 1.4352757930755615,
+ "epoch": 0.896969696969697,
+ "grad_norm": 2.070910692214966,
+ "learning_rate": 1.4033670033670034e-05,
+ "loss": 1.409663200378418,
+ "mean_token_accuracy": 0.658646821975708,
+ "num_tokens": 7274496.0,
+ "step": 444
+ },
+ {
+ "entropy": 1.350129246711731,
+ "epoch": 0.898989898989899,
+ "grad_norm": 1.8672778606414795,
+ "learning_rate": 1.4020202020202021e-05,
+ "loss": 1.3485581874847412,
+ "mean_token_accuracy": 0.6743404865264893,
+ "num_tokens": 7290880.0,
+ "step": 445
+ },
+ {
+ "entropy": 1.0900049209594727,
+ "epoch": 0.901010101010101,
+ "grad_norm": 1.8460185527801514,
+ "learning_rate": 1.4006734006734009e-05,
+ "loss": 1.074802279472351,
+ "mean_token_accuracy": 0.727161705493927,
+ "num_tokens": 7307264.0,
+ "step": 446
+ },
+ {
+ "entropy": 1.6430293321609497,
+ "epoch": 0.9030303030303031,
+ "grad_norm": 2.0190517902374268,
+ "learning_rate": 1.3993265993265994e-05,
+ "loss": 1.6505115032196045,
+ "mean_token_accuracy": 0.6073521971702576,
+ "num_tokens": 7323648.0,
+ "step": 447
+ },
+ {
+ "entropy": 1.465686559677124,
+ "epoch": 0.9050505050505051,
+ "grad_norm": 2.0139927864074707,
+ "learning_rate": 1.397979797979798e-05,
+ "loss": 1.480743408203125,
+ "mean_token_accuracy": 0.6593796014785767,
+ "num_tokens": 7340032.0,
+ "step": 448
+ },
+ {
+ "entropy": 2.152183771133423,
+ "epoch": 0.907070707070707,
+ "grad_norm": 2.2376039028167725,
+ "learning_rate": 1.3966329966329969e-05,
+ "loss": 2.117891788482666,
+ "mean_token_accuracy": 0.5404250025749207,
+ "num_tokens": 7356416.0,
+ "step": 449
+ },
+ {
+ "entropy": 1.6066724061965942,
+ "epoch": 0.9090909090909091,
+ "grad_norm": 1.895409107208252,
+ "learning_rate": 1.3952861952861953e-05,
+ "loss": 1.6247167587280273,
+ "mean_token_accuracy": 0.6281143426895142,
+ "num_tokens": 7372800.0,
+ "step": 450
+ },
+ {
+ "entropy": 1.7240369319915771,
+ "epoch": 0.9111111111111111,
+ "grad_norm": 1.9780583381652832,
+ "learning_rate": 1.3939393939393942e-05,
+ "loss": 1.7521916627883911,
+ "mean_token_accuracy": 0.6022838354110718,
+ "num_tokens": 7389184.0,
+ "step": 451
+ },
+ {
+ "entropy": 1.168492317199707,
+ "epoch": 0.9131313131313131,
+ "grad_norm": 1.676292896270752,
+ "learning_rate": 1.3925925925925928e-05,
+ "loss": 1.1678345203399658,
+ "mean_token_accuracy": 0.7136663198471069,
+ "num_tokens": 7405568.0,
+ "step": 452
+ },
+ {
+ "entropy": 1.4993922710418701,
+ "epoch": 0.9151515151515152,
+ "grad_norm": 2.051370620727539,
+ "learning_rate": 1.3912457912457913e-05,
+ "loss": 1.5273159742355347,
+ "mean_token_accuracy": 0.6378847360610962,
+ "num_tokens": 7421952.0,
+ "step": 453
+ },
+ {
+ "entropy": 1.3478108644485474,
+ "epoch": 0.9171717171717172,
+ "grad_norm": 1.9196240901947021,
+ "learning_rate": 1.3898989898989901e-05,
+ "loss": 1.3547455072402954,
+ "mean_token_accuracy": 0.666829526424408,
+ "num_tokens": 7438336.0,
+ "step": 454
+ },
+ {
+ "entropy": 1.1411141157150269,
+ "epoch": 0.9191919191919192,
+ "grad_norm": 1.9581170082092285,
+ "learning_rate": 1.3885521885521886e-05,
+ "loss": 1.1604664325714111,
+ "mean_token_accuracy": 0.7018197178840637,
+ "num_tokens": 7454720.0,
+ "step": 455
+ },
+ {
+ "entropy": 1.1814258098602295,
+ "epoch": 0.9212121212121213,
+ "grad_norm": 2.005451202392578,
+ "learning_rate": 1.3872053872053872e-05,
+ "loss": 1.1768745183944702,
+ "mean_token_accuracy": 0.7044455409049988,
+ "num_tokens": 7471104.0,
+ "step": 456
+ },
+ {
+ "entropy": 1.335093379020691,
+ "epoch": 0.9232323232323232,
+ "grad_norm": 1.928747534751892,
+ "learning_rate": 1.385858585858586e-05,
+ "loss": 1.3504598140716553,
+ "mean_token_accuracy": 0.6774548292160034,
+ "num_tokens": 7487488.0,
+ "step": 457
+ },
+ {
+ "entropy": 1.6866769790649414,
+ "epoch": 0.9252525252525252,
+ "grad_norm": 1.9078223705291748,
+ "learning_rate": 1.3845117845117845e-05,
+ "loss": 1.688689947128296,
+ "mean_token_accuracy": 0.6077796816825867,
+ "num_tokens": 7503872.0,
+ "step": 458
+ },
+ {
+ "entropy": 1.2842025756835938,
+ "epoch": 0.9272727272727272,
+ "grad_norm": 1.909472942352295,
+ "learning_rate": 1.3831649831649833e-05,
+ "loss": 1.3128941059112549,
+ "mean_token_accuracy": 0.6872252225875854,
+ "num_tokens": 7520256.0,
+ "step": 459
+ },
+ {
+ "entropy": 1.7797539234161377,
+ "epoch": 0.9292929292929293,
+ "grad_norm": 2.246411085128784,
+ "learning_rate": 1.381818181818182e-05,
+ "loss": 1.8016175031661987,
+ "mean_token_accuracy": 0.6049095988273621,
+ "num_tokens": 7536640.0,
+ "step": 460
+ },
+ {
+ "entropy": 1.3464831113815308,
+ "epoch": 0.9313131313131313,
+ "grad_norm": 1.9999858140945435,
+ "learning_rate": 1.3804713804713805e-05,
+ "loss": 1.3523657321929932,
+ "mean_token_accuracy": 0.6787371635437012,
+ "num_tokens": 7553024.0,
+ "step": 461
+ },
+ {
+ "entropy": 1.5901066064834595,
+ "epoch": 0.9333333333333333,
+ "grad_norm": 1.9632524251937866,
+ "learning_rate": 1.3791245791245793e-05,
+ "loss": 1.596254587173462,
+ "mean_token_accuracy": 0.6428309679031372,
+ "num_tokens": 7569408.0,
+ "step": 462
+ },
+ {
+ "entropy": 1.2511993646621704,
+ "epoch": 0.9353535353535354,
+ "grad_norm": 1.7678464651107788,
+ "learning_rate": 1.377777777777778e-05,
+ "loss": 1.254664659500122,
+ "mean_token_accuracy": 0.6981558203697205,
+ "num_tokens": 7585792.0,
+ "step": 463
+ },
+ {
+ "entropy": 1.546912431716919,
+ "epoch": 0.9373737373737374,
+ "grad_norm": 2.0353212356567383,
+ "learning_rate": 1.3764309764309764e-05,
+ "loss": 1.54923677444458,
+ "mean_token_accuracy": 0.6403273344039917,
+ "num_tokens": 7602176.0,
+ "step": 464
+ },
+ {
+ "entropy": 1.6535273790359497,
+ "epoch": 0.9393939393939394,
+ "grad_norm": 2.1584854125976562,
+ "learning_rate": 1.3750841750841752e-05,
+ "loss": 1.6464964151382446,
+ "mean_token_accuracy": 0.6218246221542358,
+ "num_tokens": 7618560.0,
+ "step": 465
+ },
+ {
+ "entropy": 1.1885185241699219,
+ "epoch": 0.9414141414141414,
+ "grad_norm": 1.8068963289260864,
+ "learning_rate": 1.3737373737373739e-05,
+ "loss": 1.1980109214782715,
+ "mean_token_accuracy": 0.6990718245506287,
+ "num_tokens": 7634944.0,
+ "step": 466
+ },
+ {
+ "entropy": 1.2306157350540161,
+ "epoch": 0.9434343434343434,
+ "grad_norm": 1.9044586420059204,
+ "learning_rate": 1.3723905723905725e-05,
+ "loss": 1.2291686534881592,
+ "mean_token_accuracy": 0.6996824741363525,
+ "num_tokens": 7651328.0,
+ "step": 467
+ },
+ {
+ "entropy": 1.2496092319488525,
+ "epoch": 0.9454545454545454,
+ "grad_norm": 1.7264868021011353,
+ "learning_rate": 1.3710437710437712e-05,
+ "loss": 1.2440049648284912,
+ "mean_token_accuracy": 0.6877137422561646,
+ "num_tokens": 7667712.0,
+ "step": 468
+ },
+ {
+ "entropy": 1.8231102228164673,
+ "epoch": 0.9474747474747475,
+ "grad_norm": 2.099104166030884,
+ "learning_rate": 1.3696969696969698e-05,
+ "loss": 1.84971284866333,
+ "mean_token_accuracy": 0.5869565010070801,
+ "num_tokens": 7684096.0,
+ "step": 469
+ },
+ {
+ "entropy": 1.537786841392517,
+ "epoch": 0.9494949494949495,
+ "grad_norm": 1.872962474822998,
+ "learning_rate": 1.3683501683501685e-05,
+ "loss": 1.5448193550109863,
+ "mean_token_accuracy": 0.6465559601783752,
+ "num_tokens": 7700480.0,
+ "step": 470
+ },
+ {
+ "entropy": 1.8270875215530396,
+ "epoch": 0.9515151515151515,
+ "grad_norm": 2.2747695446014404,
+ "learning_rate": 1.3670033670033671e-05,
+ "loss": 1.854095697402954,
+ "mean_token_accuracy": 0.5834758281707764,
+ "num_tokens": 7716864.0,
+ "step": 471
+ },
+ {
+ "entropy": 1.505604863166809,
+ "epoch": 0.9535353535353536,
+ "grad_norm": 1.9903687238693237,
+ "learning_rate": 1.3656565656565656e-05,
+ "loss": 1.5269451141357422,
+ "mean_token_accuracy": 0.6475940346717834,
+ "num_tokens": 7733248.0,
+ "step": 472
+ },
+ {
+ "entropy": 1.8228932619094849,
+ "epoch": 0.9555555555555556,
+ "grad_norm": 2.4003078937530518,
+ "learning_rate": 1.3643097643097644e-05,
+ "loss": 1.846016764640808,
+ "mean_token_accuracy": 0.5900097489356995,
+ "num_tokens": 7749632.0,
+ "step": 473
+ },
+ {
+ "entropy": 1.2786308526992798,
+ "epoch": 0.9575757575757575,
+ "grad_norm": 1.9355131387710571,
+ "learning_rate": 1.362962962962963e-05,
+ "loss": 1.3152062892913818,
+ "mean_token_accuracy": 0.6703712940216064,
+ "num_tokens": 7766016.0,
+ "step": 474
+ },
+ {
+ "entropy": 1.1032856702804565,
+ "epoch": 0.9595959595959596,
+ "grad_norm": 1.7720186710357666,
+ "learning_rate": 1.3616161616161619e-05,
+ "loss": 1.1242918968200684,
+ "mean_token_accuracy": 0.7250854969024658,
+ "num_tokens": 7782400.0,
+ "step": 475
+ },
+ {
+ "entropy": 1.5261021852493286,
+ "epoch": 0.9616161616161616,
+ "grad_norm": 1.8472843170166016,
+ "learning_rate": 1.3602693602693604e-05,
+ "loss": 1.54013991355896,
+ "mean_token_accuracy": 0.6471055150032043,
+ "num_tokens": 7798784.0,
+ "step": 476
+ },
+ {
+ "entropy": 0.9697252511978149,
+ "epoch": 0.9636363636363636,
+ "grad_norm": 1.8871296644210815,
+ "learning_rate": 1.358922558922559e-05,
+ "loss": 0.9874091148376465,
+ "mean_token_accuracy": 0.7430996298789978,
+ "num_tokens": 7815168.0,
+ "step": 477
+ },
+ {
+ "entropy": 1.6003227233886719,
+ "epoch": 0.9656565656565657,
+ "grad_norm": 2.371385097503662,
+ "learning_rate": 1.3575757575757578e-05,
+ "loss": 1.580587387084961,
+ "mean_token_accuracy": 0.6273815631866455,
+ "num_tokens": 7831552.0,
+ "step": 478
+ },
+ {
+ "entropy": 1.556900978088379,
+ "epoch": 0.9676767676767677,
+ "grad_norm": 1.839432954788208,
+ "learning_rate": 1.3562289562289563e-05,
+ "loss": 1.5154083967208862,
+ "mean_token_accuracy": 0.628724992275238,
+ "num_tokens": 7847936.0,
+ "step": 479
+ },
+ {
+ "entropy": 1.2394448518753052,
+ "epoch": 0.9696969696969697,
+ "grad_norm": 1.6638867855072021,
+ "learning_rate": 1.354882154882155e-05,
+ "loss": 1.2016921043395996,
+ "mean_token_accuracy": 0.6991939544677734,
+ "num_tokens": 7864320.0,
+ "step": 480
+ },
+ {
+ "entropy": 1.4200247526168823,
+ "epoch": 0.9717171717171718,
+ "grad_norm": 1.8742305040359497,
+ "learning_rate": 1.3535353535353538e-05,
+ "loss": 1.4267700910568237,
+ "mean_token_accuracy": 0.6604176759719849,
+ "num_tokens": 7880704.0,
+ "step": 481
+ },
+ {
+ "entropy": 2.0156707763671875,
+ "epoch": 0.9737373737373738,
+ "grad_norm": 2.1772451400756836,
+ "learning_rate": 1.3521885521885523e-05,
+ "loss": 2.0306758880615234,
+ "mean_token_accuracy": 0.5579506754875183,
+ "num_tokens": 7897088.0,
+ "step": 482
+ },
+ {
+ "entropy": 1.2802858352661133,
+ "epoch": 0.9757575757575757,
+ "grad_norm": 1.8951549530029297,
+ "learning_rate": 1.3508417508417509e-05,
+ "loss": 1.2761621475219727,
+ "mean_token_accuracy": 0.6825231909751892,
+ "num_tokens": 7913472.0,
+ "step": 483
+ },
+ {
+ "entropy": 1.3758989572525024,
+ "epoch": 0.9777777777777777,
+ "grad_norm": 1.8072963953018188,
+ "learning_rate": 1.3494949494949497e-05,
+ "loss": 1.3532235622406006,
+ "mean_token_accuracy": 0.6749511361122131,
+ "num_tokens": 7929856.0,
+ "step": 484
+ },
+ {
+ "entropy": 1.4488792419433594,
+ "epoch": 0.9797979797979798,
+ "grad_norm": 1.7842215299606323,
+ "learning_rate": 1.3481481481481482e-05,
+ "loss": 1.4513285160064697,
+ "mean_token_accuracy": 0.6539447903633118,
+ "num_tokens": 7946240.0,
+ "step": 485
+ },
+ {
+ "entropy": 1.2957613468170166,
+ "epoch": 0.9818181818181818,
+ "grad_norm": 1.8612068891525269,
+ "learning_rate": 1.346801346801347e-05,
+ "loss": 1.3016514778137207,
+ "mean_token_accuracy": 0.6848436594009399,
+ "num_tokens": 7962624.0,
+ "step": 486
+ },
+ {
+ "entropy": 1.4321255683898926,
+ "epoch": 0.9838383838383838,
+ "grad_norm": 1.7746661901474,
+ "learning_rate": 1.3454545454545455e-05,
+ "loss": 1.4230387210845947,
+ "mean_token_accuracy": 0.664692223072052,
+ "num_tokens": 7979008.0,
+ "step": 487
+ },
+ {
+ "entropy": 1.7615760564804077,
+ "epoch": 0.9858585858585859,
+ "grad_norm": 1.9986366033554077,
+ "learning_rate": 1.3441077441077441e-05,
+ "loss": 1.7871575355529785,
+ "mean_token_accuracy": 0.5994748473167419,
+ "num_tokens": 7995392.0,
+ "step": 488
+ },
+ {
+ "entropy": 1.349948763847351,
+ "epoch": 0.9878787878787879,
+ "grad_norm": 2.0217247009277344,
+ "learning_rate": 1.342760942760943e-05,
+ "loss": 1.352069616317749,
+ "mean_token_accuracy": 0.6762335300445557,
+ "num_tokens": 8011776.0,
+ "step": 489
+ },
+ {
+ "entropy": 1.6317073106765747,
+ "epoch": 0.98989898989899,
+ "grad_norm": 1.9479029178619385,
+ "learning_rate": 1.3414141414141414e-05,
+ "loss": 1.6507500410079956,
+ "mean_token_accuracy": 0.6186492443084717,
+ "num_tokens": 8028160.0,
+ "step": 490
+ },
+ {
+ "entropy": 2.0288875102996826,
+ "epoch": 0.9919191919191919,
+ "grad_norm": 2.1520888805389404,
+ "learning_rate": 1.3400673400673401e-05,
+ "loss": 2.06215500831604,
+ "mean_token_accuracy": 0.5530043840408325,
+ "num_tokens": 8044544.0,
+ "step": 491
+ },
+ {
+ "entropy": 1.8297137022018433,
+ "epoch": 0.9939393939393939,
+ "grad_norm": 1.949404001235962,
+ "learning_rate": 1.3387205387205389e-05,
+ "loss": 1.8859628438949585,
+ "mean_token_accuracy": 0.5876282453536987,
+ "num_tokens": 8060928.0,
+ "step": 492
+ },
+ {
+ "entropy": 1.587094783782959,
+ "epoch": 0.9959595959595959,
+ "grad_norm": 1.9205344915390015,
+ "learning_rate": 1.3373737373737374e-05,
+ "loss": 1.6053454875946045,
+ "mean_token_accuracy": 0.6257938742637634,
+ "num_tokens": 8077312.0,
+ "step": 493
+ },
+ {
+ "entropy": 1.6583648920059204,
+ "epoch": 0.997979797979798,
+ "grad_norm": 1.976946234703064,
+ "learning_rate": 1.3360269360269362e-05,
+ "loss": 1.6656805276870728,
+ "mean_token_accuracy": 0.6186492443084717,
+ "num_tokens": 8093696.0,
+ "step": 494
+ },
+ {
+ "entropy": 1.1685364246368408,
+ "epoch": 1.0,
+ "grad_norm": 1.8258967399597168,
+ "learning_rate": 1.3346801346801349e-05,
+ "loss": 1.165276288986206,
+ "mean_token_accuracy": 0.7082315683364868,
+ "num_tokens": 8110080.0,
+ "step": 495
+ },
+ {
+ "entropy": 1.1211013793945312,
+ "epoch": 1.002020202020202,
+ "grad_norm": 2.27962327003479,
+ "learning_rate": 1.3333333333333333e-05,
+ "loss": 0.9540231227874756,
+ "mean_token_accuracy": 0.747801661491394,
+ "num_tokens": 8126464.0,
+ "step": 496
+ },
+ {
+ "entropy": 1.370996356010437,
+ "epoch": 1.004040404040404,
+ "grad_norm": 2.3193519115448,
+ "learning_rate": 1.3319865319865321e-05,
+ "loss": 1.240065097808838,
+ "mean_token_accuracy": 0.6976062655448914,
+ "num_tokens": 8142848.0,
+ "step": 497
+ },
+ {
+ "entropy": 1.4673359394073486,
+ "epoch": 1.006060606060606,
+ "grad_norm": 2.071213483810425,
+ "learning_rate": 1.3306397306397308e-05,
+ "loss": 1.3351866006851196,
+ "mean_token_accuracy": 0.6921104192733765,
+ "num_tokens": 8159232.0,
+ "step": 498
+ },
+ {
+ "entropy": 1.1151381731033325,
+ "epoch": 1.0080808080808081,
+ "grad_norm": 2.1310431957244873,
+ "learning_rate": 1.3292929292929293e-05,
+ "loss": 1.0092531442642212,
+ "mean_token_accuracy": 0.7261846661567688,
+ "num_tokens": 8175616.0,
+ "step": 499
+ },
+ {
+ "entropy": 1.4734766483306885,
+ "epoch": 1.0101010101010102,
+ "grad_norm": 2.015787124633789,
+ "learning_rate": 1.3279461279461281e-05,
+ "loss": 1.3748562335968018,
+ "mean_token_accuracy": 0.6682339906692505,
+ "num_tokens": 8192000.0,
+ "step": 500
+ },
+ {
+ "epoch": 1.0101010101010102,
+ "eval_entropy": 1.3614274196086391,
+ "eval_loss": 1.5201176404953003,
+ "eval_mean_token_accuracy": 0.647282806134993,
+ "eval_num_tokens": 8192000.0,
+ "eval_runtime": 43.8232,
+ "eval_samples_per_second": 22.591,
+ "eval_steps_per_second": 2.83,
+ "step": 500
+ }
+ ],
+ "logging_steps": 1,
+ "max_steps": 1485,
+ "num_input_tokens_seen": 0,
+ "num_train_epochs": 3,
+ "save_steps": 500,
+ "stateful_callbacks": {
+ "TrainerControl": {
+ "args": {
+ "should_epoch_stop": false,
+ "should_evaluate": false,
+ "should_log": false,
+ "should_save": true,
+ "should_training_stop": false
+ },
+ "attributes": {}
+ }
+ },
+ "total_flos": 6.9275322679296e+16,
+ "train_batch_size": 4,
+ "trial_name": null,
+ "trial_params": null
+}
diff --git a/qwen3-1.7b-teutonic-2e5/checkpoint-500/training_args.bin b/qwen3-1.7b-teutonic-2e5/checkpoint-500/training_args.bin
new file mode 100644
index 0000000000000000000000000000000000000000..98c558883023185245bfac817f2ca585caa9d94f
--- /dev/null
+++ b/qwen3-1.7b-teutonic-2e5/checkpoint-500/training_args.bin
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:2c573bdac95c796f8f7368ab9af6d35c5687f53373737ea965b8102d518df7cb
+size 7121
diff --git a/qwen3-1.7b-teutonic-2e5/checkpoint-500/zero_to_fp32.py b/qwen3-1.7b-teutonic-2e5/checkpoint-500/zero_to_fp32.py
new file mode 100644
index 0000000000000000000000000000000000000000..3970548c400fd4361bd923b763db90e963ddaf8c
--- /dev/null
+++ b/qwen3-1.7b-teutonic-2e5/checkpoint-500/zero_to_fp32.py
@@ -0,0 +1,854 @@
+#!/usr/bin/env python
+
+# Copyright (c) Microsoft Corporation.
+# SPDX-License-Identifier: Apache-2.0
+
+# DeepSpeed Team
+
+# This script extracts fp32 consolidated weights from a zero 1, 2 and 3 DeepSpeed checkpoints. It gets
+# copied into the top level checkpoint dir, so the user can easily do the conversion at any point in
+# the future. Once extracted, the weights don't require DeepSpeed and can be used in any
+# application.
+#
+# example:
+# python zero_to_fp32.py . output_dir/
+# or
+# python zero_to_fp32.py . output_dir/ --safe_serialization
+
+import argparse
+import torch
+import glob
+import math
+import os
+import re
+import gc
+import json
+import numpy as np
+from tqdm import tqdm
+from collections import OrderedDict
+from dataclasses import dataclass
+
+# while this script doesn't use deepspeed to recover data, since the checkpoints are pickled with
+# DeepSpeed data structures it has to be available in the current python environment.
+from deepspeed.utils import logger
+from deepspeed.checkpoint.constants import (DS_VERSION, OPTIMIZER_STATE_DICT, SINGLE_PARTITION_OF_FP32_GROUPS,
+ FP32_FLAT_GROUPS, ZERO_STAGE, PARTITION_COUNT, PARAM_SHAPES, BUFFER_NAMES,
+ FROZEN_PARAM_SHAPES, FROZEN_PARAM_FRAGMENTS, AUTOEP_LAYERS_KEY,
+ AUTOEP_LAYERS_KEY_LEGACY, AUTOEP_ZERO3_EXPERT_STATE_FORMAT_KEY,
+ AUTOEP_ZERO3_PARTITIONED_EXPERT_STATE_FORMAT, PARAM_ALIGNMENT_PADDINGS)
+
+
+@dataclass
+class zero_model_state:
+ buffers: dict()
+ param_shapes: dict()
+ shared_params: list
+ ds_version: int
+ frozen_param_shapes: dict()
+ frozen_param_fragments: dict()
+
+
+debug = 0
+
+# load to cpu
+device = torch.device('cpu')
+
+OUTPUT_DTYPE_NAMES = {
+ 'float32': torch.float32,
+ 'fp32': torch.float32,
+ 'float16': torch.float16,
+ 'fp16': torch.float16,
+ 'bfloat16': torch.bfloat16,
+ 'bf16': torch.bfloat16,
+}
+
+
+def _resolve_output_dtype(dtype):
+ requested_dtype = dtype
+ if isinstance(dtype, str):
+ dtype_name = dtype[6:] if dtype.startswith('torch.') else dtype
+ dtype = OUTPUT_DTYPE_NAMES.get(dtype_name.lower())
+ if dtype not in set(OUTPUT_DTYPE_NAMES.values()):
+ supported = ', '.join(sorted(OUTPUT_DTYPE_NAMES))
+ raise ValueError(f"Unsupported output dtype {requested_dtype!r}. Choose one of: {supported}")
+ return dtype
+
+
+def atoi(text):
+ return int(text) if text.isdigit() else text
+
+
+def natural_keys(text):
+ '''
+ alist.sort(key=natural_keys) sorts in human order
+ http://nedbatchelder.com/blog/200712/human_sorting.html
+ (See Toothy's implementation in the comments)
+ '''
+ return [atoi(c) for c in re.split(r'(\d+)', text)]
+
+
+def get_model_state_file(checkpoint_dir, zero_stage):
+ if not os.path.isdir(checkpoint_dir):
+ raise FileNotFoundError(f"Directory '{checkpoint_dir}' doesn't exist")
+
+ # there should be only one file
+ if zero_stage <= 2:
+ file = os.path.join(checkpoint_dir, "mp_rank_00_model_states.pt")
+ elif zero_stage == 3:
+ file = os.path.join(checkpoint_dir, "zero_pp_rank_0_mp_rank_00_model_states.pt")
+
+ if not os.path.exists(file):
+ raise FileNotFoundError(f"can't find model states file at '{file}'")
+
+ return file
+
+
+def get_checkpoint_files(checkpoint_dir, glob_pattern):
+ # XXX: need to test that this simple glob rule works for multi-node setup too
+ ckpt_files = sorted(glob.glob(os.path.join(checkpoint_dir, glob_pattern)), key=natural_keys)
+
+ if len(ckpt_files) == 0:
+ raise FileNotFoundError(f"can't find {glob_pattern} files in directory '{checkpoint_dir}'")
+
+ return ckpt_files
+
+
+def get_optim_files(checkpoint_dir):
+ return get_checkpoint_files(checkpoint_dir, "*_optim_states.pt")
+
+
+def get_model_state_files(checkpoint_dir):
+ return get_checkpoint_files(checkpoint_dir, "*_model_states.pt")
+
+
+def _has_autoep_zero3_partitioned_metadata(state_dict):
+ autoep_layers = state_dict.get(AUTOEP_LAYERS_KEY)
+ if autoep_layers is None:
+ autoep_layers = state_dict.get(AUTOEP_LAYERS_KEY_LEGACY)
+ if not isinstance(autoep_layers, list):
+ return False
+ return any(
+ isinstance(entry, dict)
+ and entry.get(AUTOEP_ZERO3_EXPERT_STATE_FORMAT_KEY) == AUTOEP_ZERO3_PARTITIONED_EXPERT_STATE_FORMAT
+ for entry in autoep_layers)
+
+
+def _raise_if_autoep_zero3_partitioned_state(state_dict):
+ if _has_autoep_zero3_partitioned_metadata(state_dict):
+ raise NotImplementedError("zero_to_fp32 does not support AutoEP ZeRO-3 partition-native checkpoints. "
+ "AutoEP expert parameters are partitioned over expert replica groups, so "
+ "global data-parallel consolidation would produce incomplete expert tensors. "
+ "Use ds_to_universal.py for expert-aware conversion.")
+
+
+def _raise_if_autoep_zero3_partitioned_checkpoint(model_files):
+ for file in model_files:
+ state_dict = torch.load(file, map_location=device, weights_only=False)
+ _raise_if_autoep_zero3_partitioned_state(state_dict)
+
+
+def parse_model_states(files):
+ zero_model_states = []
+ for file in files:
+ state_dict = torch.load(file, map_location=device, weights_only=False)
+ _raise_if_autoep_zero3_partitioned_state(state_dict)
+
+ if BUFFER_NAMES not in state_dict:
+ raise ValueError(f"{file} is not a model state checkpoint")
+ buffer_names = state_dict[BUFFER_NAMES]
+ if debug:
+ print("Found buffers:", buffer_names)
+
+ # recover just the buffers while restoring them to fp32 if they were saved in fp16
+ buffers = {k: v.float() for k, v in state_dict["module"].items() if k in buffer_names}
+ param_shapes = state_dict[PARAM_SHAPES]
+
+ # collect parameters that are included in param_shapes
+ param_names = []
+ for s in param_shapes:
+ for name in s.keys():
+ param_names.append(name)
+
+ # update with frozen parameters
+ frozen_param_shapes = state_dict.get(FROZEN_PARAM_SHAPES, None)
+ if frozen_param_shapes is not None:
+ if debug:
+ print(f"Found frozen_param_shapes: {frozen_param_shapes}")
+ param_names += list(frozen_param_shapes.keys())
+
+ # handle shared params
+ shared_params = [[k, v] for k, v in state_dict["shared_params"].items()]
+
+ ds_version = state_dict.get(DS_VERSION, None)
+
+ frozen_param_fragments = state_dict.get(FROZEN_PARAM_FRAGMENTS, None)
+
+ z_model_state = zero_model_state(buffers=buffers,
+ param_shapes=param_shapes,
+ shared_params=shared_params,
+ ds_version=ds_version,
+ frozen_param_shapes=frozen_param_shapes,
+ frozen_param_fragments=frozen_param_fragments)
+ zero_model_states.append(z_model_state)
+
+ return zero_model_states
+
+
+def parse_optim_states(files, ds_checkpoint_dir):
+ total_files = len(files)
+ state_dicts = []
+ for f in tqdm(files, desc='Loading checkpoint shards'):
+ state_dict = torch.load(f, map_location=device, mmap=True, weights_only=False)
+ # immediately discard the potentially huge 2 optimizer states as we only care for fp32 master weights
+ # and also handle the case where it was already removed by another helper script
+ state_dict["optimizer_state_dict"].pop("optimizer_state_dict", None)
+ state_dicts.append(state_dict)
+
+ if ZERO_STAGE not in state_dicts[0][OPTIMIZER_STATE_DICT]:
+ raise ValueError(f"{files[0]} is not a zero checkpoint")
+ zero_stage = state_dicts[0][OPTIMIZER_STATE_DICT][ZERO_STAGE]
+ world_size = state_dicts[0][OPTIMIZER_STATE_DICT][PARTITION_COUNT]
+
+ # For ZeRO-2 each param group can have different partition_count as data parallelism for expert
+ # parameters can be different from data parallelism for non-expert parameters. So we can just
+ # use the max of the partition_count to get the dp world_size.
+
+ if type(world_size) is list:
+ world_size = max(world_size)
+
+ if world_size != total_files:
+ raise ValueError(
+ f"Expected {world_size} of '*_optim_states.pt' under '{ds_checkpoint_dir}' but found {total_files} files. "
+ "Possibly due to an overwrite of an old checkpoint, or a checkpoint didn't get saved by one or more processes."
+ )
+
+ # the groups are named differently in each stage
+ if zero_stage <= 2:
+ fp32_groups_key = SINGLE_PARTITION_OF_FP32_GROUPS
+ elif zero_stage == 3:
+ fp32_groups_key = FP32_FLAT_GROUPS
+ else:
+ raise ValueError(f"unknown zero stage {zero_stage}")
+
+ fp32_flat_groups = [state_dicts[i][OPTIMIZER_STATE_DICT][fp32_groups_key] for i in range(len(state_dicts))]
+ param_alignment_paddings = state_dicts[0][OPTIMIZER_STATE_DICT].get(PARAM_ALIGNMENT_PADDINGS)
+ return zero_stage, world_size, fp32_flat_groups, param_alignment_paddings
+
+
+def _get_fp32_state_dict_from_zero_checkpoint(ds_checkpoint_dir, exclude_frozen_parameters):
+ """
+ Returns fp32 state_dict reconstructed from ds checkpoint
+
+ Args:
+ - ``ds_checkpoint_dir``: path to the deepspeed checkpoint folder (where the optimizer files are)
+
+ """
+ print(f"Processing zero checkpoint '{ds_checkpoint_dir}'")
+
+ # parse_model_states rejects AutoEP ZeRO-3 partition-native checkpoints
+ # before the expensive optimizer-shard load below.
+ model_files = get_model_state_files(ds_checkpoint_dir)
+ zero_model_states = parse_model_states(model_files)
+ print(f'Parsing checkpoint created by deepspeed=={zero_model_states[0].ds_version}')
+
+ optim_files = get_optim_files(ds_checkpoint_dir)
+ zero_stage, world_size, fp32_flat_groups, param_alignment_paddings = parse_optim_states(
+ optim_files, ds_checkpoint_dir)
+ print(f"Detected checkpoint of type zero stage {zero_stage}, world_size: {world_size}")
+
+ if zero_stage <= 2:
+ return _get_fp32_state_dict_from_zero2_checkpoint(world_size, fp32_flat_groups, zero_model_states,
+ exclude_frozen_parameters, param_alignment_paddings)
+ elif zero_stage == 3:
+ return _get_fp32_state_dict_from_zero3_checkpoint(world_size, fp32_flat_groups, zero_model_states,
+ exclude_frozen_parameters)
+
+
+def _zero2_merge_frozen_params(state_dict, zero_model_states):
+ if zero_model_states[0].frozen_param_shapes is None or len(zero_model_states[0].frozen_param_shapes) == 0:
+ return
+
+ frozen_param_shapes = zero_model_states[0].frozen_param_shapes
+ frozen_param_fragments = zero_model_states[0].frozen_param_fragments
+
+ if debug:
+ num_elem = sum(s.numel() for s in frozen_param_shapes.values())
+ print(f'rank 0: {FROZEN_PARAM_SHAPES}.numel = {num_elem}')
+
+ wanted_params = len(frozen_param_shapes)
+ wanted_numel = sum(s.numel() for s in frozen_param_shapes.values())
+ avail_numel = sum([p.numel() for p in frozen_param_fragments.values()])
+ print(f'Frozen params: Have {avail_numel} numels to process.')
+ print(f'Frozen params: Need {wanted_numel} numels in {wanted_params} params')
+
+ total_params = 0
+ total_numel = 0
+ for name, shape in frozen_param_shapes.items():
+ total_params += 1
+ unpartitioned_numel = shape.numel()
+ total_numel += unpartitioned_numel
+
+ state_dict[name] = frozen_param_fragments[name]
+
+ if debug:
+ print(f"{name} full shape: {shape} unpartitioned numel {unpartitioned_numel} ")
+
+ print(f"Reconstructed Frozen fp32 state dict with {total_params} params {total_numel} elements")
+
+
+def _has_callable(obj, fn):
+ attr = getattr(obj, fn, None)
+ return callable(attr)
+
+
+def _zero2_merge_trainable_params(state_dict,
+ world_size,
+ fp32_flat_groups,
+ zero_model_states,
+ param_alignment_paddings=None):
+ param_shapes = zero_model_states[0].param_shapes
+
+ # Reconstruction protocol:
+ #
+ # XXX: document this
+
+ if debug:
+ for i in range(world_size):
+ for j in range(len(fp32_flat_groups[0])):
+ print(f"{FP32_FLAT_GROUPS}[{i}][{j}].shape={fp32_flat_groups[i][j].shape}")
+
+ # XXX: memory usage doubles here (zero2)
+ num_param_groups = len(fp32_flat_groups[0])
+ merged_single_partition_of_fp32_groups = []
+ for i in range(num_param_groups):
+ merged_partitions = [sd[i] for sd in fp32_flat_groups]
+ full_single_fp32_vector = torch.cat(merged_partitions, 0)
+ merged_single_partition_of_fp32_groups.append(full_single_fp32_vector)
+ avail_numel = sum(
+ [full_single_fp32_vector.numel() for full_single_fp32_vector in merged_single_partition_of_fp32_groups])
+
+ if debug:
+ wanted_params = sum([len(shapes) for shapes in param_shapes])
+ wanted_numel = sum([sum(shape.numel() for shape in shapes.values()) for shapes in param_shapes])
+ # not asserting if there is a mismatch due to possible padding
+ print(f"Have {avail_numel} numels to process.")
+ print(f"Need {wanted_numel} numels in {wanted_params} params.")
+
+ # params
+ # XXX: for huge models that can't fit into the host's RAM we will have to recode this to support
+ # out-of-core computing solution
+ total_numel = 0
+ total_params = 0
+ for group_idx, (shapes,
+ full_single_fp32_vector) in enumerate(zip(param_shapes, merged_single_partition_of_fp32_groups)):
+ offset = 0
+ avail_numel = full_single_fp32_vector.numel()
+ group_alignment_paddings = None
+ if param_alignment_paddings is not None:
+ group_alignment_paddings = param_alignment_paddings[group_idx]
+ if len(group_alignment_paddings) != len(shapes):
+ raise ValueError(f"Expected {len(shapes)} parameter alignment paddings for group {group_idx}, "
+ f"but found {len(group_alignment_paddings)}")
+
+ for param_idx, (name, shape) in enumerate(shapes.items()):
+
+ unpartitioned_numel = shape.numel() if _has_callable(shape, 'numel') else math.prod(shape)
+ total_numel += unpartitioned_numel
+ total_params += 1
+
+ if debug:
+ print(f"{name} full shape: {shape} unpartitioned numel {unpartitioned_numel} ")
+ state_dict[name] = full_single_fp32_vector.narrow(0, offset, unpartitioned_numel).view(shape)
+ offset += unpartitioned_numel
+ if group_alignment_paddings is not None:
+ offset += group_alignment_paddings[param_idx]
+
+ # Z2 started to align to 2*world_size to improve nccl performance. Therefore both offset and
+ # avail_numel can differ by anywhere between 0..2*world_size. Due to two unrelated complex
+ # paddings performed in the code it's almost impossible to predict the exact numbers w/o the
+ # live optimizer object, so we are checking that the numbers are within the right range
+ align_to = 2 * world_size
+
+ def zero2_align(x):
+ return align_to * math.ceil(x / align_to)
+
+ if debug:
+ print(f"original offset={offset}, avail_numel={avail_numel}")
+
+ offset = zero2_align(offset)
+ avail_numel = zero2_align(avail_numel)
+
+ if debug:
+ print(f"aligned offset={offset}, avail_numel={avail_numel}")
+
+ # Sanity check
+ if offset != avail_numel:
+ raise ValueError(f"consumed {offset} numels out of {avail_numel} - something is wrong")
+
+ print(f"Reconstructed fp32 state dict with {total_params} params {total_numel} elements")
+
+
+def _get_fp32_state_dict_from_zero2_checkpoint(world_size,
+ fp32_flat_groups,
+ zero_model_states,
+ exclude_frozen_parameters,
+ param_alignment_paddings=None):
+ state_dict = OrderedDict()
+
+ # buffers
+ buffers = zero_model_states[0].buffers
+ state_dict.update(buffers)
+ if debug:
+ print(f"added {len(buffers)} buffers")
+
+ if not exclude_frozen_parameters:
+ _zero2_merge_frozen_params(state_dict, zero_model_states)
+
+ _zero2_merge_trainable_params(state_dict, world_size, fp32_flat_groups, zero_model_states,
+ param_alignment_paddings)
+
+ # recover shared parameters
+ for pair in zero_model_states[0].shared_params:
+ if pair[1] in state_dict:
+ state_dict[pair[0]] = state_dict[pair[1]]
+
+ return state_dict
+
+
+def zero3_partitioned_param_info(unpartitioned_numel, world_size):
+ remainder = unpartitioned_numel % world_size
+ padding_numel = (world_size - remainder) if remainder else 0
+ partitioned_numel = math.ceil(unpartitioned_numel / world_size)
+ return partitioned_numel, padding_numel
+
+
+def _zero3_merge_frozen_params(state_dict, world_size, zero_model_states):
+ if zero_model_states[0].frozen_param_shapes is None or len(zero_model_states[0].frozen_param_shapes) == 0:
+ return
+
+ if debug:
+ for i in range(world_size):
+ num_elem = sum(s.numel() for s in zero_model_states[i].frozen_param_fragments.values())
+ print(f'rank {i}: {FROZEN_PARAM_SHAPES}.numel = {num_elem}')
+
+ frozen_param_shapes = zero_model_states[0].frozen_param_shapes
+ wanted_params = len(frozen_param_shapes)
+ wanted_numel = sum(s.numel() for s in frozen_param_shapes.values())
+ avail_numel = sum([p.numel() for p in zero_model_states[0].frozen_param_fragments.values()]) * world_size
+ print(f'Frozen params: Have {avail_numel} numels to process.')
+ print(f'Frozen params: Need {wanted_numel} numels in {wanted_params} params')
+
+ total_params = 0
+ total_numel = 0
+ for name, shape in zero_model_states[0].frozen_param_shapes.items():
+ total_params += 1
+ unpartitioned_numel = shape.numel()
+ total_numel += unpartitioned_numel
+
+ param_frags = tuple(model_state.frozen_param_fragments[name] for model_state in zero_model_states)
+ state_dict[name] = torch.cat(param_frags, 0).narrow(0, 0, unpartitioned_numel).view(shape)
+
+ partitioned_numel, partitioned_padding_numel = zero3_partitioned_param_info(unpartitioned_numel, world_size)
+
+ if debug:
+ print(
+ f"Frozen params: {total_params} {name} full shape: {shape} partition0 numel={partitioned_numel} partitioned_padding_numel={partitioned_padding_numel}"
+ )
+
+ print(f"Reconstructed Frozen fp32 state dict with {total_params} params {total_numel} elements")
+
+
+class GatheredTensor:
+ """
+ A pseudo tensor that collects partitioned weights.
+ It is more memory efficient when there are multiple groups.
+ """
+
+ def __init__(self, flat_groups, flat_groups_offset, offset, partitioned_numel, shape):
+ self.flat_groups = flat_groups
+ self.flat_groups_offset = flat_groups_offset
+ self.offset = offset
+ self.partitioned_numel = partitioned_numel
+ self.shape = shape
+ self.dtype = self.flat_groups[0][0].dtype
+
+ def contiguous(self):
+ """
+ Merge partitioned weights from flat_groups into a single tensor.
+ """
+ end_idx = self.offset + self.partitioned_numel
+ world_size = len(self.flat_groups)
+ pad_flat_param_chunks = []
+
+ for rank_i in range(world_size):
+ # for each rank, we need to collect weights from related group/groups
+ flat_groups_at_rank_i = self.flat_groups[rank_i]
+ start_group_id = None
+ end_group_id = None
+ for group_id in range(len(self.flat_groups_offset)):
+ if self.flat_groups_offset[group_id] <= self.offset < self.flat_groups_offset[group_id + 1]:
+ start_group_id = group_id
+ if self.flat_groups_offset[group_id] < end_idx <= self.flat_groups_offset[group_id + 1]:
+ end_group_id = group_id
+ break
+ # collect weights from related group/groups
+ for group_id in range(start_group_id, end_group_id + 1):
+ flat_tensor = flat_groups_at_rank_i[group_id]
+ start_offset = self.offset - self.flat_groups_offset[group_id]
+ end_offset = min(end_idx, self.flat_groups_offset[group_id + 1]) - self.flat_groups_offset[group_id]
+ pad_flat_param_chunks.append(flat_tensor[start_offset:end_offset])
+
+ # collect weights from all ranks
+ pad_flat_param = torch.cat(pad_flat_param_chunks, dim=0)
+ param = pad_flat_param[:self.shape.numel()].view(self.shape).contiguous()
+ return param
+
+
+def _zero3_merge_trainable_params(state_dict, world_size, fp32_flat_groups, zero_model_states):
+ param_shapes = zero_model_states[0].param_shapes
+ avail_numel = sum([flat_group.numel() for flat_group in fp32_flat_groups[0]]) * world_size
+
+ # Reconstruction protocol: For zero3 we need to zip the partitions together at boundary of each
+ # param, re-consolidating each param, while dealing with padding if any
+
+ # merge list of dicts, preserving order
+ param_shapes = {k: v for d in param_shapes for k, v in d.items()}
+
+ if debug:
+ for i in range(world_size):
+ print(f"{FP32_FLAT_GROUPS}[{i}].shape={fp32_flat_groups[i].shape}")
+
+ wanted_params = len(param_shapes)
+ wanted_numel = sum(shape.numel() for shape in param_shapes.values())
+ # not asserting if there is a mismatch due to possible padding
+ avail_numel = fp32_flat_groups[0].numel() * world_size
+ print(f"Trainable params: Have {avail_numel} numels to process.")
+ print(f"Trainable params: Need {wanted_numel} numels in {wanted_params} params.")
+
+ # params
+ # XXX: for huge models that can't fit into the host's RAM we will have to recode this to support
+ # out-of-core computing solution
+ offset = 0
+ total_numel = 0
+ total_params = 0
+ flat_groups_offset = [0] + list(np.cumsum([flat_tensor.numel() for flat_tensor in fp32_flat_groups[0]]))
+ for name, shape in tqdm(param_shapes.items(), desc='Gathering sharded weights'):
+ unpartitioned_numel = shape.numel()
+ total_numel += unpartitioned_numel
+ total_params += 1
+ partitioned_numel, partitioned_padding_numel = zero3_partitioned_param_info(unpartitioned_numel, world_size)
+
+ if debug:
+ print(
+ f"Trainable params: {total_params} {name} full shape: {shape} partition0 numel={partitioned_numel} partitioned_padding_numel={partitioned_padding_numel}"
+ )
+
+ # memory efficient tensor
+ tensor = GatheredTensor(fp32_flat_groups, flat_groups_offset, offset, partitioned_numel, shape)
+ state_dict[name] = tensor
+ offset += partitioned_numel
+
+ offset *= world_size
+
+ # Sanity check
+ if offset != avail_numel:
+ raise ValueError(f"consumed {offset} numels out of {avail_numel} - something is wrong")
+
+ print(f"Reconstructed Trainable fp32 state dict with {total_params} params {total_numel} elements")
+
+
+def _get_fp32_state_dict_from_zero3_checkpoint(world_size, fp32_flat_groups, zero_model_states,
+ exclude_frozen_parameters):
+ state_dict = OrderedDict()
+
+ # buffers
+ buffers = zero_model_states[0].buffers
+ state_dict.update(buffers)
+ if debug:
+ print(f"added {len(buffers)} buffers")
+
+ if not exclude_frozen_parameters:
+ _zero3_merge_frozen_params(state_dict, world_size, zero_model_states)
+
+ _zero3_merge_trainable_params(state_dict, world_size, fp32_flat_groups, zero_model_states)
+
+ # recover shared parameters
+ for pair in zero_model_states[0].shared_params:
+ if pair[1] in state_dict:
+ state_dict[pair[0]] = state_dict[pair[1]]
+
+ return state_dict
+
+
+def to_torch_tensor(state_dict, return_empty_tensor=False, dtype=None):
+ """
+ Convert state_dict of GatheredTensor to torch tensor
+ """
+ if dtype is not None:
+ dtype = _resolve_output_dtype(dtype)
+
+ torch_state_dict = {}
+ converted_tensors = {}
+ for name, tensor in state_dict.items():
+ tensor_id = id(tensor)
+ if tensor_id in converted_tensors: # shared tensors
+ shared_tensor = torch_state_dict[converted_tensors[tensor_id]]
+ torch_state_dict[name] = shared_tensor
+ else:
+ converted_tensors[tensor_id] = name
+ if return_empty_tensor:
+ torch_state_dict[name] = torch.empty(tensor.shape, dtype=dtype or tensor.dtype)
+ else:
+ contiguous_tensor = tensor.contiguous()
+ torch_state_dict[name] = contiguous_tensor.to(dtype=dtype) if dtype else contiguous_tensor
+ return torch_state_dict
+
+
+def get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir,
+ tag=None,
+ exclude_frozen_parameters=False,
+ lazy_mode=False):
+ """
+ Convert ZeRO 2 or 3 checkpoint into a single fp32 consolidated state_dict that can be loaded with
+ ``load_state_dict()`` and used for training without DeepSpeed or shared with others, for example
+ via a model hub.
+
+ Args:
+ - ``checkpoint_dir``: path to the desired checkpoint folder
+ - ``tag``: checkpoint tag used as a unique identifier for checkpoint. If not provided will attempt to load tag in 'latest' file. e.g., ``global_step14``
+ - ``exclude_frozen_parameters``: exclude frozen parameters
+ - ``lazy_mode``: get state_dict in lazy mode. It returns a dict of pesduo tensor instead of torch tensor, which is more memory efficient.
+ Convert the pesduo tensor to torch tensor by ``.contiguous()``
+
+ Returns:
+ - pytorch ``state_dict``
+
+ A typical usage might be ::
+
+ from deepspeed.utils.zero_to_fp32 import get_fp32_state_dict_from_zero_checkpoint
+ # do the training and checkpoint saving
+ state_dict = get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir) # already on cpu
+ model = model.cpu() # move to cpu
+ model.load_state_dict(state_dict)
+ # submit to model hub or save the model to share with others
+
+ In this example the ``model`` will no longer be usable in the deepspeed context of the same
+ application. i.e. you will need to re-initialize the deepspeed engine, since
+ ``model.load_state_dict(state_dict)`` will remove all the deepspeed magic from it.
+
+ If you want it all done for you, use ``load_state_dict_from_zero_checkpoint`` instead.
+
+ Note: the above usage may not work if your application doesn't have sufficient free CPU memory.
+ You may need to use the offline approach using the ``zero_to_fp32.py`` script that is saved with
+ the checkpoint. Or you can load state_dict in lazy mode ::
+
+ from deepspeed.utils.zero_to_fp32 import get_fp32_state_dict_from_zero_checkpoint
+ state_dict = get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir, lazy_mode=True) # not on cpu
+ for name, lazy_tensor in state_dict.item():
+ tensor = lazy_tensor.contiguous() # to cpu
+ print(name, tensor)
+ # del tensor to release memory if it no longer in use
+ """
+ if tag is None:
+ latest_path = os.path.join(checkpoint_dir, 'latest')
+ if os.path.isfile(latest_path):
+ with open(latest_path, 'r') as fd:
+ tag = fd.read().strip()
+ else:
+ raise ValueError(f"Unable to find 'latest' file at {latest_path}")
+
+ ds_checkpoint_dir = os.path.join(checkpoint_dir, tag)
+
+ if not os.path.isdir(ds_checkpoint_dir):
+ raise FileNotFoundError(f"Directory '{ds_checkpoint_dir}' doesn't exist")
+
+ state_dict = _get_fp32_state_dict_from_zero_checkpoint(ds_checkpoint_dir, exclude_frozen_parameters)
+ if lazy_mode:
+ return state_dict
+ else:
+ return to_torch_tensor(state_dict)
+
+
+def convert_zero_checkpoint_to_state_dict(checkpoint_dir,
+ output_dir,
+ dtype=torch.float32,
+ max_shard_size="5GB",
+ safe_serialization=False,
+ tag=None,
+ exclude_frozen_parameters=False):
+ """
+ Convert ZeRO 2 or 3 checkpoint into a consolidated ``state_dict`` file that can be
+ loaded with ``torch.load(file)`` + ``load_state_dict()`` and used for training without DeepSpeed.
+
+ Args:
+ - ``checkpoint_dir``: path to the desired checkpoint folder. (one that contains the tag-folder, like ``global_step14``)
+ - ``output_dir``: directory for the PyTorch state_dict output files
+ - ``dtype``: output tensor dtype. Supports float32, float16, and bfloat16 as strings or torch dtypes.
+ - ``max_shard_size``: the maximum size for a checkpoint before being sharded, default value is 5GB
+ - ``safe_serialization``: whether to save the model using `safetensors` or the traditional PyTorch way (that uses `pickle`).
+ - ``tag``: checkpoint tag used as a unique identifier for checkpoint. If not provided will attempt to load tag in the file named ``latest`` in the checkpoint folder, e.g., ``global_step14``
+ - ``exclude_frozen_parameters``: exclude frozen parameters
+ """
+
+ dtype = _resolve_output_dtype(dtype)
+
+ # Dependency pre-check
+ if safe_serialization:
+ try:
+ from safetensors.torch import save_file
+ except ImportError:
+ print('If you want to use `safe_serialization`, please `pip install safetensors`')
+ raise
+ if max_shard_size is not None:
+ try:
+ from huggingface_hub import split_torch_state_dict_into_shards
+ except ImportError:
+ print('If you want to use `max_shard_size`, please `pip install huggingface_hub`')
+ raise
+
+ # Convert zero checkpoint to state_dict
+ state_dict = get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir,
+ tag,
+ exclude_frozen_parameters,
+ lazy_mode=True)
+
+ # Shard the model if it is too big.
+ weights_name = "model.safetensors" if safe_serialization else "pytorch_model.bin"
+ if max_shard_size is not None:
+ filename_pattern = weights_name.replace(".bin", "{suffix}.bin").replace(".safetensors", "{suffix}.safetensors")
+ # an memory-efficient approach for sharding
+ empty_state_dict = to_torch_tensor(state_dict, return_empty_tensor=True, dtype=dtype)
+ state_dict_split = split_torch_state_dict_into_shards(empty_state_dict,
+ filename_pattern=filename_pattern,
+ max_shard_size=max_shard_size)
+ else:
+ from collections import namedtuple
+ StateDictSplit = namedtuple("StateDictSplit", ["is_sharded", "filename_to_tensors"])
+ state_dict_split = StateDictSplit(is_sharded=False,
+ filename_to_tensors={weights_name: list(state_dict.keys())})
+
+ # Save the model by shard
+ os.makedirs(output_dir, exist_ok=True)
+ filename_to_tensors = state_dict_split.filename_to_tensors.items()
+ for shard_file, tensors in tqdm(filename_to_tensors, desc="Saving checkpoint shards"):
+ shard_state_dict = {tensor_name: state_dict[tensor_name] for tensor_name in tensors}
+ shard_state_dict = to_torch_tensor(shard_state_dict, dtype=dtype)
+ output_path = os.path.join(output_dir, shard_file)
+ if safe_serialization:
+ save_file(shard_state_dict, output_path, metadata={"format": "pt"})
+ else:
+ torch.save(shard_state_dict, output_path)
+ # release the memory of current shard
+ for tensor_name in list(shard_state_dict.keys()):
+ del state_dict[tensor_name]
+ del shard_state_dict[tensor_name]
+ del shard_state_dict
+ gc.collect()
+
+ # Save index if sharded
+ if state_dict_split.is_sharded:
+ index = {
+ "metadata": state_dict_split.metadata,
+ "weight_map": state_dict_split.tensor_to_filename,
+ }
+ save_index_file = "model.safetensors.index.json" if safe_serialization else "pytorch_model.bin.index.json"
+ save_index_file = os.path.join(output_dir, save_index_file)
+ with open(save_index_file, "w", encoding="utf-8") as f:
+ content = json.dumps(index, indent=2, sort_keys=True) + "\n"
+ f.write(content)
+
+
+def convert_zero_checkpoint_to_fp32_state_dict(checkpoint_dir,
+ output_dir,
+ max_shard_size="5GB",
+ safe_serialization=False,
+ tag=None,
+ exclude_frozen_parameters=False):
+ """Backward-compatible fp32 checkpoint conversion."""
+ return convert_zero_checkpoint_to_state_dict(checkpoint_dir,
+ output_dir,
+ dtype=torch.float32,
+ max_shard_size=max_shard_size,
+ safe_serialization=safe_serialization,
+ tag=tag,
+ exclude_frozen_parameters=exclude_frozen_parameters)
+
+
+def load_state_dict_from_zero_checkpoint(model, checkpoint_dir, tag=None):
+ """
+ 1. Put the provided model to cpu
+ 2. Convert ZeRO 2 or 3 checkpoint into a single fp32 consolidated ``state_dict``
+ 3. Load it into the provided model
+
+ Args:
+ - ``model``: the model object to update
+ - ``checkpoint_dir``: path to the desired checkpoint folder. (one that contains the tag-folder, like ``global_step14``)
+ - ``tag``: checkpoint tag used as a unique identifier for checkpoint. If not provided will attempt to load tag in the file named ``latest`` in the checkpoint folder, e.g., ``global_step14``
+
+ Returns:
+ - ``model`: modified model
+
+ Make sure you have plenty of CPU memory available before you call this function. If you don't
+ have enough use the ``zero_to_fp32.py`` utility to do the conversion. You will find it
+ conveniently placed for you in the checkpoint folder.
+
+ A typical usage might be ::
+
+ from deepspeed.utils.zero_to_fp32 import load_state_dict_from_zero_checkpoint
+ model = load_state_dict_from_zero_checkpoint(trainer.model, checkpoint_dir)
+ # submit to model hub or save the model to share with others
+
+ Note, that once this was run, the ``model`` will no longer be usable in the deepspeed context
+ of the same application. i.e. you will need to re-initialize the deepspeed engine, since
+ ``model.load_state_dict(state_dict)`` will remove all the deepspeed magic from it.
+
+ """
+ logger.info("Extracting fp32 weights")
+ state_dict = get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir, tag)
+
+ logger.info("Overwriting model with fp32 weights")
+ model = model.cpu()
+ model.load_state_dict(state_dict, strict=False)
+
+ return model
+
+
+if __name__ == "__main__":
+ parser = argparse.ArgumentParser()
+ parser.add_argument("checkpoint_dir",
+ type=str,
+ help="path to the desired checkpoint folder, e.g., path/checkpoint-12")
+ parser.add_argument("output_dir",
+ type=str,
+ help="directory to the pytorch fp32 state_dict output files"
+ "(e.g. path/checkpoint-12-output/)")
+ parser.add_argument(
+ "--max_shard_size",
+ type=str,
+ default="5GB",
+ help="The maximum size for a checkpoint before being sharded. Checkpoints shard will then be each of size"
+ "lower than this size. If expressed as a string, needs to be digits followed by a unit (like `5MB`"
+ "We default it to 5GB in order for models to be able to run easily on free-tier google colab instances"
+ "without CPU OOM issues.")
+ parser.add_argument(
+ "--safe_serialization",
+ default=False,
+ action='store_true',
+ help="Whether to save the model using `safetensors` or the traditional PyTorch way (that uses `pickle`).")
+ parser.add_argument("-t",
+ "--tag",
+ type=str,
+ default=None,
+ help="checkpoint tag used as a unique identifier for checkpoint. e.g., global_step1")
+ parser.add_argument("--exclude_frozen_parameters", action='store_true', help="exclude frozen parameters")
+ parser.add_argument("-d", "--debug", action='store_true', help="enable debug")
+ args = parser.parse_args()
+
+ debug = args.debug
+
+ convert_zero_checkpoint_to_fp32_state_dict(args.checkpoint_dir,
+ args.output_dir,
+ max_shard_size=args.max_shard_size,
+ safe_serialization=args.safe_serialization,
+ tag=args.tag,
+ exclude_frozen_parameters=args.exclude_frozen_parameters)
diff --git a/qwen3-1.7b-teutonic-2e5/checkpoint-500/zero_to_torch.py b/qwen3-1.7b-teutonic-2e5/checkpoint-500/zero_to_torch.py
new file mode 100644
index 0000000000000000000000000000000000000000..81312e252400d77f03cc1a88522f8f18ebe70ee7
--- /dev/null
+++ b/qwen3-1.7b-teutonic-2e5/checkpoint-500/zero_to_torch.py
@@ -0,0 +1,54 @@
+#!/usr/bin/env python
+
+# SPDX-License-Identifier: Apache-2.0
+# DeepSpeed Team
+
+import argparse
+
+if __package__:
+ from . import zero_to_fp32
+else:
+ import zero_to_fp32
+
+
+def main(args=None):
+ parser = argparse.ArgumentParser(
+ description="Convert a DeepSpeed ZeRO checkpoint to float32, float16, or bfloat16 PyTorch weights.")
+ parser.add_argument("checkpoint_dir",
+ type=str,
+ help="path to the desired checkpoint folder, e.g., path/checkpoint-12")
+ parser.add_argument("output_dir", type=str, help="directory for the converted PyTorch state_dict files")
+ parser.add_argument("--dtype",
+ type=str,
+ choices=sorted(zero_to_fp32.OUTPUT_DTYPE_NAMES),
+ required=True,
+ help="output tensor dtype")
+ parser.add_argument("--max_shard_size",
+ type=str,
+ default="5GB",
+ help="maximum size of each checkpoint shard, such as 5GB or 500MB")
+ parser.add_argument("--safe_serialization",
+ default=False,
+ action='store_true',
+ help="save with safetensors instead of PyTorch pickle serialization")
+ parser.add_argument("-t",
+ "--tag",
+ type=str,
+ default=None,
+ help="checkpoint tag used as a unique identifier, e.g., global_step1")
+ parser.add_argument("--exclude_frozen_parameters", action='store_true', help="exclude frozen parameters")
+ parser.add_argument("-d", "--debug", action='store_true', help="enable debug output")
+ parsed_args = parser.parse_args(args)
+
+ zero_to_fp32.debug = parsed_args.debug
+ zero_to_fp32.convert_zero_checkpoint_to_state_dict(parsed_args.checkpoint_dir,
+ parsed_args.output_dir,
+ dtype=parsed_args.dtype,
+ max_shard_size=parsed_args.max_shard_size,
+ safe_serialization=parsed_args.safe_serialization,
+ tag=parsed_args.tag,
+ exclude_frozen_parameters=parsed_args.exclude_frozen_parameters)
+
+
+if __name__ == "__main__":
+ main()
diff --git a/qwen3-1.7b-teutonic-2e5/config.json b/qwen3-1.7b-teutonic-2e5/config.json
new file mode 100644
index 0000000000000000000000000000000000000000..5d9cef07396baadff5390e4508eb33025967a029
--- /dev/null
+++ b/qwen3-1.7b-teutonic-2e5/config.json
@@ -0,0 +1,63 @@
+{
+ "architectures": [
+ "Qwen3ForCausalLM"
+ ],
+ "attention_bias": false,
+ "attention_dropout": 0.0,
+ "bos_token_id": null,
+ "dtype": "bfloat16",
+ "eos_token_id": 151645,
+ "head_dim": 128,
+ "hidden_act": "silu",
+ "hidden_size": 2048,
+ "initializer_range": 0.02,
+ "intermediate_size": 6144,
+ "layer_types": [
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention"
+ ],
+ "max_position_embeddings": 40960,
+ "max_window_layers": 28,
+ "model_type": "qwen3",
+ "num_attention_heads": 16,
+ "num_hidden_layers": 28,
+ "num_key_value_heads": 8,
+ "pad_token_id": 151643,
+ "rms_norm_eps": 1e-06,
+ "rope_parameters": {
+ "rope_theta": 1000000,
+ "rope_type": "default"
+ },
+ "sliding_window": null,
+ "tie_word_embeddings": true,
+ "transformers_version": "5.17.0",
+ "use_cache": false,
+ "use_sliding_window": false,
+ "vocab_size": 151936
+}
diff --git a/qwen3-1.7b-teutonic-2e5/generation_config.json b/qwen3-1.7b-teutonic-2e5/generation_config.json
new file mode 100644
index 0000000000000000000000000000000000000000..e6941fe4b04f26113d6eef6255d005c4d7090bda
--- /dev/null
+++ b/qwen3-1.7b-teutonic-2e5/generation_config.json
@@ -0,0 +1,12 @@
+{
+ "do_sample": true,
+ "eos_token_id": [
+ 151645,
+ 151643
+ ],
+ "pad_token_id": 151643,
+ "temperature": 0.6,
+ "top_k": 20,
+ "top_p": 0.95,
+ "transformers_version": "5.17.0"
+}
diff --git a/qwen3-1.7b-teutonic-2e5/model.safetensors b/qwen3-1.7b-teutonic-2e5/model.safetensors
new file mode 100644
index 0000000000000000000000000000000000000000..9b2666228db53911589c4cd2096c38c2c35ae728
--- /dev/null
+++ b/qwen3-1.7b-teutonic-2e5/model.safetensors
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:5cdc018ad7b517ba5e4c7c4f00035390b4e85dec1fc68e477966614a72cda70c
+size 4063515640
diff --git a/qwen3-1.7b-teutonic-2e5/tokenizer.json b/qwen3-1.7b-teutonic-2e5/tokenizer.json
new file mode 100644
index 0000000000000000000000000000000000000000..c7afbed2efcdf019f88ab0572ec29d3bf595dfe2
--- /dev/null
+++ b/qwen3-1.7b-teutonic-2e5/tokenizer.json
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:be75606093db2094d7cd20f3c2f385c212750648bd6ea4fb2bf507a6a4c55506
+size 11422650
diff --git a/qwen3-1.7b-teutonic-2e5/tokenizer_config.json b/qwen3-1.7b-teutonic-2e5/tokenizer_config.json
new file mode 100644
index 0000000000000000000000000000000000000000..770e41d6c92519d525eede4cbcf3ba27f6425311
--- /dev/null
+++ b/qwen3-1.7b-teutonic-2e5/tokenizer_config.json
@@ -0,0 +1,30 @@
+{
+ "add_prefix_space": false,
+ "backend": "tokenizers",
+ "bos_token": null,
+ "clean_up_tokenization_spaces": false,
+ "eos_token": "<|im_end|>",
+ "errors": "replace",
+ "extra_special_tokens": [
+ "<|im_start|>",
+ "<|im_end|>",
+ "<|object_ref_start|>",
+ "<|object_ref_end|>",
+ "<|box_start|>",
+ "<|box_end|>",
+ "<|quad_start|>",
+ "<|quad_end|>",
+ "<|vision_start|>",
+ "<|vision_end|>",
+ "<|vision_pad|>",
+ "<|image_pad|>",
+ "<|video_pad|>"
+ ],
+ "is_local": false,
+ "local_files_only": false,
+ "model_max_length": 131072,
+ "pad_token": "<|endoftext|>",
+ "split_special_tokens": false,
+ "tokenizer_class": "Qwen2Tokenizer",
+ "unk_token": null
+}
diff --git a/qwen3-1.7b-teutonic-2e5/training_args.bin b/qwen3-1.7b-teutonic-2e5/training_args.bin
new file mode 100644
index 0000000000000000000000000000000000000000..98c558883023185245bfac817f2ca585caa9d94f
--- /dev/null
+++ b/qwen3-1.7b-teutonic-2e5/training_args.bin
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:2c573bdac95c796f8f7368ab9af6d35c5687f53373737ea965b8102d518df7cb
+size 7121
diff --git a/qwen3-1.7b-teutonic-5e6/README.md b/qwen3-1.7b-teutonic-5e6/README.md
new file mode 100644
index 0000000000000000000000000000000000000000..049e6330c65495599a6bd0129376c0e4bb70285d
--- /dev/null
+++ b/qwen3-1.7b-teutonic-5e6/README.md
@@ -0,0 +1,58 @@
+---
+base_model: Qwen/Qwen3-1.7B
+library_name: transformers
+model_name: qwen3-1.7b-teutonic-5e6
+tags:
+- generated_from_trainer
+- trl
+- sft
+licence: license
+---
+
+# Model Card for qwen3-1.7b-teutonic-5e6
+
+This model is a fine-tuned version of [Qwen/Qwen3-1.7B](https://huggingface.co/Qwen/Qwen3-1.7B).
+It has been trained using [TRL](https://github.com/huggingface/trl).
+
+## Quick start
+
+```python
+from transformers import pipeline
+
+question = "If you had a time machine, but could only go to the past or the future once and never return, which would you choose and why?"
+generator = pipeline("text-generation", model="None", device_map="auto")
+output = generator([{"role": "user", "content": question}], max_new_tokens=128, return_full_text=False)[0]
+print(output["generated_text"])
+```
+
+## Training procedure
+
+[](https://wandb.ai/digitallibrary849-adobe/tiny-teutonic-sn3/runs/70d7lwio)
+
+
+
+This model was trained with SFT.
+
+### Framework versions
+
+- TRL: 1.14.0
+- Transformers: 5.17.0
+- Pytorch: 2.14.0
+- Datasets: 5.0.1
+- Tokenizers: 0.23.2
+
+## Citations
+
+
+
+Cite TRL as:
+
+```bibtex
+@software{vonwerra2020trl,
+ title = {{TRL: Transformers Reinforcement Learning}},
+ author = {von Werra, Leandro and Belkada, Younes and Tunstall, Lewis and Beeching, Edward and Thrush, Tristan and Lambert, Nathan and Huang, Shengyi and Rasul, Kashif and Gallouédec, Quentin},
+ license = {Apache-2.0},
+ url = {https://github.com/huggingface/trl},
+ year = {2020}
+}
+```
\ No newline at end of file
diff --git a/qwen3-1.7b-teutonic-5e6/checkpoint-1000/chat_template.jinja b/qwen3-1.7b-teutonic-5e6/checkpoint-1000/chat_template.jinja
new file mode 100644
index 0000000000000000000000000000000000000000..01be9b307daa2d425f7c168c9fb145a286e0afb4
--- /dev/null
+++ b/qwen3-1.7b-teutonic-5e6/checkpoint-1000/chat_template.jinja
@@ -0,0 +1,89 @@
+{%- if tools %}
+ {{- '<|im_start|>system\n' }}
+ {%- if messages[0].role == 'system' %}
+ {{- messages[0].content + '\n\n' }}
+ {%- endif %}
+ {{- "# Tools\n\nYou may call one or more functions to assist with the user query.\n\nYou are provided with function signatures within XML tags:\n" }}
+ {%- for tool in tools %}
+ {{- "\n" }}
+ {{- tool | tojson }}
+ {%- endfor %}
+ {{- "\n\n\nFor each function call, return a json object with function name and arguments within XML tags:\n\n{\"name\": , \"arguments\": }\n<|im_end|>\n" }}
+{%- else %}
+ {%- if messages[0].role == 'system' %}
+ {{- '<|im_start|>system\n' + messages[0].content + '<|im_end|>\n' }}
+ {%- endif %}
+{%- endif %}
+{%- set ns = namespace(multi_step_tool=true, last_query_index=messages|length - 1) %}
+{%- for message in messages[::-1] %}
+ {%- set index = (messages|length - 1) - loop.index0 %}
+ {%- if ns.multi_step_tool and message.role == "user" and message.content is string and not(message.content.startswith('') and message.content.endswith('')) %}
+ {%- set ns.multi_step_tool = false %}
+ {%- set ns.last_query_index = index %}
+ {%- endif %}
+{%- endfor %}
+{%- for message in messages %}
+ {%- if message.content is string %}
+ {%- set content = message.content %}
+ {%- else %}
+ {%- set content = '' %}
+ {%- endif %}
+ {%- if (message.role == "user") or (message.role == "system" and not loop.first) %}
+ {{- '<|im_start|>' + message.role + '\n' + content + '<|im_end|>' + '\n' }}
+ {%- elif message.role == "assistant" %}
+ {%- set reasoning_content = '' %}
+ {%- if message.reasoning_content is string %}
+ {%- set reasoning_content = message.reasoning_content %}
+ {%- else %}
+ {%- if '' in content %}
+ {%- set reasoning_content = content.split('')[0].rstrip('\n').split('')[-1].lstrip('\n') %}
+ {%- set content = content.split('')[-1].lstrip('\n') %}
+ {%- endif %}
+ {%- endif %}
+ {%- if loop.index0 > ns.last_query_index %}
+ {%- if loop.last or (not loop.last and reasoning_content) %}
+ {{- '<|im_start|>' + message.role + '\n\n' + reasoning_content.strip('\n') + '\n\n\n' + content.lstrip('\n') }}
+ {%- else %}
+ {{- '<|im_start|>' + message.role + '\n' + content }}
+ {%- endif %}
+ {%- else %}
+ {{- '<|im_start|>' + message.role + '\n' + content }}
+ {%- endif %}
+ {%- if message.tool_calls %}
+ {%- for tool_call in message.tool_calls %}
+ {%- if (loop.first and content) or (not loop.first) %}
+ {{- '\n' }}
+ {%- endif %}
+ {%- if tool_call.function %}
+ {%- set tool_call = tool_call.function %}
+ {%- endif %}
+ {{- '\n{"name": "' }}
+ {{- tool_call.name }}
+ {{- '", "arguments": ' }}
+ {%- if tool_call.arguments is string %}
+ {{- tool_call.arguments }}
+ {%- else %}
+ {{- tool_call.arguments | tojson }}
+ {%- endif %}
+ {{- '}\n' }}
+ {%- endfor %}
+ {%- endif %}
+ {{- '<|im_end|>\n' }}
+ {%- elif message.role == "tool" %}
+ {%- if loop.first or (messages[loop.index0 - 1].role != "tool") %}
+ {{- '<|im_start|>user' }}
+ {%- endif %}
+ {{- '\n\n' }}
+ {{- content }}
+ {{- '\n' }}
+ {%- if loop.last or (messages[loop.index0 + 1].role != "tool") %}
+ {{- '<|im_end|>\n' }}
+ {%- endif %}
+ {%- endif %}
+{%- endfor %}
+{%- if add_generation_prompt %}
+ {{- '<|im_start|>assistant\n' }}
+ {%- if enable_thinking is defined and enable_thinking is false %}
+ {{- '\n\n\n\n' }}
+ {%- endif %}
+{%- endif %}
\ No newline at end of file
diff --git a/qwen3-1.7b-teutonic-5e6/checkpoint-1000/config.json b/qwen3-1.7b-teutonic-5e6/checkpoint-1000/config.json
new file mode 100644
index 0000000000000000000000000000000000000000..5d9cef07396baadff5390e4508eb33025967a029
--- /dev/null
+++ b/qwen3-1.7b-teutonic-5e6/checkpoint-1000/config.json
@@ -0,0 +1,63 @@
+{
+ "architectures": [
+ "Qwen3ForCausalLM"
+ ],
+ "attention_bias": false,
+ "attention_dropout": 0.0,
+ "bos_token_id": null,
+ "dtype": "bfloat16",
+ "eos_token_id": 151645,
+ "head_dim": 128,
+ "hidden_act": "silu",
+ "hidden_size": 2048,
+ "initializer_range": 0.02,
+ "intermediate_size": 6144,
+ "layer_types": [
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention"
+ ],
+ "max_position_embeddings": 40960,
+ "max_window_layers": 28,
+ "model_type": "qwen3",
+ "num_attention_heads": 16,
+ "num_hidden_layers": 28,
+ "num_key_value_heads": 8,
+ "pad_token_id": 151643,
+ "rms_norm_eps": 1e-06,
+ "rope_parameters": {
+ "rope_theta": 1000000,
+ "rope_type": "default"
+ },
+ "sliding_window": null,
+ "tie_word_embeddings": true,
+ "transformers_version": "5.17.0",
+ "use_cache": false,
+ "use_sliding_window": false,
+ "vocab_size": 151936
+}
diff --git a/qwen3-1.7b-teutonic-5e6/checkpoint-1000/generation_config.json b/qwen3-1.7b-teutonic-5e6/checkpoint-1000/generation_config.json
new file mode 100644
index 0000000000000000000000000000000000000000..e6941fe4b04f26113d6eef6255d005c4d7090bda
--- /dev/null
+++ b/qwen3-1.7b-teutonic-5e6/checkpoint-1000/generation_config.json
@@ -0,0 +1,12 @@
+{
+ "do_sample": true,
+ "eos_token_id": [
+ 151645,
+ 151643
+ ],
+ "pad_token_id": 151643,
+ "temperature": 0.6,
+ "top_k": 20,
+ "top_p": 0.95,
+ "transformers_version": "5.17.0"
+}
diff --git a/qwen3-1.7b-teutonic-5e6/checkpoint-1000/global_step1000/bf16_zero_pp_rank_0_mp_rank_00_optim_states.pt b/qwen3-1.7b-teutonic-5e6/checkpoint-1000/global_step1000/bf16_zero_pp_rank_0_mp_rank_00_optim_states.pt
new file mode 100644
index 0000000000000000000000000000000000000000..b6bdc16d21b0820cfac803813745f1fd70bd3dd5
--- /dev/null
+++ b/qwen3-1.7b-teutonic-5e6/checkpoint-1000/global_step1000/bf16_zero_pp_rank_0_mp_rank_00_optim_states.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c8a370f3e99985781f582f8b41979f9fb2f97d41e1aacb7f03a31f7cd78e752d
+size 10323466465
diff --git a/qwen3-1.7b-teutonic-5e6/checkpoint-1000/global_step1000/bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt b/qwen3-1.7b-teutonic-5e6/checkpoint-1000/global_step1000/bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt
new file mode 100644
index 0000000000000000000000000000000000000000..ea3d0e0691f656fe4f8c6179177b0d34e185fe10
--- /dev/null
+++ b/qwen3-1.7b-teutonic-5e6/checkpoint-1000/global_step1000/bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:16039ddb7318658adea232db5745823afa53d087fd4be4ac9994cf224c807c74
+size 10323469601
diff --git a/qwen3-1.7b-teutonic-5e6/checkpoint-1000/global_step1000/mp_rank_00_model_states.pt b/qwen3-1.7b-teutonic-5e6/checkpoint-1000/global_step1000/mp_rank_00_model_states.pt
new file mode 100644
index 0000000000000000000000000000000000000000..f4b2827711ab1aa283747f66268977cbe724e546
--- /dev/null
+++ b/qwen3-1.7b-teutonic-5e6/checkpoint-1000/global_step1000/mp_rank_00_model_states.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:3f32f1006b04fdd13f53abf608168a5e5d22360ee55e8bd05fca00ada9c8ffd1
+size 3441239653
diff --git a/qwen3-1.7b-teutonic-5e6/checkpoint-1000/latest b/qwen3-1.7b-teutonic-5e6/checkpoint-1000/latest
new file mode 100644
index 0000000000000000000000000000000000000000..e2d3435fb1acf8913e6bd6c51b01adfc69b11ac6
--- /dev/null
+++ b/qwen3-1.7b-teutonic-5e6/checkpoint-1000/latest
@@ -0,0 +1 @@
+global_step1000
\ No newline at end of file
diff --git a/qwen3-1.7b-teutonic-5e6/checkpoint-1000/model.safetensors b/qwen3-1.7b-teutonic-5e6/checkpoint-1000/model.safetensors
new file mode 100644
index 0000000000000000000000000000000000000000..270cca23599e7bb5dc06edf449356db5d39422f0
--- /dev/null
+++ b/qwen3-1.7b-teutonic-5e6/checkpoint-1000/model.safetensors
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f2f35f184f0c01dfdf959680db354c78568f33701e69fc8a3fe596e3f07e5bb3
+size 4063515640
diff --git a/qwen3-1.7b-teutonic-5e6/checkpoint-1000/rng_state_0.pth b/qwen3-1.7b-teutonic-5e6/checkpoint-1000/rng_state_0.pth
new file mode 100644
index 0000000000000000000000000000000000000000..725d3197bef5258977c915442fc51d4b4ea7a0e4
--- /dev/null
+++ b/qwen3-1.7b-teutonic-5e6/checkpoint-1000/rng_state_0.pth
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:fc2024de3977e14d8eb49138e464852d3139a95403b42712dff426b122dbd9a8
+size 14917
diff --git a/qwen3-1.7b-teutonic-5e6/checkpoint-1000/rng_state_1.pth b/qwen3-1.7b-teutonic-5e6/checkpoint-1000/rng_state_1.pth
new file mode 100644
index 0000000000000000000000000000000000000000..51be189cbea3691d1ea86ad41ebb86675d39e2ab
--- /dev/null
+++ b/qwen3-1.7b-teutonic-5e6/checkpoint-1000/rng_state_1.pth
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9f3ccf71b831b178a658a6e1b5e409218184e44f34db2ecfb332baac716953af
+size 14917
diff --git a/qwen3-1.7b-teutonic-5e6/checkpoint-1000/scheduler.pt b/qwen3-1.7b-teutonic-5e6/checkpoint-1000/scheduler.pt
new file mode 100644
index 0000000000000000000000000000000000000000..751fad2d2e6a75d120f9bb22b0601b725a62ee4a
--- /dev/null
+++ b/qwen3-1.7b-teutonic-5e6/checkpoint-1000/scheduler.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d6ffa1830cc30c9d315b1b2a0bbeac93f0e07a42e9742eaad163664d9c593061
+size 1465
diff --git a/qwen3-1.7b-teutonic-5e6/checkpoint-1000/tokenizer.json b/qwen3-1.7b-teutonic-5e6/checkpoint-1000/tokenizer.json
new file mode 100644
index 0000000000000000000000000000000000000000..c7afbed2efcdf019f88ab0572ec29d3bf595dfe2
--- /dev/null
+++ b/qwen3-1.7b-teutonic-5e6/checkpoint-1000/tokenizer.json
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:be75606093db2094d7cd20f3c2f385c212750648bd6ea4fb2bf507a6a4c55506
+size 11422650
diff --git a/qwen3-1.7b-teutonic-5e6/checkpoint-1000/tokenizer_config.json b/qwen3-1.7b-teutonic-5e6/checkpoint-1000/tokenizer_config.json
new file mode 100644
index 0000000000000000000000000000000000000000..770e41d6c92519d525eede4cbcf3ba27f6425311
--- /dev/null
+++ b/qwen3-1.7b-teutonic-5e6/checkpoint-1000/tokenizer_config.json
@@ -0,0 +1,30 @@
+{
+ "add_prefix_space": false,
+ "backend": "tokenizers",
+ "bos_token": null,
+ "clean_up_tokenization_spaces": false,
+ "eos_token": "<|im_end|>",
+ "errors": "replace",
+ "extra_special_tokens": [
+ "<|im_start|>",
+ "<|im_end|>",
+ "<|object_ref_start|>",
+ "<|object_ref_end|>",
+ "<|box_start|>",
+ "<|box_end|>",
+ "<|quad_start|>",
+ "<|quad_end|>",
+ "<|vision_start|>",
+ "<|vision_end|>",
+ "<|vision_pad|>",
+ "<|image_pad|>",
+ "<|video_pad|>"
+ ],
+ "is_local": false,
+ "local_files_only": false,
+ "model_max_length": 131072,
+ "pad_token": "<|endoftext|>",
+ "split_special_tokens": false,
+ "tokenizer_class": "Qwen2Tokenizer",
+ "unk_token": null
+}
diff --git a/qwen3-1.7b-teutonic-5e6/checkpoint-1000/trainer_state.json b/qwen3-1.7b-teutonic-5e6/checkpoint-1000/trainer_state.json
new file mode 100644
index 0000000000000000000000000000000000000000..eac30f84c823b21ebe46ad5e624789c45a60803e
--- /dev/null
+++ b/qwen3-1.7b-teutonic-5e6/checkpoint-1000/trainer_state.json
@@ -0,0 +1,10254 @@
+{
+ "best_global_step": 950,
+ "best_metric": 1.5272752046585083,
+ "best_model_checkpoint": "./checkpoints/qwen3-1.7b-teutonic-5e6/checkpoint-500",
+ "epoch": 2.0202020202020203,
+ "eval_steps": 50,
+ "global_step": 1000,
+ "is_hyper_param_search": false,
+ "is_local_process_zero": true,
+ "is_world_process_zero": true,
+ "log_history": [
+ {
+ "entropy": 0.8295683860778809,
+ "epoch": 0.00202020202020202,
+ "grad_norm": 10.886772155761719,
+ "learning_rate": 5e-06,
+ "loss": 1.5185801982879639,
+ "mean_token_accuracy": 0.672874927520752,
+ "num_tokens": 16384.0,
+ "step": 1
+ },
+ {
+ "entropy": 1.1781222820281982,
+ "epoch": 0.00404040404040404,
+ "grad_norm": 13.80163288116455,
+ "learning_rate": 4.997979797979798e-06,
+ "loss": 2.18377685546875,
+ "mean_token_accuracy": 0.5700415372848511,
+ "num_tokens": 32768.0,
+ "step": 2
+ },
+ {
+ "entropy": 0.7972303032875061,
+ "epoch": 0.006060606060606061,
+ "grad_norm": 13.71261215209961,
+ "learning_rate": 4.995959595959596e-06,
+ "loss": 1.5810277462005615,
+ "mean_token_accuracy": 0.6782486438751221,
+ "num_tokens": 49152.0,
+ "step": 3
+ },
+ {
+ "entropy": 0.9709298610687256,
+ "epoch": 0.00808080808080808,
+ "grad_norm": 9.18797779083252,
+ "learning_rate": 4.993939393939394e-06,
+ "loss": 1.6923656463623047,
+ "mean_token_accuracy": 0.6447850465774536,
+ "num_tokens": 65536.0,
+ "step": 4
+ },
+ {
+ "entropy": 1.2161898612976074,
+ "epoch": 0.010101010101010102,
+ "grad_norm": 8.97508430480957,
+ "learning_rate": 4.991919191919192e-06,
+ "loss": 1.9361608028411865,
+ "mean_token_accuracy": 0.5922691822052002,
+ "num_tokens": 81920.0,
+ "step": 5
+ },
+ {
+ "entropy": 0.9950039982795715,
+ "epoch": 0.012121212121212121,
+ "grad_norm": 6.453213214874268,
+ "learning_rate": 4.98989898989899e-06,
+ "loss": 1.5012480020523071,
+ "mean_token_accuracy": 0.6671959161758423,
+ "num_tokens": 98304.0,
+ "step": 6
+ },
+ {
+ "entropy": 1.452682614326477,
+ "epoch": 0.014141414141414142,
+ "grad_norm": 8.50411605834961,
+ "learning_rate": 4.987878787878789e-06,
+ "loss": 2.1297783851623535,
+ "mean_token_accuracy": 0.5807889699935913,
+ "num_tokens": 114688.0,
+ "step": 7
+ },
+ {
+ "entropy": 1.4661014080047607,
+ "epoch": 0.01616161616161616,
+ "grad_norm": 4.290146827697754,
+ "learning_rate": 4.9858585858585865e-06,
+ "loss": 1.7391963005065918,
+ "mean_token_accuracy": 0.611993134021759,
+ "num_tokens": 131072.0,
+ "step": 8
+ },
+ {
+ "entropy": 1.9030903577804565,
+ "epoch": 0.01818181818181818,
+ "grad_norm": 4.503584384918213,
+ "learning_rate": 4.983838383838384e-06,
+ "loss": 2.283596992492676,
+ "mean_token_accuracy": 0.5261968970298767,
+ "num_tokens": 147456.0,
+ "step": 9
+ },
+ {
+ "entropy": 1.2784137725830078,
+ "epoch": 0.020202020202020204,
+ "grad_norm": 3.427410840988159,
+ "learning_rate": 4.981818181818182e-06,
+ "loss": 1.5032392740249634,
+ "mean_token_accuracy": 0.6578529477119446,
+ "num_tokens": 163840.0,
+ "step": 10
+ },
+ {
+ "entropy": 1.7487188577651978,
+ "epoch": 0.022222222222222223,
+ "grad_norm": 3.20536470413208,
+ "learning_rate": 4.97979797979798e-06,
+ "loss": 1.9184643030166626,
+ "mean_token_accuracy": 0.6093673706054688,
+ "num_tokens": 180224.0,
+ "step": 11
+ },
+ {
+ "entropy": 1.6585394144058228,
+ "epoch": 0.024242424242424242,
+ "grad_norm": 3.2463598251342773,
+ "learning_rate": 4.977777777777778e-06,
+ "loss": 1.8364639282226562,
+ "mean_token_accuracy": 0.6014899611473083,
+ "num_tokens": 196608.0,
+ "step": 12
+ },
+ {
+ "entropy": 1.4869519472122192,
+ "epoch": 0.026262626262626262,
+ "grad_norm": 2.8681719303131104,
+ "learning_rate": 4.975757575757576e-06,
+ "loss": 1.6356265544891357,
+ "mean_token_accuracy": 0.6361138224601746,
+ "num_tokens": 212992.0,
+ "step": 13
+ },
+ {
+ "entropy": 1.5179094076156616,
+ "epoch": 0.028282828282828285,
+ "grad_norm": 3.0470707416534424,
+ "learning_rate": 4.973737373737374e-06,
+ "loss": 1.6317660808563232,
+ "mean_token_accuracy": 0.6334269642829895,
+ "num_tokens": 229376.0,
+ "step": 14
+ },
+ {
+ "entropy": 1.7384330034255981,
+ "epoch": 0.030303030303030304,
+ "grad_norm": 2.6149742603302,
+ "learning_rate": 4.971717171717172e-06,
+ "loss": 1.67923903465271,
+ "mean_token_accuracy": 0.6469223499298096,
+ "num_tokens": 245760.0,
+ "step": 15
+ },
+ {
+ "entropy": 2.108799457550049,
+ "epoch": 0.03232323232323232,
+ "grad_norm": 3.694795608520508,
+ "learning_rate": 4.9696969696969696e-06,
+ "loss": 1.9052371978759766,
+ "mean_token_accuracy": 0.5927577018737793,
+ "num_tokens": 262144.0,
+ "step": 16
+ },
+ {
+ "entropy": 1.7532848119735718,
+ "epoch": 0.03434343434343434,
+ "grad_norm": 3.1510567665100098,
+ "learning_rate": 4.9676767676767675e-06,
+ "loss": 1.6382691860198975,
+ "mean_token_accuracy": 0.6311675906181335,
+ "num_tokens": 278528.0,
+ "step": 17
+ },
+ {
+ "entropy": 1.723548412322998,
+ "epoch": 0.03636363636363636,
+ "grad_norm": 3.093825340270996,
+ "learning_rate": 4.965656565656566e-06,
+ "loss": 1.6039624214172363,
+ "mean_token_accuracy": 0.6313507556915283,
+ "num_tokens": 294912.0,
+ "step": 18
+ },
+ {
+ "entropy": 1.4790765047073364,
+ "epoch": 0.03838383838383838,
+ "grad_norm": 2.7150535583496094,
+ "learning_rate": 4.963636363636364e-06,
+ "loss": 1.4173667430877686,
+ "mean_token_accuracy": 0.6627381443977356,
+ "num_tokens": 311296.0,
+ "step": 19
+ },
+ {
+ "entropy": 1.5827536582946777,
+ "epoch": 0.04040404040404041,
+ "grad_norm": 2.610966444015503,
+ "learning_rate": 4.961616161616162e-06,
+ "loss": 1.4604644775390625,
+ "mean_token_accuracy": 0.656509518623352,
+ "num_tokens": 327680.0,
+ "step": 20
+ },
+ {
+ "entropy": 2.054673194885254,
+ "epoch": 0.04242424242424243,
+ "grad_norm": 2.612940788269043,
+ "learning_rate": 4.95959595959596e-06,
+ "loss": 1.962029218673706,
+ "mean_token_accuracy": 0.5719956159591675,
+ "num_tokens": 344064.0,
+ "step": 21
+ },
+ {
+ "entropy": 1.6684198379516602,
+ "epoch": 0.044444444444444446,
+ "grad_norm": 2.5426013469696045,
+ "learning_rate": 4.957575757575758e-06,
+ "loss": 1.591176986694336,
+ "mean_token_accuracy": 0.6519907116889954,
+ "num_tokens": 360448.0,
+ "step": 22
+ },
+ {
+ "entropy": 1.9070855379104614,
+ "epoch": 0.046464646464646465,
+ "grad_norm": 2.5853357315063477,
+ "learning_rate": 4.9555555555555565e-06,
+ "loss": 1.930276870727539,
+ "mean_token_accuracy": 0.5895823240280151,
+ "num_tokens": 376832.0,
+ "step": 23
+ },
+ {
+ "entropy": 1.716689109802246,
+ "epoch": 0.048484848484848485,
+ "grad_norm": 2.7608494758605957,
+ "learning_rate": 4.953535353535354e-06,
+ "loss": 1.739159345626831,
+ "mean_token_accuracy": 0.6028944849967957,
+ "num_tokens": 393216.0,
+ "step": 24
+ },
+ {
+ "entropy": 1.5666695833206177,
+ "epoch": 0.050505050505050504,
+ "grad_norm": 2.5973074436187744,
+ "learning_rate": 4.951515151515152e-06,
+ "loss": 1.6016302108764648,
+ "mean_token_accuracy": 0.626099169254303,
+ "num_tokens": 409600.0,
+ "step": 25
+ },
+ {
+ "entropy": 1.4267652034759521,
+ "epoch": 0.052525252525252523,
+ "grad_norm": 2.163451910018921,
+ "learning_rate": 4.94949494949495e-06,
+ "loss": 1.5093598365783691,
+ "mean_token_accuracy": 0.6530288457870483,
+ "num_tokens": 425984.0,
+ "step": 26
+ },
+ {
+ "entropy": 1.8537150621414185,
+ "epoch": 0.05454545454545454,
+ "grad_norm": 2.6983258724212646,
+ "learning_rate": 4.947474747474748e-06,
+ "loss": 1.8831868171691895,
+ "mean_token_accuracy": 0.5802393555641174,
+ "num_tokens": 442368.0,
+ "step": 27
+ },
+ {
+ "entropy": 1.410015344619751,
+ "epoch": 0.05656565656565657,
+ "grad_norm": 2.4777796268463135,
+ "learning_rate": 4.945454545454546e-06,
+ "loss": 1.4835734367370605,
+ "mean_token_accuracy": 0.6519907116889954,
+ "num_tokens": 458752.0,
+ "step": 28
+ },
+ {
+ "entropy": 1.2399017810821533,
+ "epoch": 0.05858585858585859,
+ "grad_norm": 2.1676478385925293,
+ "learning_rate": 4.943434343434344e-06,
+ "loss": 1.330633282661438,
+ "mean_token_accuracy": 0.6802638173103333,
+ "num_tokens": 475136.0,
+ "step": 29
+ },
+ {
+ "entropy": 1.6478898525238037,
+ "epoch": 0.06060606060606061,
+ "grad_norm": 2.5325536727905273,
+ "learning_rate": 4.941414141414142e-06,
+ "loss": 1.771777868270874,
+ "mean_token_accuracy": 0.6128480434417725,
+ "num_tokens": 491520.0,
+ "step": 30
+ },
+ {
+ "entropy": 1.7915360927581787,
+ "epoch": 0.06262626262626263,
+ "grad_norm": 2.2419557571411133,
+ "learning_rate": 4.93939393939394e-06,
+ "loss": 1.870645523071289,
+ "mean_token_accuracy": 0.6074743270874023,
+ "num_tokens": 507904.0,
+ "step": 31
+ },
+ {
+ "entropy": 1.7638899087905884,
+ "epoch": 0.06464646464646465,
+ "grad_norm": 2.8063058853149414,
+ "learning_rate": 4.937373737373738e-06,
+ "loss": 1.83987557888031,
+ "mean_token_accuracy": 0.5836589932441711,
+ "num_tokens": 524288.0,
+ "step": 32
+ },
+ {
+ "entropy": 1.4097516536712646,
+ "epoch": 0.06666666666666667,
+ "grad_norm": 2.1857714653015137,
+ "learning_rate": 4.935353535353536e-06,
+ "loss": 1.477668285369873,
+ "mean_token_accuracy": 0.6507083773612976,
+ "num_tokens": 540672.0,
+ "step": 33
+ },
+ {
+ "entropy": 1.4447426795959473,
+ "epoch": 0.06868686868686869,
+ "grad_norm": 2.3771870136260986,
+ "learning_rate": 4.933333333333334e-06,
+ "loss": 1.536318063735962,
+ "mean_token_accuracy": 0.6383732557296753,
+ "num_tokens": 557056.0,
+ "step": 34
+ },
+ {
+ "entropy": 1.9658164978027344,
+ "epoch": 0.0707070707070707,
+ "grad_norm": 2.4784936904907227,
+ "learning_rate": 4.931313131313132e-06,
+ "loss": 2.0565035343170166,
+ "mean_token_accuracy": 0.5581949353218079,
+ "num_tokens": 573440.0,
+ "step": 35
+ },
+ {
+ "entropy": 1.6045317649841309,
+ "epoch": 0.07272727272727272,
+ "grad_norm": 2.1128504276275635,
+ "learning_rate": 4.92929292929293e-06,
+ "loss": 1.6234952211380005,
+ "mean_token_accuracy": 0.6293356418609619,
+ "num_tokens": 589824.0,
+ "step": 36
+ },
+ {
+ "entropy": 1.5491269826889038,
+ "epoch": 0.07474747474747474,
+ "grad_norm": 2.1672937870025635,
+ "learning_rate": 4.927272727272728e-06,
+ "loss": 1.5809822082519531,
+ "mean_token_accuracy": 0.6419760584831238,
+ "num_tokens": 606208.0,
+ "step": 37
+ },
+ {
+ "entropy": 1.545534610748291,
+ "epoch": 0.07676767676767676,
+ "grad_norm": 2.2526934146881104,
+ "learning_rate": 4.925252525252526e-06,
+ "loss": 1.5707473754882812,
+ "mean_token_accuracy": 0.6344650983810425,
+ "num_tokens": 622592.0,
+ "step": 38
+ },
+ {
+ "entropy": 1.579596757888794,
+ "epoch": 0.07878787878787878,
+ "grad_norm": 2.3988492488861084,
+ "learning_rate": 4.9232323232323235e-06,
+ "loss": 1.6725983619689941,
+ "mean_token_accuracy": 0.6243282556533813,
+ "num_tokens": 638976.0,
+ "step": 39
+ },
+ {
+ "entropy": 2.0438785552978516,
+ "epoch": 0.08080808080808081,
+ "grad_norm": 2.3314359188079834,
+ "learning_rate": 4.9212121212121214e-06,
+ "loss": 2.0424888134002686,
+ "mean_token_accuracy": 0.570713222026825,
+ "num_tokens": 655360.0,
+ "step": 40
+ },
+ {
+ "entropy": 1.8690773248672485,
+ "epoch": 0.08282828282828283,
+ "grad_norm": 2.130401134490967,
+ "learning_rate": 4.919191919191919e-06,
+ "loss": 1.8796970844268799,
+ "mean_token_accuracy": 0.5882999300956726,
+ "num_tokens": 671744.0,
+ "step": 41
+ },
+ {
+ "entropy": 1.756626009941101,
+ "epoch": 0.08484848484848485,
+ "grad_norm": 2.342318534851074,
+ "learning_rate": 4.917171717171717e-06,
+ "loss": 1.71901535987854,
+ "mean_token_accuracy": 0.6173668503761292,
+ "num_tokens": 688128.0,
+ "step": 42
+ },
+ {
+ "entropy": 1.4642508029937744,
+ "epoch": 0.08686868686868687,
+ "grad_norm": 1.993338704109192,
+ "learning_rate": 4.915151515151516e-06,
+ "loss": 1.4660165309906006,
+ "mean_token_accuracy": 0.6687225103378296,
+ "num_tokens": 704512.0,
+ "step": 43
+ },
+ {
+ "entropy": 1.790807843208313,
+ "epoch": 0.08888888888888889,
+ "grad_norm": 2.3186943531036377,
+ "learning_rate": 4.913131313131314e-06,
+ "loss": 1.7102060317993164,
+ "mean_token_accuracy": 0.6238397359848022,
+ "num_tokens": 720896.0,
+ "step": 44
+ },
+ {
+ "entropy": 1.5970485210418701,
+ "epoch": 0.09090909090909091,
+ "grad_norm": 2.299307346343994,
+ "learning_rate": 4.911111111111112e-06,
+ "loss": 1.5170013904571533,
+ "mean_token_accuracy": 0.652662456035614,
+ "num_tokens": 737280.0,
+ "step": 45
+ },
+ {
+ "entropy": 1.3955466747283936,
+ "epoch": 0.09292929292929293,
+ "grad_norm": 2.171952962875366,
+ "learning_rate": 4.90909090909091e-06,
+ "loss": 1.4059661626815796,
+ "mean_token_accuracy": 0.6654250025749207,
+ "num_tokens": 753664.0,
+ "step": 46
+ },
+ {
+ "entropy": 1.7198575735092163,
+ "epoch": 0.09494949494949495,
+ "grad_norm": 2.385092258453369,
+ "learning_rate": 4.9070707070707075e-06,
+ "loss": 1.730346918106079,
+ "mean_token_accuracy": 0.6191987991333008,
+ "num_tokens": 770048.0,
+ "step": 47
+ },
+ {
+ "entropy": 1.3542555570602417,
+ "epoch": 0.09696969696969697,
+ "grad_norm": 2.1463189125061035,
+ "learning_rate": 4.905050505050505e-06,
+ "loss": 1.346085786819458,
+ "mean_token_accuracy": 0.680446982383728,
+ "num_tokens": 786432.0,
+ "step": 48
+ },
+ {
+ "entropy": 1.8084443807601929,
+ "epoch": 0.09898989898989899,
+ "grad_norm": 2.1505849361419678,
+ "learning_rate": 4.903030303030303e-06,
+ "loss": 1.847151756286621,
+ "mean_token_accuracy": 0.5926355719566345,
+ "num_tokens": 802816.0,
+ "step": 49
+ },
+ {
+ "entropy": 1.751160979270935,
+ "epoch": 0.10101010101010101,
+ "grad_norm": 2.1436259746551514,
+ "learning_rate": 4.901010101010101e-06,
+ "loss": 1.7802404165267944,
+ "mean_token_accuracy": 0.5996580123901367,
+ "num_tokens": 819200.0,
+ "step": 50
+ },
+ {
+ "epoch": 0.10101010101010101,
+ "eval_entropy": 1.6292865045609013,
+ "eval_loss": 1.6725393533706665,
+ "eval_mean_token_accuracy": 0.6230050469598463,
+ "eval_num_tokens": 819200.0,
+ "eval_runtime": 43.9148,
+ "eval_samples_per_second": 22.544,
+ "eval_steps_per_second": 2.824,
+ "step": 50
+ },
+ {
+ "entropy": 1.4428319931030273,
+ "epoch": 0.10303030303030303,
+ "grad_norm": 2.0954954624176025,
+ "learning_rate": 4.898989898989899e-06,
+ "loss": 1.4927232265472412,
+ "mean_token_accuracy": 0.6522349715232849,
+ "num_tokens": 835584.0,
+ "step": 51
+ },
+ {
+ "entropy": 1.7493115663528442,
+ "epoch": 0.10505050505050505,
+ "grad_norm": 2.300030469894409,
+ "learning_rate": 4.896969696969697e-06,
+ "loss": 1.737417221069336,
+ "mean_token_accuracy": 0.6213361024856567,
+ "num_tokens": 851968.0,
+ "step": 52
+ },
+ {
+ "entropy": 1.7801647186279297,
+ "epoch": 0.10707070707070707,
+ "grad_norm": 2.3947081565856934,
+ "learning_rate": 4.894949494949495e-06,
+ "loss": 1.8372564315795898,
+ "mean_token_accuracy": 0.5931240916252136,
+ "num_tokens": 868352.0,
+ "step": 53
+ },
+ {
+ "entropy": 1.6934887170791626,
+ "epoch": 0.10909090909090909,
+ "grad_norm": 2.1981089115142822,
+ "learning_rate": 4.8929292929292936e-06,
+ "loss": 1.734646201133728,
+ "mean_token_accuracy": 0.611932098865509,
+ "num_tokens": 884736.0,
+ "step": 54
+ },
+ {
+ "entropy": 1.3401941061019897,
+ "epoch": 0.1111111111111111,
+ "grad_norm": 2.193511724472046,
+ "learning_rate": 4.8909090909090914e-06,
+ "loss": 1.399888038635254,
+ "mean_token_accuracy": 0.6745847463607788,
+ "num_tokens": 901120.0,
+ "step": 55
+ },
+ {
+ "entropy": 1.8244661092758179,
+ "epoch": 0.11313131313131314,
+ "grad_norm": 2.4358489513397217,
+ "learning_rate": 4.888888888888889e-06,
+ "loss": 1.8716282844543457,
+ "mean_token_accuracy": 0.5867122411727905,
+ "num_tokens": 917504.0,
+ "step": 56
+ },
+ {
+ "entropy": 1.5019619464874268,
+ "epoch": 0.11515151515151516,
+ "grad_norm": 2.1135261058807373,
+ "learning_rate": 4.886868686868687e-06,
+ "loss": 1.523103952407837,
+ "mean_token_accuracy": 0.6414265036582947,
+ "num_tokens": 933888.0,
+ "step": 57
+ },
+ {
+ "entropy": 1.6132855415344238,
+ "epoch": 0.11717171717171718,
+ "grad_norm": 2.026301145553589,
+ "learning_rate": 4.884848484848485e-06,
+ "loss": 1.6233609914779663,
+ "mean_token_accuracy": 0.6279311180114746,
+ "num_tokens": 950272.0,
+ "step": 58
+ },
+ {
+ "entropy": 1.738538384437561,
+ "epoch": 0.1191919191919192,
+ "grad_norm": 2.1539394855499268,
+ "learning_rate": 4.882828282828283e-06,
+ "loss": 1.8147332668304443,
+ "mean_token_accuracy": 0.6124816536903381,
+ "num_tokens": 966656.0,
+ "step": 59
+ },
+ {
+ "entropy": 1.5533764362335205,
+ "epoch": 0.12121212121212122,
+ "grad_norm": 2.2603628635406494,
+ "learning_rate": 4.880808080808081e-06,
+ "loss": 1.562873125076294,
+ "mean_token_accuracy": 0.6398388147354126,
+ "num_tokens": 983040.0,
+ "step": 60
+ },
+ {
+ "entropy": 1.722406268119812,
+ "epoch": 0.12323232323232323,
+ "grad_norm": 2.3630757331848145,
+ "learning_rate": 4.878787878787879e-06,
+ "loss": 1.7461689710617065,
+ "mean_token_accuracy": 0.6023448705673218,
+ "num_tokens": 999424.0,
+ "step": 61
+ },
+ {
+ "entropy": 1.6533517837524414,
+ "epoch": 0.12525252525252525,
+ "grad_norm": 2.2165415287017822,
+ "learning_rate": 4.876767676767677e-06,
+ "loss": 1.676560640335083,
+ "mean_token_accuracy": 0.6437469720840454,
+ "num_tokens": 1015808.0,
+ "step": 62
+ },
+ {
+ "entropy": 1.749995470046997,
+ "epoch": 0.12727272727272726,
+ "grad_norm": 2.1572678089141846,
+ "learning_rate": 4.8747474747474745e-06,
+ "loss": 1.784087896347046,
+ "mean_token_accuracy": 0.5884831547737122,
+ "num_tokens": 1032192.0,
+ "step": 63
+ },
+ {
+ "entropy": 1.442152738571167,
+ "epoch": 0.1292929292929293,
+ "grad_norm": 2.163490056991577,
+ "learning_rate": 4.872727272727273e-06,
+ "loss": 1.4307503700256348,
+ "mean_token_accuracy": 0.6618832349777222,
+ "num_tokens": 1048576.0,
+ "step": 64
+ },
+ {
+ "entropy": 1.2657029628753662,
+ "epoch": 0.13131313131313133,
+ "grad_norm": 2.1225337982177734,
+ "learning_rate": 4.870707070707071e-06,
+ "loss": 1.2731966972351074,
+ "mean_token_accuracy": 0.688568651676178,
+ "num_tokens": 1064960.0,
+ "step": 65
+ },
+ {
+ "entropy": 1.1613880395889282,
+ "epoch": 0.13333333333333333,
+ "grad_norm": 1.9527196884155273,
+ "learning_rate": 4.868686868686869e-06,
+ "loss": 1.13922119140625,
+ "mean_token_accuracy": 0.7389472126960754,
+ "num_tokens": 1081344.0,
+ "step": 66
+ },
+ {
+ "entropy": 1.6936380863189697,
+ "epoch": 0.13535353535353536,
+ "grad_norm": 2.004284620285034,
+ "learning_rate": 4.866666666666667e-06,
+ "loss": 1.6982238292694092,
+ "mean_token_accuracy": 0.6191987991333008,
+ "num_tokens": 1097728.0,
+ "step": 67
+ },
+ {
+ "entropy": 1.750565528869629,
+ "epoch": 0.13737373737373737,
+ "grad_norm": 2.225955009460449,
+ "learning_rate": 4.864646464646466e-06,
+ "loss": 1.796521782875061,
+ "mean_token_accuracy": 0.5934293866157532,
+ "num_tokens": 1114112.0,
+ "step": 68
+ },
+ {
+ "entropy": 1.6608220338821411,
+ "epoch": 0.1393939393939394,
+ "grad_norm": 2.127035617828369,
+ "learning_rate": 4.8626262626262636e-06,
+ "loss": 1.6633095741271973,
+ "mean_token_accuracy": 0.6356253027915955,
+ "num_tokens": 1130496.0,
+ "step": 69
+ },
+ {
+ "entropy": 1.4848661422729492,
+ "epoch": 0.1414141414141414,
+ "grad_norm": 2.0338215827941895,
+ "learning_rate": 4.8606060606060615e-06,
+ "loss": 1.4789674282073975,
+ "mean_token_accuracy": 0.6680508255958557,
+ "num_tokens": 1146880.0,
+ "step": 70
+ },
+ {
+ "entropy": 1.2310466766357422,
+ "epoch": 0.14343434343434344,
+ "grad_norm": 2.105898141860962,
+ "learning_rate": 4.858585858585859e-06,
+ "loss": 1.2301937341690063,
+ "mean_token_accuracy": 0.7040791511535645,
+ "num_tokens": 1163264.0,
+ "step": 71
+ },
+ {
+ "entropy": 1.6310514211654663,
+ "epoch": 0.14545454545454545,
+ "grad_norm": 2.152125358581543,
+ "learning_rate": 4.856565656565657e-06,
+ "loss": 1.643636703491211,
+ "mean_token_accuracy": 0.6221299171447754,
+ "num_tokens": 1179648.0,
+ "step": 72
+ },
+ {
+ "entropy": 1.4747940301895142,
+ "epoch": 0.14747474747474748,
+ "grad_norm": 2.096461296081543,
+ "learning_rate": 4.854545454545455e-06,
+ "loss": 1.42953360080719,
+ "mean_token_accuracy": 0.6651807427406311,
+ "num_tokens": 1196032.0,
+ "step": 73
+ },
+ {
+ "entropy": 1.819259524345398,
+ "epoch": 0.1494949494949495,
+ "grad_norm": 2.2852063179016113,
+ "learning_rate": 4.852525252525253e-06,
+ "loss": 1.845325231552124,
+ "mean_token_accuracy": 0.6036272644996643,
+ "num_tokens": 1212416.0,
+ "step": 74
+ },
+ {
+ "entropy": 1.4233598709106445,
+ "epoch": 0.15151515151515152,
+ "grad_norm": 2.1157381534576416,
+ "learning_rate": 4.850505050505051e-06,
+ "loss": 1.4565438032150269,
+ "mean_token_accuracy": 0.6607230305671692,
+ "num_tokens": 1228800.0,
+ "step": 75
+ },
+ {
+ "entropy": 1.6441459655761719,
+ "epoch": 0.15353535353535352,
+ "grad_norm": 2.2092180252075195,
+ "learning_rate": 4.848484848484849e-06,
+ "loss": 1.6467639207839966,
+ "mean_token_accuracy": 0.6285417675971985,
+ "num_tokens": 1245184.0,
+ "step": 76
+ },
+ {
+ "entropy": 1.6124308109283447,
+ "epoch": 0.15555555555555556,
+ "grad_norm": 2.231876850128174,
+ "learning_rate": 4.846464646464647e-06,
+ "loss": 1.617384672164917,
+ "mean_token_accuracy": 0.6340987086296082,
+ "num_tokens": 1261568.0,
+ "step": 77
+ },
+ {
+ "entropy": 1.828405499458313,
+ "epoch": 0.15757575757575756,
+ "grad_norm": 2.1372313499450684,
+ "learning_rate": 4.8444444444444446e-06,
+ "loss": 1.8333203792572021,
+ "mean_token_accuracy": 0.5979481935501099,
+ "num_tokens": 1277952.0,
+ "step": 78
+ },
+ {
+ "entropy": 1.855564832687378,
+ "epoch": 0.1595959595959596,
+ "grad_norm": 2.1422762870788574,
+ "learning_rate": 4.842424242424243e-06,
+ "loss": 1.9133609533309937,
+ "mean_token_accuracy": 0.570652186870575,
+ "num_tokens": 1294336.0,
+ "step": 79
+ },
+ {
+ "entropy": 1.7908183336257935,
+ "epoch": 0.16161616161616163,
+ "grad_norm": 2.172368049621582,
+ "learning_rate": 4.840404040404041e-06,
+ "loss": 1.8109419345855713,
+ "mean_token_accuracy": 0.60332190990448,
+ "num_tokens": 1310720.0,
+ "step": 80
+ },
+ {
+ "entropy": 1.9537721872329712,
+ "epoch": 0.16363636363636364,
+ "grad_norm": 2.199505090713501,
+ "learning_rate": 4.838383838383839e-06,
+ "loss": 2.0001401901245117,
+ "mean_token_accuracy": 0.5585613250732422,
+ "num_tokens": 1327104.0,
+ "step": 81
+ },
+ {
+ "entropy": 1.5365772247314453,
+ "epoch": 0.16565656565656567,
+ "grad_norm": 2.0778913497924805,
+ "learning_rate": 4.836363636363637e-06,
+ "loss": 1.575089931488037,
+ "mean_token_accuracy": 0.6375183463096619,
+ "num_tokens": 1343488.0,
+ "step": 82
+ },
+ {
+ "entropy": 1.7177143096923828,
+ "epoch": 0.16767676767676767,
+ "grad_norm": 2.1010894775390625,
+ "learning_rate": 4.834343434343435e-06,
+ "loss": 1.7428388595581055,
+ "mean_token_accuracy": 0.6113824844360352,
+ "num_tokens": 1359872.0,
+ "step": 83
+ },
+ {
+ "entropy": 1.5080527067184448,
+ "epoch": 0.1696969696969697,
+ "grad_norm": 2.021969795227051,
+ "learning_rate": 4.832323232323233e-06,
+ "loss": 1.5361201763153076,
+ "mean_token_accuracy": 0.656509518623352,
+ "num_tokens": 1376256.0,
+ "step": 84
+ },
+ {
+ "entropy": 1.5042120218276978,
+ "epoch": 0.1717171717171717,
+ "grad_norm": 2.121314287185669,
+ "learning_rate": 4.830303030303031e-06,
+ "loss": 1.4912033081054688,
+ "mean_token_accuracy": 0.6621274948120117,
+ "num_tokens": 1392640.0,
+ "step": 85
+ },
+ {
+ "entropy": 1.9523948431015015,
+ "epoch": 0.17373737373737375,
+ "grad_norm": 2.1702609062194824,
+ "learning_rate": 4.8282828282828285e-06,
+ "loss": 1.9369449615478516,
+ "mean_token_accuracy": 0.5727283954620361,
+ "num_tokens": 1409024.0,
+ "step": 86
+ },
+ {
+ "entropy": 1.6174770593643188,
+ "epoch": 0.17575757575757575,
+ "grad_norm": 2.222412109375,
+ "learning_rate": 4.826262626262626e-06,
+ "loss": 1.6794973611831665,
+ "mean_token_accuracy": 0.6151685118675232,
+ "num_tokens": 1425408.0,
+ "step": 87
+ },
+ {
+ "entropy": 1.1480307579040527,
+ "epoch": 0.17777777777777778,
+ "grad_norm": 1.98936128616333,
+ "learning_rate": 4.824242424242424e-06,
+ "loss": 1.1324063539505005,
+ "mean_token_accuracy": 0.7193453907966614,
+ "num_tokens": 1441792.0,
+ "step": 88
+ },
+ {
+ "entropy": 1.5290263891220093,
+ "epoch": 0.1797979797979798,
+ "grad_norm": 2.098860263824463,
+ "learning_rate": 4.822222222222222e-06,
+ "loss": 1.556044340133667,
+ "mean_token_accuracy": 0.6415486335754395,
+ "num_tokens": 1458176.0,
+ "step": 89
+ },
+ {
+ "entropy": 1.1366127729415894,
+ "epoch": 0.18181818181818182,
+ "grad_norm": 1.9387420415878296,
+ "learning_rate": 4.820202020202021e-06,
+ "loss": 1.1635141372680664,
+ "mean_token_accuracy": 0.7168416976928711,
+ "num_tokens": 1474560.0,
+ "step": 90
+ },
+ {
+ "entropy": 1.5142875909805298,
+ "epoch": 0.18383838383838383,
+ "grad_norm": 2.259131908416748,
+ "learning_rate": 4.818181818181819e-06,
+ "loss": 1.5329954624176025,
+ "mean_token_accuracy": 0.6471666097640991,
+ "num_tokens": 1490944.0,
+ "step": 91
+ },
+ {
+ "entropy": 1.5197135210037231,
+ "epoch": 0.18585858585858586,
+ "grad_norm": 2.1230807304382324,
+ "learning_rate": 4.816161616161617e-06,
+ "loss": 1.541062593460083,
+ "mean_token_accuracy": 0.6398388147354126,
+ "num_tokens": 1507328.0,
+ "step": 92
+ },
+ {
+ "entropy": 1.5469954013824463,
+ "epoch": 0.18787878787878787,
+ "grad_norm": 2.0568583011627197,
+ "learning_rate": 4.8141414141414146e-06,
+ "loss": 1.5564078092575073,
+ "mean_token_accuracy": 0.6384953856468201,
+ "num_tokens": 1523712.0,
+ "step": 93
+ },
+ {
+ "entropy": 1.5429692268371582,
+ "epoch": 0.1898989898989899,
+ "grad_norm": 2.200144052505493,
+ "learning_rate": 4.8121212121212125e-06,
+ "loss": 1.5292989015579224,
+ "mean_token_accuracy": 0.6373351216316223,
+ "num_tokens": 1540096.0,
+ "step": 94
+ },
+ {
+ "entropy": 1.4556076526641846,
+ "epoch": 0.1919191919191919,
+ "grad_norm": 2.250291585922241,
+ "learning_rate": 4.81010101010101e-06,
+ "loss": 1.426419734954834,
+ "mean_token_accuracy": 0.6604176759719849,
+ "num_tokens": 1556480.0,
+ "step": 95
+ },
+ {
+ "entropy": 1.4021440744400024,
+ "epoch": 0.19393939393939394,
+ "grad_norm": 2.287038803100586,
+ "learning_rate": 4.808080808080808e-06,
+ "loss": 1.4377000331878662,
+ "mean_token_accuracy": 0.6572422981262207,
+ "num_tokens": 1572864.0,
+ "step": 96
+ },
+ {
+ "entropy": 1.658683180809021,
+ "epoch": 0.19595959595959597,
+ "grad_norm": 2.1817963123321533,
+ "learning_rate": 4.806060606060606e-06,
+ "loss": 1.6514620780944824,
+ "mean_token_accuracy": 0.6278700828552246,
+ "num_tokens": 1589248.0,
+ "step": 97
+ },
+ {
+ "entropy": 1.5182844400405884,
+ "epoch": 0.19797979797979798,
+ "grad_norm": 2.144071340560913,
+ "learning_rate": 4.804040404040404e-06,
+ "loss": 1.5554628372192383,
+ "mean_token_accuracy": 0.6311064958572388,
+ "num_tokens": 1605632.0,
+ "step": 98
+ },
+ {
+ "entropy": 1.5997719764709473,
+ "epoch": 0.2,
+ "grad_norm": 2.0921664237976074,
+ "learning_rate": 4.802020202020202e-06,
+ "loss": 1.6185517311096191,
+ "mean_token_accuracy": 0.6286028623580933,
+ "num_tokens": 1622016.0,
+ "step": 99
+ },
+ {
+ "entropy": 1.4011279344558716,
+ "epoch": 0.20202020202020202,
+ "grad_norm": 2.2240700721740723,
+ "learning_rate": 4.800000000000001e-06,
+ "loss": 1.4112927913665771,
+ "mean_token_accuracy": 0.6696995496749878,
+ "num_tokens": 1638400.0,
+ "step": 100
+ },
+ {
+ "epoch": 0.20202020202020202,
+ "eval_entropy": 1.606662715634992,
+ "eval_loss": 1.6283859014511108,
+ "eval_mean_token_accuracy": 0.6293055717983553,
+ "eval_num_tokens": 1638400.0,
+ "eval_runtime": 43.7782,
+ "eval_samples_per_second": 22.614,
+ "eval_steps_per_second": 2.832,
+ "step": 100
+ },
+ {
+ "entropy": 1.6743865013122559,
+ "epoch": 0.20404040404040405,
+ "grad_norm": 2.025153398513794,
+ "learning_rate": 4.7979797979797985e-06,
+ "loss": 1.6404725313186646,
+ "mean_token_accuracy": 0.656509518623352,
+ "num_tokens": 1654784.0,
+ "step": 101
+ },
+ {
+ "entropy": 1.6196308135986328,
+ "epoch": 0.20606060606060606,
+ "grad_norm": 2.037229299545288,
+ "learning_rate": 4.795959595959596e-06,
+ "loss": 1.647303581237793,
+ "mean_token_accuracy": 0.6203590631484985,
+ "num_tokens": 1671168.0,
+ "step": 102
+ },
+ {
+ "entropy": 1.5600125789642334,
+ "epoch": 0.2080808080808081,
+ "grad_norm": 2.144221782684326,
+ "learning_rate": 4.793939393939394e-06,
+ "loss": 1.6199731826782227,
+ "mean_token_accuracy": 0.6249389052391052,
+ "num_tokens": 1687552.0,
+ "step": 103
+ },
+ {
+ "entropy": 2.064497947692871,
+ "epoch": 0.2101010101010101,
+ "grad_norm": 2.3956470489501953,
+ "learning_rate": 4.791919191919192e-06,
+ "loss": 2.048987627029419,
+ "mean_token_accuracy": 0.5507450103759766,
+ "num_tokens": 1703936.0,
+ "step": 104
+ },
+ {
+ "entropy": 1.7934812307357788,
+ "epoch": 0.21212121212121213,
+ "grad_norm": 2.487302780151367,
+ "learning_rate": 4.78989898989899e-06,
+ "loss": 1.8285956382751465,
+ "mean_token_accuracy": 0.5749877691268921,
+ "num_tokens": 1720320.0,
+ "step": 105
+ },
+ {
+ "entropy": 1.365216851234436,
+ "epoch": 0.21414141414141413,
+ "grad_norm": 2.0714964866638184,
+ "learning_rate": 4.787878787878788e-06,
+ "loss": 1.3870220184326172,
+ "mean_token_accuracy": 0.6692721247673035,
+ "num_tokens": 1736704.0,
+ "step": 106
+ },
+ {
+ "entropy": 1.4983874559402466,
+ "epoch": 0.21616161616161617,
+ "grad_norm": 1.9995853900909424,
+ "learning_rate": 4.785858585858586e-06,
+ "loss": 1.4949266910552979,
+ "mean_token_accuracy": 0.6554103493690491,
+ "num_tokens": 1753088.0,
+ "step": 107
+ },
+ {
+ "entropy": 2.0503509044647217,
+ "epoch": 0.21818181818181817,
+ "grad_norm": 2.190884590148926,
+ "learning_rate": 4.783838383838385e-06,
+ "loss": 2.079286813735962,
+ "mean_token_accuracy": 0.5657669901847839,
+ "num_tokens": 1769472.0,
+ "step": 108
+ },
+ {
+ "entropy": 1.5557374954223633,
+ "epoch": 0.2202020202020202,
+ "grad_norm": 2.044100761413574,
+ "learning_rate": 4.7818181818181825e-06,
+ "loss": 1.5735318660736084,
+ "mean_token_accuracy": 0.6359916925430298,
+ "num_tokens": 1785856.0,
+ "step": 109
+ },
+ {
+ "entropy": 1.5567635297775269,
+ "epoch": 0.2222222222222222,
+ "grad_norm": 2.3714160919189453,
+ "learning_rate": 4.77979797979798e-06,
+ "loss": 1.6026921272277832,
+ "mean_token_accuracy": 0.6290302872657776,
+ "num_tokens": 1802240.0,
+ "step": 110
+ },
+ {
+ "entropy": 1.8457536697387695,
+ "epoch": 0.22424242424242424,
+ "grad_norm": 2.202939987182617,
+ "learning_rate": 4.777777777777778e-06,
+ "loss": 1.7986081838607788,
+ "mean_token_accuracy": 0.5952613353729248,
+ "num_tokens": 1818624.0,
+ "step": 111
+ },
+ {
+ "entropy": 1.8822020292282104,
+ "epoch": 0.22626262626262628,
+ "grad_norm": 2.2674992084503174,
+ "learning_rate": 4.775757575757576e-06,
+ "loss": 1.9095954895019531,
+ "mean_token_accuracy": 0.5769418478012085,
+ "num_tokens": 1835008.0,
+ "step": 112
+ },
+ {
+ "entropy": 1.5445998907089233,
+ "epoch": 0.22828282828282828,
+ "grad_norm": 2.2848100662231445,
+ "learning_rate": 4.773737373737374e-06,
+ "loss": 1.5832195281982422,
+ "mean_token_accuracy": 0.6337933540344238,
+ "num_tokens": 1851392.0,
+ "step": 113
+ },
+ {
+ "entropy": 1.934509515762329,
+ "epoch": 0.23030303030303031,
+ "grad_norm": 2.3585174083709717,
+ "learning_rate": 4.771717171717172e-06,
+ "loss": 1.9809319972991943,
+ "mean_token_accuracy": 0.5789570212364197,
+ "num_tokens": 1867776.0,
+ "step": 114
+ },
+ {
+ "entropy": 1.7354214191436768,
+ "epoch": 0.23232323232323232,
+ "grad_norm": 2.13913631439209,
+ "learning_rate": 4.769696969696971e-06,
+ "loss": 1.7527806758880615,
+ "mean_token_accuracy": 0.6061308979988098,
+ "num_tokens": 1884160.0,
+ "step": 115
+ },
+ {
+ "entropy": 1.7714784145355225,
+ "epoch": 0.23434343434343435,
+ "grad_norm": 1.9915403127670288,
+ "learning_rate": 4.7676767676767685e-06,
+ "loss": 1.8065431118011475,
+ "mean_token_accuracy": 0.5986199378967285,
+ "num_tokens": 1900544.0,
+ "step": 116
+ },
+ {
+ "entropy": 1.5272125005722046,
+ "epoch": 0.23636363636363636,
+ "grad_norm": 2.004565715789795,
+ "learning_rate": 4.765656565656566e-06,
+ "loss": 1.5074517726898193,
+ "mean_token_accuracy": 0.6601123809814453,
+ "num_tokens": 1916928.0,
+ "step": 117
+ },
+ {
+ "entropy": 1.5393218994140625,
+ "epoch": 0.2383838383838384,
+ "grad_norm": 2.018089532852173,
+ "learning_rate": 4.763636363636364e-06,
+ "loss": 1.5607573986053467,
+ "mean_token_accuracy": 0.6540058851242065,
+ "num_tokens": 1933312.0,
+ "step": 118
+ },
+ {
+ "entropy": 1.7827534675598145,
+ "epoch": 0.2404040404040404,
+ "grad_norm": 2.062368392944336,
+ "learning_rate": 4.761616161616162e-06,
+ "loss": 1.7844001054763794,
+ "mean_token_accuracy": 0.6187102794647217,
+ "num_tokens": 1949696.0,
+ "step": 119
+ },
+ {
+ "entropy": 1.416417121887207,
+ "epoch": 0.24242424242424243,
+ "grad_norm": 2.0980024337768555,
+ "learning_rate": 4.75959595959596e-06,
+ "loss": 1.4041050672531128,
+ "mean_token_accuracy": 0.6561431288719177,
+ "num_tokens": 1966080.0,
+ "step": 120
+ },
+ {
+ "entropy": 1.3325153589248657,
+ "epoch": 0.24444444444444444,
+ "grad_norm": 1.9985002279281616,
+ "learning_rate": 4.757575757575758e-06,
+ "loss": 1.3197274208068848,
+ "mean_token_accuracy": 0.6806912422180176,
+ "num_tokens": 1982464.0,
+ "step": 121
+ },
+ {
+ "entropy": 1.7306798696517944,
+ "epoch": 0.24646464646464647,
+ "grad_norm": 2.2098441123962402,
+ "learning_rate": 4.755555555555556e-06,
+ "loss": 1.7595295906066895,
+ "mean_token_accuracy": 0.6110160946846008,
+ "num_tokens": 1998848.0,
+ "step": 122
+ },
+ {
+ "entropy": 1.3733264207839966,
+ "epoch": 0.24848484848484848,
+ "grad_norm": 1.9827327728271484,
+ "learning_rate": 4.753535353535354e-06,
+ "loss": 1.4026854038238525,
+ "mean_token_accuracy": 0.65486079454422,
+ "num_tokens": 2015232.0,
+ "step": 123
+ },
+ {
+ "entropy": 1.5910528898239136,
+ "epoch": 0.2505050505050505,
+ "grad_norm": 1.9615319967269897,
+ "learning_rate": 4.751515151515152e-06,
+ "loss": 1.6071114540100098,
+ "mean_token_accuracy": 0.648937463760376,
+ "num_tokens": 2031616.0,
+ "step": 124
+ },
+ {
+ "entropy": 1.3300938606262207,
+ "epoch": 0.25252525252525254,
+ "grad_norm": 1.9878504276275635,
+ "learning_rate": 4.7494949494949495e-06,
+ "loss": 1.3623383045196533,
+ "mean_token_accuracy": 0.6873473525047302,
+ "num_tokens": 2048000.0,
+ "step": 125
+ },
+ {
+ "entropy": 1.6087368726730347,
+ "epoch": 0.2545454545454545,
+ "grad_norm": 2.264647960662842,
+ "learning_rate": 4.747474747474748e-06,
+ "loss": 1.634058952331543,
+ "mean_token_accuracy": 0.6211528778076172,
+ "num_tokens": 2064384.0,
+ "step": 126
+ },
+ {
+ "entropy": 1.536401391029358,
+ "epoch": 0.25656565656565655,
+ "grad_norm": 1.9561539888381958,
+ "learning_rate": 4.745454545454546e-06,
+ "loss": 1.5699501037597656,
+ "mean_token_accuracy": 0.6248167753219604,
+ "num_tokens": 2080768.0,
+ "step": 127
+ },
+ {
+ "entropy": 1.656266212463379,
+ "epoch": 0.2585858585858586,
+ "grad_norm": 2.1124353408813477,
+ "learning_rate": 4.743434343434344e-06,
+ "loss": 1.6612167358398438,
+ "mean_token_accuracy": 0.623900830745697,
+ "num_tokens": 2097152.0,
+ "step": 128
+ },
+ {
+ "entropy": 1.170629620552063,
+ "epoch": 0.2606060606060606,
+ "grad_norm": 1.917840600013733,
+ "learning_rate": 4.741414141414142e-06,
+ "loss": 1.1992300748825073,
+ "mean_token_accuracy": 0.6995603442192078,
+ "num_tokens": 2113536.0,
+ "step": 129
+ },
+ {
+ "entropy": 1.4126694202423096,
+ "epoch": 0.26262626262626265,
+ "grad_norm": 1.9041539430618286,
+ "learning_rate": 4.73939393939394e-06,
+ "loss": 1.4334447383880615,
+ "mean_token_accuracy": 0.6637151837348938,
+ "num_tokens": 2129920.0,
+ "step": 130
+ },
+ {
+ "entropy": 1.8490277528762817,
+ "epoch": 0.26464646464646463,
+ "grad_norm": 2.1440138816833496,
+ "learning_rate": 4.737373737373738e-06,
+ "loss": 1.9226353168487549,
+ "mean_token_accuracy": 0.5805447101593018,
+ "num_tokens": 2146304.0,
+ "step": 131
+ },
+ {
+ "entropy": 1.5000964403152466,
+ "epoch": 0.26666666666666666,
+ "grad_norm": 1.9615508317947388,
+ "learning_rate": 4.735353535353536e-06,
+ "loss": 1.495596170425415,
+ "mean_token_accuracy": 0.6542501449584961,
+ "num_tokens": 2162688.0,
+ "step": 132
+ },
+ {
+ "entropy": 1.9371142387390137,
+ "epoch": 0.2686868686868687,
+ "grad_norm": 2.202200412750244,
+ "learning_rate": 4.7333333333333335e-06,
+ "loss": 1.94151771068573,
+ "mean_token_accuracy": 0.5810332298278809,
+ "num_tokens": 2179072.0,
+ "step": 133
+ },
+ {
+ "entropy": 1.4535126686096191,
+ "epoch": 0.27070707070707073,
+ "grad_norm": 1.9804027080535889,
+ "learning_rate": 4.731313131313131e-06,
+ "loss": 1.4401545524597168,
+ "mean_token_accuracy": 0.6585246920585632,
+ "num_tokens": 2195456.0,
+ "step": 134
+ },
+ {
+ "entropy": 2.0871424674987793,
+ "epoch": 0.2727272727272727,
+ "grad_norm": 2.1417081356048584,
+ "learning_rate": 4.729292929292929e-06,
+ "loss": 2.117375373840332,
+ "mean_token_accuracy": 0.5600268840789795,
+ "num_tokens": 2211840.0,
+ "step": 135
+ },
+ {
+ "entropy": 1.2986469268798828,
+ "epoch": 0.27474747474747474,
+ "grad_norm": 2.0299136638641357,
+ "learning_rate": 4.727272727272728e-06,
+ "loss": 1.3631991147994995,
+ "mean_token_accuracy": 0.6750732660293579,
+ "num_tokens": 2228224.0,
+ "step": 136
+ },
+ {
+ "entropy": 1.163429617881775,
+ "epoch": 0.2767676767676768,
+ "grad_norm": 1.8368210792541504,
+ "learning_rate": 4.725252525252526e-06,
+ "loss": 1.1603795289993286,
+ "mean_token_accuracy": 0.7150708436965942,
+ "num_tokens": 2244608.0,
+ "step": 137
+ },
+ {
+ "entropy": 1.4301519393920898,
+ "epoch": 0.2787878787878788,
+ "grad_norm": 1.996767520904541,
+ "learning_rate": 4.723232323232324e-06,
+ "loss": 1.422170639038086,
+ "mean_token_accuracy": 0.6816682815551758,
+ "num_tokens": 2260992.0,
+ "step": 138
+ },
+ {
+ "entropy": 1.5160106420516968,
+ "epoch": 0.2808080808080808,
+ "grad_norm": 2.004770278930664,
+ "learning_rate": 4.721212121212122e-06,
+ "loss": 1.5385751724243164,
+ "mean_token_accuracy": 0.640693724155426,
+ "num_tokens": 2277376.0,
+ "step": 139
+ },
+ {
+ "entropy": 1.2483867406845093,
+ "epoch": 0.2828282828282828,
+ "grad_norm": 1.8488364219665527,
+ "learning_rate": 4.7191919191919195e-06,
+ "loss": 1.2563843727111816,
+ "mean_token_accuracy": 0.6998046040534973,
+ "num_tokens": 2293760.0,
+ "step": 140
+ },
+ {
+ "entropy": 1.3214129209518433,
+ "epoch": 0.28484848484848485,
+ "grad_norm": 2.1334660053253174,
+ "learning_rate": 4.717171717171717e-06,
+ "loss": 1.3392664194107056,
+ "mean_token_accuracy": 0.6822789311408997,
+ "num_tokens": 2310144.0,
+ "step": 141
+ },
+ {
+ "entropy": 1.611849069595337,
+ "epoch": 0.2868686868686869,
+ "grad_norm": 2.0539803504943848,
+ "learning_rate": 4.715151515151515e-06,
+ "loss": 1.6335396766662598,
+ "mean_token_accuracy": 0.6235954761505127,
+ "num_tokens": 2326528.0,
+ "step": 142
+ },
+ {
+ "entropy": 1.5847896337509155,
+ "epoch": 0.28888888888888886,
+ "grad_norm": 2.1658759117126465,
+ "learning_rate": 4.713131313131313e-06,
+ "loss": 1.603764533996582,
+ "mean_token_accuracy": 0.633671224117279,
+ "num_tokens": 2342912.0,
+ "step": 143
+ },
+ {
+ "entropy": 1.5731406211853027,
+ "epoch": 0.2909090909090909,
+ "grad_norm": 2.0830390453338623,
+ "learning_rate": 4.711111111111111e-06,
+ "loss": 1.558933973312378,
+ "mean_token_accuracy": 0.6392892003059387,
+ "num_tokens": 2359296.0,
+ "step": 144
+ },
+ {
+ "entropy": 1.8901628255844116,
+ "epoch": 0.29292929292929293,
+ "grad_norm": 2.1436922550201416,
+ "learning_rate": 4.709090909090909e-06,
+ "loss": 1.878631353378296,
+ "mean_token_accuracy": 0.57333904504776,
+ "num_tokens": 2375680.0,
+ "step": 145
+ },
+ {
+ "entropy": 1.9344228506088257,
+ "epoch": 0.29494949494949496,
+ "grad_norm": 2.1245176792144775,
+ "learning_rate": 4.707070707070707e-06,
+ "loss": 1.9546704292297363,
+ "mean_token_accuracy": 0.5987420678138733,
+ "num_tokens": 2392064.0,
+ "step": 146
+ },
+ {
+ "entropy": 1.2455096244812012,
+ "epoch": 0.296969696969697,
+ "grad_norm": 1.896581768989563,
+ "learning_rate": 4.705050505050506e-06,
+ "loss": 1.2738728523254395,
+ "mean_token_accuracy": 0.7025524973869324,
+ "num_tokens": 2408448.0,
+ "step": 147
+ },
+ {
+ "entropy": 1.4235948324203491,
+ "epoch": 0.298989898989899,
+ "grad_norm": 2.522636890411377,
+ "learning_rate": 4.7030303030303035e-06,
+ "loss": 1.4524480104446411,
+ "mean_token_accuracy": 0.6441133618354797,
+ "num_tokens": 2424832.0,
+ "step": 148
+ },
+ {
+ "entropy": 1.764552116394043,
+ "epoch": 0.301010101010101,
+ "grad_norm": 2.0232255458831787,
+ "learning_rate": 4.701010101010101e-06,
+ "loss": 1.7811740636825562,
+ "mean_token_accuracy": 0.6229848265647888,
+ "num_tokens": 2441216.0,
+ "step": 149
+ },
+ {
+ "entropy": 1.74843168258667,
+ "epoch": 0.30303030303030304,
+ "grad_norm": 2.1589369773864746,
+ "learning_rate": 4.698989898989899e-06,
+ "loss": 1.7337679862976074,
+ "mean_token_accuracy": 0.5975207686424255,
+ "num_tokens": 2457600.0,
+ "step": 150
+ },
+ {
+ "epoch": 0.30303030303030304,
+ "eval_entropy": 1.6033287221385586,
+ "eval_loss": 1.6036633253097534,
+ "eval_mean_token_accuracy": 0.6329842450157288,
+ "eval_num_tokens": 2457600.0,
+ "eval_runtime": 43.7655,
+ "eval_samples_per_second": 22.621,
+ "eval_steps_per_second": 2.833,
+ "step": 150
+ },
+ {
+ "entropy": 2.0572290420532227,
+ "epoch": 0.30505050505050507,
+ "grad_norm": 2.0511579513549805,
+ "learning_rate": 4.696969696969698e-06,
+ "loss": 2.0196030139923096,
+ "mean_token_accuracy": 0.5519052147865295,
+ "num_tokens": 2473984.0,
+ "step": 151
+ },
+ {
+ "entropy": 1.5792397260665894,
+ "epoch": 0.30707070707070705,
+ "grad_norm": 2.048396587371826,
+ "learning_rate": 4.694949494949496e-06,
+ "loss": 1.575985074043274,
+ "mean_token_accuracy": 0.623961865901947,
+ "num_tokens": 2490368.0,
+ "step": 152
+ },
+ {
+ "entropy": 1.487528681755066,
+ "epoch": 0.3090909090909091,
+ "grad_norm": 2.1903791427612305,
+ "learning_rate": 4.692929292929294e-06,
+ "loss": 1.479973316192627,
+ "mean_token_accuracy": 0.6595017313957214,
+ "num_tokens": 2506752.0,
+ "step": 153
+ },
+ {
+ "entropy": 1.766797423362732,
+ "epoch": 0.3111111111111111,
+ "grad_norm": 2.1843011379241943,
+ "learning_rate": 4.690909090909092e-06,
+ "loss": 1.7993412017822266,
+ "mean_token_accuracy": 0.5965437293052673,
+ "num_tokens": 2523136.0,
+ "step": 154
+ },
+ {
+ "entropy": 1.7058254480361938,
+ "epoch": 0.31313131313131315,
+ "grad_norm": 2.204461097717285,
+ "learning_rate": 4.6888888888888895e-06,
+ "loss": 1.7346007823944092,
+ "mean_token_accuracy": 0.6077185869216919,
+ "num_tokens": 2539520.0,
+ "step": 155
+ },
+ {
+ "entropy": 1.4929591417312622,
+ "epoch": 0.3151515151515151,
+ "grad_norm": 2.1739237308502197,
+ "learning_rate": 4.6868686868686874e-06,
+ "loss": 1.4708621501922607,
+ "mean_token_accuracy": 0.6427088379859924,
+ "num_tokens": 2555904.0,
+ "step": 156
+ },
+ {
+ "entropy": 1.5797587633132935,
+ "epoch": 0.31717171717171716,
+ "grad_norm": 2.150561571121216,
+ "learning_rate": 4.684848484848485e-06,
+ "loss": 1.5921857357025146,
+ "mean_token_accuracy": 0.6262823939323425,
+ "num_tokens": 2572288.0,
+ "step": 157
+ },
+ {
+ "entropy": 1.6198230981826782,
+ "epoch": 0.3191919191919192,
+ "grad_norm": 2.061169385910034,
+ "learning_rate": 4.682828282828283e-06,
+ "loss": 1.6601009368896484,
+ "mean_token_accuracy": 0.619015634059906,
+ "num_tokens": 2588672.0,
+ "step": 158
+ },
+ {
+ "entropy": 1.4026126861572266,
+ "epoch": 0.3212121212121212,
+ "grad_norm": 2.066208839416504,
+ "learning_rate": 4.680808080808081e-06,
+ "loss": 1.4062483310699463,
+ "mean_token_accuracy": 0.6681729555130005,
+ "num_tokens": 2605056.0,
+ "step": 159
+ },
+ {
+ "entropy": 1.4608066082000732,
+ "epoch": 0.32323232323232326,
+ "grad_norm": 2.022627353668213,
+ "learning_rate": 4.678787878787879e-06,
+ "loss": 1.4644970893859863,
+ "mean_token_accuracy": 0.6533341407775879,
+ "num_tokens": 2621440.0,
+ "step": 160
+ },
+ {
+ "entropy": 1.222448468208313,
+ "epoch": 0.32525252525252524,
+ "grad_norm": 2.0399329662323,
+ "learning_rate": 4.676767676767677e-06,
+ "loss": 1.2547852993011475,
+ "mean_token_accuracy": 0.6925378441810608,
+ "num_tokens": 2637824.0,
+ "step": 161
+ },
+ {
+ "entropy": 1.8702255487442017,
+ "epoch": 0.32727272727272727,
+ "grad_norm": 2.177307367324829,
+ "learning_rate": 4.674747474747476e-06,
+ "loss": 1.9041712284088135,
+ "mean_token_accuracy": 0.5754152536392212,
+ "num_tokens": 2654208.0,
+ "step": 162
+ },
+ {
+ "entropy": 1.6861947774887085,
+ "epoch": 0.3292929292929293,
+ "grad_norm": 2.009544610977173,
+ "learning_rate": 4.6727272727272735e-06,
+ "loss": 1.7079355716705322,
+ "mean_token_accuracy": 0.615290641784668,
+ "num_tokens": 2670592.0,
+ "step": 163
+ },
+ {
+ "entropy": 1.3880327939987183,
+ "epoch": 0.33131313131313134,
+ "grad_norm": 2.16359543800354,
+ "learning_rate": 4.670707070707071e-06,
+ "loss": 1.4220571517944336,
+ "mean_token_accuracy": 0.6604787707328796,
+ "num_tokens": 2686976.0,
+ "step": 164
+ },
+ {
+ "entropy": 1.5760643482208252,
+ "epoch": 0.3333333333333333,
+ "grad_norm": 2.09773325920105,
+ "learning_rate": 4.668686868686869e-06,
+ "loss": 1.6036784648895264,
+ "mean_token_accuracy": 0.6267098188400269,
+ "num_tokens": 2703360.0,
+ "step": 165
+ },
+ {
+ "entropy": 1.078303575515747,
+ "epoch": 0.33535353535353535,
+ "grad_norm": 1.7760599851608276,
+ "learning_rate": 4.666666666666667e-06,
+ "loss": 1.0612695217132568,
+ "mean_token_accuracy": 0.7349169254302979,
+ "num_tokens": 2719744.0,
+ "step": 166
+ },
+ {
+ "entropy": 1.3722130060195923,
+ "epoch": 0.3373737373737374,
+ "grad_norm": 2.0816409587860107,
+ "learning_rate": 4.664646464646465e-06,
+ "loss": 1.3693504333496094,
+ "mean_token_accuracy": 0.6682950854301453,
+ "num_tokens": 2736128.0,
+ "step": 167
+ },
+ {
+ "entropy": 1.8752760887145996,
+ "epoch": 0.3393939393939394,
+ "grad_norm": 2.1670358180999756,
+ "learning_rate": 4.662626262626263e-06,
+ "loss": 1.8874578475952148,
+ "mean_token_accuracy": 0.5881778001785278,
+ "num_tokens": 2752512.0,
+ "step": 168
+ },
+ {
+ "entropy": 1.7384581565856934,
+ "epoch": 0.3414141414141414,
+ "grad_norm": 2.0072429180145264,
+ "learning_rate": 4.660606060606061e-06,
+ "loss": 1.7491583824157715,
+ "mean_token_accuracy": 0.6041157841682434,
+ "num_tokens": 2768896.0,
+ "step": 169
+ },
+ {
+ "entropy": 1.6651691198349,
+ "epoch": 0.3434343434343434,
+ "grad_norm": 2.0812578201293945,
+ "learning_rate": 4.658585858585859e-06,
+ "loss": 1.6808438301086426,
+ "mean_token_accuracy": 0.6273204684257507,
+ "num_tokens": 2785280.0,
+ "step": 170
+ },
+ {
+ "entropy": 1.3426616191864014,
+ "epoch": 0.34545454545454546,
+ "grad_norm": 1.9108822345733643,
+ "learning_rate": 4.656565656565657e-06,
+ "loss": 1.3404264450073242,
+ "mean_token_accuracy": 0.6703101992607117,
+ "num_tokens": 2801664.0,
+ "step": 171
+ },
+ {
+ "entropy": 1.631722092628479,
+ "epoch": 0.3474747474747475,
+ "grad_norm": 2.07317852973938,
+ "learning_rate": 4.654545454545455e-06,
+ "loss": 1.6545813083648682,
+ "mean_token_accuracy": 0.6143136024475098,
+ "num_tokens": 2818048.0,
+ "step": 172
+ },
+ {
+ "entropy": 1.3678289651870728,
+ "epoch": 0.34949494949494947,
+ "grad_norm": 1.8517454862594604,
+ "learning_rate": 4.652525252525253e-06,
+ "loss": 1.3759769201278687,
+ "mean_token_accuracy": 0.6693942546844482,
+ "num_tokens": 2834432.0,
+ "step": 173
+ },
+ {
+ "entropy": 1.3927773237228394,
+ "epoch": 0.3515151515151515,
+ "grad_norm": 2.1304867267608643,
+ "learning_rate": 4.650505050505051e-06,
+ "loss": 1.4104211330413818,
+ "mean_token_accuracy": 0.6612725853919983,
+ "num_tokens": 2850816.0,
+ "step": 174
+ },
+ {
+ "entropy": 1.3352863788604736,
+ "epoch": 0.35353535353535354,
+ "grad_norm": 2.2788777351379395,
+ "learning_rate": 4.648484848484849e-06,
+ "loss": 1.3665719032287598,
+ "mean_token_accuracy": 0.6640205383300781,
+ "num_tokens": 2867200.0,
+ "step": 175
+ },
+ {
+ "entropy": 1.6800754070281982,
+ "epoch": 0.35555555555555557,
+ "grad_norm": 2.030787944793701,
+ "learning_rate": 4.646464646464647e-06,
+ "loss": 1.7239181995391846,
+ "mean_token_accuracy": 0.6193209290504456,
+ "num_tokens": 2883584.0,
+ "step": 176
+ },
+ {
+ "entropy": 1.2852731943130493,
+ "epoch": 0.3575757575757576,
+ "grad_norm": 1.9148248434066772,
+ "learning_rate": 4.644444444444445e-06,
+ "loss": 1.2934880256652832,
+ "mean_token_accuracy": 0.6798363327980042,
+ "num_tokens": 2899968.0,
+ "step": 177
+ },
+ {
+ "entropy": 1.6366593837738037,
+ "epoch": 0.3595959595959596,
+ "grad_norm": 2.2264015674591064,
+ "learning_rate": 4.642424242424243e-06,
+ "loss": 1.632002592086792,
+ "mean_token_accuracy": 0.6180385947227478,
+ "num_tokens": 2916352.0,
+ "step": 178
+ },
+ {
+ "entropy": 1.3935565948486328,
+ "epoch": 0.3616161616161616,
+ "grad_norm": 2.0410475730895996,
+ "learning_rate": 4.6404040404040406e-06,
+ "loss": 1.407370686531067,
+ "mean_token_accuracy": 0.6722032427787781,
+ "num_tokens": 2932736.0,
+ "step": 179
+ },
+ {
+ "entropy": 1.0952510833740234,
+ "epoch": 0.36363636363636365,
+ "grad_norm": 1.937270164489746,
+ "learning_rate": 4.6383838383838384e-06,
+ "loss": 1.1101973056793213,
+ "mean_token_accuracy": 0.730947732925415,
+ "num_tokens": 2949120.0,
+ "step": 180
+ },
+ {
+ "entropy": 1.184926152229309,
+ "epoch": 0.3656565656565657,
+ "grad_norm": 1.9867528676986694,
+ "learning_rate": 4.636363636363636e-06,
+ "loss": 1.1932196617126465,
+ "mean_token_accuracy": 0.7085368633270264,
+ "num_tokens": 2965504.0,
+ "step": 181
+ },
+ {
+ "entropy": 1.4552072286605835,
+ "epoch": 0.36767676767676766,
+ "grad_norm": 1.9395766258239746,
+ "learning_rate": 4.634343434343434e-06,
+ "loss": 1.4559956789016724,
+ "mean_token_accuracy": 0.6534562706947327,
+ "num_tokens": 2981888.0,
+ "step": 182
+ },
+ {
+ "entropy": 1.6437443494796753,
+ "epoch": 0.3696969696969697,
+ "grad_norm": 2.1257567405700684,
+ "learning_rate": 4.632323232323233e-06,
+ "loss": 1.6260980367660522,
+ "mean_token_accuracy": 0.625,
+ "num_tokens": 2998272.0,
+ "step": 183
+ },
+ {
+ "entropy": 1.8264633417129517,
+ "epoch": 0.3717171717171717,
+ "grad_norm": 2.187041759490967,
+ "learning_rate": 4.630303030303031e-06,
+ "loss": 1.8431676626205444,
+ "mean_token_accuracy": 0.60332190990448,
+ "num_tokens": 3014656.0,
+ "step": 184
+ },
+ {
+ "entropy": 1.1121351718902588,
+ "epoch": 0.37373737373737376,
+ "grad_norm": 1.8460402488708496,
+ "learning_rate": 4.628282828282829e-06,
+ "loss": 1.1054425239562988,
+ "mean_token_accuracy": 0.7254518866539001,
+ "num_tokens": 3031040.0,
+ "step": 185
+ },
+ {
+ "entropy": 1.3531955480575562,
+ "epoch": 0.37575757575757573,
+ "grad_norm": 2.3463470935821533,
+ "learning_rate": 4.626262626262627e-06,
+ "loss": 1.359381914138794,
+ "mean_token_accuracy": 0.6610894203186035,
+ "num_tokens": 3047424.0,
+ "step": 186
+ },
+ {
+ "entropy": 1.2131577730178833,
+ "epoch": 0.37777777777777777,
+ "grad_norm": 1.9595171213150024,
+ "learning_rate": 4.6242424242424245e-06,
+ "loss": 1.2312005758285522,
+ "mean_token_accuracy": 0.6979115605354309,
+ "num_tokens": 3063808.0,
+ "step": 187
+ },
+ {
+ "entropy": 1.5441125631332397,
+ "epoch": 0.3797979797979798,
+ "grad_norm": 2.2971343994140625,
+ "learning_rate": 4.622222222222222e-06,
+ "loss": 1.5923339128494263,
+ "mean_token_accuracy": 0.6297020316123962,
+ "num_tokens": 3080192.0,
+ "step": 188
+ },
+ {
+ "entropy": 1.7998323440551758,
+ "epoch": 0.38181818181818183,
+ "grad_norm": 2.010552167892456,
+ "learning_rate": 4.62020202020202e-06,
+ "loss": 1.8257204294204712,
+ "mean_token_accuracy": 0.5997191071510315,
+ "num_tokens": 3096576.0,
+ "step": 189
+ },
+ {
+ "entropy": 1.389051914215088,
+ "epoch": 0.3838383838383838,
+ "grad_norm": 2.1183855533599854,
+ "learning_rate": 4.618181818181818e-06,
+ "loss": 1.431575059890747,
+ "mean_token_accuracy": 0.6537005305290222,
+ "num_tokens": 3112960.0,
+ "step": 190
+ },
+ {
+ "entropy": 2.0066142082214355,
+ "epoch": 0.38585858585858585,
+ "grad_norm": 2.061399221420288,
+ "learning_rate": 4.616161616161616e-06,
+ "loss": 2.0477406978607178,
+ "mean_token_accuracy": 0.5454934239387512,
+ "num_tokens": 3129344.0,
+ "step": 191
+ },
+ {
+ "entropy": 1.3063198328018188,
+ "epoch": 0.3878787878787879,
+ "grad_norm": 1.9490301609039307,
+ "learning_rate": 4.614141414141414e-06,
+ "loss": 1.3293564319610596,
+ "mean_token_accuracy": 0.6797142028808594,
+ "num_tokens": 3145728.0,
+ "step": 192
+ },
+ {
+ "entropy": 1.3636538982391357,
+ "epoch": 0.3898989898989899,
+ "grad_norm": 2.025458574295044,
+ "learning_rate": 4.612121212121212e-06,
+ "loss": 1.3736209869384766,
+ "mean_token_accuracy": 0.679286777973175,
+ "num_tokens": 3162112.0,
+ "step": 193
+ },
+ {
+ "entropy": 1.5669925212860107,
+ "epoch": 0.39191919191919194,
+ "grad_norm": 2.064033269882202,
+ "learning_rate": 4.6101010101010106e-06,
+ "loss": 1.5587083101272583,
+ "mean_token_accuracy": 0.6267709136009216,
+ "num_tokens": 3178496.0,
+ "step": 194
+ },
+ {
+ "entropy": 1.4489716291427612,
+ "epoch": 0.3939393939393939,
+ "grad_norm": 2.0174880027770996,
+ "learning_rate": 4.6080808080808085e-06,
+ "loss": 1.4331417083740234,
+ "mean_token_accuracy": 0.6571812629699707,
+ "num_tokens": 3194880.0,
+ "step": 195
+ },
+ {
+ "entropy": 2.026416778564453,
+ "epoch": 0.39595959595959596,
+ "grad_norm": 2.1820878982543945,
+ "learning_rate": 4.606060606060606e-06,
+ "loss": 2.029003143310547,
+ "mean_token_accuracy": 0.563568651676178,
+ "num_tokens": 3211264.0,
+ "step": 196
+ },
+ {
+ "entropy": 2.1844022274017334,
+ "epoch": 0.397979797979798,
+ "grad_norm": 2.175349473953247,
+ "learning_rate": 4.604040404040404e-06,
+ "loss": 2.2053825855255127,
+ "mean_token_accuracy": 0.5553248524665833,
+ "num_tokens": 3227648.0,
+ "step": 197
+ },
+ {
+ "entropy": 1.6750906705856323,
+ "epoch": 0.4,
+ "grad_norm": 1.8300689458847046,
+ "learning_rate": 4.602020202020203e-06,
+ "loss": 1.692289113998413,
+ "mean_token_accuracy": 0.6345261335372925,
+ "num_tokens": 3244032.0,
+ "step": 198
+ },
+ {
+ "entropy": 1.6192772388458252,
+ "epoch": 0.402020202020202,
+ "grad_norm": 1.9788341522216797,
+ "learning_rate": 4.600000000000001e-06,
+ "loss": 1.628842830657959,
+ "mean_token_accuracy": 0.615229606628418,
+ "num_tokens": 3260416.0,
+ "step": 199
+ },
+ {
+ "entropy": 1.444870948791504,
+ "epoch": 0.40404040404040403,
+ "grad_norm": 2.05141544342041,
+ "learning_rate": 4.597979797979799e-06,
+ "loss": 1.4365839958190918,
+ "mean_token_accuracy": 0.652723491191864,
+ "num_tokens": 3276800.0,
+ "step": 200
+ },
+ {
+ "epoch": 0.40404040404040403,
+ "eval_entropy": 1.5766179907706477,
+ "eval_loss": 1.5878442525863647,
+ "eval_mean_token_accuracy": 0.6355829551335304,
+ "eval_num_tokens": 3276800.0,
+ "eval_runtime": 43.7272,
+ "eval_samples_per_second": 22.64,
+ "eval_steps_per_second": 2.836,
+ "step": 200
+ },
+ {
+ "entropy": 1.3354029655456543,
+ "epoch": 0.40606060606060607,
+ "grad_norm": 2.012895345687866,
+ "learning_rate": 4.595959595959597e-06,
+ "loss": 1.3289990425109863,
+ "mean_token_accuracy": 0.685271143913269,
+ "num_tokens": 3293184.0,
+ "step": 201
+ },
+ {
+ "entropy": 1.1990011930465698,
+ "epoch": 0.4080808080808081,
+ "grad_norm": 1.9462409019470215,
+ "learning_rate": 4.5939393939393945e-06,
+ "loss": 1.224869728088379,
+ "mean_token_accuracy": 0.6976673007011414,
+ "num_tokens": 3309568.0,
+ "step": 202
+ },
+ {
+ "entropy": 1.6442710161209106,
+ "epoch": 0.4101010101010101,
+ "grad_norm": 2.064532995223999,
+ "learning_rate": 4.591919191919192e-06,
+ "loss": 1.6800963878631592,
+ "mean_token_accuracy": 0.6111993193626404,
+ "num_tokens": 3325952.0,
+ "step": 203
+ },
+ {
+ "entropy": 1.8336306810379028,
+ "epoch": 0.4121212121212121,
+ "grad_norm": 2.478731393814087,
+ "learning_rate": 4.58989898989899e-06,
+ "loss": 1.8518702983856201,
+ "mean_token_accuracy": 0.6021006107330322,
+ "num_tokens": 3342336.0,
+ "step": 204
+ },
+ {
+ "entropy": 1.6767220497131348,
+ "epoch": 0.41414141414141414,
+ "grad_norm": 2.0676512718200684,
+ "learning_rate": 4.587878787878788e-06,
+ "loss": 1.674522042274475,
+ "mean_token_accuracy": 0.6178553700447083,
+ "num_tokens": 3358720.0,
+ "step": 205
+ },
+ {
+ "entropy": 1.7236497402191162,
+ "epoch": 0.4161616161616162,
+ "grad_norm": 2.057074785232544,
+ "learning_rate": 4.585858585858586e-06,
+ "loss": 1.725832462310791,
+ "mean_token_accuracy": 0.620114803314209,
+ "num_tokens": 3375104.0,
+ "step": 206
+ },
+ {
+ "entropy": 1.6493065357208252,
+ "epoch": 0.41818181818181815,
+ "grad_norm": 2.0946099758148193,
+ "learning_rate": 4.583838383838384e-06,
+ "loss": 1.6826295852661133,
+ "mean_token_accuracy": 0.6251221299171448,
+ "num_tokens": 3391488.0,
+ "step": 207
+ },
+ {
+ "entropy": 1.3565926551818848,
+ "epoch": 0.4202020202020202,
+ "grad_norm": 1.9611284732818604,
+ "learning_rate": 4.581818181818183e-06,
+ "loss": 1.346787452697754,
+ "mean_token_accuracy": 0.685332179069519,
+ "num_tokens": 3407872.0,
+ "step": 208
+ },
+ {
+ "entropy": 1.4401211738586426,
+ "epoch": 0.4222222222222222,
+ "grad_norm": 1.8614766597747803,
+ "learning_rate": 4.579797979797981e-06,
+ "loss": 1.4717891216278076,
+ "mean_token_accuracy": 0.650036633014679,
+ "num_tokens": 3424256.0,
+ "step": 209
+ },
+ {
+ "entropy": 1.194276213645935,
+ "epoch": 0.42424242424242425,
+ "grad_norm": 2.0625810623168945,
+ "learning_rate": 4.5777777777777785e-06,
+ "loss": 1.2106804847717285,
+ "mean_token_accuracy": 0.700537383556366,
+ "num_tokens": 3440640.0,
+ "step": 210
+ },
+ {
+ "entropy": 1.7622313499450684,
+ "epoch": 0.4262626262626263,
+ "grad_norm": 2.330610752105713,
+ "learning_rate": 4.575757575757576e-06,
+ "loss": 1.7872710227966309,
+ "mean_token_accuracy": 0.598436713218689,
+ "num_tokens": 3457024.0,
+ "step": 211
+ },
+ {
+ "entropy": 1.409427523612976,
+ "epoch": 0.42828282828282827,
+ "grad_norm": 2.0988259315490723,
+ "learning_rate": 4.573737373737374e-06,
+ "loss": 1.416553020477295,
+ "mean_token_accuracy": 0.6682950854301453,
+ "num_tokens": 3473408.0,
+ "step": 212
+ },
+ {
+ "entropy": 1.6856107711791992,
+ "epoch": 0.4303030303030303,
+ "grad_norm": 2.341475009918213,
+ "learning_rate": 4.571717171717172e-06,
+ "loss": 1.692287802696228,
+ "mean_token_accuracy": 0.6024059653282166,
+ "num_tokens": 3489792.0,
+ "step": 213
+ },
+ {
+ "entropy": 1.997343897819519,
+ "epoch": 0.43232323232323233,
+ "grad_norm": 2.200498104095459,
+ "learning_rate": 4.56969696969697e-06,
+ "loss": 2.026289939880371,
+ "mean_token_accuracy": 0.5781631469726562,
+ "num_tokens": 3506176.0,
+ "step": 214
+ },
+ {
+ "entropy": 1.5175039768218994,
+ "epoch": 0.43434343434343436,
+ "grad_norm": 2.105257272720337,
+ "learning_rate": 4.567676767676768e-06,
+ "loss": 1.521841049194336,
+ "mean_token_accuracy": 0.6404494643211365,
+ "num_tokens": 3522560.0,
+ "step": 215
+ },
+ {
+ "entropy": 1.3616528511047363,
+ "epoch": 0.43636363636363634,
+ "grad_norm": 2.0035297870635986,
+ "learning_rate": 4.565656565656566e-06,
+ "loss": 1.3674360513687134,
+ "mean_token_accuracy": 0.6771494746208191,
+ "num_tokens": 3538944.0,
+ "step": 216
+ },
+ {
+ "entropy": 1.42499840259552,
+ "epoch": 0.4383838383838384,
+ "grad_norm": 1.9115629196166992,
+ "learning_rate": 4.563636363636364e-06,
+ "loss": 1.4129266738891602,
+ "mean_token_accuracy": 0.6686614751815796,
+ "num_tokens": 3555328.0,
+ "step": 217
+ },
+ {
+ "entropy": 1.2844709157943726,
+ "epoch": 0.4404040404040404,
+ "grad_norm": 2.3313121795654297,
+ "learning_rate": 4.5616161616161616e-06,
+ "loss": 1.315006971359253,
+ "mean_token_accuracy": 0.681546151638031,
+ "num_tokens": 3571712.0,
+ "step": 218
+ },
+ {
+ "entropy": 1.5227075815200806,
+ "epoch": 0.44242424242424244,
+ "grad_norm": 2.1849124431610107,
+ "learning_rate": 4.55959595959596e-06,
+ "loss": 1.517989158630371,
+ "mean_token_accuracy": 0.6466169953346252,
+ "num_tokens": 3588096.0,
+ "step": 219
+ },
+ {
+ "entropy": 1.494642734527588,
+ "epoch": 0.4444444444444444,
+ "grad_norm": 2.3109116554260254,
+ "learning_rate": 4.557575757575758e-06,
+ "loss": 1.525421142578125,
+ "mean_token_accuracy": 0.6392281651496887,
+ "num_tokens": 3604480.0,
+ "step": 220
+ },
+ {
+ "entropy": 1.6428948640823364,
+ "epoch": 0.44646464646464645,
+ "grad_norm": 2.1182680130004883,
+ "learning_rate": 4.555555555555556e-06,
+ "loss": 1.6634926795959473,
+ "mean_token_accuracy": 0.6207864880561829,
+ "num_tokens": 3620864.0,
+ "step": 221
+ },
+ {
+ "entropy": 1.7323675155639648,
+ "epoch": 0.4484848484848485,
+ "grad_norm": 2.2620837688446045,
+ "learning_rate": 4.553535353535354e-06,
+ "loss": 1.7312631607055664,
+ "mean_token_accuracy": 0.6138250827789307,
+ "num_tokens": 3637248.0,
+ "step": 222
+ },
+ {
+ "entropy": 1.7863894701004028,
+ "epoch": 0.4505050505050505,
+ "grad_norm": 2.1416971683502197,
+ "learning_rate": 4.551515151515152e-06,
+ "loss": 1.799318552017212,
+ "mean_token_accuracy": 0.5906203985214233,
+ "num_tokens": 3653632.0,
+ "step": 223
+ },
+ {
+ "entropy": 1.7404330968856812,
+ "epoch": 0.45252525252525255,
+ "grad_norm": 2.0814504623413086,
+ "learning_rate": 4.54949494949495e-06,
+ "loss": 1.742197036743164,
+ "mean_token_accuracy": 0.6050317287445068,
+ "num_tokens": 3670016.0,
+ "step": 224
+ },
+ {
+ "entropy": 1.4387869834899902,
+ "epoch": 0.45454545454545453,
+ "grad_norm": 2.1386022567749023,
+ "learning_rate": 4.547474747474748e-06,
+ "loss": 1.4602317810058594,
+ "mean_token_accuracy": 0.6502198576927185,
+ "num_tokens": 3686400.0,
+ "step": 225
+ },
+ {
+ "entropy": 1.2733348608016968,
+ "epoch": 0.45656565656565656,
+ "grad_norm": 2.02687668800354,
+ "learning_rate": 4.5454545454545455e-06,
+ "loss": 1.2683714628219604,
+ "mean_token_accuracy": 0.6966902613639832,
+ "num_tokens": 3702784.0,
+ "step": 226
+ },
+ {
+ "entropy": 1.4554595947265625,
+ "epoch": 0.4585858585858586,
+ "grad_norm": 2.169268846511841,
+ "learning_rate": 4.543434343434343e-06,
+ "loss": 1.4541833400726318,
+ "mean_token_accuracy": 0.6522960662841797,
+ "num_tokens": 3719168.0,
+ "step": 227
+ },
+ {
+ "entropy": 1.487573266029358,
+ "epoch": 0.46060606060606063,
+ "grad_norm": 1.9726165533065796,
+ "learning_rate": 4.541414141414141e-06,
+ "loss": 1.5024409294128418,
+ "mean_token_accuracy": 0.6509526371955872,
+ "num_tokens": 3735552.0,
+ "step": 228
+ },
+ {
+ "entropy": 1.3991138935089111,
+ "epoch": 0.4626262626262626,
+ "grad_norm": 2.0992283821105957,
+ "learning_rate": 4.539393939393939e-06,
+ "loss": 1.4016860723495483,
+ "mean_token_accuracy": 0.6639594435691833,
+ "num_tokens": 3751936.0,
+ "step": 229
+ },
+ {
+ "entropy": 1.6564134359359741,
+ "epoch": 0.46464646464646464,
+ "grad_norm": 2.047656297683716,
+ "learning_rate": 4.537373737373738e-06,
+ "loss": 1.6598721742630005,
+ "mean_token_accuracy": 0.6166951656341553,
+ "num_tokens": 3768320.0,
+ "step": 230
+ },
+ {
+ "entropy": 1.5165014266967773,
+ "epoch": 0.4666666666666667,
+ "grad_norm": 2.079996109008789,
+ "learning_rate": 4.535353535353536e-06,
+ "loss": 1.4980010986328125,
+ "mean_token_accuracy": 0.6370908617973328,
+ "num_tokens": 3784704.0,
+ "step": 231
+ },
+ {
+ "entropy": 1.5409225225448608,
+ "epoch": 0.4686868686868687,
+ "grad_norm": 2.0996923446655273,
+ "learning_rate": 4.533333333333334e-06,
+ "loss": 1.5504257678985596,
+ "mean_token_accuracy": 0.6465559601783752,
+ "num_tokens": 3801088.0,
+ "step": 232
+ },
+ {
+ "entropy": 1.3590043783187866,
+ "epoch": 0.4707070707070707,
+ "grad_norm": 2.14617919921875,
+ "learning_rate": 4.531313131313132e-06,
+ "loss": 1.3581812381744385,
+ "mean_token_accuracy": 0.6683561205863953,
+ "num_tokens": 3817472.0,
+ "step": 233
+ },
+ {
+ "entropy": 1.641785740852356,
+ "epoch": 0.4727272727272727,
+ "grad_norm": 1.9499926567077637,
+ "learning_rate": 4.5292929292929295e-06,
+ "loss": 1.6653470993041992,
+ "mean_token_accuracy": 0.6319614052772522,
+ "num_tokens": 3833856.0,
+ "step": 234
+ },
+ {
+ "entropy": 1.4700758457183838,
+ "epoch": 0.47474747474747475,
+ "grad_norm": 1.9411437511444092,
+ "learning_rate": 4.527272727272727e-06,
+ "loss": 1.4762463569641113,
+ "mean_token_accuracy": 0.6631656289100647,
+ "num_tokens": 3850240.0,
+ "step": 235
+ },
+ {
+ "entropy": 1.5418941974639893,
+ "epoch": 0.4767676767676768,
+ "grad_norm": 2.3191940784454346,
+ "learning_rate": 4.525252525252526e-06,
+ "loss": 1.5151214599609375,
+ "mean_token_accuracy": 0.6326331496238708,
+ "num_tokens": 3866624.0,
+ "step": 236
+ },
+ {
+ "entropy": 2.1102776527404785,
+ "epoch": 0.47878787878787876,
+ "grad_norm": 2.2287707328796387,
+ "learning_rate": 4.523232323232324e-06,
+ "loss": 2.106567859649658,
+ "mean_token_accuracy": 0.5506839156150818,
+ "num_tokens": 3883008.0,
+ "step": 237
+ },
+ {
+ "entropy": 1.7448827028274536,
+ "epoch": 0.4808080808080808,
+ "grad_norm": 2.2352893352508545,
+ "learning_rate": 4.521212121212122e-06,
+ "loss": 1.7495017051696777,
+ "mean_token_accuracy": 0.60326087474823,
+ "num_tokens": 3899392.0,
+ "step": 238
+ },
+ {
+ "entropy": 1.315240740776062,
+ "epoch": 0.48282828282828283,
+ "grad_norm": 1.9933631420135498,
+ "learning_rate": 4.51919191919192e-06,
+ "loss": 1.315302848815918,
+ "mean_token_accuracy": 0.686431348323822,
+ "num_tokens": 3915776.0,
+ "step": 239
+ },
+ {
+ "entropy": 1.5851637125015259,
+ "epoch": 0.48484848484848486,
+ "grad_norm": 2.153303623199463,
+ "learning_rate": 4.517171717171718e-06,
+ "loss": 1.5947662591934204,
+ "mean_token_accuracy": 0.6297020316123962,
+ "num_tokens": 3932160.0,
+ "step": 240
+ },
+ {
+ "entropy": 1.8050944805145264,
+ "epoch": 0.4868686868686869,
+ "grad_norm": 2.025022506713867,
+ "learning_rate": 4.5151515151515155e-06,
+ "loss": 1.8051010370254517,
+ "mean_token_accuracy": 0.6055202484130859,
+ "num_tokens": 3948544.0,
+ "step": 241
+ },
+ {
+ "entropy": 1.518296718597412,
+ "epoch": 0.4888888888888889,
+ "grad_norm": 2.1021833419799805,
+ "learning_rate": 4.513131313131313e-06,
+ "loss": 1.5396809577941895,
+ "mean_token_accuracy": 0.6359916925430298,
+ "num_tokens": 3964928.0,
+ "step": 242
+ },
+ {
+ "entropy": 1.518903136253357,
+ "epoch": 0.4909090909090909,
+ "grad_norm": 2.0960140228271484,
+ "learning_rate": 4.511111111111111e-06,
+ "loss": 1.5528055429458618,
+ "mean_token_accuracy": 0.6224963068962097,
+ "num_tokens": 3981312.0,
+ "step": 243
+ },
+ {
+ "entropy": 1.6960705518722534,
+ "epoch": 0.49292929292929294,
+ "grad_norm": 1.8580718040466309,
+ "learning_rate": 4.50909090909091e-06,
+ "loss": 1.7192862033843994,
+ "mean_token_accuracy": 0.6259770393371582,
+ "num_tokens": 3997696.0,
+ "step": 244
+ },
+ {
+ "entropy": 1.4633033275604248,
+ "epoch": 0.494949494949495,
+ "grad_norm": 1.8983049392700195,
+ "learning_rate": 4.507070707070708e-06,
+ "loss": 1.451701283454895,
+ "mean_token_accuracy": 0.6669516563415527,
+ "num_tokens": 4014080.0,
+ "step": 245
+ },
+ {
+ "entropy": 1.6508095264434814,
+ "epoch": 0.49696969696969695,
+ "grad_norm": 2.298679828643799,
+ "learning_rate": 4.505050505050506e-06,
+ "loss": 1.6596897840499878,
+ "mean_token_accuracy": 0.6115046143531799,
+ "num_tokens": 4030464.0,
+ "step": 246
+ },
+ {
+ "entropy": 1.313779354095459,
+ "epoch": 0.498989898989899,
+ "grad_norm": 1.9894335269927979,
+ "learning_rate": 4.503030303030304e-06,
+ "loss": 1.3285409212112427,
+ "mean_token_accuracy": 0.6802027225494385,
+ "num_tokens": 4046848.0,
+ "step": 247
+ },
+ {
+ "entropy": 1.5139521360397339,
+ "epoch": 0.501010101010101,
+ "grad_norm": 2.028320789337158,
+ "learning_rate": 4.501010101010102e-06,
+ "loss": 1.5366487503051758,
+ "mean_token_accuracy": 0.6416707634925842,
+ "num_tokens": 4063232.0,
+ "step": 248
+ },
+ {
+ "entropy": 1.5277045965194702,
+ "epoch": 0.503030303030303,
+ "grad_norm": 2.3160979747772217,
+ "learning_rate": 4.4989898989898995e-06,
+ "loss": 1.5571036338806152,
+ "mean_token_accuracy": 0.6436858773231506,
+ "num_tokens": 4079616.0,
+ "step": 249
+ },
+ {
+ "entropy": 1.6084188222885132,
+ "epoch": 0.5050505050505051,
+ "grad_norm": 2.20550274848938,
+ "learning_rate": 4.496969696969697e-06,
+ "loss": 1.622127652168274,
+ "mean_token_accuracy": 0.6284196376800537,
+ "num_tokens": 4096000.0,
+ "step": 250
+ },
+ {
+ "epoch": 0.5050505050505051,
+ "eval_entropy": 1.5547234262189558,
+ "eval_loss": 1.5764786005020142,
+ "eval_mean_token_accuracy": 0.6375306306346771,
+ "eval_num_tokens": 4096000.0,
+ "eval_runtime": 43.7607,
+ "eval_samples_per_second": 22.623,
+ "eval_steps_per_second": 2.834,
+ "step": 250
+ },
+ {
+ "entropy": 1.4309135675430298,
+ "epoch": 0.5070707070707071,
+ "grad_norm": 2.080864906311035,
+ "learning_rate": 4.494949494949495e-06,
+ "loss": 1.4481091499328613,
+ "mean_token_accuracy": 0.6694552898406982,
+ "num_tokens": 4112384.0,
+ "step": 251
+ },
+ {
+ "entropy": 1.4067270755767822,
+ "epoch": 0.509090909090909,
+ "grad_norm": 2.036475419998169,
+ "learning_rate": 4.492929292929293e-06,
+ "loss": 1.4035298824310303,
+ "mean_token_accuracy": 0.6547996997833252,
+ "num_tokens": 4128768.0,
+ "step": 252
+ },
+ {
+ "entropy": 2.1751608848571777,
+ "epoch": 0.5111111111111111,
+ "grad_norm": 1.953995943069458,
+ "learning_rate": 4.490909090909091e-06,
+ "loss": 2.1616692543029785,
+ "mean_token_accuracy": 0.5592941045761108,
+ "num_tokens": 4145152.0,
+ "step": 253
+ },
+ {
+ "entropy": 1.45353102684021,
+ "epoch": 0.5131313131313131,
+ "grad_norm": 2.084437370300293,
+ "learning_rate": 4.488888888888889e-06,
+ "loss": 1.4633586406707764,
+ "mean_token_accuracy": 0.6620664596557617,
+ "num_tokens": 4161536.0,
+ "step": 254
+ },
+ {
+ "entropy": 1.64310884475708,
+ "epoch": 0.5151515151515151,
+ "grad_norm": 2.0700111389160156,
+ "learning_rate": 4.486868686868688e-06,
+ "loss": 1.6868078708648682,
+ "mean_token_accuracy": 0.616328775882721,
+ "num_tokens": 4177920.0,
+ "step": 255
+ },
+ {
+ "entropy": 1.5113472938537598,
+ "epoch": 0.5171717171717172,
+ "grad_norm": 2.102180242538452,
+ "learning_rate": 4.4848484848484855e-06,
+ "loss": 1.5240120887756348,
+ "mean_token_accuracy": 0.638067901134491,
+ "num_tokens": 4194304.0,
+ "step": 256
+ },
+ {
+ "entropy": 1.5752851963043213,
+ "epoch": 0.5191919191919192,
+ "grad_norm": 2.026737689971924,
+ "learning_rate": 4.4828282828282834e-06,
+ "loss": 1.6041791439056396,
+ "mean_token_accuracy": 0.6275647282600403,
+ "num_tokens": 4210688.0,
+ "step": 257
+ },
+ {
+ "entropy": 1.548423171043396,
+ "epoch": 0.5212121212121212,
+ "grad_norm": 2.0578935146331787,
+ "learning_rate": 4.480808080808081e-06,
+ "loss": 1.563529372215271,
+ "mean_token_accuracy": 0.6348925232887268,
+ "num_tokens": 4227072.0,
+ "step": 258
+ },
+ {
+ "entropy": 1.5324457883834839,
+ "epoch": 0.5232323232323233,
+ "grad_norm": 2.216235637664795,
+ "learning_rate": 4.478787878787879e-06,
+ "loss": 1.5559678077697754,
+ "mean_token_accuracy": 0.6300073266029358,
+ "num_tokens": 4243456.0,
+ "step": 259
+ },
+ {
+ "entropy": 1.5275540351867676,
+ "epoch": 0.5252525252525253,
+ "grad_norm": 2.037306070327759,
+ "learning_rate": 4.476767676767677e-06,
+ "loss": 1.541567087173462,
+ "mean_token_accuracy": 0.6424035429954529,
+ "num_tokens": 4259840.0,
+ "step": 260
+ },
+ {
+ "entropy": 1.6842185258865356,
+ "epoch": 0.5272727272727272,
+ "grad_norm": 2.0241005420684814,
+ "learning_rate": 4.474747474747475e-06,
+ "loss": 1.6819056272506714,
+ "mean_token_accuracy": 0.6102222800254822,
+ "num_tokens": 4276224.0,
+ "step": 261
+ },
+ {
+ "entropy": 1.5656248331069946,
+ "epoch": 0.5292929292929293,
+ "grad_norm": 1.993054986000061,
+ "learning_rate": 4.472727272727273e-06,
+ "loss": 1.6132277250289917,
+ "mean_token_accuracy": 0.634709358215332,
+ "num_tokens": 4292608.0,
+ "step": 262
+ },
+ {
+ "entropy": 1.6354466676712036,
+ "epoch": 0.5313131313131313,
+ "grad_norm": 2.06508731842041,
+ "learning_rate": 4.470707070707071e-06,
+ "loss": 1.6475149393081665,
+ "mean_token_accuracy": 0.6121763586997986,
+ "num_tokens": 4308992.0,
+ "step": 263
+ },
+ {
+ "entropy": 1.5267232656478882,
+ "epoch": 0.5333333333333333,
+ "grad_norm": 2.1227569580078125,
+ "learning_rate": 4.468686868686869e-06,
+ "loss": 1.5468671321868896,
+ "mean_token_accuracy": 0.6365413069725037,
+ "num_tokens": 4325376.0,
+ "step": 264
+ },
+ {
+ "entropy": 1.2252761125564575,
+ "epoch": 0.5353535353535354,
+ "grad_norm": 2.0596134662628174,
+ "learning_rate": 4.4666666666666665e-06,
+ "loss": 1.2242389917373657,
+ "mean_token_accuracy": 0.6973620057106018,
+ "num_tokens": 4341760.0,
+ "step": 265
+ },
+ {
+ "entropy": 1.6007431745529175,
+ "epoch": 0.5373737373737374,
+ "grad_norm": 1.9341918230056763,
+ "learning_rate": 4.464646464646465e-06,
+ "loss": 1.5989094972610474,
+ "mean_token_accuracy": 0.6284196376800537,
+ "num_tokens": 4358144.0,
+ "step": 266
+ },
+ {
+ "entropy": 1.2708780765533447,
+ "epoch": 0.5393939393939394,
+ "grad_norm": 4.594091415405273,
+ "learning_rate": 4.462626262626263e-06,
+ "loss": 1.3484312295913696,
+ "mean_token_accuracy": 0.6790425181388855,
+ "num_tokens": 4374528.0,
+ "step": 267
+ },
+ {
+ "entropy": 1.5423723459243774,
+ "epoch": 0.5414141414141415,
+ "grad_norm": 1.933602213859558,
+ "learning_rate": 4.460606060606061e-06,
+ "loss": 1.594527244567871,
+ "mean_token_accuracy": 0.6386785507202148,
+ "num_tokens": 4390912.0,
+ "step": 268
+ },
+ {
+ "entropy": 1.7213952541351318,
+ "epoch": 0.5434343434343434,
+ "grad_norm": 2.195904016494751,
+ "learning_rate": 4.458585858585859e-06,
+ "loss": 1.724353551864624,
+ "mean_token_accuracy": 0.6008182764053345,
+ "num_tokens": 4407296.0,
+ "step": 269
+ },
+ {
+ "entropy": 1.4772557020187378,
+ "epoch": 0.5454545454545454,
+ "grad_norm": 2.0990796089172363,
+ "learning_rate": 4.456565656565657e-06,
+ "loss": 1.486825942993164,
+ "mean_token_accuracy": 0.6507083773612976,
+ "num_tokens": 4423680.0,
+ "step": 270
+ },
+ {
+ "entropy": 1.6876367330551147,
+ "epoch": 0.5474747474747474,
+ "grad_norm": 2.1944479942321777,
+ "learning_rate": 4.454545454545455e-06,
+ "loss": 1.71107816696167,
+ "mean_token_accuracy": 0.6027112603187561,
+ "num_tokens": 4440064.0,
+ "step": 271
+ },
+ {
+ "entropy": 1.5480726957321167,
+ "epoch": 0.5494949494949495,
+ "grad_norm": 2.1579341888427734,
+ "learning_rate": 4.452525252525253e-06,
+ "loss": 1.544647216796875,
+ "mean_token_accuracy": 0.6345261335372925,
+ "num_tokens": 4456448.0,
+ "step": 272
+ },
+ {
+ "entropy": 1.6161645650863647,
+ "epoch": 0.5515151515151515,
+ "grad_norm": 1.981154203414917,
+ "learning_rate": 4.4505050505050505e-06,
+ "loss": 1.6109068393707275,
+ "mean_token_accuracy": 0.6318392753601074,
+ "num_tokens": 4472832.0,
+ "step": 273
+ },
+ {
+ "entropy": 1.2576326131820679,
+ "epoch": 0.5535353535353535,
+ "grad_norm": 1.95668625831604,
+ "learning_rate": 4.448484848484848e-06,
+ "loss": 1.2614656686782837,
+ "mean_token_accuracy": 0.7025524973869324,
+ "num_tokens": 4489216.0,
+ "step": 274
+ },
+ {
+ "entropy": 1.405206561088562,
+ "epoch": 0.5555555555555556,
+ "grad_norm": 2.0302884578704834,
+ "learning_rate": 4.446464646464646e-06,
+ "loss": 1.416546106338501,
+ "mean_token_accuracy": 0.6753786206245422,
+ "num_tokens": 4505600.0,
+ "step": 275
+ },
+ {
+ "entropy": 1.9038625955581665,
+ "epoch": 0.5575757575757576,
+ "grad_norm": 2.06475567817688,
+ "learning_rate": 4.444444444444444e-06,
+ "loss": 1.9273614883422852,
+ "mean_token_accuracy": 0.5813996195793152,
+ "num_tokens": 4521984.0,
+ "step": 276
+ },
+ {
+ "entropy": 1.7208465337753296,
+ "epoch": 0.5595959595959596,
+ "grad_norm": 2.0136189460754395,
+ "learning_rate": 4.442424242424243e-06,
+ "loss": 1.7260158061981201,
+ "mean_token_accuracy": 0.6006350517272949,
+ "num_tokens": 4538368.0,
+ "step": 277
+ },
+ {
+ "entropy": 1.7570570707321167,
+ "epoch": 0.5616161616161616,
+ "grad_norm": 2.148594379425049,
+ "learning_rate": 4.440404040404041e-06,
+ "loss": 1.7799286842346191,
+ "mean_token_accuracy": 0.5943453907966614,
+ "num_tokens": 4554752.0,
+ "step": 278
+ },
+ {
+ "entropy": 1.6388157606124878,
+ "epoch": 0.5636363636363636,
+ "grad_norm": 2.1301987171173096,
+ "learning_rate": 4.438383838383839e-06,
+ "loss": 1.63419508934021,
+ "mean_token_accuracy": 0.6405715942382812,
+ "num_tokens": 4571136.0,
+ "step": 279
+ },
+ {
+ "entropy": 1.5902295112609863,
+ "epoch": 0.5656565656565656,
+ "grad_norm": 2.0526790618896484,
+ "learning_rate": 4.436363636363637e-06,
+ "loss": 1.6278276443481445,
+ "mean_token_accuracy": 0.615229606628418,
+ "num_tokens": 4587520.0,
+ "step": 280
+ },
+ {
+ "entropy": 1.5494027137756348,
+ "epoch": 0.5676767676767677,
+ "grad_norm": 1.9863859415054321,
+ "learning_rate": 4.434343434343435e-06,
+ "loss": 1.5622975826263428,
+ "mean_token_accuracy": 0.6261602640151978,
+ "num_tokens": 4603904.0,
+ "step": 281
+ },
+ {
+ "entropy": 1.2140685319900513,
+ "epoch": 0.5696969696969697,
+ "grad_norm": 2.4206902980804443,
+ "learning_rate": 4.432323232323233e-06,
+ "loss": 1.2389612197875977,
+ "mean_token_accuracy": 0.6943697929382324,
+ "num_tokens": 4620288.0,
+ "step": 282
+ },
+ {
+ "entropy": 1.4347270727157593,
+ "epoch": 0.5717171717171717,
+ "grad_norm": 2.0198237895965576,
+ "learning_rate": 4.430303030303031e-06,
+ "loss": 1.4648659229278564,
+ "mean_token_accuracy": 0.6573033928871155,
+ "num_tokens": 4636672.0,
+ "step": 283
+ },
+ {
+ "entropy": 1.3239331245422363,
+ "epoch": 0.5737373737373738,
+ "grad_norm": 1.9862704277038574,
+ "learning_rate": 4.428282828282829e-06,
+ "loss": 1.3520365953445435,
+ "mean_token_accuracy": 0.6852100491523743,
+ "num_tokens": 4653056.0,
+ "step": 284
+ },
+ {
+ "entropy": 1.7534631490707397,
+ "epoch": 0.5757575757575758,
+ "grad_norm": 2.0964176654815674,
+ "learning_rate": 4.426262626262627e-06,
+ "loss": 1.7659757137298584,
+ "mean_token_accuracy": 0.5975207686424255,
+ "num_tokens": 4669440.0,
+ "step": 285
+ },
+ {
+ "entropy": 1.1573433876037598,
+ "epoch": 0.5777777777777777,
+ "grad_norm": 2.0671567916870117,
+ "learning_rate": 4.424242424242425e-06,
+ "loss": 1.1479461193084717,
+ "mean_token_accuracy": 0.7143380641937256,
+ "num_tokens": 4685824.0,
+ "step": 286
+ },
+ {
+ "entropy": 1.2439504861831665,
+ "epoch": 0.5797979797979798,
+ "grad_norm": 2.0317327976226807,
+ "learning_rate": 4.422222222222223e-06,
+ "loss": 1.255782961845398,
+ "mean_token_accuracy": 0.691255509853363,
+ "num_tokens": 4702208.0,
+ "step": 287
+ },
+ {
+ "entropy": 1.5569473505020142,
+ "epoch": 0.5818181818181818,
+ "grad_norm": 2.0759670734405518,
+ "learning_rate": 4.4202020202020205e-06,
+ "loss": 1.5564974546432495,
+ "mean_token_accuracy": 0.6284807324409485,
+ "num_tokens": 4718592.0,
+ "step": 288
+ },
+ {
+ "entropy": 1.478676199913025,
+ "epoch": 0.5838383838383838,
+ "grad_norm": 1.9528205394744873,
+ "learning_rate": 4.418181818181818e-06,
+ "loss": 1.491654396057129,
+ "mean_token_accuracy": 0.6520518064498901,
+ "num_tokens": 4734976.0,
+ "step": 289
+ },
+ {
+ "entropy": 1.2454503774642944,
+ "epoch": 0.5858585858585859,
+ "grad_norm": 1.9008079767227173,
+ "learning_rate": 4.416161616161616e-06,
+ "loss": 1.253904938697815,
+ "mean_token_accuracy": 0.6971787810325623,
+ "num_tokens": 4751360.0,
+ "step": 290
+ },
+ {
+ "entropy": 1.1740810871124268,
+ "epoch": 0.5878787878787879,
+ "grad_norm": 1.8888787031173706,
+ "learning_rate": 4.414141414141415e-06,
+ "loss": 1.195070505142212,
+ "mean_token_accuracy": 0.7048119306564331,
+ "num_tokens": 4767744.0,
+ "step": 291
+ },
+ {
+ "entropy": 1.6064486503601074,
+ "epoch": 0.5898989898989899,
+ "grad_norm": 2.073559284210205,
+ "learning_rate": 4.412121212121213e-06,
+ "loss": 1.6180689334869385,
+ "mean_token_accuracy": 0.6242061257362366,
+ "num_tokens": 4784128.0,
+ "step": 292
+ },
+ {
+ "entropy": 1.6119383573532104,
+ "epoch": 0.591919191919192,
+ "grad_norm": 1.8773425817489624,
+ "learning_rate": 4.410101010101011e-06,
+ "loss": 1.5875662565231323,
+ "mean_token_accuracy": 0.647838294506073,
+ "num_tokens": 4800512.0,
+ "step": 293
+ },
+ {
+ "entropy": 1.392905592918396,
+ "epoch": 0.593939393939394,
+ "grad_norm": 2.1699368953704834,
+ "learning_rate": 4.408080808080809e-06,
+ "loss": 1.3949127197265625,
+ "mean_token_accuracy": 0.6538837552070618,
+ "num_tokens": 4816896.0,
+ "step": 294
+ },
+ {
+ "entropy": 1.5057002305984497,
+ "epoch": 0.5959595959595959,
+ "grad_norm": 2.2730712890625,
+ "learning_rate": 4.4060606060606066e-06,
+ "loss": 1.4755051136016846,
+ "mean_token_accuracy": 0.6498534679412842,
+ "num_tokens": 4833280.0,
+ "step": 295
+ },
+ {
+ "entropy": 1.5468902587890625,
+ "epoch": 0.597979797979798,
+ "grad_norm": 1.9911075830459595,
+ "learning_rate": 4.4040404040404044e-06,
+ "loss": 1.539963722229004,
+ "mean_token_accuracy": 0.6520518064498901,
+ "num_tokens": 4849664.0,
+ "step": 296
+ },
+ {
+ "entropy": 1.5196901559829712,
+ "epoch": 0.6,
+ "grad_norm": 1.9859540462493896,
+ "learning_rate": 4.402020202020202e-06,
+ "loss": 1.523442029953003,
+ "mean_token_accuracy": 0.6417317986488342,
+ "num_tokens": 4866048.0,
+ "step": 297
+ },
+ {
+ "entropy": 1.8251866102218628,
+ "epoch": 0.602020202020202,
+ "grad_norm": 2.2566516399383545,
+ "learning_rate": 4.4e-06,
+ "loss": 1.8667771816253662,
+ "mean_token_accuracy": 0.5872007608413696,
+ "num_tokens": 4882432.0,
+ "step": 298
+ },
+ {
+ "entropy": 1.6816744804382324,
+ "epoch": 0.604040404040404,
+ "grad_norm": 2.0107715129852295,
+ "learning_rate": 4.397979797979798e-06,
+ "loss": 1.6907548904418945,
+ "mean_token_accuracy": 0.6176722049713135,
+ "num_tokens": 4898816.0,
+ "step": 299
+ },
+ {
+ "entropy": 1.4992223978042603,
+ "epoch": 0.6060606060606061,
+ "grad_norm": 2.1236329078674316,
+ "learning_rate": 4.395959595959596e-06,
+ "loss": 1.5015790462493896,
+ "mean_token_accuracy": 0.6372129917144775,
+ "num_tokens": 4915200.0,
+ "step": 300
+ },
+ {
+ "epoch": 0.6060606060606061,
+ "eval_entropy": 1.5544315297757425,
+ "eval_loss": 1.5676765441894531,
+ "eval_mean_token_accuracy": 0.6389380799185845,
+ "eval_num_tokens": 4915200.0,
+ "eval_runtime": 43.7857,
+ "eval_samples_per_second": 22.61,
+ "eval_steps_per_second": 2.832,
+ "step": 300
+ },
+ {
+ "entropy": 1.518556833267212,
+ "epoch": 0.6080808080808081,
+ "grad_norm": 2.88371205329895,
+ "learning_rate": 4.393939393939394e-06,
+ "loss": 1.5310916900634766,
+ "mean_token_accuracy": 0.6551660895347595,
+ "num_tokens": 4931584.0,
+ "step": 301
+ },
+ {
+ "entropy": 1.4650386571884155,
+ "epoch": 0.6101010101010101,
+ "grad_norm": 2.1849780082702637,
+ "learning_rate": 4.391919191919193e-06,
+ "loss": 1.4405598640441895,
+ "mean_token_accuracy": 0.6531509757041931,
+ "num_tokens": 4947968.0,
+ "step": 302
+ },
+ {
+ "entropy": 1.5209505558013916,
+ "epoch": 0.6121212121212121,
+ "grad_norm": 2.1135010719299316,
+ "learning_rate": 4.3898989898989905e-06,
+ "loss": 1.5324647426605225,
+ "mean_token_accuracy": 0.6561431288719177,
+ "num_tokens": 4964352.0,
+ "step": 303
+ },
+ {
+ "entropy": 1.448391318321228,
+ "epoch": 0.6141414141414141,
+ "grad_norm": 1.9959306716918945,
+ "learning_rate": 4.387878787878788e-06,
+ "loss": 1.4721965789794922,
+ "mean_token_accuracy": 0.6578529477119446,
+ "num_tokens": 4980736.0,
+ "step": 304
+ },
+ {
+ "entropy": 1.4609358310699463,
+ "epoch": 0.6161616161616161,
+ "grad_norm": 2.1308915615081787,
+ "learning_rate": 4.385858585858586e-06,
+ "loss": 1.4707520008087158,
+ "mean_token_accuracy": 0.6482046842575073,
+ "num_tokens": 4997120.0,
+ "step": 305
+ },
+ {
+ "entropy": 1.7125155925750732,
+ "epoch": 0.6181818181818182,
+ "grad_norm": 2.2170841693878174,
+ "learning_rate": 4.383838383838384e-06,
+ "loss": 1.7630269527435303,
+ "mean_token_accuracy": 0.5999022722244263,
+ "num_tokens": 5013504.0,
+ "step": 306
+ },
+ {
+ "entropy": 1.6740268468856812,
+ "epoch": 0.6202020202020202,
+ "grad_norm": 1.983644723892212,
+ "learning_rate": 4.381818181818182e-06,
+ "loss": 1.6849563121795654,
+ "mean_token_accuracy": 0.6237176060676575,
+ "num_tokens": 5029888.0,
+ "step": 307
+ },
+ {
+ "entropy": 1.3754971027374268,
+ "epoch": 0.6222222222222222,
+ "grad_norm": 2.005277395248413,
+ "learning_rate": 4.37979797979798e-06,
+ "loss": 1.385213851928711,
+ "mean_token_accuracy": 0.6638984084129333,
+ "num_tokens": 5046272.0,
+ "step": 308
+ },
+ {
+ "entropy": 1.5476733446121216,
+ "epoch": 0.6242424242424243,
+ "grad_norm": 2.4317610263824463,
+ "learning_rate": 4.377777777777778e-06,
+ "loss": 1.5417041778564453,
+ "mean_token_accuracy": 0.6323888897895813,
+ "num_tokens": 5062656.0,
+ "step": 309
+ },
+ {
+ "entropy": 1.4437031745910645,
+ "epoch": 0.6262626262626263,
+ "grad_norm": 2.1464109420776367,
+ "learning_rate": 4.375757575757576e-06,
+ "loss": 1.4704627990722656,
+ "mean_token_accuracy": 0.6772105693817139,
+ "num_tokens": 5079040.0,
+ "step": 310
+ },
+ {
+ "entropy": 1.4859641790390015,
+ "epoch": 0.6282828282828283,
+ "grad_norm": 2.0800492763519287,
+ "learning_rate": 4.373737373737374e-06,
+ "loss": 1.5072834491729736,
+ "mean_token_accuracy": 0.658707857131958,
+ "num_tokens": 5095424.0,
+ "step": 311
+ },
+ {
+ "entropy": 1.8733419179916382,
+ "epoch": 0.6303030303030303,
+ "grad_norm": 2.120965003967285,
+ "learning_rate": 4.3717171717171715e-06,
+ "loss": 1.8567254543304443,
+ "mean_token_accuracy": 0.5847581624984741,
+ "num_tokens": 5111808.0,
+ "step": 312
+ },
+ {
+ "entropy": 1.4578243494033813,
+ "epoch": 0.6323232323232323,
+ "grad_norm": 1.9577823877334595,
+ "learning_rate": 4.36969696969697e-06,
+ "loss": 1.4758052825927734,
+ "mean_token_accuracy": 0.6590742468833923,
+ "num_tokens": 5128192.0,
+ "step": 313
+ },
+ {
+ "entropy": 1.8680084943771362,
+ "epoch": 0.6343434343434343,
+ "grad_norm": 2.170994520187378,
+ "learning_rate": 4.367676767676768e-06,
+ "loss": 1.904400110244751,
+ "mean_token_accuracy": 0.5857962965965271,
+ "num_tokens": 5144576.0,
+ "step": 314
+ },
+ {
+ "entropy": 1.6488304138183594,
+ "epoch": 0.6363636363636364,
+ "grad_norm": 1.959490418434143,
+ "learning_rate": 4.365656565656566e-06,
+ "loss": 1.6570477485656738,
+ "mean_token_accuracy": 0.6257327795028687,
+ "num_tokens": 5160960.0,
+ "step": 315
+ },
+ {
+ "entropy": 1.5038025379180908,
+ "epoch": 0.6383838383838384,
+ "grad_norm": 2.072697401046753,
+ "learning_rate": 4.363636363636364e-06,
+ "loss": 1.5234860181808472,
+ "mean_token_accuracy": 0.6494259834289551,
+ "num_tokens": 5177344.0,
+ "step": 316
+ },
+ {
+ "entropy": 1.4154654741287231,
+ "epoch": 0.6404040404040404,
+ "grad_norm": 1.894587755203247,
+ "learning_rate": 4.361616161616162e-06,
+ "loss": 1.4131592512130737,
+ "mean_token_accuracy": 0.6667073965072632,
+ "num_tokens": 5193728.0,
+ "step": 317
+ },
+ {
+ "entropy": 1.5813250541687012,
+ "epoch": 0.6424242424242425,
+ "grad_norm": 2.4088737964630127,
+ "learning_rate": 4.35959595959596e-06,
+ "loss": 1.6236622333526611,
+ "mean_token_accuracy": 0.6313507556915283,
+ "num_tokens": 5210112.0,
+ "step": 318
+ },
+ {
+ "entropy": 1.4140045642852783,
+ "epoch": 0.6444444444444445,
+ "grad_norm": 1.991557240486145,
+ "learning_rate": 4.3575757575757576e-06,
+ "loss": 1.4242517948150635,
+ "mean_token_accuracy": 0.6569980382919312,
+ "num_tokens": 5226496.0,
+ "step": 319
+ },
+ {
+ "entropy": 1.615628957748413,
+ "epoch": 0.6464646464646465,
+ "grad_norm": 1.8819077014923096,
+ "learning_rate": 4.3555555555555555e-06,
+ "loss": 1.6331532001495361,
+ "mean_token_accuracy": 0.6299462914466858,
+ "num_tokens": 5242880.0,
+ "step": 320
+ },
+ {
+ "entropy": 1.3118394613265991,
+ "epoch": 0.6484848484848484,
+ "grad_norm": 2.0589284896850586,
+ "learning_rate": 4.353535353535353e-06,
+ "loss": 1.2880034446716309,
+ "mean_token_accuracy": 0.6842941045761108,
+ "num_tokens": 5259264.0,
+ "step": 321
+ },
+ {
+ "entropy": 1.2786612510681152,
+ "epoch": 0.6505050505050505,
+ "grad_norm": 2.033839225769043,
+ "learning_rate": 4.351515151515152e-06,
+ "loss": 1.2829055786132812,
+ "mean_token_accuracy": 0.6910112500190735,
+ "num_tokens": 5275648.0,
+ "step": 322
+ },
+ {
+ "entropy": 1.5632988214492798,
+ "epoch": 0.6525252525252525,
+ "grad_norm": 2.1970419883728027,
+ "learning_rate": 4.34949494949495e-06,
+ "loss": 1.5865097045898438,
+ "mean_token_accuracy": 0.642892062664032,
+ "num_tokens": 5292032.0,
+ "step": 323
+ },
+ {
+ "entropy": 1.1542725563049316,
+ "epoch": 0.6545454545454545,
+ "grad_norm": 1.9750654697418213,
+ "learning_rate": 4.347474747474748e-06,
+ "loss": 1.1401035785675049,
+ "mean_token_accuracy": 0.7061553597450256,
+ "num_tokens": 5308416.0,
+ "step": 324
+ },
+ {
+ "entropy": 1.6879942417144775,
+ "epoch": 0.6565656565656566,
+ "grad_norm": 2.0022354125976562,
+ "learning_rate": 4.345454545454546e-06,
+ "loss": 1.70950448513031,
+ "mean_token_accuracy": 0.6089398860931396,
+ "num_tokens": 5324800.0,
+ "step": 325
+ },
+ {
+ "entropy": 1.7589699029922485,
+ "epoch": 0.6585858585858586,
+ "grad_norm": 1.925520420074463,
+ "learning_rate": 4.343434343434344e-06,
+ "loss": 1.7749860286712646,
+ "mean_token_accuracy": 0.6015510559082031,
+ "num_tokens": 5341184.0,
+ "step": 326
+ },
+ {
+ "entropy": 1.4720325469970703,
+ "epoch": 0.6606060606060606,
+ "grad_norm": 1.9487124681472778,
+ "learning_rate": 4.341414141414142e-06,
+ "loss": 1.4983797073364258,
+ "mean_token_accuracy": 0.6537005305290222,
+ "num_tokens": 5357568.0,
+ "step": 327
+ },
+ {
+ "entropy": 1.676164150238037,
+ "epoch": 0.6626262626262627,
+ "grad_norm": 2.13053035736084,
+ "learning_rate": 4.33939393939394e-06,
+ "loss": 1.699425458908081,
+ "mean_token_accuracy": 0.6100390553474426,
+ "num_tokens": 5373952.0,
+ "step": 328
+ },
+ {
+ "entropy": 1.3424437046051025,
+ "epoch": 0.6646464646464646,
+ "grad_norm": 2.0245954990386963,
+ "learning_rate": 4.337373737373738e-06,
+ "loss": 1.3393046855926514,
+ "mean_token_accuracy": 0.677760124206543,
+ "num_tokens": 5390336.0,
+ "step": 329
+ },
+ {
+ "entropy": 1.4926583766937256,
+ "epoch": 0.6666666666666666,
+ "grad_norm": 1.893343448638916,
+ "learning_rate": 4.335353535353536e-06,
+ "loss": 1.4779269695281982,
+ "mean_token_accuracy": 0.6713483333587646,
+ "num_tokens": 5406720.0,
+ "step": 330
+ },
+ {
+ "entropy": 1.5753449201583862,
+ "epoch": 0.6686868686868687,
+ "grad_norm": 2.051647424697876,
+ "learning_rate": 4.333333333333334e-06,
+ "loss": 1.622597336769104,
+ "mean_token_accuracy": 0.6436248421669006,
+ "num_tokens": 5423104.0,
+ "step": 331
+ },
+ {
+ "entropy": 1.4573988914489746,
+ "epoch": 0.6707070707070707,
+ "grad_norm": 1.8709567785263062,
+ "learning_rate": 4.331313131313132e-06,
+ "loss": 1.4902422428131104,
+ "mean_token_accuracy": 0.6682950854301453,
+ "num_tokens": 5439488.0,
+ "step": 332
+ },
+ {
+ "entropy": 1.378867745399475,
+ "epoch": 0.6727272727272727,
+ "grad_norm": 2.235928773880005,
+ "learning_rate": 4.32929292929293e-06,
+ "loss": 1.4067103862762451,
+ "mean_token_accuracy": 0.662432849407196,
+ "num_tokens": 5455872.0,
+ "step": 333
+ },
+ {
+ "entropy": 1.7553350925445557,
+ "epoch": 0.6747474747474748,
+ "grad_norm": 2.0335066318511963,
+ "learning_rate": 4.327272727272728e-06,
+ "loss": 1.772943139076233,
+ "mean_token_accuracy": 0.5953834652900696,
+ "num_tokens": 5472256.0,
+ "step": 334
+ },
+ {
+ "entropy": 1.1587097644805908,
+ "epoch": 0.6767676767676768,
+ "grad_norm": 1.8764615058898926,
+ "learning_rate": 4.3252525252525255e-06,
+ "loss": 1.1396210193634033,
+ "mean_token_accuracy": 0.7197117805480957,
+ "num_tokens": 5488640.0,
+ "step": 335
+ },
+ {
+ "entropy": 1.4819872379302979,
+ "epoch": 0.6787878787878788,
+ "grad_norm": 2.0907511711120605,
+ "learning_rate": 4.323232323232323e-06,
+ "loss": 1.460550308227539,
+ "mean_token_accuracy": 0.6520518064498901,
+ "num_tokens": 5505024.0,
+ "step": 336
+ },
+ {
+ "entropy": 1.0744472742080688,
+ "epoch": 0.6808080808080809,
+ "grad_norm": 2.227606773376465,
+ "learning_rate": 4.321212121212121e-06,
+ "loss": 1.0909301042556763,
+ "mean_token_accuracy": 0.7258182764053345,
+ "num_tokens": 5521408.0,
+ "step": 337
+ },
+ {
+ "entropy": 1.507999300956726,
+ "epoch": 0.6828282828282828,
+ "grad_norm": 2.0332415103912354,
+ "learning_rate": 4.31919191919192e-06,
+ "loss": 1.5173046588897705,
+ "mean_token_accuracy": 0.6374572515487671,
+ "num_tokens": 5537792.0,
+ "step": 338
+ },
+ {
+ "entropy": 1.6792720556259155,
+ "epoch": 0.6848484848484848,
+ "grad_norm": 2.0194997787475586,
+ "learning_rate": 4.317171717171718e-06,
+ "loss": 1.679445743560791,
+ "mean_token_accuracy": 0.6138250827789307,
+ "num_tokens": 5554176.0,
+ "step": 339
+ },
+ {
+ "entropy": 1.6470589637756348,
+ "epoch": 0.6868686868686869,
+ "grad_norm": 2.09116268157959,
+ "learning_rate": 4.315151515151516e-06,
+ "loss": 1.6386632919311523,
+ "mean_token_accuracy": 0.6146799921989441,
+ "num_tokens": 5570560.0,
+ "step": 340
+ },
+ {
+ "entropy": 1.3612488508224487,
+ "epoch": 0.6888888888888889,
+ "grad_norm": 2.1586217880249023,
+ "learning_rate": 4.313131313131314e-06,
+ "loss": 1.3698725700378418,
+ "mean_token_accuracy": 0.6696995496749878,
+ "num_tokens": 5586944.0,
+ "step": 341
+ },
+ {
+ "entropy": 1.400327205657959,
+ "epoch": 0.6909090909090909,
+ "grad_norm": 2.082094192504883,
+ "learning_rate": 4.3111111111111115e-06,
+ "loss": 1.396535873413086,
+ "mean_token_accuracy": 0.6780654788017273,
+ "num_tokens": 5603328.0,
+ "step": 342
+ },
+ {
+ "entropy": 1.389290690422058,
+ "epoch": 0.692929292929293,
+ "grad_norm": 2.0780303478240967,
+ "learning_rate": 4.309090909090909e-06,
+ "loss": 1.4275782108306885,
+ "mean_token_accuracy": 0.6656082272529602,
+ "num_tokens": 5619712.0,
+ "step": 343
+ },
+ {
+ "entropy": 1.5133870840072632,
+ "epoch": 0.694949494949495,
+ "grad_norm": 1.9819531440734863,
+ "learning_rate": 4.307070707070707e-06,
+ "loss": 1.5309596061706543,
+ "mean_token_accuracy": 0.6373351216316223,
+ "num_tokens": 5636096.0,
+ "step": 344
+ },
+ {
+ "entropy": 1.483219861984253,
+ "epoch": 0.696969696969697,
+ "grad_norm": 1.9794325828552246,
+ "learning_rate": 4.305050505050505e-06,
+ "loss": 1.4693087339401245,
+ "mean_token_accuracy": 0.6550439596176147,
+ "num_tokens": 5652480.0,
+ "step": 345
+ },
+ {
+ "entropy": 1.381489634513855,
+ "epoch": 0.6989898989898989,
+ "grad_norm": 2.0637686252593994,
+ "learning_rate": 4.303030303030303e-06,
+ "loss": 1.3862476348876953,
+ "mean_token_accuracy": 0.6631045341491699,
+ "num_tokens": 5668864.0,
+ "step": 346
+ },
+ {
+ "entropy": 1.9860024452209473,
+ "epoch": 0.701010101010101,
+ "grad_norm": 2.1626803874969482,
+ "learning_rate": 4.301010101010101e-06,
+ "loss": 1.9815950393676758,
+ "mean_token_accuracy": 0.5698583126068115,
+ "num_tokens": 5685248.0,
+ "step": 347
+ },
+ {
+ "entropy": 1.5044004917144775,
+ "epoch": 0.703030303030303,
+ "grad_norm": 2.060976266860962,
+ "learning_rate": 4.298989898989899e-06,
+ "loss": 1.4937127828598022,
+ "mean_token_accuracy": 0.6524792313575745,
+ "num_tokens": 5701632.0,
+ "step": 348
+ },
+ {
+ "entropy": 1.3397772312164307,
+ "epoch": 0.705050505050505,
+ "grad_norm": 2.0162901878356934,
+ "learning_rate": 4.296969696969698e-06,
+ "loss": 1.3358572721481323,
+ "mean_token_accuracy": 0.6949804425239563,
+ "num_tokens": 5718016.0,
+ "step": 349
+ },
+ {
+ "entropy": 1.145772099494934,
+ "epoch": 0.7070707070707071,
+ "grad_norm": 2.097634792327881,
+ "learning_rate": 4.2949494949494955e-06,
+ "loss": 1.1833416223526,
+ "mean_token_accuracy": 0.7104909420013428,
+ "num_tokens": 5734400.0,
+ "step": 350
+ },
+ {
+ "epoch": 0.7070707070707071,
+ "eval_entropy": 1.5476290602837839,
+ "eval_loss": 1.5603480339050293,
+ "eval_mean_token_accuracy": 0.640111118555069,
+ "eval_num_tokens": 5734400.0,
+ "eval_runtime": 43.7844,
+ "eval_samples_per_second": 22.611,
+ "eval_steps_per_second": 2.832,
+ "step": 350
+ },
+ {
+ "entropy": 1.4311926364898682,
+ "epoch": 0.7090909090909091,
+ "grad_norm": 2.002321720123291,
+ "learning_rate": 4.292929292929293e-06,
+ "loss": 1.4534825086593628,
+ "mean_token_accuracy": 0.6614558100700378,
+ "num_tokens": 5750784.0,
+ "step": 351
+ },
+ {
+ "entropy": 1.3983922004699707,
+ "epoch": 0.7111111111111111,
+ "grad_norm": 2.1724867820739746,
+ "learning_rate": 4.290909090909091e-06,
+ "loss": 1.3969402313232422,
+ "mean_token_accuracy": 0.6652418375015259,
+ "num_tokens": 5767168.0,
+ "step": 352
+ },
+ {
+ "entropy": 1.8218920230865479,
+ "epoch": 0.7131313131313132,
+ "grad_norm": 2.1629281044006348,
+ "learning_rate": 4.288888888888889e-06,
+ "loss": 1.8196980953216553,
+ "mean_token_accuracy": 0.6027112603187561,
+ "num_tokens": 5783552.0,
+ "step": 353
+ },
+ {
+ "entropy": 1.5322320461273193,
+ "epoch": 0.7151515151515152,
+ "grad_norm": 1.8486647605895996,
+ "learning_rate": 4.286868686868687e-06,
+ "loss": 1.5504610538482666,
+ "mean_token_accuracy": 0.6533341407775879,
+ "num_tokens": 5799936.0,
+ "step": 354
+ },
+ {
+ "entropy": 1.935966968536377,
+ "epoch": 0.7171717171717171,
+ "grad_norm": 2.252814292907715,
+ "learning_rate": 4.284848484848485e-06,
+ "loss": 1.9300384521484375,
+ "mean_token_accuracy": 0.5656448602676392,
+ "num_tokens": 5816320.0,
+ "step": 355
+ },
+ {
+ "entropy": 1.6711398363113403,
+ "epoch": 0.7191919191919192,
+ "grad_norm": 2.023379325866699,
+ "learning_rate": 4.282828282828283e-06,
+ "loss": 1.6584455966949463,
+ "mean_token_accuracy": 0.6235954761505127,
+ "num_tokens": 5832704.0,
+ "step": 356
+ },
+ {
+ "entropy": 1.3922803401947021,
+ "epoch": 0.7212121212121212,
+ "grad_norm": 2.0487611293792725,
+ "learning_rate": 4.280808080808081e-06,
+ "loss": 1.4220250844955444,
+ "mean_token_accuracy": 0.6591963768005371,
+ "num_tokens": 5849088.0,
+ "step": 357
+ },
+ {
+ "entropy": 1.59521484375,
+ "epoch": 0.7232323232323232,
+ "grad_norm": 1.8598614931106567,
+ "learning_rate": 4.278787878787879e-06,
+ "loss": 1.5979329347610474,
+ "mean_token_accuracy": 0.6237176060676575,
+ "num_tokens": 5865472.0,
+ "step": 358
+ },
+ {
+ "entropy": 1.5810115337371826,
+ "epoch": 0.7252525252525253,
+ "grad_norm": 2.140115737915039,
+ "learning_rate": 4.276767676767677e-06,
+ "loss": 1.5774705410003662,
+ "mean_token_accuracy": 0.6274425983428955,
+ "num_tokens": 5881856.0,
+ "step": 359
+ },
+ {
+ "entropy": 1.5839109420776367,
+ "epoch": 0.7272727272727273,
+ "grad_norm": 2.0947749614715576,
+ "learning_rate": 4.274747474747475e-06,
+ "loss": 1.601511001586914,
+ "mean_token_accuracy": 0.630984365940094,
+ "num_tokens": 5898240.0,
+ "step": 360
+ },
+ {
+ "entropy": 1.195770263671875,
+ "epoch": 0.7292929292929293,
+ "grad_norm": 1.8740363121032715,
+ "learning_rate": 4.272727272727273e-06,
+ "loss": 1.195652961730957,
+ "mean_token_accuracy": 0.7077430486679077,
+ "num_tokens": 5914624.0,
+ "step": 361
+ },
+ {
+ "entropy": 1.375698208808899,
+ "epoch": 0.7313131313131314,
+ "grad_norm": 1.9580703973770142,
+ "learning_rate": 4.270707070707071e-06,
+ "loss": 1.3746864795684814,
+ "mean_token_accuracy": 0.6775158643722534,
+ "num_tokens": 5931008.0,
+ "step": 362
+ },
+ {
+ "entropy": 1.7259451150894165,
+ "epoch": 0.7333333333333333,
+ "grad_norm": 2.3127365112304688,
+ "learning_rate": 4.268686868686869e-06,
+ "loss": 1.745998501777649,
+ "mean_token_accuracy": 0.6040546894073486,
+ "num_tokens": 5947392.0,
+ "step": 363
+ },
+ {
+ "entropy": 1.209228754043579,
+ "epoch": 0.7353535353535353,
+ "grad_norm": 1.9004793167114258,
+ "learning_rate": 4.266666666666668e-06,
+ "loss": 1.232427716255188,
+ "mean_token_accuracy": 0.6925989389419556,
+ "num_tokens": 5963776.0,
+ "step": 364
+ },
+ {
+ "entropy": 1.2811263799667358,
+ "epoch": 0.7373737373737373,
+ "grad_norm": 1.9568246603012085,
+ "learning_rate": 4.2646464646464655e-06,
+ "loss": 1.291853427886963,
+ "mean_token_accuracy": 0.6884465217590332,
+ "num_tokens": 5980160.0,
+ "step": 365
+ },
+ {
+ "entropy": 1.843044638633728,
+ "epoch": 0.7393939393939394,
+ "grad_norm": 2.132735252380371,
+ "learning_rate": 4.262626262626263e-06,
+ "loss": 1.8818857669830322,
+ "mean_token_accuracy": 0.5785295367240906,
+ "num_tokens": 5996544.0,
+ "step": 366
+ },
+ {
+ "entropy": 1.3353440761566162,
+ "epoch": 0.7414141414141414,
+ "grad_norm": 1.8893013000488281,
+ "learning_rate": 4.260606060606061e-06,
+ "loss": 1.354011058807373,
+ "mean_token_accuracy": 0.6797752976417542,
+ "num_tokens": 6012928.0,
+ "step": 367
+ },
+ {
+ "entropy": 1.3889728784561157,
+ "epoch": 0.7434343434343434,
+ "grad_norm": 1.980757236480713,
+ "learning_rate": 4.258585858585859e-06,
+ "loss": 1.3671875,
+ "mean_token_accuracy": 0.6627381443977356,
+ "num_tokens": 6029312.0,
+ "step": 368
+ },
+ {
+ "entropy": 1.4605348110198975,
+ "epoch": 0.7454545454545455,
+ "grad_norm": 2.1033780574798584,
+ "learning_rate": 4.256565656565657e-06,
+ "loss": 1.455024003982544,
+ "mean_token_accuracy": 0.6451514363288879,
+ "num_tokens": 6045696.0,
+ "step": 369
+ },
+ {
+ "entropy": 1.4382058382034302,
+ "epoch": 0.7474747474747475,
+ "grad_norm": 2.1068074703216553,
+ "learning_rate": 4.254545454545455e-06,
+ "loss": 1.4422768354415894,
+ "mean_token_accuracy": 0.6630434989929199,
+ "num_tokens": 6062080.0,
+ "step": 370
+ },
+ {
+ "entropy": 1.8046759366989136,
+ "epoch": 0.7494949494949495,
+ "grad_norm": 2.214466094970703,
+ "learning_rate": 4.252525252525253e-06,
+ "loss": 1.8247106075286865,
+ "mean_token_accuracy": 0.6003297567367554,
+ "num_tokens": 6078464.0,
+ "step": 371
+ },
+ {
+ "entropy": 1.671690583229065,
+ "epoch": 0.7515151515151515,
+ "grad_norm": 1.9790785312652588,
+ "learning_rate": 4.250505050505051e-06,
+ "loss": 1.6907753944396973,
+ "mean_token_accuracy": 0.6232290863990784,
+ "num_tokens": 6094848.0,
+ "step": 372
+ },
+ {
+ "entropy": 1.6881897449493408,
+ "epoch": 0.7535353535353535,
+ "grad_norm": 1.8638463020324707,
+ "learning_rate": 4.248484848484849e-06,
+ "loss": 1.7132716178894043,
+ "mean_token_accuracy": 0.6357474327087402,
+ "num_tokens": 6111232.0,
+ "step": 373
+ },
+ {
+ "entropy": 1.2555122375488281,
+ "epoch": 0.7555555555555555,
+ "grad_norm": 2.082378387451172,
+ "learning_rate": 4.246464646464647e-06,
+ "loss": 1.240688681602478,
+ "mean_token_accuracy": 0.6965070962905884,
+ "num_tokens": 6127616.0,
+ "step": 374
+ },
+ {
+ "entropy": 1.847135305404663,
+ "epoch": 0.7575757575757576,
+ "grad_norm": 2.142962694168091,
+ "learning_rate": 4.244444444444445e-06,
+ "loss": 1.8840911388397217,
+ "mean_token_accuracy": 0.5822545289993286,
+ "num_tokens": 6144000.0,
+ "step": 375
+ },
+ {
+ "entropy": 1.239328384399414,
+ "epoch": 0.7595959595959596,
+ "grad_norm": 2.0589468479156494,
+ "learning_rate": 4.242424242424243e-06,
+ "loss": 1.256324052810669,
+ "mean_token_accuracy": 0.6845383644104004,
+ "num_tokens": 6160384.0,
+ "step": 376
+ },
+ {
+ "entropy": 1.5047202110290527,
+ "epoch": 0.7616161616161616,
+ "grad_norm": 1.7951308488845825,
+ "learning_rate": 4.240404040404041e-06,
+ "loss": 1.5063304901123047,
+ "mean_token_accuracy": 0.6583414673805237,
+ "num_tokens": 6176768.0,
+ "step": 377
+ },
+ {
+ "entropy": 1.4072566032409668,
+ "epoch": 0.7636363636363637,
+ "grad_norm": 2.1670594215393066,
+ "learning_rate": 4.238383838383839e-06,
+ "loss": 1.4205389022827148,
+ "mean_token_accuracy": 0.658707857131958,
+ "num_tokens": 6193152.0,
+ "step": 378
+ },
+ {
+ "entropy": 1.3207885026931763,
+ "epoch": 0.7656565656565657,
+ "grad_norm": 1.9017083644866943,
+ "learning_rate": 4.236363636363637e-06,
+ "loss": 1.3169896602630615,
+ "mean_token_accuracy": 0.6954079270362854,
+ "num_tokens": 6209536.0,
+ "step": 379
+ },
+ {
+ "entropy": 1.4762436151504517,
+ "epoch": 0.7676767676767676,
+ "grad_norm": 2.0023069381713867,
+ "learning_rate": 4.234343434343435e-06,
+ "loss": 1.4725041389465332,
+ "mean_token_accuracy": 0.6620053648948669,
+ "num_tokens": 6225920.0,
+ "step": 380
+ },
+ {
+ "entropy": 1.4895304441452026,
+ "epoch": 0.7696969696969697,
+ "grad_norm": 1.9528017044067383,
+ "learning_rate": 4.2323232323232325e-06,
+ "loss": 1.49650239944458,
+ "mean_token_accuracy": 0.6508915424346924,
+ "num_tokens": 6242304.0,
+ "step": 381
+ },
+ {
+ "entropy": 1.3024516105651855,
+ "epoch": 0.7717171717171717,
+ "grad_norm": 1.9855588674545288,
+ "learning_rate": 4.2303030303030304e-06,
+ "loss": 1.34218168258667,
+ "mean_token_accuracy": 0.6838055849075317,
+ "num_tokens": 6258688.0,
+ "step": 382
+ },
+ {
+ "entropy": 1.6005626916885376,
+ "epoch": 0.7737373737373737,
+ "grad_norm": 2.028286933898926,
+ "learning_rate": 4.228282828282828e-06,
+ "loss": 1.6299283504486084,
+ "mean_token_accuracy": 0.6315950155258179,
+ "num_tokens": 6275072.0,
+ "step": 383
+ },
+ {
+ "entropy": 1.7050484418869019,
+ "epoch": 0.7757575757575758,
+ "grad_norm": 2.474047899246216,
+ "learning_rate": 4.226262626262626e-06,
+ "loss": 1.777151346206665,
+ "mean_token_accuracy": 0.5919027924537659,
+ "num_tokens": 6291456.0,
+ "step": 384
+ },
+ {
+ "entropy": 1.5014543533325195,
+ "epoch": 0.7777777777777778,
+ "grad_norm": 1.9866594076156616,
+ "learning_rate": 4.224242424242425e-06,
+ "loss": 1.5308949947357178,
+ "mean_token_accuracy": 0.639106035232544,
+ "num_tokens": 6307840.0,
+ "step": 385
+ },
+ {
+ "entropy": 1.6957359313964844,
+ "epoch": 0.7797979797979798,
+ "grad_norm": 2.229820966720581,
+ "learning_rate": 4.222222222222223e-06,
+ "loss": 1.728761911392212,
+ "mean_token_accuracy": 0.6242061257362366,
+ "num_tokens": 6324224.0,
+ "step": 386
+ },
+ {
+ "entropy": 1.6823521852493286,
+ "epoch": 0.7818181818181819,
+ "grad_norm": 2.033383369445801,
+ "learning_rate": 4.220202020202021e-06,
+ "loss": 1.7310154438018799,
+ "mean_token_accuracy": 0.6257938742637634,
+ "num_tokens": 6340608.0,
+ "step": 387
+ },
+ {
+ "entropy": 1.710307240486145,
+ "epoch": 0.7838383838383839,
+ "grad_norm": 2.061861038208008,
+ "learning_rate": 4.218181818181819e-06,
+ "loss": 1.7066943645477295,
+ "mean_token_accuracy": 0.6123595237731934,
+ "num_tokens": 6356992.0,
+ "step": 388
+ },
+ {
+ "entropy": 1.241638422012329,
+ "epoch": 0.7858585858585858,
+ "grad_norm": 1.9743852615356445,
+ "learning_rate": 4.2161616161616165e-06,
+ "loss": 1.2520272731781006,
+ "mean_token_accuracy": 0.692415714263916,
+ "num_tokens": 6373376.0,
+ "step": 389
+ },
+ {
+ "entropy": 1.5918776988983154,
+ "epoch": 0.7878787878787878,
+ "grad_norm": 2.156675100326538,
+ "learning_rate": 4.214141414141414e-06,
+ "loss": 1.6060255765914917,
+ "mean_token_accuracy": 0.6178553700447083,
+ "num_tokens": 6389760.0,
+ "step": 390
+ },
+ {
+ "entropy": 1.7307862043380737,
+ "epoch": 0.7898989898989899,
+ "grad_norm": 2.171247959136963,
+ "learning_rate": 4.212121212121212e-06,
+ "loss": 1.7327581644058228,
+ "mean_token_accuracy": 0.6502808928489685,
+ "num_tokens": 6406144.0,
+ "step": 391
+ },
+ {
+ "entropy": 1.6725553274154663,
+ "epoch": 0.7919191919191919,
+ "grad_norm": 2.020228624343872,
+ "learning_rate": 4.21010101010101e-06,
+ "loss": 1.6803646087646484,
+ "mean_token_accuracy": 0.6383732557296753,
+ "num_tokens": 6422528.0,
+ "step": 392
+ },
+ {
+ "entropy": 1.4120928049087524,
+ "epoch": 0.793939393939394,
+ "grad_norm": 1.9896639585494995,
+ "learning_rate": 4.208080808080808e-06,
+ "loss": 1.3974279165267944,
+ "mean_token_accuracy": 0.6661577820777893,
+ "num_tokens": 6438912.0,
+ "step": 393
+ },
+ {
+ "entropy": 0.933545708656311,
+ "epoch": 0.795959595959596,
+ "grad_norm": 1.6884223222732544,
+ "learning_rate": 4.206060606060606e-06,
+ "loss": 0.9287986755371094,
+ "mean_token_accuracy": 0.7636175155639648,
+ "num_tokens": 6455296.0,
+ "step": 394
+ },
+ {
+ "entropy": 1.419004201889038,
+ "epoch": 0.797979797979798,
+ "grad_norm": 2.2590551376342773,
+ "learning_rate": 4.204040404040405e-06,
+ "loss": 1.4652538299560547,
+ "mean_token_accuracy": 0.6589521169662476,
+ "num_tokens": 6471680.0,
+ "step": 395
+ },
+ {
+ "entropy": 1.750221610069275,
+ "epoch": 0.8,
+ "grad_norm": 2.195030450820923,
+ "learning_rate": 4.2020202020202026e-06,
+ "loss": 1.7328863143920898,
+ "mean_token_accuracy": 0.6008182764053345,
+ "num_tokens": 6488064.0,
+ "step": 396
+ },
+ {
+ "entropy": 1.7440015077590942,
+ "epoch": 0.802020202020202,
+ "grad_norm": 1.9833927154541016,
+ "learning_rate": 4.2000000000000004e-06,
+ "loss": 1.7617835998535156,
+ "mean_token_accuracy": 0.5992916226387024,
+ "num_tokens": 6504448.0,
+ "step": 397
+ },
+ {
+ "entropy": 1.1636452674865723,
+ "epoch": 0.804040404040404,
+ "grad_norm": 1.9506802558898926,
+ "learning_rate": 4.197979797979798e-06,
+ "loss": 1.1344032287597656,
+ "mean_token_accuracy": 0.7122618556022644,
+ "num_tokens": 6520832.0,
+ "step": 398
+ },
+ {
+ "entropy": 1.6173542737960815,
+ "epoch": 0.806060606060606,
+ "grad_norm": 2.0997231006622314,
+ "learning_rate": 4.195959595959596e-06,
+ "loss": 1.6303956508636475,
+ "mean_token_accuracy": 0.6215192675590515,
+ "num_tokens": 6537216.0,
+ "step": 399
+ },
+ {
+ "entropy": 1.4391542673110962,
+ "epoch": 0.8080808080808081,
+ "grad_norm": 2.356828212738037,
+ "learning_rate": 4.193939393939394e-06,
+ "loss": 1.4465923309326172,
+ "mean_token_accuracy": 0.6594406366348267,
+ "num_tokens": 6553600.0,
+ "step": 400
+ },
+ {
+ "epoch": 0.8080808080808081,
+ "eval_entropy": 1.5504949785047961,
+ "eval_loss": 1.5544542074203491,
+ "eval_mean_token_accuracy": 0.6409837569921247,
+ "eval_num_tokens": 6553600.0,
+ "eval_runtime": 43.7986,
+ "eval_samples_per_second": 22.603,
+ "eval_steps_per_second": 2.831,
+ "step": 400
+ },
+ {
+ "entropy": 1.60548996925354,
+ "epoch": 0.8101010101010101,
+ "grad_norm": 2.1894404888153076,
+ "learning_rate": 4.191919191919192e-06,
+ "loss": 1.6116085052490234,
+ "mean_token_accuracy": 0.6276868581771851,
+ "num_tokens": 6569984.0,
+ "step": 401
+ },
+ {
+ "entropy": 1.436041235923767,
+ "epoch": 0.8121212121212121,
+ "grad_norm": 2.1180264949798584,
+ "learning_rate": 4.18989898989899e-06,
+ "loss": 1.4203698635101318,
+ "mean_token_accuracy": 0.6554714441299438,
+ "num_tokens": 6586368.0,
+ "step": 402
+ },
+ {
+ "entropy": 1.1348106861114502,
+ "epoch": 0.8141414141414142,
+ "grad_norm": 2.0420851707458496,
+ "learning_rate": 4.187878787878788e-06,
+ "loss": 1.1424527168273926,
+ "mean_token_accuracy": 0.7154372334480286,
+ "num_tokens": 6602752.0,
+ "step": 403
+ },
+ {
+ "entropy": 1.4039727449417114,
+ "epoch": 0.8161616161616162,
+ "grad_norm": 2.148340940475464,
+ "learning_rate": 4.185858585858586e-06,
+ "loss": 1.4246132373809814,
+ "mean_token_accuracy": 0.6723864078521729,
+ "num_tokens": 6619136.0,
+ "step": 404
+ },
+ {
+ "entropy": 1.4222626686096191,
+ "epoch": 0.8181818181818182,
+ "grad_norm": 1.9436827898025513,
+ "learning_rate": 4.1838383838383835e-06,
+ "loss": 1.4235990047454834,
+ "mean_token_accuracy": 0.6759281754493713,
+ "num_tokens": 6635520.0,
+ "step": 405
+ },
+ {
+ "entropy": 1.4771310091018677,
+ "epoch": 0.8202020202020202,
+ "grad_norm": 2.0953726768493652,
+ "learning_rate": 4.181818181818182e-06,
+ "loss": 1.501934289932251,
+ "mean_token_accuracy": 0.6535173654556274,
+ "num_tokens": 6651904.0,
+ "step": 406
+ },
+ {
+ "entropy": 1.526256799697876,
+ "epoch": 0.8222222222222222,
+ "grad_norm": 2.245994806289673,
+ "learning_rate": 4.17979797979798e-06,
+ "loss": 1.539383888244629,
+ "mean_token_accuracy": 0.6417317986488342,
+ "num_tokens": 6668288.0,
+ "step": 407
+ },
+ {
+ "entropy": 1.1716097593307495,
+ "epoch": 0.8242424242424242,
+ "grad_norm": 1.8283652067184448,
+ "learning_rate": 4.177777777777778e-06,
+ "loss": 1.1798359155654907,
+ "mean_token_accuracy": 0.7123839855194092,
+ "num_tokens": 6684672.0,
+ "step": 408
+ },
+ {
+ "entropy": 1.4581540822982788,
+ "epoch": 0.8262626262626263,
+ "grad_norm": 1.9325168132781982,
+ "learning_rate": 4.175757575757576e-06,
+ "loss": 1.4526585340499878,
+ "mean_token_accuracy": 0.662432849407196,
+ "num_tokens": 6701056.0,
+ "step": 409
+ },
+ {
+ "entropy": 1.4073102474212646,
+ "epoch": 0.8282828282828283,
+ "grad_norm": 2.1543467044830322,
+ "learning_rate": 4.173737373737375e-06,
+ "loss": 1.437713861465454,
+ "mean_token_accuracy": 0.6725696325302124,
+ "num_tokens": 6717440.0,
+ "step": 410
+ },
+ {
+ "entropy": 1.5970062017440796,
+ "epoch": 0.8303030303030303,
+ "grad_norm": 2.1714792251586914,
+ "learning_rate": 4.1717171717171726e-06,
+ "loss": 1.613295078277588,
+ "mean_token_accuracy": 0.6229848265647888,
+ "num_tokens": 6733824.0,
+ "step": 411
+ },
+ {
+ "entropy": 1.449837327003479,
+ "epoch": 0.8323232323232324,
+ "grad_norm": 2.2499871253967285,
+ "learning_rate": 4.1696969696969705e-06,
+ "loss": 1.489758014678955,
+ "mean_token_accuracy": 0.645639955997467,
+ "num_tokens": 6750208.0,
+ "step": 412
+ },
+ {
+ "entropy": 1.7696173191070557,
+ "epoch": 0.8343434343434344,
+ "grad_norm": 2.3637073040008545,
+ "learning_rate": 4.167676767676768e-06,
+ "loss": 1.8147599697113037,
+ "mean_token_accuracy": 0.5821323990821838,
+ "num_tokens": 6766592.0,
+ "step": 413
+ },
+ {
+ "entropy": 1.99376380443573,
+ "epoch": 0.8363636363636363,
+ "grad_norm": 2.265683174133301,
+ "learning_rate": 4.165656565656566e-06,
+ "loss": 2.009024143218994,
+ "mean_token_accuracy": 0.5591108798980713,
+ "num_tokens": 6782976.0,
+ "step": 414
+ },
+ {
+ "entropy": 1.7276980876922607,
+ "epoch": 0.8383838383838383,
+ "grad_norm": 1.9407477378845215,
+ "learning_rate": 4.163636363636364e-06,
+ "loss": 1.7130229473114014,
+ "mean_token_accuracy": 0.6202979683876038,
+ "num_tokens": 6799360.0,
+ "step": 415
+ },
+ {
+ "entropy": 1.5418223142623901,
+ "epoch": 0.8404040404040404,
+ "grad_norm": 1.9765743017196655,
+ "learning_rate": 4.161616161616162e-06,
+ "loss": 1.5627632141113281,
+ "mean_token_accuracy": 0.6433194875717163,
+ "num_tokens": 6815744.0,
+ "step": 416
+ },
+ {
+ "entropy": 1.6040345430374146,
+ "epoch": 0.8424242424242424,
+ "grad_norm": 2.1942877769470215,
+ "learning_rate": 4.15959595959596e-06,
+ "loss": 1.6044622659683228,
+ "mean_token_accuracy": 0.6278700828552246,
+ "num_tokens": 6832128.0,
+ "step": 417
+ },
+ {
+ "entropy": 1.8461577892303467,
+ "epoch": 0.8444444444444444,
+ "grad_norm": 2.289196729660034,
+ "learning_rate": 4.157575757575758e-06,
+ "loss": 1.8679372072219849,
+ "mean_token_accuracy": 0.5887884497642517,
+ "num_tokens": 6848512.0,
+ "step": 418
+ },
+ {
+ "entropy": 1.4079350233078003,
+ "epoch": 0.8464646464646465,
+ "grad_norm": 1.9526047706604004,
+ "learning_rate": 4.155555555555556e-06,
+ "loss": 1.4145114421844482,
+ "mean_token_accuracy": 0.6731802821159363,
+ "num_tokens": 6864896.0,
+ "step": 419
+ },
+ {
+ "entropy": 1.1973973512649536,
+ "epoch": 0.8484848484848485,
+ "grad_norm": 2.0716679096221924,
+ "learning_rate": 4.1535353535353536e-06,
+ "loss": 1.260810136795044,
+ "mean_token_accuracy": 0.701697587966919,
+ "num_tokens": 6881280.0,
+ "step": 420
+ },
+ {
+ "entropy": 1.507702350616455,
+ "epoch": 0.8505050505050505,
+ "grad_norm": 2.0233314037323,
+ "learning_rate": 4.151515151515152e-06,
+ "loss": 1.513720154762268,
+ "mean_token_accuracy": 0.6497313380241394,
+ "num_tokens": 6897664.0,
+ "step": 421
+ },
+ {
+ "entropy": 1.54402756690979,
+ "epoch": 0.8525252525252526,
+ "grad_norm": 2.23366379737854,
+ "learning_rate": 4.14949494949495e-06,
+ "loss": 1.5434964895248413,
+ "mean_token_accuracy": 0.6502198576927185,
+ "num_tokens": 6914048.0,
+ "step": 422
+ },
+ {
+ "entropy": 1.820296049118042,
+ "epoch": 0.8545454545454545,
+ "grad_norm": 2.066905975341797,
+ "learning_rate": 4.147474747474748e-06,
+ "loss": 1.8356924057006836,
+ "mean_token_accuracy": 0.5886663198471069,
+ "num_tokens": 6930432.0,
+ "step": 423
+ },
+ {
+ "entropy": 1.4747720956802368,
+ "epoch": 0.8565656565656565,
+ "grad_norm": 2.040022850036621,
+ "learning_rate": 4.145454545454546e-06,
+ "loss": 1.4495999813079834,
+ "mean_token_accuracy": 0.6532731056213379,
+ "num_tokens": 6946816.0,
+ "step": 424
+ },
+ {
+ "entropy": 1.7536696195602417,
+ "epoch": 0.8585858585858586,
+ "grad_norm": 2.0884320735931396,
+ "learning_rate": 4.143434343434344e-06,
+ "loss": 1.7520158290863037,
+ "mean_token_accuracy": 0.6083903312683105,
+ "num_tokens": 6963200.0,
+ "step": 425
+ },
+ {
+ "entropy": 1.715582251548767,
+ "epoch": 0.8606060606060606,
+ "grad_norm": 2.2991514205932617,
+ "learning_rate": 4.141414141414142e-06,
+ "loss": 1.7138090133666992,
+ "mean_token_accuracy": 0.607107937335968,
+ "num_tokens": 6979584.0,
+ "step": 426
+ },
+ {
+ "entropy": 1.8116382360458374,
+ "epoch": 0.8626262626262626,
+ "grad_norm": 2.289909839630127,
+ "learning_rate": 4.13939393939394e-06,
+ "loss": 1.7755894660949707,
+ "mean_token_accuracy": 0.5776746273040771,
+ "num_tokens": 6995968.0,
+ "step": 427
+ },
+ {
+ "entropy": 1.487213373184204,
+ "epoch": 0.8646464646464647,
+ "grad_norm": 1.8834803104400635,
+ "learning_rate": 4.1373737373737375e-06,
+ "loss": 1.511157751083374,
+ "mean_token_accuracy": 0.6546165347099304,
+ "num_tokens": 7012352.0,
+ "step": 428
+ },
+ {
+ "entropy": 1.2769891023635864,
+ "epoch": 0.8666666666666667,
+ "grad_norm": 1.892616629600525,
+ "learning_rate": 4.135353535353535e-06,
+ "loss": 1.2772598266601562,
+ "mean_token_accuracy": 0.6984611749649048,
+ "num_tokens": 7028736.0,
+ "step": 429
+ },
+ {
+ "entropy": 1.579519271850586,
+ "epoch": 0.8686868686868687,
+ "grad_norm": 1.9801563024520874,
+ "learning_rate": 4.133333333333333e-06,
+ "loss": 1.5647528171539307,
+ "mean_token_accuracy": 0.6482046842575073,
+ "num_tokens": 7045120.0,
+ "step": 430
+ },
+ {
+ "entropy": 1.407600998878479,
+ "epoch": 0.8707070707070707,
+ "grad_norm": 2.1737446784973145,
+ "learning_rate": 4.131313131313132e-06,
+ "loss": 1.4184494018554688,
+ "mean_token_accuracy": 0.6640815734863281,
+ "num_tokens": 7061504.0,
+ "step": 431
+ },
+ {
+ "entropy": 1.9118441343307495,
+ "epoch": 0.8727272727272727,
+ "grad_norm": 1.9541205167770386,
+ "learning_rate": 4.12929292929293e-06,
+ "loss": 1.9581422805786133,
+ "mean_token_accuracy": 0.564667820930481,
+ "num_tokens": 7077888.0,
+ "step": 432
+ },
+ {
+ "entropy": 1.925604224205017,
+ "epoch": 0.8747474747474747,
+ "grad_norm": 1.9696223735809326,
+ "learning_rate": 4.127272727272728e-06,
+ "loss": 1.9226163625717163,
+ "mean_token_accuracy": 0.5870786309242249,
+ "num_tokens": 7094272.0,
+ "step": 433
+ },
+ {
+ "entropy": 1.3994735479354858,
+ "epoch": 0.8767676767676768,
+ "grad_norm": 2.003532886505127,
+ "learning_rate": 4.125252525252526e-06,
+ "loss": 1.4291752576828003,
+ "mean_token_accuracy": 0.669516384601593,
+ "num_tokens": 7110656.0,
+ "step": 434
+ },
+ {
+ "entropy": 1.5550929307937622,
+ "epoch": 0.8787878787878788,
+ "grad_norm": 2.2658586502075195,
+ "learning_rate": 4.1232323232323236e-06,
+ "loss": 1.5533335208892822,
+ "mean_token_accuracy": 0.6242672204971313,
+ "num_tokens": 7127040.0,
+ "step": 435
+ },
+ {
+ "entropy": 1.6622785329818726,
+ "epoch": 0.8808080808080808,
+ "grad_norm": 2.0748393535614014,
+ "learning_rate": 4.1212121212121215e-06,
+ "loss": 1.700000524520874,
+ "mean_token_accuracy": 0.6221299171447754,
+ "num_tokens": 7143424.0,
+ "step": 436
+ },
+ {
+ "entropy": 1.1223996877670288,
+ "epoch": 0.8828282828282829,
+ "grad_norm": 2.0348756313323975,
+ "learning_rate": 4.119191919191919e-06,
+ "loss": 1.143293857574463,
+ "mean_token_accuracy": 0.7045676708221436,
+ "num_tokens": 7159808.0,
+ "step": 437
+ },
+ {
+ "entropy": 1.7156380414962769,
+ "epoch": 0.8848484848484849,
+ "grad_norm": 2.306549549102783,
+ "learning_rate": 4.117171717171717e-06,
+ "loss": 1.7625794410705566,
+ "mean_token_accuracy": 0.6049706935882568,
+ "num_tokens": 7176192.0,
+ "step": 438
+ },
+ {
+ "entropy": 1.8507202863693237,
+ "epoch": 0.8868686868686869,
+ "grad_norm": 2.2955853939056396,
+ "learning_rate": 4.115151515151515e-06,
+ "loss": 1.8923052549362183,
+ "mean_token_accuracy": 0.5938568711280823,
+ "num_tokens": 7192576.0,
+ "step": 439
+ },
+ {
+ "entropy": 1.813754916191101,
+ "epoch": 0.8888888888888888,
+ "grad_norm": 2.095700263977051,
+ "learning_rate": 4.113131313131313e-06,
+ "loss": 1.8375307321548462,
+ "mean_token_accuracy": 0.5848192572593689,
+ "num_tokens": 7208960.0,
+ "step": 440
+ },
+ {
+ "entropy": 1.4185007810592651,
+ "epoch": 0.8909090909090909,
+ "grad_norm": 2.118213176727295,
+ "learning_rate": 4.111111111111111e-06,
+ "loss": 1.4548171758651733,
+ "mean_token_accuracy": 0.657608687877655,
+ "num_tokens": 7225344.0,
+ "step": 441
+ },
+ {
+ "entropy": 1.1041690111160278,
+ "epoch": 0.8929292929292929,
+ "grad_norm": 1.9638988971710205,
+ "learning_rate": 4.10909090909091e-06,
+ "loss": 1.1232322454452515,
+ "mean_token_accuracy": 0.7068881392478943,
+ "num_tokens": 7241728.0,
+ "step": 442
+ },
+ {
+ "entropy": 1.6423180103302002,
+ "epoch": 0.8949494949494949,
+ "grad_norm": 2.32987117767334,
+ "learning_rate": 4.1070707070707075e-06,
+ "loss": 1.665462613105774,
+ "mean_token_accuracy": 0.6133365631103516,
+ "num_tokens": 7258112.0,
+ "step": 443
+ },
+ {
+ "entropy": 1.4778310060501099,
+ "epoch": 0.896969696969697,
+ "grad_norm": 2.1247661113739014,
+ "learning_rate": 4.105050505050505e-06,
+ "loss": 1.4554922580718994,
+ "mean_token_accuracy": 0.6474108695983887,
+ "num_tokens": 7274496.0,
+ "step": 444
+ },
+ {
+ "entropy": 1.3777755498886108,
+ "epoch": 0.898989898989899,
+ "grad_norm": 1.9243263006210327,
+ "learning_rate": 4.103030303030303e-06,
+ "loss": 1.3942883014678955,
+ "mean_token_accuracy": 0.6656692624092102,
+ "num_tokens": 7290880.0,
+ "step": 445
+ },
+ {
+ "entropy": 1.1036415100097656,
+ "epoch": 0.901010101010101,
+ "grad_norm": 1.9711178541183472,
+ "learning_rate": 4.101010101010101e-06,
+ "loss": 1.1070351600646973,
+ "mean_token_accuracy": 0.7213605046272278,
+ "num_tokens": 7307264.0,
+ "step": 446
+ },
+ {
+ "entropy": 1.695487380027771,
+ "epoch": 0.9030303030303031,
+ "grad_norm": 2.1113150119781494,
+ "learning_rate": 4.098989898989899e-06,
+ "loss": 1.6993653774261475,
+ "mean_token_accuracy": 0.5992305874824524,
+ "num_tokens": 7323648.0,
+ "step": 447
+ },
+ {
+ "entropy": 1.496254801750183,
+ "epoch": 0.9050505050505051,
+ "grad_norm": 2.072986602783203,
+ "learning_rate": 4.096969696969697e-06,
+ "loss": 1.5046567916870117,
+ "mean_token_accuracy": 0.6558378338813782,
+ "num_tokens": 7340032.0,
+ "step": 448
+ },
+ {
+ "entropy": 2.1845033168792725,
+ "epoch": 0.907070707070707,
+ "grad_norm": 2.3246262073516846,
+ "learning_rate": 4.094949494949495e-06,
+ "loss": 2.1508708000183105,
+ "mean_token_accuracy": 0.5376160144805908,
+ "num_tokens": 7356416.0,
+ "step": 449
+ },
+ {
+ "entropy": 1.6329164505004883,
+ "epoch": 0.9090909090909091,
+ "grad_norm": 1.997310757637024,
+ "learning_rate": 4.092929292929294e-06,
+ "loss": 1.6422264575958252,
+ "mean_token_accuracy": 0.6232290863990784,
+ "num_tokens": 7372800.0,
+ "step": 450
+ },
+ {
+ "epoch": 0.9090909090909091,
+ "eval_entropy": 1.5515337480652718,
+ "eval_loss": 1.5485161542892456,
+ "eval_mean_token_accuracy": 0.6417716929028111,
+ "eval_num_tokens": 7372800.0,
+ "eval_runtime": 43.7901,
+ "eval_samples_per_second": 22.608,
+ "eval_steps_per_second": 2.832,
+ "step": 450
+ },
+ {
+ "entropy": 1.7854291200637817,
+ "epoch": 0.9111111111111111,
+ "grad_norm": 2.0212173461914062,
+ "learning_rate": 4.0909090909090915e-06,
+ "loss": 1.7920666933059692,
+ "mean_token_accuracy": 0.5950170755386353,
+ "num_tokens": 7389184.0,
+ "step": 451
+ },
+ {
+ "entropy": 1.216304898262024,
+ "epoch": 0.9131313131313131,
+ "grad_norm": 1.8800705671310425,
+ "learning_rate": 4.088888888888889e-06,
+ "loss": 1.2014267444610596,
+ "mean_token_accuracy": 0.7076819539070129,
+ "num_tokens": 7405568.0,
+ "step": 452
+ },
+ {
+ "entropy": 1.5593703985214233,
+ "epoch": 0.9151515151515152,
+ "grad_norm": 2.101381301879883,
+ "learning_rate": 4.086868686868687e-06,
+ "loss": 1.559610366821289,
+ "mean_token_accuracy": 0.6337933540344238,
+ "num_tokens": 7421952.0,
+ "step": 453
+ },
+ {
+ "entropy": 1.4199841022491455,
+ "epoch": 0.9171717171717172,
+ "grad_norm": 2.0038692951202393,
+ "learning_rate": 4.084848484848485e-06,
+ "loss": 1.4040653705596924,
+ "mean_token_accuracy": 0.660845160484314,
+ "num_tokens": 7438336.0,
+ "step": 454
+ },
+ {
+ "entropy": 1.2010453939437866,
+ "epoch": 0.9191919191919192,
+ "grad_norm": 2.004110336303711,
+ "learning_rate": 4.082828282828283e-06,
+ "loss": 1.194115400314331,
+ "mean_token_accuracy": 0.6955910921096802,
+ "num_tokens": 7454720.0,
+ "step": 455
+ },
+ {
+ "entropy": 1.2295804023742676,
+ "epoch": 0.9212121212121213,
+ "grad_norm": 2.130387544631958,
+ "learning_rate": 4.080808080808081e-06,
+ "loss": 1.2125787734985352,
+ "mean_token_accuracy": 0.6993771195411682,
+ "num_tokens": 7471104.0,
+ "step": 456
+ },
+ {
+ "entropy": 1.3706485033035278,
+ "epoch": 0.9232323232323232,
+ "grad_norm": 2.1667706966400146,
+ "learning_rate": 4.07878787878788e-06,
+ "loss": 1.3886957168579102,
+ "mean_token_accuracy": 0.6686614751815796,
+ "num_tokens": 7487488.0,
+ "step": 457
+ },
+ {
+ "entropy": 1.714059591293335,
+ "epoch": 0.9252525252525252,
+ "grad_norm": 2.0084264278411865,
+ "learning_rate": 4.0767676767676775e-06,
+ "loss": 1.724153757095337,
+ "mean_token_accuracy": 0.602161705493927,
+ "num_tokens": 7503872.0,
+ "step": 458
+ },
+ {
+ "entropy": 1.3170617818832397,
+ "epoch": 0.9272727272727272,
+ "grad_norm": 1.9923994541168213,
+ "learning_rate": 4.0747474747474754e-06,
+ "loss": 1.3518096208572388,
+ "mean_token_accuracy": 0.6801416873931885,
+ "num_tokens": 7520256.0,
+ "step": 459
+ },
+ {
+ "entropy": 1.7946380376815796,
+ "epoch": 0.9292929292929293,
+ "grad_norm": 2.3475260734558105,
+ "learning_rate": 4.072727272727273e-06,
+ "loss": 1.824514389038086,
+ "mean_token_accuracy": 0.6005740165710449,
+ "num_tokens": 7536640.0,
+ "step": 460
+ },
+ {
+ "entropy": 1.3695449829101562,
+ "epoch": 0.9313131313131313,
+ "grad_norm": 2.0983943939208984,
+ "learning_rate": 4.070707070707071e-06,
+ "loss": 1.3934273719787598,
+ "mean_token_accuracy": 0.6723253726959229,
+ "num_tokens": 7553024.0,
+ "step": 461
+ },
+ {
+ "entropy": 1.5890663862228394,
+ "epoch": 0.9333333333333333,
+ "grad_norm": 2.1465413570404053,
+ "learning_rate": 4.068686868686869e-06,
+ "loss": 1.611652135848999,
+ "mean_token_accuracy": 0.6408158540725708,
+ "num_tokens": 7569408.0,
+ "step": 462
+ },
+ {
+ "entropy": 1.2593727111816406,
+ "epoch": 0.9353535353535354,
+ "grad_norm": 1.7885241508483887,
+ "learning_rate": 4.066666666666667e-06,
+ "loss": 1.2781705856323242,
+ "mean_token_accuracy": 0.6946751475334167,
+ "num_tokens": 7585792.0,
+ "step": 463
+ },
+ {
+ "entropy": 1.5680429935455322,
+ "epoch": 0.9373737373737374,
+ "grad_norm": 2.1466636657714844,
+ "learning_rate": 4.064646464646465e-06,
+ "loss": 1.60218346118927,
+ "mean_token_accuracy": 0.6334269642829895,
+ "num_tokens": 7602176.0,
+ "step": 464
+ },
+ {
+ "entropy": 1.681670069694519,
+ "epoch": 0.9393939393939394,
+ "grad_norm": 2.2211875915527344,
+ "learning_rate": 4.062626262626263e-06,
+ "loss": 1.6819908618927002,
+ "mean_token_accuracy": 0.6177943348884583,
+ "num_tokens": 7618560.0,
+ "step": 465
+ },
+ {
+ "entropy": 1.2239924669265747,
+ "epoch": 0.9414141414141414,
+ "grad_norm": 1.9997022151947021,
+ "learning_rate": 4.060606060606061e-06,
+ "loss": 1.2383348941802979,
+ "mean_token_accuracy": 0.6922325491905212,
+ "num_tokens": 7634944.0,
+ "step": 466
+ },
+ {
+ "entropy": 1.291772723197937,
+ "epoch": 0.9434343434343434,
+ "grad_norm": 2.0868117809295654,
+ "learning_rate": 4.058585858585859e-06,
+ "loss": 1.2879221439361572,
+ "mean_token_accuracy": 0.6883854269981384,
+ "num_tokens": 7651328.0,
+ "step": 467
+ },
+ {
+ "entropy": 1.2860186100006104,
+ "epoch": 0.9454545454545454,
+ "grad_norm": 1.8586393594741821,
+ "learning_rate": 4.056565656565657e-06,
+ "loss": 1.2820203304290771,
+ "mean_token_accuracy": 0.6802638173103333,
+ "num_tokens": 7667712.0,
+ "step": 468
+ },
+ {
+ "entropy": 1.860795259475708,
+ "epoch": 0.9474747474747475,
+ "grad_norm": 2.364405393600464,
+ "learning_rate": 4.054545454545455e-06,
+ "loss": 1.8880727291107178,
+ "mean_token_accuracy": 0.5821323990821838,
+ "num_tokens": 7684096.0,
+ "step": 469
+ },
+ {
+ "entropy": 1.6017589569091797,
+ "epoch": 0.9494949494949495,
+ "grad_norm": 2.023054599761963,
+ "learning_rate": 4.052525252525253e-06,
+ "loss": 1.5993852615356445,
+ "mean_token_accuracy": 0.6366634368896484,
+ "num_tokens": 7700480.0,
+ "step": 470
+ },
+ {
+ "entropy": 1.8498003482818604,
+ "epoch": 0.9515151515151515,
+ "grad_norm": 2.4474003314971924,
+ "learning_rate": 4.050505050505051e-06,
+ "loss": 1.868700623512268,
+ "mean_token_accuracy": 0.5826209187507629,
+ "num_tokens": 7716864.0,
+ "step": 471
+ },
+ {
+ "entropy": 1.5625540018081665,
+ "epoch": 0.9535353535353536,
+ "grad_norm": 2.1018362045288086,
+ "learning_rate": 4.048484848484849e-06,
+ "loss": 1.571077585220337,
+ "mean_token_accuracy": 0.6433805823326111,
+ "num_tokens": 7733248.0,
+ "step": 472
+ },
+ {
+ "entropy": 1.8534579277038574,
+ "epoch": 0.9555555555555556,
+ "grad_norm": 2.407588243484497,
+ "learning_rate": 4.046464646464647e-06,
+ "loss": 1.8789153099060059,
+ "mean_token_accuracy": 0.5859794616699219,
+ "num_tokens": 7749632.0,
+ "step": 473
+ },
+ {
+ "entropy": 1.3264559507369995,
+ "epoch": 0.9575757575757575,
+ "grad_norm": 2.007199764251709,
+ "learning_rate": 4.044444444444445e-06,
+ "loss": 1.3489338159561157,
+ "mean_token_accuracy": 0.6650586128234863,
+ "num_tokens": 7766016.0,
+ "step": 474
+ },
+ {
+ "entropy": 1.1361761093139648,
+ "epoch": 0.9595959595959596,
+ "grad_norm": 1.9226998090744019,
+ "learning_rate": 4.0424242424242425e-06,
+ "loss": 1.156738519668579,
+ "mean_token_accuracy": 0.7226428985595703,
+ "num_tokens": 7782400.0,
+ "step": 475
+ },
+ {
+ "entropy": 1.550872802734375,
+ "epoch": 0.9616161616161616,
+ "grad_norm": 2.0082473754882812,
+ "learning_rate": 4.04040404040404e-06,
+ "loss": 1.5657379627227783,
+ "mean_token_accuracy": 0.642953097820282,
+ "num_tokens": 7798784.0,
+ "step": 476
+ },
+ {
+ "entropy": 1.0122721195220947,
+ "epoch": 0.9636363636363636,
+ "grad_norm": 1.9946776628494263,
+ "learning_rate": 4.038383838383838e-06,
+ "loss": 1.0301119089126587,
+ "mean_token_accuracy": 0.7361993193626404,
+ "num_tokens": 7815168.0,
+ "step": 477
+ },
+ {
+ "entropy": 1.61614990234375,
+ "epoch": 0.9656565656565657,
+ "grad_norm": 2.5009427070617676,
+ "learning_rate": 4.036363636363637e-06,
+ "loss": 1.6194994449615479,
+ "mean_token_accuracy": 0.620175838470459,
+ "num_tokens": 7831552.0,
+ "step": 478
+ },
+ {
+ "entropy": 1.5723443031311035,
+ "epoch": 0.9676767676767677,
+ "grad_norm": 1.9101917743682861,
+ "learning_rate": 4.034343434343435e-06,
+ "loss": 1.5586073398590088,
+ "mean_token_accuracy": 0.6223741769790649,
+ "num_tokens": 7847936.0,
+ "step": 479
+ },
+ {
+ "entropy": 1.271834373474121,
+ "epoch": 0.9696969696969697,
+ "grad_norm": 1.768438458442688,
+ "learning_rate": 4.032323232323233e-06,
+ "loss": 1.2466087341308594,
+ "mean_token_accuracy": 0.692354679107666,
+ "num_tokens": 7864320.0,
+ "step": 480
+ },
+ {
+ "entropy": 1.4550020694732666,
+ "epoch": 0.9717171717171718,
+ "grad_norm": 1.9753917455673218,
+ "learning_rate": 4.030303030303031e-06,
+ "loss": 1.4689980745315552,
+ "mean_token_accuracy": 0.6530288457870483,
+ "num_tokens": 7880704.0,
+ "step": 481
+ },
+ {
+ "entropy": 2.023750066757202,
+ "epoch": 0.9737373737373738,
+ "grad_norm": 2.3014674186706543,
+ "learning_rate": 4.0282828282828285e-06,
+ "loss": 2.0601019859313965,
+ "mean_token_accuracy": 0.5528212189674377,
+ "num_tokens": 7897088.0,
+ "step": 482
+ },
+ {
+ "entropy": 1.3225218057632446,
+ "epoch": 0.9757575757575757,
+ "grad_norm": 1.9937688112258911,
+ "learning_rate": 4.0262626262626264e-06,
+ "loss": 1.3186583518981934,
+ "mean_token_accuracy": 0.6751343607902527,
+ "num_tokens": 7913472.0,
+ "step": 483
+ },
+ {
+ "entropy": 1.4357911348342896,
+ "epoch": 0.9777777777777777,
+ "grad_norm": 1.9591492414474487,
+ "learning_rate": 4.024242424242424e-06,
+ "loss": 1.4157384634017944,
+ "mean_token_accuracy": 0.6618832349777222,
+ "num_tokens": 7929856.0,
+ "step": 484
+ },
+ {
+ "entropy": 1.470274567604065,
+ "epoch": 0.9797979797979798,
+ "grad_norm": 1.9318779706954956,
+ "learning_rate": 4.022222222222222e-06,
+ "loss": 1.4754853248596191,
+ "mean_token_accuracy": 0.6497923731803894,
+ "num_tokens": 7946240.0,
+ "step": 485
+ },
+ {
+ "entropy": 1.345750331878662,
+ "epoch": 0.9818181818181818,
+ "grad_norm": 1.9079619646072388,
+ "learning_rate": 4.02020202020202e-06,
+ "loss": 1.336526870727539,
+ "mean_token_accuracy": 0.6800805926322937,
+ "num_tokens": 7962624.0,
+ "step": 486
+ },
+ {
+ "entropy": 1.4641839265823364,
+ "epoch": 0.9838383838383838,
+ "grad_norm": 1.8413265943527222,
+ "learning_rate": 4.018181818181818e-06,
+ "loss": 1.4556326866149902,
+ "mean_token_accuracy": 0.6591963768005371,
+ "num_tokens": 7979008.0,
+ "step": 487
+ },
+ {
+ "entropy": 1.7966911792755127,
+ "epoch": 0.9858585858585859,
+ "grad_norm": 2.138899564743042,
+ "learning_rate": 4.016161616161616e-06,
+ "loss": 1.822898507118225,
+ "mean_token_accuracy": 0.5959941148757935,
+ "num_tokens": 7995392.0,
+ "step": 488
+ },
+ {
+ "entropy": 1.4334503412246704,
+ "epoch": 0.9878787878787879,
+ "grad_norm": 2.206122636795044,
+ "learning_rate": 4.014141414141415e-06,
+ "loss": 1.4127681255340576,
+ "mean_token_accuracy": 0.6609672904014587,
+ "num_tokens": 8011776.0,
+ "step": 489
+ },
+ {
+ "entropy": 1.684112548828125,
+ "epoch": 0.98989898989899,
+ "grad_norm": 2.0637400150299072,
+ "learning_rate": 4.0121212121212125e-06,
+ "loss": 1.6790317296981812,
+ "mean_token_accuracy": 0.6159012913703918,
+ "num_tokens": 8028160.0,
+ "step": 490
+ },
+ {
+ "entropy": 2.0716898441314697,
+ "epoch": 0.9919191919191919,
+ "grad_norm": 3.3912627696990967,
+ "learning_rate": 4.01010101010101e-06,
+ "loss": 2.0814826488494873,
+ "mean_token_accuracy": 0.5522105693817139,
+ "num_tokens": 8044544.0,
+ "step": 491
+ },
+ {
+ "entropy": 1.8764609098434448,
+ "epoch": 0.9939393939393939,
+ "grad_norm": 2.00569748878479,
+ "learning_rate": 4.008080808080808e-06,
+ "loss": 1.9150068759918213,
+ "mean_token_accuracy": 0.5823766589164734,
+ "num_tokens": 8060928.0,
+ "step": 492
+ },
+ {
+ "entropy": 1.630242943763733,
+ "epoch": 0.9959595959595959,
+ "grad_norm": 2.0648858547210693,
+ "learning_rate": 4.006060606060607e-06,
+ "loss": 1.6354784965515137,
+ "mean_token_accuracy": 0.621213972568512,
+ "num_tokens": 8077312.0,
+ "step": 493
+ },
+ {
+ "entropy": 1.7174078226089478,
+ "epoch": 0.997979797979798,
+ "grad_norm": 2.1376583576202393,
+ "learning_rate": 4.004040404040405e-06,
+ "loss": 1.7092773914337158,
+ "mean_token_accuracy": 0.6129701733589172,
+ "num_tokens": 8093696.0,
+ "step": 494
+ },
+ {
+ "entropy": 1.2112717628479004,
+ "epoch": 1.0,
+ "grad_norm": 1.9164371490478516,
+ "learning_rate": 4.002020202020203e-06,
+ "loss": 1.2106354236602783,
+ "mean_token_accuracy": 0.7040180563926697,
+ "num_tokens": 8110080.0,
+ "step": 495
+ },
+ {
+ "entropy": 1.2168891429901123,
+ "epoch": 1.002020202020202,
+ "grad_norm": 1.9845075607299805,
+ "learning_rate": 4.000000000000001e-06,
+ "loss": 1.1398870944976807,
+ "mean_token_accuracy": 0.7053004503250122,
+ "num_tokens": 8126464.0,
+ "step": 496
+ },
+ {
+ "entropy": 1.4670923948287964,
+ "epoch": 1.004040404040404,
+ "grad_norm": 2.0864014625549316,
+ "learning_rate": 3.9979797979797986e-06,
+ "loss": 1.4241242408752441,
+ "mean_token_accuracy": 0.6605398058891296,
+ "num_tokens": 8142848.0,
+ "step": 497
+ },
+ {
+ "entropy": 1.560593843460083,
+ "epoch": 1.006060606060606,
+ "grad_norm": 1.8882137537002563,
+ "learning_rate": 3.9959595959595964e-06,
+ "loss": 1.5164835453033447,
+ "mean_token_accuracy": 0.6610894203186035,
+ "num_tokens": 8159232.0,
+ "step": 498
+ },
+ {
+ "entropy": 1.2573609352111816,
+ "epoch": 1.0080808080808081,
+ "grad_norm": 2.011486530303955,
+ "learning_rate": 3.993939393939394e-06,
+ "loss": 1.2119636535644531,
+ "mean_token_accuracy": 0.6832559704780579,
+ "num_tokens": 8175616.0,
+ "step": 499
+ },
+ {
+ "entropy": 1.6119122505187988,
+ "epoch": 1.0101010101010102,
+ "grad_norm": 2.0089032649993896,
+ "learning_rate": 3.991919191919192e-06,
+ "loss": 1.5869622230529785,
+ "mean_token_accuracy": 0.6301905512809753,
+ "num_tokens": 8192000.0,
+ "step": 500
+ },
+ {
+ "epoch": 1.0101010101010102,
+ "eval_entropy": 1.4907484121860997,
+ "eval_loss": 1.5444872379302979,
+ "eval_mean_token_accuracy": 0.6426531960887294,
+ "eval_num_tokens": 8192000.0,
+ "eval_runtime": 43.729,
+ "eval_samples_per_second": 22.639,
+ "eval_steps_per_second": 2.836,
+ "step": 500
+ },
+ {
+ "entropy": 1.5645418167114258,
+ "epoch": 1.0121212121212122,
+ "grad_norm": 2.171133279800415,
+ "learning_rate": 3.98989898989899e-06,
+ "loss": 1.5252666473388672,
+ "mean_token_accuracy": 0.639594554901123,
+ "num_tokens": 8208384.0,
+ "step": 501
+ },
+ {
+ "entropy": 1.2985379695892334,
+ "epoch": 1.0141414141414142,
+ "grad_norm": 1.9075100421905518,
+ "learning_rate": 3.987878787878788e-06,
+ "loss": 1.2863168716430664,
+ "mean_token_accuracy": 0.6797142028808594,
+ "num_tokens": 8224768.0,
+ "step": 502
+ },
+ {
+ "entropy": 1.4055086374282837,
+ "epoch": 1.0161616161616163,
+ "grad_norm": 1.8511557579040527,
+ "learning_rate": 3.985858585858587e-06,
+ "loss": 1.440205454826355,
+ "mean_token_accuracy": 0.6671348214149475,
+ "num_tokens": 8241152.0,
+ "step": 503
+ },
+ {
+ "entropy": 1.0818580389022827,
+ "epoch": 1.018181818181818,
+ "grad_norm": 1.8912067413330078,
+ "learning_rate": 3.983838383838385e-06,
+ "loss": 1.0614542961120605,
+ "mean_token_accuracy": 0.7345505356788635,
+ "num_tokens": 8257536.0,
+ "step": 504
+ },
+ {
+ "entropy": 0.9176992177963257,
+ "epoch": 1.02020202020202,
+ "grad_norm": 1.7737077474594116,
+ "learning_rate": 3.9818181818181825e-06,
+ "loss": 0.9281337261199951,
+ "mean_token_accuracy": 0.7594040036201477,
+ "num_tokens": 8273920.0,
+ "step": 505
+ },
+ {
+ "entropy": 1.8801840543746948,
+ "epoch": 1.0222222222222221,
+ "grad_norm": 2.191689968109131,
+ "learning_rate": 3.97979797979798e-06,
+ "loss": 1.9127342700958252,
+ "mean_token_accuracy": 0.5732169151306152,
+ "num_tokens": 8290304.0,
+ "step": 506
+ },
+ {
+ "entropy": 1.2336418628692627,
+ "epoch": 1.0242424242424242,
+ "grad_norm": 2.0369555950164795,
+ "learning_rate": 3.977777777777778e-06,
+ "loss": 1.2513363361358643,
+ "mean_token_accuracy": 0.6852100491523743,
+ "num_tokens": 8306688.0,
+ "step": 507
+ },
+ {
+ "entropy": 1.8379974365234375,
+ "epoch": 1.0262626262626262,
+ "grad_norm": 2.176361083984375,
+ "learning_rate": 3.975757575757576e-06,
+ "loss": 1.8472888469696045,
+ "mean_token_accuracy": 0.5863458514213562,
+ "num_tokens": 8323072.0,
+ "step": 508
+ },
+ {
+ "entropy": 1.4421337842941284,
+ "epoch": 1.0282828282828282,
+ "grad_norm": 2.305441379547119,
+ "learning_rate": 3.973737373737374e-06,
+ "loss": 1.4835591316223145,
+ "mean_token_accuracy": 0.6530288457870483,
+ "num_tokens": 8339456.0,
+ "step": 509
+ },
+ {
+ "entropy": 1.535184621810913,
+ "epoch": 1.0303030303030303,
+ "grad_norm": 2.0329015254974365,
+ "learning_rate": 3.971717171717172e-06,
+ "loss": 1.547795057296753,
+ "mean_token_accuracy": 0.6268319487571716,
+ "num_tokens": 8355840.0,
+ "step": 510
+ },
+ {
+ "entropy": 1.295592188835144,
+ "epoch": 1.0323232323232323,
+ "grad_norm": 2.0321359634399414,
+ "learning_rate": 3.96969696969697e-06,
+ "loss": 1.2885974645614624,
+ "mean_token_accuracy": 0.681485116481781,
+ "num_tokens": 8372224.0,
+ "step": 511
+ },
+ {
+ "entropy": 1.345553994178772,
+ "epoch": 1.0343434343434343,
+ "grad_norm": 1.9151290655136108,
+ "learning_rate": 3.967676767676768e-06,
+ "loss": 1.3364320993423462,
+ "mean_token_accuracy": 0.6849657893180847,
+ "num_tokens": 8388608.0,
+ "step": 512
+ },
+ {
+ "entropy": 1.3150815963745117,
+ "epoch": 1.0363636363636364,
+ "grad_norm": 1.9379258155822754,
+ "learning_rate": 3.965656565656566e-06,
+ "loss": 1.317258596420288,
+ "mean_token_accuracy": 0.6873473525047302,
+ "num_tokens": 8404992.0,
+ "step": 513
+ },
+ {
+ "entropy": 1.46832275390625,
+ "epoch": 1.0383838383838384,
+ "grad_norm": 2.1009161472320557,
+ "learning_rate": 3.963636363636364e-06,
+ "loss": 1.4794366359710693,
+ "mean_token_accuracy": 0.6513800621032715,
+ "num_tokens": 8421376.0,
+ "step": 514
+ },
+ {
+ "entropy": 1.389290452003479,
+ "epoch": 1.0404040404040404,
+ "grad_norm": 1.8813941478729248,
+ "learning_rate": 3.961616161616162e-06,
+ "loss": 1.3930776119232178,
+ "mean_token_accuracy": 0.6799584627151489,
+ "num_tokens": 8437760.0,
+ "step": 515
+ },
+ {
+ "entropy": 1.2919796705245972,
+ "epoch": 1.0424242424242425,
+ "grad_norm": 2.0765745639801025,
+ "learning_rate": 3.95959595959596e-06,
+ "loss": 1.2943801879882812,
+ "mean_token_accuracy": 0.6929653286933899,
+ "num_tokens": 8454144.0,
+ "step": 516
+ },
+ {
+ "entropy": 1.172440767288208,
+ "epoch": 1.0444444444444445,
+ "grad_norm": 2.126800298690796,
+ "learning_rate": 3.957575757575758e-06,
+ "loss": 1.1923961639404297,
+ "mean_token_accuracy": 0.7098192572593689,
+ "num_tokens": 8470528.0,
+ "step": 517
+ },
+ {
+ "entropy": 1.8239458799362183,
+ "epoch": 1.0464646464646465,
+ "grad_norm": 1.96744966506958,
+ "learning_rate": 3.955555555555556e-06,
+ "loss": 1.840916633605957,
+ "mean_token_accuracy": 0.6033830046653748,
+ "num_tokens": 8486912.0,
+ "step": 518
+ },
+ {
+ "entropy": 1.4117076396942139,
+ "epoch": 1.0484848484848486,
+ "grad_norm": 2.0560126304626465,
+ "learning_rate": 3.953535353535354e-06,
+ "loss": 1.421530842781067,
+ "mean_token_accuracy": 0.6581583023071289,
+ "num_tokens": 8503296.0,
+ "step": 519
+ },
+ {
+ "entropy": 1.7085175514221191,
+ "epoch": 1.0505050505050506,
+ "grad_norm": 2.1386539936065674,
+ "learning_rate": 3.951515151515152e-06,
+ "loss": 1.7334365844726562,
+ "mean_token_accuracy": 0.6282975077629089,
+ "num_tokens": 8519680.0,
+ "step": 520
+ },
+ {
+ "entropy": 1.0607928037643433,
+ "epoch": 1.0525252525252524,
+ "grad_norm": 1.981020212173462,
+ "learning_rate": 3.9494949494949496e-06,
+ "loss": 1.0639690160751343,
+ "mean_token_accuracy": 0.722337543964386,
+ "num_tokens": 8536064.0,
+ "step": 521
+ },
+ {
+ "entropy": 1.2227518558502197,
+ "epoch": 1.0545454545454545,
+ "grad_norm": 1.9116015434265137,
+ "learning_rate": 3.9474747474747474e-06,
+ "loss": 1.2139748334884644,
+ "mean_token_accuracy": 0.704384446144104,
+ "num_tokens": 8552448.0,
+ "step": 522
+ },
+ {
+ "entropy": 1.2655551433563232,
+ "epoch": 1.0565656565656565,
+ "grad_norm": 2.176706075668335,
+ "learning_rate": 3.945454545454545e-06,
+ "loss": 1.2869982719421387,
+ "mean_token_accuracy": 0.678798258304596,
+ "num_tokens": 8568832.0,
+ "step": 523
+ },
+ {
+ "entropy": 1.436476707458496,
+ "epoch": 1.0585858585858585,
+ "grad_norm": 2.034846544265747,
+ "learning_rate": 3.943434343434343e-06,
+ "loss": 1.431445837020874,
+ "mean_token_accuracy": 0.6650586128234863,
+ "num_tokens": 8585216.0,
+ "step": 524
+ },
+ {
+ "entropy": 1.4563549757003784,
+ "epoch": 1.0606060606060606,
+ "grad_norm": 2.0301241874694824,
+ "learning_rate": 3.941414141414142e-06,
+ "loss": 1.468353033065796,
+ "mean_token_accuracy": 0.6550439596176147,
+ "num_tokens": 8601600.0,
+ "step": 525
+ },
+ {
+ "entropy": 1.4636993408203125,
+ "epoch": 1.0626262626262626,
+ "grad_norm": 2.092679023742676,
+ "learning_rate": 3.93939393939394e-06,
+ "loss": 1.4488987922668457,
+ "mean_token_accuracy": 0.652723491191864,
+ "num_tokens": 8617984.0,
+ "step": 526
+ },
+ {
+ "entropy": 1.1217654943466187,
+ "epoch": 1.0646464646464646,
+ "grad_norm": 1.953627347946167,
+ "learning_rate": 3.937373737373738e-06,
+ "loss": 1.1403257846832275,
+ "mean_token_accuracy": 0.7172080874443054,
+ "num_tokens": 8634368.0,
+ "step": 527
+ },
+ {
+ "entropy": 1.7785910367965698,
+ "epoch": 1.0666666666666667,
+ "grad_norm": 1.8496789932250977,
+ "learning_rate": 3.935353535353536e-06,
+ "loss": 1.7961615324020386,
+ "mean_token_accuracy": 0.6036272644996643,
+ "num_tokens": 8650752.0,
+ "step": 528
+ },
+ {
+ "entropy": 1.0998575687408447,
+ "epoch": 1.0686868686868687,
+ "grad_norm": 1.8732962608337402,
+ "learning_rate": 3.9333333333333335e-06,
+ "loss": 1.0960031747817993,
+ "mean_token_accuracy": 0.7223986387252808,
+ "num_tokens": 8667136.0,
+ "step": 529
+ },
+ {
+ "entropy": 1.6707724332809448,
+ "epoch": 1.0707070707070707,
+ "grad_norm": 2.220453977584839,
+ "learning_rate": 3.931313131313131e-06,
+ "loss": 1.6837453842163086,
+ "mean_token_accuracy": 0.6148021221160889,
+ "num_tokens": 8683520.0,
+ "step": 530
+ },
+ {
+ "entropy": 1.6108022928237915,
+ "epoch": 1.0727272727272728,
+ "grad_norm": 2.2464118003845215,
+ "learning_rate": 3.929292929292929e-06,
+ "loss": 1.633517861366272,
+ "mean_token_accuracy": 0.6195651888847351,
+ "num_tokens": 8699904.0,
+ "step": 531
+ },
+ {
+ "entropy": 1.1888173818588257,
+ "epoch": 1.0747474747474748,
+ "grad_norm": 1.8803986310958862,
+ "learning_rate": 3.927272727272727e-06,
+ "loss": 1.1904420852661133,
+ "mean_token_accuracy": 0.7134220600128174,
+ "num_tokens": 8716288.0,
+ "step": 532
+ },
+ {
+ "entropy": 1.0656445026397705,
+ "epoch": 1.0767676767676768,
+ "grad_norm": 2.038243532180786,
+ "learning_rate": 3.925252525252525e-06,
+ "loss": 1.0505216121673584,
+ "mean_token_accuracy": 0.7284440398216248,
+ "num_tokens": 8732672.0,
+ "step": 533
+ },
+ {
+ "entropy": 1.1473655700683594,
+ "epoch": 1.0787878787878789,
+ "grad_norm": 1.981107234954834,
+ "learning_rate": 3.923232323232323e-06,
+ "loss": 1.136179804801941,
+ "mean_token_accuracy": 0.7085368633270264,
+ "num_tokens": 8749056.0,
+ "step": 534
+ },
+ {
+ "entropy": 1.340027928352356,
+ "epoch": 1.0808080808080809,
+ "grad_norm": 2.2797436714172363,
+ "learning_rate": 3.921212121212122e-06,
+ "loss": 1.334214210510254,
+ "mean_token_accuracy": 0.6749511361122131,
+ "num_tokens": 8765440.0,
+ "step": 535
+ },
+ {
+ "entropy": 1.1464864015579224,
+ "epoch": 1.082828282828283,
+ "grad_norm": 1.9428092241287231,
+ "learning_rate": 3.9191919191919196e-06,
+ "loss": 1.1649314165115356,
+ "mean_token_accuracy": 0.72013920545578,
+ "num_tokens": 8781824.0,
+ "step": 536
+ },
+ {
+ "entropy": 1.784925103187561,
+ "epoch": 1.084848484848485,
+ "grad_norm": 2.157468795776367,
+ "learning_rate": 3.9171717171717175e-06,
+ "loss": 1.7725508213043213,
+ "mean_token_accuracy": 0.6014899611473083,
+ "num_tokens": 8798208.0,
+ "step": 537
+ },
+ {
+ "entropy": 1.362166166305542,
+ "epoch": 1.0868686868686868,
+ "grad_norm": 2.109646797180176,
+ "learning_rate": 3.915151515151515e-06,
+ "loss": 1.331969976425171,
+ "mean_token_accuracy": 0.6720200181007385,
+ "num_tokens": 8814592.0,
+ "step": 538
+ },
+ {
+ "entropy": 1.5614176988601685,
+ "epoch": 1.0888888888888888,
+ "grad_norm": 2.1047098636627197,
+ "learning_rate": 3.913131313131314e-06,
+ "loss": 1.534292459487915,
+ "mean_token_accuracy": 0.6446629166603088,
+ "num_tokens": 8830976.0,
+ "step": 539
+ },
+ {
+ "entropy": 1.5551490783691406,
+ "epoch": 1.0909090909090908,
+ "grad_norm": 2.10304594039917,
+ "learning_rate": 3.911111111111112e-06,
+ "loss": 1.5706363916397095,
+ "mean_token_accuracy": 0.6322056651115417,
+ "num_tokens": 8847360.0,
+ "step": 540
+ },
+ {
+ "entropy": 1.7433172464370728,
+ "epoch": 1.0929292929292929,
+ "grad_norm": 1.8621257543563843,
+ "learning_rate": 3.90909090909091e-06,
+ "loss": 1.7367552518844604,
+ "mean_token_accuracy": 0.6162066459655762,
+ "num_tokens": 8863744.0,
+ "step": 541
+ },
+ {
+ "entropy": 1.4073128700256348,
+ "epoch": 1.094949494949495,
+ "grad_norm": 1.939588189125061,
+ "learning_rate": 3.907070707070708e-06,
+ "loss": 1.3844857215881348,
+ "mean_token_accuracy": 0.6634709239006042,
+ "num_tokens": 8880128.0,
+ "step": 542
+ },
+ {
+ "entropy": 1.479836344718933,
+ "epoch": 1.096969696969697,
+ "grad_norm": 2.06921648979187,
+ "learning_rate": 3.905050505050506e-06,
+ "loss": 1.4933744668960571,
+ "mean_token_accuracy": 0.6510136723518372,
+ "num_tokens": 8896512.0,
+ "step": 543
+ },
+ {
+ "entropy": 1.3934576511383057,
+ "epoch": 1.098989898989899,
+ "grad_norm": 2.0697920322418213,
+ "learning_rate": 3.9030303030303035e-06,
+ "loss": 1.3886666297912598,
+ "mean_token_accuracy": 0.6775158643722534,
+ "num_tokens": 8912896.0,
+ "step": 544
+ },
+ {
+ "entropy": 1.5223709344863892,
+ "epoch": 1.101010101010101,
+ "grad_norm": 2.1627066135406494,
+ "learning_rate": 3.901010101010101e-06,
+ "loss": 1.5601061582565308,
+ "mean_token_accuracy": 0.6392892003059387,
+ "num_tokens": 8929280.0,
+ "step": 545
+ },
+ {
+ "entropy": 1.4462933540344238,
+ "epoch": 1.103030303030303,
+ "grad_norm": 1.9871046543121338,
+ "learning_rate": 3.898989898989899e-06,
+ "loss": 1.463792085647583,
+ "mean_token_accuracy": 0.6518075466156006,
+ "num_tokens": 8945664.0,
+ "step": 546
+ },
+ {
+ "entropy": 1.325921654701233,
+ "epoch": 1.105050505050505,
+ "grad_norm": 1.9081087112426758,
+ "learning_rate": 3.896969696969697e-06,
+ "loss": 1.3303362131118774,
+ "mean_token_accuracy": 0.7009648084640503,
+ "num_tokens": 8962048.0,
+ "step": 547
+ },
+ {
+ "entropy": 1.415048360824585,
+ "epoch": 1.107070707070707,
+ "grad_norm": 2.3306756019592285,
+ "learning_rate": 3.894949494949495e-06,
+ "loss": 1.4139145612716675,
+ "mean_token_accuracy": 0.6849047541618347,
+ "num_tokens": 8978432.0,
+ "step": 548
+ },
+ {
+ "entropy": 1.5863295793533325,
+ "epoch": 1.1090909090909091,
+ "grad_norm": 1.993159294128418,
+ "learning_rate": 3.892929292929293e-06,
+ "loss": 1.6010534763336182,
+ "mean_token_accuracy": 0.6424035429954529,
+ "num_tokens": 8994816.0,
+ "step": 549
+ },
+ {
+ "entropy": 1.5526721477508545,
+ "epoch": 1.1111111111111112,
+ "grad_norm": 2.0528876781463623,
+ "learning_rate": 3.890909090909092e-06,
+ "loss": 1.5465538501739502,
+ "mean_token_accuracy": 0.659807026386261,
+ "num_tokens": 9011200.0,
+ "step": 550
+ },
+ {
+ "epoch": 1.1111111111111112,
+ "eval_entropy": 1.4479231247978825,
+ "eval_loss": 1.5459802150726318,
+ "eval_mean_token_accuracy": 0.6425906530311031,
+ "eval_num_tokens": 9011200.0,
+ "eval_runtime": 43.7828,
+ "eval_samples_per_second": 22.612,
+ "eval_steps_per_second": 2.832,
+ "step": 550
+ },
+ {
+ "entropy": 1.4322376251220703,
+ "epoch": 1.1131313131313132,
+ "grad_norm": 1.933549404144287,
+ "learning_rate": 3.88888888888889e-06,
+ "loss": 1.4371025562286377,
+ "mean_token_accuracy": 0.670676589012146,
+ "num_tokens": 9027584.0,
+ "step": 551
+ },
+ {
+ "entropy": 1.5050671100616455,
+ "epoch": 1.1151515151515152,
+ "grad_norm": 1.9736099243164062,
+ "learning_rate": 3.8868686868686875e-06,
+ "loss": 1.5362370014190674,
+ "mean_token_accuracy": 0.651624321937561,
+ "num_tokens": 9043968.0,
+ "step": 552
+ },
+ {
+ "entropy": 1.4335887432098389,
+ "epoch": 1.1171717171717173,
+ "grad_norm": 1.8896421194076538,
+ "learning_rate": 3.884848484848485e-06,
+ "loss": 1.4439888000488281,
+ "mean_token_accuracy": 0.670615553855896,
+ "num_tokens": 9060352.0,
+ "step": 553
+ },
+ {
+ "entropy": 1.5979355573654175,
+ "epoch": 1.1191919191919193,
+ "grad_norm": 2.1710526943206787,
+ "learning_rate": 3.882828282828283e-06,
+ "loss": 1.629098653793335,
+ "mean_token_accuracy": 0.6226184368133545,
+ "num_tokens": 9076736.0,
+ "step": 554
+ },
+ {
+ "entropy": 1.2901698350906372,
+ "epoch": 1.121212121212121,
+ "grad_norm": 2.129443407058716,
+ "learning_rate": 3.880808080808081e-06,
+ "loss": 1.2824913263320923,
+ "mean_token_accuracy": 0.6816682815551758,
+ "num_tokens": 9093120.0,
+ "step": 555
+ },
+ {
+ "entropy": 1.403212308883667,
+ "epoch": 1.1232323232323231,
+ "grad_norm": 1.883240818977356,
+ "learning_rate": 3.878787878787879e-06,
+ "loss": 1.3960213661193848,
+ "mean_token_accuracy": 0.6651197075843811,
+ "num_tokens": 9109504.0,
+ "step": 556
+ },
+ {
+ "entropy": 1.3959479331970215,
+ "epoch": 1.1252525252525252,
+ "grad_norm": 2.1145691871643066,
+ "learning_rate": 3.876767676767677e-06,
+ "loss": 1.4046590328216553,
+ "mean_token_accuracy": 0.6607230305671692,
+ "num_tokens": 9125888.0,
+ "step": 557
+ },
+ {
+ "entropy": 1.0899873971939087,
+ "epoch": 1.1272727272727272,
+ "grad_norm": 1.8862192630767822,
+ "learning_rate": 3.874747474747475e-06,
+ "loss": 1.0965509414672852,
+ "mean_token_accuracy": 0.7245969772338867,
+ "num_tokens": 9142272.0,
+ "step": 558
+ },
+ {
+ "entropy": 1.0380184650421143,
+ "epoch": 1.1292929292929292,
+ "grad_norm": 1.946702241897583,
+ "learning_rate": 3.872727272727273e-06,
+ "loss": 1.0539875030517578,
+ "mean_token_accuracy": 0.7423058152198792,
+ "num_tokens": 9158656.0,
+ "step": 559
+ },
+ {
+ "entropy": 1.1173628568649292,
+ "epoch": 1.1313131313131313,
+ "grad_norm": 2.002847909927368,
+ "learning_rate": 3.8707070707070706e-06,
+ "loss": 1.128816843032837,
+ "mean_token_accuracy": 0.7112848162651062,
+ "num_tokens": 9175040.0,
+ "step": 560
+ },
+ {
+ "entropy": 1.6536206007003784,
+ "epoch": 1.1333333333333333,
+ "grad_norm": 2.212761640548706,
+ "learning_rate": 3.868686868686869e-06,
+ "loss": 1.6286437511444092,
+ "mean_token_accuracy": 0.6277479529380798,
+ "num_tokens": 9191424.0,
+ "step": 561
+ },
+ {
+ "entropy": 1.7187970876693726,
+ "epoch": 1.1353535353535353,
+ "grad_norm": 2.207310199737549,
+ "learning_rate": 3.866666666666667e-06,
+ "loss": 1.7072829008102417,
+ "mean_token_accuracy": 0.6180996298789978,
+ "num_tokens": 9207808.0,
+ "step": 562
+ },
+ {
+ "entropy": 1.7057682275772095,
+ "epoch": 1.1373737373737374,
+ "grad_norm": 2.1582961082458496,
+ "learning_rate": 3.864646464646465e-06,
+ "loss": 1.7216498851776123,
+ "mean_token_accuracy": 0.6105886697769165,
+ "num_tokens": 9224192.0,
+ "step": 563
+ },
+ {
+ "entropy": 1.8016951084136963,
+ "epoch": 1.1393939393939394,
+ "grad_norm": 2.035249948501587,
+ "learning_rate": 3.862626262626263e-06,
+ "loss": 1.8101240396499634,
+ "mean_token_accuracy": 0.6199315786361694,
+ "num_tokens": 9240576.0,
+ "step": 564
+ },
+ {
+ "entropy": 1.094889760017395,
+ "epoch": 1.1414141414141414,
+ "grad_norm": 2.119861364364624,
+ "learning_rate": 3.860606060606061e-06,
+ "loss": 1.0990344285964966,
+ "mean_token_accuracy": 0.7071323990821838,
+ "num_tokens": 9256960.0,
+ "step": 565
+ },
+ {
+ "entropy": 1.537480115890503,
+ "epoch": 1.1434343434343435,
+ "grad_norm": 2.167386770248413,
+ "learning_rate": 3.858585858585859e-06,
+ "loss": 1.5262541770935059,
+ "mean_token_accuracy": 0.6343429684638977,
+ "num_tokens": 9273344.0,
+ "step": 566
+ },
+ {
+ "entropy": 1.2940728664398193,
+ "epoch": 1.1454545454545455,
+ "grad_norm": 1.941097617149353,
+ "learning_rate": 3.856565656565657e-06,
+ "loss": 1.3145123720169067,
+ "mean_token_accuracy": 0.6822789311408997,
+ "num_tokens": 9289728.0,
+ "step": 567
+ },
+ {
+ "entropy": 1.3154000043869019,
+ "epoch": 1.1474747474747475,
+ "grad_norm": 2.088576078414917,
+ "learning_rate": 3.8545454545454545e-06,
+ "loss": 1.3154902458190918,
+ "mean_token_accuracy": 0.6703101992607117,
+ "num_tokens": 9306112.0,
+ "step": 568
+ },
+ {
+ "entropy": 1.3576427698135376,
+ "epoch": 1.1494949494949496,
+ "grad_norm": 1.9895246028900146,
+ "learning_rate": 3.852525252525252e-06,
+ "loss": 1.3295378684997559,
+ "mean_token_accuracy": 0.6762335300445557,
+ "num_tokens": 9322496.0,
+ "step": 569
+ },
+ {
+ "entropy": 1.6067878007888794,
+ "epoch": 1.1515151515151516,
+ "grad_norm": 2.021191358566284,
+ "learning_rate": 3.85050505050505e-06,
+ "loss": 1.6273870468139648,
+ "mean_token_accuracy": 0.6329995393753052,
+ "num_tokens": 9338880.0,
+ "step": 570
+ },
+ {
+ "entropy": 1.4831246137619019,
+ "epoch": 1.1535353535353536,
+ "grad_norm": 2.114612340927124,
+ "learning_rate": 3.848484848484848e-06,
+ "loss": 1.4925904273986816,
+ "mean_token_accuracy": 0.6566316485404968,
+ "num_tokens": 9355264.0,
+ "step": 571
+ },
+ {
+ "entropy": 1.4476258754730225,
+ "epoch": 1.1555555555555554,
+ "grad_norm": 2.3435230255126953,
+ "learning_rate": 3.846464646464647e-06,
+ "loss": 1.438629388809204,
+ "mean_token_accuracy": 0.656020998954773,
+ "num_tokens": 9371648.0,
+ "step": 572
+ },
+ {
+ "entropy": 1.7930315732955933,
+ "epoch": 1.1575757575757575,
+ "grad_norm": 2.1700010299682617,
+ "learning_rate": 3.844444444444445e-06,
+ "loss": 1.786928653717041,
+ "mean_token_accuracy": 0.6028333902359009,
+ "num_tokens": 9388032.0,
+ "step": 573
+ },
+ {
+ "entropy": 1.3800512552261353,
+ "epoch": 1.1595959595959595,
+ "grad_norm": 2.2036688327789307,
+ "learning_rate": 3.842424242424243e-06,
+ "loss": 1.3767224550247192,
+ "mean_token_accuracy": 0.6692110300064087,
+ "num_tokens": 9404416.0,
+ "step": 574
+ },
+ {
+ "entropy": 0.8562329411506653,
+ "epoch": 1.1616161616161615,
+ "grad_norm": 1.765388011932373,
+ "learning_rate": 3.840404040404041e-06,
+ "loss": 0.8366047143936157,
+ "mean_token_accuracy": 0.783707857131958,
+ "num_tokens": 9420800.0,
+ "step": 575
+ },
+ {
+ "entropy": 1.4267486333847046,
+ "epoch": 1.1636363636363636,
+ "grad_norm": 2.0768239498138428,
+ "learning_rate": 3.8383838383838385e-06,
+ "loss": 1.4165449142456055,
+ "mean_token_accuracy": 0.656020998954773,
+ "num_tokens": 9437184.0,
+ "step": 576
+ },
+ {
+ "entropy": 1.4123388528823853,
+ "epoch": 1.1656565656565656,
+ "grad_norm": 2.028716802597046,
+ "learning_rate": 3.836363636363636e-06,
+ "loss": 1.420403003692627,
+ "mean_token_accuracy": 0.6729360222816467,
+ "num_tokens": 9453568.0,
+ "step": 577
+ },
+ {
+ "entropy": 1.1549286842346191,
+ "epoch": 1.1676767676767676,
+ "grad_norm": 2.1142919063568115,
+ "learning_rate": 3.834343434343435e-06,
+ "loss": 1.1487064361572266,
+ "mean_token_accuracy": 0.7040180563926697,
+ "num_tokens": 9469952.0,
+ "step": 578
+ },
+ {
+ "entropy": 1.4779953956604004,
+ "epoch": 1.1696969696969697,
+ "grad_norm": 2.166046142578125,
+ "learning_rate": 3.832323232323233e-06,
+ "loss": 1.4788683652877808,
+ "mean_token_accuracy": 0.6436858773231506,
+ "num_tokens": 9486336.0,
+ "step": 579
+ },
+ {
+ "entropy": 1.568039059638977,
+ "epoch": 1.1717171717171717,
+ "grad_norm": 2.067704200744629,
+ "learning_rate": 3.830303030303031e-06,
+ "loss": 1.5830929279327393,
+ "mean_token_accuracy": 0.6461895704269409,
+ "num_tokens": 9502720.0,
+ "step": 580
+ },
+ {
+ "entropy": 1.1785298585891724,
+ "epoch": 1.1737373737373737,
+ "grad_norm": 1.9789953231811523,
+ "learning_rate": 3.828282828282829e-06,
+ "loss": 1.2083481550216675,
+ "mean_token_accuracy": 0.7120786309242249,
+ "num_tokens": 9519104.0,
+ "step": 581
+ },
+ {
+ "entropy": 1.3783847093582153,
+ "epoch": 1.1757575757575758,
+ "grad_norm": 2.1503992080688477,
+ "learning_rate": 3.826262626262627e-06,
+ "loss": 1.3721052408218384,
+ "mean_token_accuracy": 0.6669516563415527,
+ "num_tokens": 9535488.0,
+ "step": 582
+ },
+ {
+ "entropy": 1.120093584060669,
+ "epoch": 1.1777777777777778,
+ "grad_norm": 1.9822942018508911,
+ "learning_rate": 3.8242424242424245e-06,
+ "loss": 1.1215628385543823,
+ "mean_token_accuracy": 0.7122618556022644,
+ "num_tokens": 9551872.0,
+ "step": 583
+ },
+ {
+ "entropy": 1.4621506929397583,
+ "epoch": 1.1797979797979798,
+ "grad_norm": 2.159489631652832,
+ "learning_rate": 3.8222222222222224e-06,
+ "loss": 1.4614722728729248,
+ "mean_token_accuracy": 0.6535173654556274,
+ "num_tokens": 9568256.0,
+ "step": 584
+ },
+ {
+ "entropy": 1.015797734260559,
+ "epoch": 1.1818181818181819,
+ "grad_norm": 1.846848726272583,
+ "learning_rate": 3.82020202020202e-06,
+ "loss": 1.0082862377166748,
+ "mean_token_accuracy": 0.7394968271255493,
+ "num_tokens": 9584640.0,
+ "step": 585
+ },
+ {
+ "entropy": 1.364890217781067,
+ "epoch": 1.183838383838384,
+ "grad_norm": 2.175546646118164,
+ "learning_rate": 3.818181818181819e-06,
+ "loss": 1.3496818542480469,
+ "mean_token_accuracy": 0.6665241718292236,
+ "num_tokens": 9601024.0,
+ "step": 586
+ },
+ {
+ "entropy": 1.6056870222091675,
+ "epoch": 1.185858585858586,
+ "grad_norm": 2.1069748401641846,
+ "learning_rate": 3.816161616161617e-06,
+ "loss": 1.6048622131347656,
+ "mean_token_accuracy": 0.642892062664032,
+ "num_tokens": 9617408.0,
+ "step": 587
+ },
+ {
+ "entropy": 1.415107250213623,
+ "epoch": 1.187878787878788,
+ "grad_norm": 2.184544324874878,
+ "learning_rate": 3.8141414141414144e-06,
+ "loss": 1.4305826425552368,
+ "mean_token_accuracy": 0.6619443297386169,
+ "num_tokens": 9633792.0,
+ "step": 588
+ },
+ {
+ "entropy": 1.6554296016693115,
+ "epoch": 1.1898989898989898,
+ "grad_norm": 2.109793186187744,
+ "learning_rate": 3.8121212121212127e-06,
+ "loss": 1.6850776672363281,
+ "mean_token_accuracy": 0.6116267442703247,
+ "num_tokens": 9650176.0,
+ "step": 589
+ },
+ {
+ "entropy": 1.681472659111023,
+ "epoch": 1.1919191919191918,
+ "grad_norm": 2.3179714679718018,
+ "learning_rate": 3.8101010101010106e-06,
+ "loss": 1.699328064918518,
+ "mean_token_accuracy": 0.6105275750160217,
+ "num_tokens": 9666560.0,
+ "step": 590
+ },
+ {
+ "entropy": 1.113366961479187,
+ "epoch": 1.1939393939393939,
+ "grad_norm": 1.9885324239730835,
+ "learning_rate": 3.8080808080808085e-06,
+ "loss": 1.1357836723327637,
+ "mean_token_accuracy": 0.7157425284385681,
+ "num_tokens": 9682944.0,
+ "step": 591
+ },
+ {
+ "entropy": 1.1010088920593262,
+ "epoch": 1.195959595959596,
+ "grad_norm": 1.9745922088623047,
+ "learning_rate": 3.8060606060606064e-06,
+ "loss": 1.094293475151062,
+ "mean_token_accuracy": 0.7214215993881226,
+ "num_tokens": 9699328.0,
+ "step": 592
+ },
+ {
+ "entropy": 1.562911033630371,
+ "epoch": 1.197979797979798,
+ "grad_norm": 2.0526134967803955,
+ "learning_rate": 3.8040404040404043e-06,
+ "loss": 1.5747783184051514,
+ "mean_token_accuracy": 0.6342818737030029,
+ "num_tokens": 9715712.0,
+ "step": 593
+ },
+ {
+ "entropy": 1.6541608572006226,
+ "epoch": 1.2,
+ "grad_norm": 2.1268227100372314,
+ "learning_rate": 3.8020202020202026e-06,
+ "loss": 1.696983814239502,
+ "mean_token_accuracy": 0.6127259135246277,
+ "num_tokens": 9732096.0,
+ "step": 594
+ },
+ {
+ "entropy": 1.5259482860565186,
+ "epoch": 1.202020202020202,
+ "grad_norm": 2.240692377090454,
+ "learning_rate": 3.8000000000000005e-06,
+ "loss": 1.5582221746444702,
+ "mean_token_accuracy": 0.644052267074585,
+ "num_tokens": 9748480.0,
+ "step": 595
+ },
+ {
+ "entropy": 1.455372929573059,
+ "epoch": 1.204040404040404,
+ "grad_norm": 2.0934698581695557,
+ "learning_rate": 3.7979797979797984e-06,
+ "loss": 1.464172124862671,
+ "mean_token_accuracy": 0.6555935740470886,
+ "num_tokens": 9764864.0,
+ "step": 596
+ },
+ {
+ "entropy": 1.5074836015701294,
+ "epoch": 1.206060606060606,
+ "grad_norm": 1.9779568910598755,
+ "learning_rate": 3.7959595959595962e-06,
+ "loss": 1.5280466079711914,
+ "mean_token_accuracy": 0.6460063457489014,
+ "num_tokens": 9781248.0,
+ "step": 597
+ },
+ {
+ "entropy": 1.3286548852920532,
+ "epoch": 1.208080808080808,
+ "grad_norm": 1.9357717037200928,
+ "learning_rate": 3.793939393939394e-06,
+ "loss": 1.3448848724365234,
+ "mean_token_accuracy": 0.6816072463989258,
+ "num_tokens": 9797632.0,
+ "step": 598
+ },
+ {
+ "entropy": 1.5161375999450684,
+ "epoch": 1.2101010101010101,
+ "grad_norm": 2.224217414855957,
+ "learning_rate": 3.791919191919192e-06,
+ "loss": 1.5168235301971436,
+ "mean_token_accuracy": 0.6453346610069275,
+ "num_tokens": 9814016.0,
+ "step": 599
+ },
+ {
+ "entropy": 1.3337587118148804,
+ "epoch": 1.2121212121212122,
+ "grad_norm": 1.9308290481567383,
+ "learning_rate": 3.7898989898989903e-06,
+ "loss": 1.3429791927337646,
+ "mean_token_accuracy": 0.6827064156532288,
+ "num_tokens": 9830400.0,
+ "step": 600
+ },
+ {
+ "epoch": 1.2121212121212122,
+ "eval_entropy": 1.4592116455877981,
+ "eval_loss": 1.5424447059631348,
+ "eval_mean_token_accuracy": 0.6431668299821115,
+ "eval_num_tokens": 9830400.0,
+ "eval_runtime": 43.8289,
+ "eval_samples_per_second": 22.588,
+ "eval_steps_per_second": 2.829,
+ "step": 600
+ },
+ {
+ "entropy": 1.6697322130203247,
+ "epoch": 1.2141414141414142,
+ "grad_norm": 1.9462894201278687,
+ "learning_rate": 3.7878787878787882e-06,
+ "loss": 1.6434354782104492,
+ "mean_token_accuracy": 0.635808527469635,
+ "num_tokens": 9846784.0,
+ "step": 601
+ },
+ {
+ "entropy": 1.427184820175171,
+ "epoch": 1.2161616161616162,
+ "grad_norm": 2.196017026901245,
+ "learning_rate": 3.785858585858586e-06,
+ "loss": 1.4024749994277954,
+ "mean_token_accuracy": 0.6720200181007385,
+ "num_tokens": 9863168.0,
+ "step": 602
+ },
+ {
+ "entropy": 1.4623222351074219,
+ "epoch": 1.2181818181818183,
+ "grad_norm": 2.160585403442383,
+ "learning_rate": 3.783838383838384e-06,
+ "loss": 1.4455416202545166,
+ "mean_token_accuracy": 0.6557767391204834,
+ "num_tokens": 9879552.0,
+ "step": 603
+ },
+ {
+ "entropy": 1.348331093788147,
+ "epoch": 1.2202020202020203,
+ "grad_norm": 2.079458713531494,
+ "learning_rate": 3.781818181818182e-06,
+ "loss": 1.3672473430633545,
+ "mean_token_accuracy": 0.6834391951560974,
+ "num_tokens": 9895936.0,
+ "step": 604
+ },
+ {
+ "entropy": 1.362277865409851,
+ "epoch": 1.2222222222222223,
+ "grad_norm": 2.10718035697937,
+ "learning_rate": 3.77979797979798e-06,
+ "loss": 1.3567216396331787,
+ "mean_token_accuracy": 0.6659135222434998,
+ "num_tokens": 9912320.0,
+ "step": 605
+ },
+ {
+ "entropy": 1.2743226289749146,
+ "epoch": 1.2242424242424241,
+ "grad_norm": 2.116549491882324,
+ "learning_rate": 3.777777777777778e-06,
+ "loss": 1.303470253944397,
+ "mean_token_accuracy": 0.6791035532951355,
+ "num_tokens": 9928704.0,
+ "step": 606
+ },
+ {
+ "entropy": 1.079649567604065,
+ "epoch": 1.2262626262626264,
+ "grad_norm": 1.8429855108261108,
+ "learning_rate": 3.775757575757576e-06,
+ "loss": 1.0616915225982666,
+ "mean_token_accuracy": 0.7321690320968628,
+ "num_tokens": 9945088.0,
+ "step": 607
+ },
+ {
+ "entropy": 1.363136649131775,
+ "epoch": 1.2282828282828282,
+ "grad_norm": 2.171295166015625,
+ "learning_rate": 3.773737373737374e-06,
+ "loss": 1.3525331020355225,
+ "mean_token_accuracy": 0.6732413172721863,
+ "num_tokens": 9961472.0,
+ "step": 608
+ },
+ {
+ "entropy": 1.324924349784851,
+ "epoch": 1.2303030303030302,
+ "grad_norm": 2.0872693061828613,
+ "learning_rate": 3.7717171717171717e-06,
+ "loss": 1.3282644748687744,
+ "mean_token_accuracy": 0.674462616443634,
+ "num_tokens": 9977856.0,
+ "step": 609
+ },
+ {
+ "entropy": 1.0672967433929443,
+ "epoch": 1.2323232323232323,
+ "grad_norm": 2.0819947719573975,
+ "learning_rate": 3.76969696969697e-06,
+ "loss": 1.0929367542266846,
+ "mean_token_accuracy": 0.715254008769989,
+ "num_tokens": 9994240.0,
+ "step": 610
+ },
+ {
+ "entropy": 1.4465869665145874,
+ "epoch": 1.2343434343434343,
+ "grad_norm": 2.1786868572235107,
+ "learning_rate": 3.767676767676768e-06,
+ "loss": 1.436307668685913,
+ "mean_token_accuracy": 0.6642037034034729,
+ "num_tokens": 10010624.0,
+ "step": 611
+ },
+ {
+ "entropy": 1.6424144506454468,
+ "epoch": 1.2363636363636363,
+ "grad_norm": 2.2582032680511475,
+ "learning_rate": 3.765656565656566e-06,
+ "loss": 1.6621832847595215,
+ "mean_token_accuracy": 0.6218246221542358,
+ "num_tokens": 10027008.0,
+ "step": 612
+ },
+ {
+ "entropy": 1.546712875366211,
+ "epoch": 1.2383838383838384,
+ "grad_norm": 2.1685879230499268,
+ "learning_rate": 3.7636363636363637e-06,
+ "loss": 1.5617464780807495,
+ "mean_token_accuracy": 0.6325110197067261,
+ "num_tokens": 10043392.0,
+ "step": 613
+ },
+ {
+ "entropy": 1.5758557319641113,
+ "epoch": 1.2404040404040404,
+ "grad_norm": 2.0911128520965576,
+ "learning_rate": 3.7616161616161616e-06,
+ "loss": 1.5830271244049072,
+ "mean_token_accuracy": 0.6349536180496216,
+ "num_tokens": 10059776.0,
+ "step": 614
+ },
+ {
+ "entropy": 1.5223995447158813,
+ "epoch": 1.2424242424242424,
+ "grad_norm": 2.0333359241485596,
+ "learning_rate": 3.7595959595959595e-06,
+ "loss": 1.5132982730865479,
+ "mean_token_accuracy": 0.6518075466156006,
+ "num_tokens": 10076160.0,
+ "step": 615
+ },
+ {
+ "entropy": 1.0950185060501099,
+ "epoch": 1.2444444444444445,
+ "grad_norm": 2.0048129558563232,
+ "learning_rate": 3.757575757575758e-06,
+ "loss": 1.0901896953582764,
+ "mean_token_accuracy": 0.723375678062439,
+ "num_tokens": 10092544.0,
+ "step": 616
+ },
+ {
+ "entropy": 1.340641736984253,
+ "epoch": 1.2464646464646465,
+ "grad_norm": 2.079256534576416,
+ "learning_rate": 3.7555555555555557e-06,
+ "loss": 1.3583415746688843,
+ "mean_token_accuracy": 0.671775758266449,
+ "num_tokens": 10108928.0,
+ "step": 617
+ },
+ {
+ "entropy": 1.1836222410202026,
+ "epoch": 1.2484848484848485,
+ "grad_norm": 1.9266347885131836,
+ "learning_rate": 3.7535353535353536e-06,
+ "loss": 1.171565294265747,
+ "mean_token_accuracy": 0.7046287059783936,
+ "num_tokens": 10125312.0,
+ "step": 618
+ },
+ {
+ "entropy": 1.6007641553878784,
+ "epoch": 1.2505050505050506,
+ "grad_norm": 2.25252103805542,
+ "learning_rate": 3.7515151515151515e-06,
+ "loss": 1.5955660343170166,
+ "mean_token_accuracy": 0.6306790709495544,
+ "num_tokens": 10141696.0,
+ "step": 619
+ },
+ {
+ "entropy": 1.8159959316253662,
+ "epoch": 1.2525252525252526,
+ "grad_norm": 2.255959987640381,
+ "learning_rate": 3.74949494949495e-06,
+ "loss": 1.8223116397857666,
+ "mean_token_accuracy": 0.5867122411727905,
+ "num_tokens": 10158080.0,
+ "step": 620
+ },
+ {
+ "entropy": 1.6469950675964355,
+ "epoch": 1.2545454545454544,
+ "grad_norm": 1.9183777570724487,
+ "learning_rate": 3.747474747474748e-06,
+ "loss": 1.6623587608337402,
+ "mean_token_accuracy": 0.6283586025238037,
+ "num_tokens": 10174464.0,
+ "step": 621
+ },
+ {
+ "entropy": 1.3507136106491089,
+ "epoch": 1.2565656565656567,
+ "grad_norm": 1.9393805265426636,
+ "learning_rate": 3.745454545454546e-06,
+ "loss": 1.3371413946151733,
+ "mean_token_accuracy": 0.6808744668960571,
+ "num_tokens": 10190848.0,
+ "step": 622
+ },
+ {
+ "entropy": 1.5176761150360107,
+ "epoch": 1.2585858585858585,
+ "grad_norm": 2.0721757411956787,
+ "learning_rate": 3.743434343434344e-06,
+ "loss": 1.5010406970977783,
+ "mean_token_accuracy": 0.6603566408157349,
+ "num_tokens": 10207232.0,
+ "step": 623
+ },
+ {
+ "entropy": 1.1409586668014526,
+ "epoch": 1.2606060606060607,
+ "grad_norm": 1.90805983543396,
+ "learning_rate": 3.7414141414141418e-06,
+ "loss": 1.1339551210403442,
+ "mean_token_accuracy": 0.7151318788528442,
+ "num_tokens": 10223616.0,
+ "step": 624
+ },
+ {
+ "entropy": 1.2258127927780151,
+ "epoch": 1.2626262626262625,
+ "grad_norm": 1.9276432991027832,
+ "learning_rate": 3.73939393939394e-06,
+ "loss": 1.2232249975204468,
+ "mean_token_accuracy": 0.7053614854812622,
+ "num_tokens": 10240000.0,
+ "step": 625
+ },
+ {
+ "entropy": 1.236358642578125,
+ "epoch": 1.2646464646464646,
+ "grad_norm": 2.0792832374572754,
+ "learning_rate": 3.737373737373738e-06,
+ "loss": 1.2570738792419434,
+ "mean_token_accuracy": 0.6869809627532959,
+ "num_tokens": 10256384.0,
+ "step": 626
+ },
+ {
+ "entropy": 1.0995185375213623,
+ "epoch": 1.2666666666666666,
+ "grad_norm": 2.035024881362915,
+ "learning_rate": 3.735353535353536e-06,
+ "loss": 1.10158109664917,
+ "mean_token_accuracy": 0.7248412370681763,
+ "num_tokens": 10272768.0,
+ "step": 627
+ },
+ {
+ "entropy": 1.8980706930160522,
+ "epoch": 1.2686868686868686,
+ "grad_norm": 2.040414333343506,
+ "learning_rate": 3.7333333333333337e-06,
+ "loss": 1.9260807037353516,
+ "mean_token_accuracy": 0.5953834652900696,
+ "num_tokens": 10289152.0,
+ "step": 628
+ },
+ {
+ "entropy": 1.4265168905258179,
+ "epoch": 1.2707070707070707,
+ "grad_norm": 2.418179750442505,
+ "learning_rate": 3.7313131313131316e-06,
+ "loss": 1.4415385723114014,
+ "mean_token_accuracy": 0.6535784006118774,
+ "num_tokens": 10305536.0,
+ "step": 629
+ },
+ {
+ "entropy": 1.217471957206726,
+ "epoch": 1.2727272727272727,
+ "grad_norm": 2.072441577911377,
+ "learning_rate": 3.72929292929293e-06,
+ "loss": 1.2361960411071777,
+ "mean_token_accuracy": 0.6947972774505615,
+ "num_tokens": 10321920.0,
+ "step": 630
+ },
+ {
+ "entropy": 1.0714137554168701,
+ "epoch": 1.2747474747474747,
+ "grad_norm": 3.0428647994995117,
+ "learning_rate": 3.727272727272728e-06,
+ "loss": 1.112711787223816,
+ "mean_token_accuracy": 0.7243527173995972,
+ "num_tokens": 10338304.0,
+ "step": 631
+ },
+ {
+ "entropy": 1.5873851776123047,
+ "epoch": 1.2767676767676768,
+ "grad_norm": 1.945098638534546,
+ "learning_rate": 3.7252525252525257e-06,
+ "loss": 1.595362663269043,
+ "mean_token_accuracy": 0.639594554901123,
+ "num_tokens": 10354688.0,
+ "step": 632
+ },
+ {
+ "entropy": 1.9688408374786377,
+ "epoch": 1.2787878787878788,
+ "grad_norm": 2.0450353622436523,
+ "learning_rate": 3.7232323232323236e-06,
+ "loss": 1.9989259243011475,
+ "mean_token_accuracy": 0.5685759782791138,
+ "num_tokens": 10371072.0,
+ "step": 633
+ },
+ {
+ "entropy": 1.2927459478378296,
+ "epoch": 1.2808080808080808,
+ "grad_norm": 1.959108591079712,
+ "learning_rate": 3.7212121212121215e-06,
+ "loss": 1.3025646209716797,
+ "mean_token_accuracy": 0.6853932738304138,
+ "num_tokens": 10387456.0,
+ "step": 634
+ },
+ {
+ "entropy": 1.6645994186401367,
+ "epoch": 1.2828282828282829,
+ "grad_norm": 2.041923999786377,
+ "learning_rate": 3.7191919191919194e-06,
+ "loss": 1.6959854364395142,
+ "mean_token_accuracy": 0.6319003701210022,
+ "num_tokens": 10403840.0,
+ "step": 635
+ },
+ {
+ "entropy": 1.401423692703247,
+ "epoch": 1.284848484848485,
+ "grad_norm": 2.040456533432007,
+ "learning_rate": 3.7171717171717177e-06,
+ "loss": 1.3859784603118896,
+ "mean_token_accuracy": 0.6740962266921997,
+ "num_tokens": 10420224.0,
+ "step": 636
+ },
+ {
+ "entropy": 1.0597256422042847,
+ "epoch": 1.286868686868687,
+ "grad_norm": 2.053690195083618,
+ "learning_rate": 3.7151515151515156e-06,
+ "loss": 1.052672266960144,
+ "mean_token_accuracy": 0.7290546894073486,
+ "num_tokens": 10436608.0,
+ "step": 637
+ },
+ {
+ "entropy": 1.280874252319336,
+ "epoch": 1.2888888888888888,
+ "grad_norm": 2.0566959381103516,
+ "learning_rate": 3.7131313131313135e-06,
+ "loss": 1.2883412837982178,
+ "mean_token_accuracy": 0.6822178959846497,
+ "num_tokens": 10452992.0,
+ "step": 638
+ },
+ {
+ "entropy": 1.299896240234375,
+ "epoch": 1.290909090909091,
+ "grad_norm": 1.8896070718765259,
+ "learning_rate": 3.7111111111111113e-06,
+ "loss": 1.3038103580474854,
+ "mean_token_accuracy": 0.6712872385978699,
+ "num_tokens": 10469376.0,
+ "step": 639
+ },
+ {
+ "entropy": 1.1713769435882568,
+ "epoch": 1.2929292929292928,
+ "grad_norm": 2.1309149265289307,
+ "learning_rate": 3.7090909090909092e-06,
+ "loss": 1.1899120807647705,
+ "mean_token_accuracy": 0.700537383556366,
+ "num_tokens": 10485760.0,
+ "step": 640
+ },
+ {
+ "entropy": 1.3056974411010742,
+ "epoch": 1.294949494949495,
+ "grad_norm": 1.9990391731262207,
+ "learning_rate": 3.7070707070707075e-06,
+ "loss": 1.3287105560302734,
+ "mean_token_accuracy": 0.6773326992988586,
+ "num_tokens": 10502144.0,
+ "step": 641
+ },
+ {
+ "entropy": 1.7352585792541504,
+ "epoch": 1.2969696969696969,
+ "grad_norm": 2.100043535232544,
+ "learning_rate": 3.7050505050505054e-06,
+ "loss": 1.7806944847106934,
+ "mean_token_accuracy": 0.60332190990448,
+ "num_tokens": 10518528.0,
+ "step": 642
+ },
+ {
+ "entropy": 1.3748377561569214,
+ "epoch": 1.298989898989899,
+ "grad_norm": 2.1280689239501953,
+ "learning_rate": 3.7030303030303033e-06,
+ "loss": 1.379891037940979,
+ "mean_token_accuracy": 0.6617611050605774,
+ "num_tokens": 10534912.0,
+ "step": 643
+ },
+ {
+ "entropy": 0.9991571307182312,
+ "epoch": 1.301010101010101,
+ "grad_norm": 1.7343134880065918,
+ "learning_rate": 3.701010101010101e-06,
+ "loss": 1.0082780122756958,
+ "mean_token_accuracy": 0.740229606628418,
+ "num_tokens": 10551296.0,
+ "step": 644
+ },
+ {
+ "entropy": 1.6313605308532715,
+ "epoch": 1.303030303030303,
+ "grad_norm": 2.216167449951172,
+ "learning_rate": 3.698989898989899e-06,
+ "loss": 1.6183781623840332,
+ "mean_token_accuracy": 0.6164509057998657,
+ "num_tokens": 10567680.0,
+ "step": 645
+ },
+ {
+ "entropy": 1.4174962043762207,
+ "epoch": 1.305050505050505,
+ "grad_norm": 2.0921854972839355,
+ "learning_rate": 3.6969696969696974e-06,
+ "loss": 1.4087945222854614,
+ "mean_token_accuracy": 0.6705544590950012,
+ "num_tokens": 10584064.0,
+ "step": 646
+ },
+ {
+ "entropy": 1.8302873373031616,
+ "epoch": 1.307070707070707,
+ "grad_norm": 2.0742204189300537,
+ "learning_rate": 3.6949494949494953e-06,
+ "loss": 1.8456642627716064,
+ "mean_token_accuracy": 0.5914142727851868,
+ "num_tokens": 10600448.0,
+ "step": 647
+ },
+ {
+ "entropy": 1.29628324508667,
+ "epoch": 1.309090909090909,
+ "grad_norm": 2.1446709632873535,
+ "learning_rate": 3.692929292929293e-06,
+ "loss": 1.3066375255584717,
+ "mean_token_accuracy": 0.6798363327980042,
+ "num_tokens": 10616832.0,
+ "step": 648
+ },
+ {
+ "entropy": 1.396378993988037,
+ "epoch": 1.3111111111111111,
+ "grad_norm": 1.9578899145126343,
+ "learning_rate": 3.690909090909091e-06,
+ "loss": 1.3943686485290527,
+ "mean_token_accuracy": 0.6772105693817139,
+ "num_tokens": 10633216.0,
+ "step": 649
+ },
+ {
+ "entropy": 1.6457722187042236,
+ "epoch": 1.3131313131313131,
+ "grad_norm": 1.9999926090240479,
+ "learning_rate": 3.688888888888889e-06,
+ "loss": 1.6266872882843018,
+ "mean_token_accuracy": 0.630923330783844,
+ "num_tokens": 10649600.0,
+ "step": 650
+ },
+ {
+ "epoch": 1.3131313131313131,
+ "eval_entropy": 1.456459879875183,
+ "eval_loss": 1.539737582206726,
+ "eval_mean_token_accuracy": 0.643713459853203,
+ "eval_num_tokens": 10649600.0,
+ "eval_runtime": 43.7637,
+ "eval_samples_per_second": 22.621,
+ "eval_steps_per_second": 2.833,
+ "step": 650
+ },
+ {
+ "entropy": 1.1152859926223755,
+ "epoch": 1.3151515151515152,
+ "grad_norm": 2.9232723712921143,
+ "learning_rate": 3.686868686868687e-06,
+ "loss": 1.099653959274292,
+ "mean_token_accuracy": 0.7227039337158203,
+ "num_tokens": 10665984.0,
+ "step": 651
+ },
+ {
+ "entropy": 1.6141386032104492,
+ "epoch": 1.3171717171717172,
+ "grad_norm": 2.1557929515838623,
+ "learning_rate": 3.684848484848485e-06,
+ "loss": 1.6156055927276611,
+ "mean_token_accuracy": 0.6288471221923828,
+ "num_tokens": 10682368.0,
+ "step": 652
+ },
+ {
+ "entropy": 1.282158374786377,
+ "epoch": 1.3191919191919192,
+ "grad_norm": 2.061983346939087,
+ "learning_rate": 3.682828282828283e-06,
+ "loss": 1.2867733240127563,
+ "mean_token_accuracy": 0.6869198679924011,
+ "num_tokens": 10698752.0,
+ "step": 653
+ },
+ {
+ "entropy": 0.9986366033554077,
+ "epoch": 1.3212121212121213,
+ "grad_norm": 2.0446274280548096,
+ "learning_rate": 3.680808080808081e-06,
+ "loss": 1.019629716873169,
+ "mean_token_accuracy": 0.7427943348884583,
+ "num_tokens": 10715136.0,
+ "step": 654
+ },
+ {
+ "entropy": 1.4256011247634888,
+ "epoch": 1.3232323232323233,
+ "grad_norm": 2.1024749279022217,
+ "learning_rate": 3.678787878787879e-06,
+ "loss": 1.4269766807556152,
+ "mean_token_accuracy": 0.6674401760101318,
+ "num_tokens": 10731520.0,
+ "step": 655
+ },
+ {
+ "entropy": 1.4913274049758911,
+ "epoch": 1.3252525252525253,
+ "grad_norm": 2.068432569503784,
+ "learning_rate": 3.6767676767676767e-06,
+ "loss": 1.4930779933929443,
+ "mean_token_accuracy": 0.6474719047546387,
+ "num_tokens": 10747904.0,
+ "step": 656
+ },
+ {
+ "entropy": 1.2957285642623901,
+ "epoch": 1.3272727272727272,
+ "grad_norm": 2.2480008602142334,
+ "learning_rate": 3.674747474747475e-06,
+ "loss": 1.3001320362091064,
+ "mean_token_accuracy": 0.679286777973175,
+ "num_tokens": 10764288.0,
+ "step": 657
+ },
+ {
+ "entropy": 1.328946828842163,
+ "epoch": 1.3292929292929294,
+ "grad_norm": 2.311176300048828,
+ "learning_rate": 3.672727272727273e-06,
+ "loss": 1.3286024332046509,
+ "mean_token_accuracy": 0.662432849407196,
+ "num_tokens": 10780672.0,
+ "step": 658
+ },
+ {
+ "entropy": 1.458139419555664,
+ "epoch": 1.3313131313131312,
+ "grad_norm": 2.1967899799346924,
+ "learning_rate": 3.670707070707071e-06,
+ "loss": 1.4384279251098633,
+ "mean_token_accuracy": 0.6435637474060059,
+ "num_tokens": 10797056.0,
+ "step": 659
+ },
+ {
+ "entropy": 1.43666672706604,
+ "epoch": 1.3333333333333333,
+ "grad_norm": 2.163083076477051,
+ "learning_rate": 3.6686868686868687e-06,
+ "loss": 1.4396371841430664,
+ "mean_token_accuracy": 0.6531509757041931,
+ "num_tokens": 10813440.0,
+ "step": 660
+ },
+ {
+ "entropy": 1.6210501194000244,
+ "epoch": 1.3353535353535353,
+ "grad_norm": 2.3601770401000977,
+ "learning_rate": 3.6666666666666666e-06,
+ "loss": 1.6370227336883545,
+ "mean_token_accuracy": 0.6149242520332336,
+ "num_tokens": 10829824.0,
+ "step": 661
+ },
+ {
+ "entropy": 0.9602832794189453,
+ "epoch": 1.3373737373737373,
+ "grad_norm": 1.8452314138412476,
+ "learning_rate": 3.664646464646465e-06,
+ "loss": 0.9445997476577759,
+ "mean_token_accuracy": 0.759709358215332,
+ "num_tokens": 10846208.0,
+ "step": 662
+ },
+ {
+ "entropy": 1.2215200662612915,
+ "epoch": 1.3393939393939394,
+ "grad_norm": 2.019989252090454,
+ "learning_rate": 3.662626262626263e-06,
+ "loss": 1.2451637983322144,
+ "mean_token_accuracy": 0.7051172256469727,
+ "num_tokens": 10862592.0,
+ "step": 663
+ },
+ {
+ "entropy": 1.3253310918807983,
+ "epoch": 1.3414141414141414,
+ "grad_norm": 1.9700425863265991,
+ "learning_rate": 3.660606060606061e-06,
+ "loss": 1.3407385349273682,
+ "mean_token_accuracy": 0.6988885998725891,
+ "num_tokens": 10878976.0,
+ "step": 664
+ },
+ {
+ "entropy": 1.1397737264633179,
+ "epoch": 1.3434343434343434,
+ "grad_norm": 2.1194040775299072,
+ "learning_rate": 3.658585858585859e-06,
+ "loss": 1.1545679569244385,
+ "mean_token_accuracy": 0.7071323990821838,
+ "num_tokens": 10895360.0,
+ "step": 665
+ },
+ {
+ "entropy": 1.1714493036270142,
+ "epoch": 1.3454545454545455,
+ "grad_norm": 2.078051805496216,
+ "learning_rate": 3.6565656565656573e-06,
+ "loss": 1.1751537322998047,
+ "mean_token_accuracy": 0.6963238716125488,
+ "num_tokens": 10911744.0,
+ "step": 666
+ },
+ {
+ "entropy": 1.2954860925674438,
+ "epoch": 1.3474747474747475,
+ "grad_norm": 1.9348593950271606,
+ "learning_rate": 3.654545454545455e-06,
+ "loss": 1.2961801290512085,
+ "mean_token_accuracy": 0.6831338405609131,
+ "num_tokens": 10928128.0,
+ "step": 667
+ },
+ {
+ "entropy": 1.401628851890564,
+ "epoch": 1.3494949494949495,
+ "grad_norm": 2.1757614612579346,
+ "learning_rate": 3.652525252525253e-06,
+ "loss": 1.4025013446807861,
+ "mean_token_accuracy": 0.6590132117271423,
+ "num_tokens": 10944512.0,
+ "step": 668
+ },
+ {
+ "entropy": 1.4164557456970215,
+ "epoch": 1.3515151515151516,
+ "grad_norm": 2.0192055702209473,
+ "learning_rate": 3.650505050505051e-06,
+ "loss": 1.43379545211792,
+ "mean_token_accuracy": 0.6554714441299438,
+ "num_tokens": 10960896.0,
+ "step": 669
+ },
+ {
+ "entropy": 1.4888815879821777,
+ "epoch": 1.3535353535353536,
+ "grad_norm": 2.2166643142700195,
+ "learning_rate": 3.648484848484849e-06,
+ "loss": 1.494372844696045,
+ "mean_token_accuracy": 0.6402052044868469,
+ "num_tokens": 10977280.0,
+ "step": 670
+ },
+ {
+ "entropy": 1.215625286102295,
+ "epoch": 1.3555555555555556,
+ "grad_norm": 1.8059836626052856,
+ "learning_rate": 3.6464646464646467e-06,
+ "loss": 1.2074902057647705,
+ "mean_token_accuracy": 0.7123228907585144,
+ "num_tokens": 10993664.0,
+ "step": 671
+ },
+ {
+ "entropy": 0.9577685594558716,
+ "epoch": 1.3575757575757577,
+ "grad_norm": 1.8803631067276,
+ "learning_rate": 3.644444444444445e-06,
+ "loss": 0.956764817237854,
+ "mean_token_accuracy": 0.759709358215332,
+ "num_tokens": 11010048.0,
+ "step": 672
+ },
+ {
+ "entropy": 1.7215807437896729,
+ "epoch": 1.3595959595959597,
+ "grad_norm": 2.1421561241149902,
+ "learning_rate": 3.642424242424243e-06,
+ "loss": 1.7220706939697266,
+ "mean_token_accuracy": 0.6187102794647217,
+ "num_tokens": 11026432.0,
+ "step": 673
+ },
+ {
+ "entropy": 1.057277798652649,
+ "epoch": 1.3616161616161615,
+ "grad_norm": 1.9014254808425903,
+ "learning_rate": 3.640404040404041e-06,
+ "loss": 1.0582892894744873,
+ "mean_token_accuracy": 0.7327185869216919,
+ "num_tokens": 11042816.0,
+ "step": 674
+ },
+ {
+ "entropy": 1.220097541809082,
+ "epoch": 1.3636363636363638,
+ "grad_norm": 2.1406402587890625,
+ "learning_rate": 3.6383838383838387e-06,
+ "loss": 1.2152808904647827,
+ "mean_token_accuracy": 0.6976673007011414,
+ "num_tokens": 11059200.0,
+ "step": 675
+ },
+ {
+ "entropy": 1.4171489477157593,
+ "epoch": 1.3656565656565656,
+ "grad_norm": 2.1809890270233154,
+ "learning_rate": 3.6363636363636366e-06,
+ "loss": 1.4846035242080688,
+ "mean_token_accuracy": 0.6681118607521057,
+ "num_tokens": 11075584.0,
+ "step": 676
+ },
+ {
+ "entropy": 1.2915124893188477,
+ "epoch": 1.3676767676767676,
+ "grad_norm": 2.0533218383789062,
+ "learning_rate": 3.634343434343435e-06,
+ "loss": 1.3094089031219482,
+ "mean_token_accuracy": 0.6841719746589661,
+ "num_tokens": 11091968.0,
+ "step": 677
+ },
+ {
+ "entropy": 1.4717603921890259,
+ "epoch": 1.3696969696969696,
+ "grad_norm": 2.125450849533081,
+ "learning_rate": 3.6323232323232328e-06,
+ "loss": 1.479896903038025,
+ "mean_token_accuracy": 0.6601734161376953,
+ "num_tokens": 11108352.0,
+ "step": 678
+ },
+ {
+ "entropy": 1.0841981172561646,
+ "epoch": 1.3717171717171717,
+ "grad_norm": 1.936905026435852,
+ "learning_rate": 3.6303030303030307e-06,
+ "loss": 1.0895963907241821,
+ "mean_token_accuracy": 0.7307645082473755,
+ "num_tokens": 11124736.0,
+ "step": 679
+ },
+ {
+ "entropy": 0.85239177942276,
+ "epoch": 1.3737373737373737,
+ "grad_norm": 1.7117112874984741,
+ "learning_rate": 3.6282828282828286e-06,
+ "loss": 0.8716775178909302,
+ "mean_token_accuracy": 0.7721666097640991,
+ "num_tokens": 11141120.0,
+ "step": 680
+ },
+ {
+ "entropy": 1.4737738370895386,
+ "epoch": 1.3757575757575757,
+ "grad_norm": 2.0140671730041504,
+ "learning_rate": 3.6262626262626264e-06,
+ "loss": 1.5029723644256592,
+ "mean_token_accuracy": 0.653822660446167,
+ "num_tokens": 11157504.0,
+ "step": 681
+ },
+ {
+ "entropy": 1.260698914527893,
+ "epoch": 1.3777777777777778,
+ "grad_norm": 1.9868851900100708,
+ "learning_rate": 3.6242424242424248e-06,
+ "loss": 1.256290316581726,
+ "mean_token_accuracy": 0.6891182065010071,
+ "num_tokens": 11173888.0,
+ "step": 682
+ },
+ {
+ "entropy": 1.7418453693389893,
+ "epoch": 1.3797979797979798,
+ "grad_norm": 2.0905439853668213,
+ "learning_rate": 3.6222222222222226e-06,
+ "loss": 1.7568984031677246,
+ "mean_token_accuracy": 0.6291524171829224,
+ "num_tokens": 11190272.0,
+ "step": 683
+ },
+ {
+ "entropy": 1.5552523136138916,
+ "epoch": 1.3818181818181818,
+ "grad_norm": 2.2071683406829834,
+ "learning_rate": 3.6202020202020205e-06,
+ "loss": 1.555542230606079,
+ "mean_token_accuracy": 0.6403883695602417,
+ "num_tokens": 11206656.0,
+ "step": 684
+ },
+ {
+ "entropy": 1.4404902458190918,
+ "epoch": 1.3838383838383839,
+ "grad_norm": 2.276245594024658,
+ "learning_rate": 3.6181818181818184e-06,
+ "loss": 1.4155148267745972,
+ "mean_token_accuracy": 0.6630434989929199,
+ "num_tokens": 11223040.0,
+ "step": 685
+ },
+ {
+ "entropy": 1.1282018423080444,
+ "epoch": 1.385858585858586,
+ "grad_norm": 1.9824057817459106,
+ "learning_rate": 3.6161616161616163e-06,
+ "loss": 1.099952220916748,
+ "mean_token_accuracy": 0.7234978079795837,
+ "num_tokens": 11239424.0,
+ "step": 686
+ },
+ {
+ "entropy": 1.5559885501861572,
+ "epoch": 1.387878787878788,
+ "grad_norm": 2.243121862411499,
+ "learning_rate": 3.614141414141414e-06,
+ "loss": 1.520850658416748,
+ "mean_token_accuracy": 0.6340987086296082,
+ "num_tokens": 11255808.0,
+ "step": 687
+ },
+ {
+ "entropy": 1.322407841682434,
+ "epoch": 1.38989898989899,
+ "grad_norm": 2.049233913421631,
+ "learning_rate": 3.6121212121212125e-06,
+ "loss": 1.322263479232788,
+ "mean_token_accuracy": 0.6778212189674377,
+ "num_tokens": 11272192.0,
+ "step": 688
+ },
+ {
+ "entropy": 1.3112859725952148,
+ "epoch": 1.391919191919192,
+ "grad_norm": 2.0666465759277344,
+ "learning_rate": 3.6101010101010104e-06,
+ "loss": 1.3103243112564087,
+ "mean_token_accuracy": 0.6828285455703735,
+ "num_tokens": 11288576.0,
+ "step": 689
+ },
+ {
+ "entropy": 1.3321104049682617,
+ "epoch": 1.393939393939394,
+ "grad_norm": 2.5464279651641846,
+ "learning_rate": 3.6080808080808083e-06,
+ "loss": 1.3247697353363037,
+ "mean_token_accuracy": 0.6785539984703064,
+ "num_tokens": 11304960.0,
+ "step": 690
+ },
+ {
+ "entropy": 1.0779680013656616,
+ "epoch": 1.3959595959595958,
+ "grad_norm": 2.1386117935180664,
+ "learning_rate": 3.606060606060606e-06,
+ "loss": 1.0994013547897339,
+ "mean_token_accuracy": 0.7129946351051331,
+ "num_tokens": 11321344.0,
+ "step": 691
+ },
+ {
+ "entropy": 1.0701864957809448,
+ "epoch": 1.397979797979798,
+ "grad_norm": 1.9387527704238892,
+ "learning_rate": 3.604040404040404e-06,
+ "loss": 1.0633184909820557,
+ "mean_token_accuracy": 0.7378480434417725,
+ "num_tokens": 11337728.0,
+ "step": 692
+ },
+ {
+ "entropy": 1.2258752584457397,
+ "epoch": 1.4,
+ "grad_norm": 2.0486111640930176,
+ "learning_rate": 3.6020202020202024e-06,
+ "loss": 1.2098207473754883,
+ "mean_token_accuracy": 0.6999877691268921,
+ "num_tokens": 11354112.0,
+ "step": 693
+ },
+ {
+ "entropy": 1.4652774333953857,
+ "epoch": 1.402020202020202,
+ "grad_norm": 2.043079376220703,
+ "learning_rate": 3.6000000000000003e-06,
+ "loss": 1.4780974388122559,
+ "mean_token_accuracy": 0.6681729555130005,
+ "num_tokens": 11370496.0,
+ "step": 694
+ },
+ {
+ "entropy": 1.3215711116790771,
+ "epoch": 1.404040404040404,
+ "grad_norm": 2.1327438354492188,
+ "learning_rate": 3.597979797979798e-06,
+ "loss": 1.3116662502288818,
+ "mean_token_accuracy": 0.6757450103759766,
+ "num_tokens": 11386880.0,
+ "step": 695
+ },
+ {
+ "entropy": 1.624266266822815,
+ "epoch": 1.406060606060606,
+ "grad_norm": 2.114398717880249,
+ "learning_rate": 3.595959595959596e-06,
+ "loss": 1.628610372543335,
+ "mean_token_accuracy": 0.6487542986869812,
+ "num_tokens": 11403264.0,
+ "step": 696
+ },
+ {
+ "entropy": 1.6059489250183105,
+ "epoch": 1.408080808080808,
+ "grad_norm": 1.952991008758545,
+ "learning_rate": 3.593939393939394e-06,
+ "loss": 1.6145906448364258,
+ "mean_token_accuracy": 0.6257938742637634,
+ "num_tokens": 11419648.0,
+ "step": 697
+ },
+ {
+ "entropy": 1.0662391185760498,
+ "epoch": 1.41010101010101,
+ "grad_norm": 2.0804851055145264,
+ "learning_rate": 3.5919191919191922e-06,
+ "loss": 1.0772000551223755,
+ "mean_token_accuracy": 0.7187347412109375,
+ "num_tokens": 11436032.0,
+ "step": 698
+ },
+ {
+ "entropy": 1.221283197402954,
+ "epoch": 1.412121212121212,
+ "grad_norm": 1.9502744674682617,
+ "learning_rate": 3.58989898989899e-06,
+ "loss": 1.2352337837219238,
+ "mean_token_accuracy": 0.6918050646781921,
+ "num_tokens": 11452416.0,
+ "step": 699
+ },
+ {
+ "entropy": 1.4205158948898315,
+ "epoch": 1.4141414141414141,
+ "grad_norm": 2.2222518920898438,
+ "learning_rate": 3.587878787878788e-06,
+ "loss": 1.451093077659607,
+ "mean_token_accuracy": 0.6648754477500916,
+ "num_tokens": 11468800.0,
+ "step": 700
+ },
+ {
+ "epoch": 1.4141414141414141,
+ "eval_entropy": 1.4370074204860195,
+ "eval_loss": 1.5383458137512207,
+ "eval_mean_token_accuracy": 0.6441497793120723,
+ "eval_num_tokens": 11468800.0,
+ "eval_runtime": 43.744,
+ "eval_samples_per_second": 22.632,
+ "eval_steps_per_second": 2.835,
+ "step": 700
+ },
+ {
+ "entropy": 1.5432499647140503,
+ "epoch": 1.4161616161616162,
+ "grad_norm": 2.084463596343994,
+ "learning_rate": 3.585858585858586e-06,
+ "loss": 1.5958356857299805,
+ "mean_token_accuracy": 0.6377015113830566,
+ "num_tokens": 11485184.0,
+ "step": 701
+ },
+ {
+ "entropy": 1.297119379043579,
+ "epoch": 1.4181818181818182,
+ "grad_norm": 2.1996524333953857,
+ "learning_rate": 3.5838383838383838e-06,
+ "loss": 1.3346054553985596,
+ "mean_token_accuracy": 0.6801416873931885,
+ "num_tokens": 11501568.0,
+ "step": 702
+ },
+ {
+ "entropy": 1.3905766010284424,
+ "epoch": 1.4202020202020202,
+ "grad_norm": 2.1683545112609863,
+ "learning_rate": 3.5818181818181817e-06,
+ "loss": 1.40826416015625,
+ "mean_token_accuracy": 0.6653639674186707,
+ "num_tokens": 11517952.0,
+ "step": 703
+ },
+ {
+ "entropy": 0.9598695635795593,
+ "epoch": 1.4222222222222223,
+ "grad_norm": 1.9240537881851196,
+ "learning_rate": 3.57979797979798e-06,
+ "loss": 0.9487459659576416,
+ "mean_token_accuracy": 0.7556179761886597,
+ "num_tokens": 11534336.0,
+ "step": 704
+ },
+ {
+ "entropy": 1.2008687257766724,
+ "epoch": 1.4242424242424243,
+ "grad_norm": 1.9687083959579468,
+ "learning_rate": 3.577777777777778e-06,
+ "loss": 1.2234617471694946,
+ "mean_token_accuracy": 0.6982779502868652,
+ "num_tokens": 11550720.0,
+ "step": 705
+ },
+ {
+ "entropy": 1.1168856620788574,
+ "epoch": 1.4262626262626263,
+ "grad_norm": 2.1796388626098633,
+ "learning_rate": 3.575757575757576e-06,
+ "loss": 1.1320589780807495,
+ "mean_token_accuracy": 0.7057889699935913,
+ "num_tokens": 11567104.0,
+ "step": 706
+ },
+ {
+ "entropy": 1.3408682346343994,
+ "epoch": 1.4282828282828284,
+ "grad_norm": 1.9048689603805542,
+ "learning_rate": 3.573737373737374e-06,
+ "loss": 1.3476686477661133,
+ "mean_token_accuracy": 0.6839887499809265,
+ "num_tokens": 11583488.0,
+ "step": 707
+ },
+ {
+ "entropy": 1.4392427206039429,
+ "epoch": 1.4303030303030302,
+ "grad_norm": 2.183016777038574,
+ "learning_rate": 3.5717171717171724e-06,
+ "loss": 1.4443378448486328,
+ "mean_token_accuracy": 0.6546775698661804,
+ "num_tokens": 11599872.0,
+ "step": 708
+ },
+ {
+ "entropy": 1.1428641080856323,
+ "epoch": 1.4323232323232324,
+ "grad_norm": 2.0477187633514404,
+ "learning_rate": 3.5696969696969703e-06,
+ "loss": 1.1438124179840088,
+ "mean_token_accuracy": 0.7126282453536987,
+ "num_tokens": 11616256.0,
+ "step": 709
+ },
+ {
+ "entropy": 1.6266967058181763,
+ "epoch": 1.4343434343434343,
+ "grad_norm": 2.322098731994629,
+ "learning_rate": 3.567676767676768e-06,
+ "loss": 1.6161916255950928,
+ "mean_token_accuracy": 0.6202979683876038,
+ "num_tokens": 11632640.0,
+ "step": 710
+ },
+ {
+ "entropy": 1.349686622619629,
+ "epoch": 1.4363636363636363,
+ "grad_norm": 2.021871328353882,
+ "learning_rate": 3.565656565656566e-06,
+ "loss": 1.3713490962982178,
+ "mean_token_accuracy": 0.6596238613128662,
+ "num_tokens": 11649024.0,
+ "step": 711
+ },
+ {
+ "entropy": 1.4476866722106934,
+ "epoch": 1.4383838383838383,
+ "grad_norm": 2.194718837738037,
+ "learning_rate": 3.563636363636364e-06,
+ "loss": 1.4593651294708252,
+ "mean_token_accuracy": 0.6476551294326782,
+ "num_tokens": 11665408.0,
+ "step": 712
+ },
+ {
+ "entropy": 1.625157356262207,
+ "epoch": 1.4404040404040404,
+ "grad_norm": 2.0966169834136963,
+ "learning_rate": 3.5616161616161622e-06,
+ "loss": 1.6374845504760742,
+ "mean_token_accuracy": 0.6293356418609619,
+ "num_tokens": 11681792.0,
+ "step": 713
+ },
+ {
+ "entropy": 1.985729455947876,
+ "epoch": 1.4424242424242424,
+ "grad_norm": 2.129556655883789,
+ "learning_rate": 3.55959595959596e-06,
+ "loss": 2.0454561710357666,
+ "mean_token_accuracy": 0.5728505253791809,
+ "num_tokens": 11698176.0,
+ "step": 714
+ },
+ {
+ "entropy": 1.546321153640747,
+ "epoch": 1.4444444444444444,
+ "grad_norm": 2.1565628051757812,
+ "learning_rate": 3.557575757575758e-06,
+ "loss": 1.5467555522918701,
+ "mean_token_accuracy": 0.6351978778839111,
+ "num_tokens": 11714560.0,
+ "step": 715
+ },
+ {
+ "entropy": 1.3690776824951172,
+ "epoch": 1.4464646464646465,
+ "grad_norm": 2.206984519958496,
+ "learning_rate": 3.555555555555556e-06,
+ "loss": 1.3877894878387451,
+ "mean_token_accuracy": 0.6911333799362183,
+ "num_tokens": 11730944.0,
+ "step": 716
+ },
+ {
+ "entropy": 1.3643333911895752,
+ "epoch": 1.4484848484848485,
+ "grad_norm": 2.0049662590026855,
+ "learning_rate": 3.553535353535354e-06,
+ "loss": 1.364513635635376,
+ "mean_token_accuracy": 0.6781876087188721,
+ "num_tokens": 11747328.0,
+ "step": 717
+ },
+ {
+ "entropy": 1.337018370628357,
+ "epoch": 1.4505050505050505,
+ "grad_norm": 2.0919127464294434,
+ "learning_rate": 3.551515151515152e-06,
+ "loss": 1.3578065633773804,
+ "mean_token_accuracy": 0.672874927520752,
+ "num_tokens": 11763712.0,
+ "step": 718
+ },
+ {
+ "entropy": 1.3421508073806763,
+ "epoch": 1.4525252525252526,
+ "grad_norm": 2.0651159286499023,
+ "learning_rate": 3.54949494949495e-06,
+ "loss": 1.33928382396698,
+ "mean_token_accuracy": 0.6866145730018616,
+ "num_tokens": 11780096.0,
+ "step": 719
+ },
+ {
+ "entropy": 1.2841005325317383,
+ "epoch": 1.4545454545454546,
+ "grad_norm": 2.124370813369751,
+ "learning_rate": 3.547474747474748e-06,
+ "loss": 1.258314609527588,
+ "mean_token_accuracy": 0.6806302070617676,
+ "num_tokens": 11796480.0,
+ "step": 720
+ },
+ {
+ "entropy": 1.56289803981781,
+ "epoch": 1.4565656565656566,
+ "grad_norm": 1.9271100759506226,
+ "learning_rate": 3.5454545454545458e-06,
+ "loss": 1.5867946147918701,
+ "mean_token_accuracy": 0.6430752277374268,
+ "num_tokens": 11812864.0,
+ "step": 721
+ },
+ {
+ "entropy": 1.3374838829040527,
+ "epoch": 1.4585858585858587,
+ "grad_norm": 2.1113364696502686,
+ "learning_rate": 3.5434343434343437e-06,
+ "loss": 1.327169418334961,
+ "mean_token_accuracy": 0.6806912422180176,
+ "num_tokens": 11829248.0,
+ "step": 722
+ },
+ {
+ "entropy": 1.7233415842056274,
+ "epoch": 1.4606060606060607,
+ "grad_norm": 2.1862399578094482,
+ "learning_rate": 3.5414141414141416e-06,
+ "loss": 1.7290821075439453,
+ "mean_token_accuracy": 0.6056423783302307,
+ "num_tokens": 11845632.0,
+ "step": 723
+ },
+ {
+ "entropy": 1.28484046459198,
+ "epoch": 1.4626262626262627,
+ "grad_norm": 2.0059030055999756,
+ "learning_rate": 3.53939393939394e-06,
+ "loss": 1.2958555221557617,
+ "mean_token_accuracy": 0.6774548292160034,
+ "num_tokens": 11862016.0,
+ "step": 724
+ },
+ {
+ "entropy": 1.2592295408248901,
+ "epoch": 1.4646464646464645,
+ "grad_norm": 1.9422065019607544,
+ "learning_rate": 3.5373737373737378e-06,
+ "loss": 1.2501411437988281,
+ "mean_token_accuracy": 0.700537383556366,
+ "num_tokens": 11878400.0,
+ "step": 725
+ },
+ {
+ "entropy": 1.3708571195602417,
+ "epoch": 1.4666666666666668,
+ "grad_norm": 2.164499044418335,
+ "learning_rate": 3.5353535353535356e-06,
+ "loss": 1.3901662826538086,
+ "mean_token_accuracy": 0.6618832349777222,
+ "num_tokens": 11894784.0,
+ "step": 726
+ },
+ {
+ "entropy": 1.239045262336731,
+ "epoch": 1.4686868686868686,
+ "grad_norm": 1.9581060409545898,
+ "learning_rate": 3.5333333333333335e-06,
+ "loss": 1.2436164617538452,
+ "mean_token_accuracy": 0.6939423680305481,
+ "num_tokens": 11911168.0,
+ "step": 727
+ },
+ {
+ "entropy": 1.5965995788574219,
+ "epoch": 1.4707070707070706,
+ "grad_norm": 2.343669891357422,
+ "learning_rate": 3.5313131313131314e-06,
+ "loss": 1.5918347835540771,
+ "mean_token_accuracy": 0.6274425983428955,
+ "num_tokens": 11927552.0,
+ "step": 728
+ },
+ {
+ "entropy": 1.1196811199188232,
+ "epoch": 1.4727272727272727,
+ "grad_norm": 2.181840181350708,
+ "learning_rate": 3.5292929292929297e-06,
+ "loss": 1.1142621040344238,
+ "mean_token_accuracy": 0.7192842960357666,
+ "num_tokens": 11943936.0,
+ "step": 729
+ },
+ {
+ "entropy": 1.129361629486084,
+ "epoch": 1.4747474747474747,
+ "grad_norm": 2.065730571746826,
+ "learning_rate": 3.5272727272727276e-06,
+ "loss": 1.143613576889038,
+ "mean_token_accuracy": 0.7145212292671204,
+ "num_tokens": 11960320.0,
+ "step": 730
+ },
+ {
+ "entropy": 1.4940218925476074,
+ "epoch": 1.4767676767676767,
+ "grad_norm": 2.083453893661499,
+ "learning_rate": 3.5252525252525255e-06,
+ "loss": 1.4957321882247925,
+ "mean_token_accuracy": 0.6481436491012573,
+ "num_tokens": 11976704.0,
+ "step": 731
+ },
+ {
+ "entropy": 1.3427786827087402,
+ "epoch": 1.4787878787878788,
+ "grad_norm": 2.0897183418273926,
+ "learning_rate": 3.5232323232323234e-06,
+ "loss": 1.354066014289856,
+ "mean_token_accuracy": 0.6660356521606445,
+ "num_tokens": 11993088.0,
+ "step": 732
+ },
+ {
+ "entropy": 1.3300065994262695,
+ "epoch": 1.4808080808080808,
+ "grad_norm": 2.0537912845611572,
+ "learning_rate": 3.5212121212121213e-06,
+ "loss": 1.3332751989364624,
+ "mean_token_accuracy": 0.6817293763160706,
+ "num_tokens": 12009472.0,
+ "step": 733
+ },
+ {
+ "entropy": 1.0657827854156494,
+ "epoch": 1.4828282828282828,
+ "grad_norm": 1.9099130630493164,
+ "learning_rate": 3.5191919191919196e-06,
+ "loss": 1.067232608795166,
+ "mean_token_accuracy": 0.734245240688324,
+ "num_tokens": 12025856.0,
+ "step": 734
+ },
+ {
+ "entropy": 1.5265862941741943,
+ "epoch": 1.4848484848484849,
+ "grad_norm": 2.100236177444458,
+ "learning_rate": 3.5171717171717175e-06,
+ "loss": 1.5169599056243896,
+ "mean_token_accuracy": 0.6382511258125305,
+ "num_tokens": 12042240.0,
+ "step": 735
+ },
+ {
+ "entropy": 1.4428536891937256,
+ "epoch": 1.486868686868687,
+ "grad_norm": 1.9902187585830688,
+ "learning_rate": 3.5151515151515154e-06,
+ "loss": 1.4398434162139893,
+ "mean_token_accuracy": 0.675500750541687,
+ "num_tokens": 12058624.0,
+ "step": 736
+ },
+ {
+ "entropy": 1.528032660484314,
+ "epoch": 1.488888888888889,
+ "grad_norm": 2.043238878250122,
+ "learning_rate": 3.5131313131313133e-06,
+ "loss": 1.5280101299285889,
+ "mean_token_accuracy": 0.6431363224983215,
+ "num_tokens": 12075008.0,
+ "step": 737
+ },
+ {
+ "entropy": 1.4250329732894897,
+ "epoch": 1.490909090909091,
+ "grad_norm": 1.791640281677246,
+ "learning_rate": 3.511111111111111e-06,
+ "loss": 1.426466464996338,
+ "mean_token_accuracy": 0.6765388250350952,
+ "num_tokens": 12091392.0,
+ "step": 738
+ },
+ {
+ "entropy": 1.1264820098876953,
+ "epoch": 1.492929292929293,
+ "grad_norm": 2.0549607276916504,
+ "learning_rate": 3.509090909090909e-06,
+ "loss": 1.1271553039550781,
+ "mean_token_accuracy": 0.6996213793754578,
+ "num_tokens": 12107776.0,
+ "step": 739
+ },
+ {
+ "entropy": 1.538375735282898,
+ "epoch": 1.494949494949495,
+ "grad_norm": 2.22322940826416,
+ "learning_rate": 3.5070707070707073e-06,
+ "loss": 1.5588887929916382,
+ "mean_token_accuracy": 0.6384953856468201,
+ "num_tokens": 12124160.0,
+ "step": 740
+ },
+ {
+ "entropy": 1.2698255777359009,
+ "epoch": 1.496969696969697,
+ "grad_norm": 2.17710542678833,
+ "learning_rate": 3.5050505050505052e-06,
+ "loss": 1.29567289352417,
+ "mean_token_accuracy": 0.6762945652008057,
+ "num_tokens": 12140544.0,
+ "step": 741
+ },
+ {
+ "entropy": 1.523453950881958,
+ "epoch": 1.4989898989898989,
+ "grad_norm": 2.0350148677825928,
+ "learning_rate": 3.503030303030303e-06,
+ "loss": 1.5341038703918457,
+ "mean_token_accuracy": 0.6367855668067932,
+ "num_tokens": 12156928.0,
+ "step": 742
+ },
+ {
+ "entropy": 1.4250539541244507,
+ "epoch": 1.5010101010101011,
+ "grad_norm": 2.1306707859039307,
+ "learning_rate": 3.501010101010101e-06,
+ "loss": 1.4353611469268799,
+ "mean_token_accuracy": 0.6626160144805908,
+ "num_tokens": 12173312.0,
+ "step": 743
+ },
+ {
+ "entropy": 1.1440296173095703,
+ "epoch": 1.503030303030303,
+ "grad_norm": 2.053969383239746,
+ "learning_rate": 3.498989898989899e-06,
+ "loss": 1.1151821613311768,
+ "mean_token_accuracy": 0.7080483436584473,
+ "num_tokens": 12189696.0,
+ "step": 744
+ },
+ {
+ "entropy": 1.5290073156356812,
+ "epoch": 1.5050505050505052,
+ "grad_norm": 2.0550003051757812,
+ "learning_rate": 3.496969696969697e-06,
+ "loss": 1.5260624885559082,
+ "mean_token_accuracy": 0.6493038535118103,
+ "num_tokens": 12206080.0,
+ "step": 745
+ },
+ {
+ "entropy": 1.1096605062484741,
+ "epoch": 1.507070707070707,
+ "grad_norm": 1.8131766319274902,
+ "learning_rate": 3.494949494949495e-06,
+ "loss": 1.1221846342086792,
+ "mean_token_accuracy": 0.7263678312301636,
+ "num_tokens": 12222464.0,
+ "step": 746
+ },
+ {
+ "entropy": 1.282676339149475,
+ "epoch": 1.509090909090909,
+ "grad_norm": 2.3051960468292236,
+ "learning_rate": 3.492929292929293e-06,
+ "loss": 1.2863187789916992,
+ "mean_token_accuracy": 0.6765388250350952,
+ "num_tokens": 12238848.0,
+ "step": 747
+ },
+ {
+ "entropy": 1.2971303462982178,
+ "epoch": 1.511111111111111,
+ "grad_norm": 2.0721139907836914,
+ "learning_rate": 3.4909090909090913e-06,
+ "loss": 1.3010127544403076,
+ "mean_token_accuracy": 0.6753175258636475,
+ "num_tokens": 12255232.0,
+ "step": 748
+ },
+ {
+ "entropy": 1.3711751699447632,
+ "epoch": 1.513131313131313,
+ "grad_norm": 1.9657210111618042,
+ "learning_rate": 3.4888888888888896e-06,
+ "loss": 1.3703930377960205,
+ "mean_token_accuracy": 0.6686614751815796,
+ "num_tokens": 12271616.0,
+ "step": 749
+ },
+ {
+ "entropy": 1.512154459953308,
+ "epoch": 1.5151515151515151,
+ "grad_norm": 2.214172124862671,
+ "learning_rate": 3.4868686868686875e-06,
+ "loss": 1.5243256092071533,
+ "mean_token_accuracy": 0.6485100388526917,
+ "num_tokens": 12288000.0,
+ "step": 750
+ },
+ {
+ "epoch": 1.5151515151515151,
+ "eval_entropy": 1.4458443743567313,
+ "eval_loss": 1.535510540008545,
+ "eval_mean_token_accuracy": 0.6444821915318889,
+ "eval_num_tokens": 12288000.0,
+ "eval_runtime": 43.8043,
+ "eval_samples_per_second": 22.601,
+ "eval_steps_per_second": 2.831,
+ "step": 750
+ },
+ {
+ "entropy": 1.2052820920944214,
+ "epoch": 1.5171717171717172,
+ "grad_norm": 2.104870319366455,
+ "learning_rate": 3.4848484848484854e-06,
+ "loss": 1.2207554578781128,
+ "mean_token_accuracy": 0.695713222026825,
+ "num_tokens": 12304384.0,
+ "step": 751
+ },
+ {
+ "entropy": 1.732282280921936,
+ "epoch": 1.5191919191919192,
+ "grad_norm": 2.101760149002075,
+ "learning_rate": 3.4828282828282833e-06,
+ "loss": 1.7461241483688354,
+ "mean_token_accuracy": 0.6505251526832581,
+ "num_tokens": 12320768.0,
+ "step": 752
+ },
+ {
+ "entropy": 1.5805878639221191,
+ "epoch": 1.5212121212121212,
+ "grad_norm": 2.139244318008423,
+ "learning_rate": 3.480808080808081e-06,
+ "loss": 1.5813164710998535,
+ "mean_token_accuracy": 0.6281143426895142,
+ "num_tokens": 12337152.0,
+ "step": 753
+ },
+ {
+ "entropy": 1.1882269382476807,
+ "epoch": 1.5232323232323233,
+ "grad_norm": 2.187206745147705,
+ "learning_rate": 3.4787878787878795e-06,
+ "loss": 1.1732902526855469,
+ "mean_token_accuracy": 0.7063385248184204,
+ "num_tokens": 12353536.0,
+ "step": 754
+ },
+ {
+ "entropy": 1.4743424654006958,
+ "epoch": 1.5252525252525253,
+ "grad_norm": 2.12687349319458,
+ "learning_rate": 3.4767676767676774e-06,
+ "loss": 1.4681663513183594,
+ "mean_token_accuracy": 0.6558378338813782,
+ "num_tokens": 12369920.0,
+ "step": 755
+ },
+ {
+ "entropy": 1.5533041954040527,
+ "epoch": 1.5272727272727273,
+ "grad_norm": 2.1306610107421875,
+ "learning_rate": 3.4747474747474752e-06,
+ "loss": 1.5599431991577148,
+ "mean_token_accuracy": 0.6399609446525574,
+ "num_tokens": 12386304.0,
+ "step": 756
+ },
+ {
+ "entropy": 0.8933209180831909,
+ "epoch": 1.5292929292929291,
+ "grad_norm": 2.425602912902832,
+ "learning_rate": 3.472727272727273e-06,
+ "loss": 0.8840203285217285,
+ "mean_token_accuracy": 0.7638006806373596,
+ "num_tokens": 12402688.0,
+ "step": 757
+ },
+ {
+ "entropy": 1.6764189004898071,
+ "epoch": 1.5313131313131314,
+ "grad_norm": 2.0684356689453125,
+ "learning_rate": 3.470707070707071e-06,
+ "loss": 1.6948609352111816,
+ "mean_token_accuracy": 0.6087567210197449,
+ "num_tokens": 12419072.0,
+ "step": 758
+ },
+ {
+ "entropy": 1.1875263452529907,
+ "epoch": 1.5333333333333332,
+ "grad_norm": 2.0625953674316406,
+ "learning_rate": 3.468686868686869e-06,
+ "loss": 1.1840041875839233,
+ "mean_token_accuracy": 0.7067660093307495,
+ "num_tokens": 12435456.0,
+ "step": 759
+ },
+ {
+ "entropy": 1.1589192152023315,
+ "epoch": 1.5353535353535355,
+ "grad_norm": 1.9212846755981445,
+ "learning_rate": 3.4666666666666672e-06,
+ "loss": 1.1537678241729736,
+ "mean_token_accuracy": 0.7107962965965271,
+ "num_tokens": 12451840.0,
+ "step": 760
+ },
+ {
+ "entropy": 0.9687032699584961,
+ "epoch": 1.5373737373737373,
+ "grad_norm": 1.9402192831039429,
+ "learning_rate": 3.464646464646465e-06,
+ "loss": 0.9954442977905273,
+ "mean_token_accuracy": 0.7412676811218262,
+ "num_tokens": 12468224.0,
+ "step": 761
+ },
+ {
+ "entropy": 1.4129019975662231,
+ "epoch": 1.5393939393939395,
+ "grad_norm": 2.0903310775756836,
+ "learning_rate": 3.462626262626263e-06,
+ "loss": 1.4131944179534912,
+ "mean_token_accuracy": 0.6676844358444214,
+ "num_tokens": 12484608.0,
+ "step": 762
+ },
+ {
+ "entropy": 1.5170351266860962,
+ "epoch": 1.5414141414141413,
+ "grad_norm": 2.12807035446167,
+ "learning_rate": 3.460606060606061e-06,
+ "loss": 1.5342246294021606,
+ "mean_token_accuracy": 0.6621885895729065,
+ "num_tokens": 12500992.0,
+ "step": 763
+ },
+ {
+ "entropy": 1.1815718412399292,
+ "epoch": 1.5434343434343434,
+ "grad_norm": 2.2491719722747803,
+ "learning_rate": 3.4585858585858588e-06,
+ "loss": 1.1669261455535889,
+ "mean_token_accuracy": 0.694614052772522,
+ "num_tokens": 12517376.0,
+ "step": 764
+ },
+ {
+ "entropy": 1.547598958015442,
+ "epoch": 1.5454545454545454,
+ "grad_norm": 2.136096954345703,
+ "learning_rate": 3.456565656565657e-06,
+ "loss": 1.5515470504760742,
+ "mean_token_accuracy": 0.648876428604126,
+ "num_tokens": 12533760.0,
+ "step": 765
+ },
+ {
+ "entropy": 0.7156143188476562,
+ "epoch": 1.5474747474747474,
+ "grad_norm": 1.8139498233795166,
+ "learning_rate": 3.454545454545455e-06,
+ "loss": 0.7052675485610962,
+ "mean_token_accuracy": 0.8103932738304138,
+ "num_tokens": 12550144.0,
+ "step": 766
+ },
+ {
+ "entropy": 1.400156855583191,
+ "epoch": 1.5494949494949495,
+ "grad_norm": 2.14924693107605,
+ "learning_rate": 3.452525252525253e-06,
+ "loss": 1.4047740697860718,
+ "mean_token_accuracy": 0.6719589829444885,
+ "num_tokens": 12566528.0,
+ "step": 767
+ },
+ {
+ "entropy": 1.8254657983779907,
+ "epoch": 1.5515151515151515,
+ "grad_norm": 2.1882269382476807,
+ "learning_rate": 3.4505050505050507e-06,
+ "loss": 1.8615440130233765,
+ "mean_token_accuracy": 0.5776746273040771,
+ "num_tokens": 12582912.0,
+ "step": 768
+ },
+ {
+ "entropy": 1.5947710275650024,
+ "epoch": 1.5535353535353535,
+ "grad_norm": 2.3221426010131836,
+ "learning_rate": 3.4484848484848486e-06,
+ "loss": 1.6178712844848633,
+ "mean_token_accuracy": 0.6210918426513672,
+ "num_tokens": 12599296.0,
+ "step": 769
+ },
+ {
+ "entropy": 1.2678714990615845,
+ "epoch": 1.5555555555555556,
+ "grad_norm": 1.970064640045166,
+ "learning_rate": 3.446464646464647e-06,
+ "loss": 1.2646088600158691,
+ "mean_token_accuracy": 0.6990107297897339,
+ "num_tokens": 12615680.0,
+ "step": 770
+ },
+ {
+ "entropy": 1.3723604679107666,
+ "epoch": 1.5575757575757576,
+ "grad_norm": 2.171534776687622,
+ "learning_rate": 3.444444444444445e-06,
+ "loss": 1.3890197277069092,
+ "mean_token_accuracy": 0.6709819436073303,
+ "num_tokens": 12632064.0,
+ "step": 771
+ },
+ {
+ "entropy": 1.0273096561431885,
+ "epoch": 1.5595959595959596,
+ "grad_norm": 2.072798490524292,
+ "learning_rate": 3.4424242424242427e-06,
+ "loss": 1.029193639755249,
+ "mean_token_accuracy": 0.7348558902740479,
+ "num_tokens": 12648448.0,
+ "step": 772
+ },
+ {
+ "entropy": 1.4004037380218506,
+ "epoch": 1.5616161616161617,
+ "grad_norm": 2.111480474472046,
+ "learning_rate": 3.4404040404040406e-06,
+ "loss": 1.4213340282440186,
+ "mean_token_accuracy": 0.660906195640564,
+ "num_tokens": 12664832.0,
+ "step": 773
+ },
+ {
+ "entropy": 1.533424735069275,
+ "epoch": 1.5636363636363635,
+ "grad_norm": 2.0949435234069824,
+ "learning_rate": 3.4383838383838385e-06,
+ "loss": 1.5259253978729248,
+ "mean_token_accuracy": 0.6544333100318909,
+ "num_tokens": 12681216.0,
+ "step": 774
+ },
+ {
+ "entropy": 1.5620275735855103,
+ "epoch": 1.5656565656565657,
+ "grad_norm": 2.339731454849243,
+ "learning_rate": 3.4363636363636364e-06,
+ "loss": 1.5596071481704712,
+ "mean_token_accuracy": 0.6290302872657776,
+ "num_tokens": 12697600.0,
+ "step": 775
+ },
+ {
+ "entropy": 1.3272082805633545,
+ "epoch": 1.5676767676767676,
+ "grad_norm": 2.0871188640594482,
+ "learning_rate": 3.4343434343434347e-06,
+ "loss": 1.3413164615631104,
+ "mean_token_accuracy": 0.6809965968132019,
+ "num_tokens": 12713984.0,
+ "step": 776
+ },
+ {
+ "entropy": 1.202813982963562,
+ "epoch": 1.5696969696969698,
+ "grad_norm": 2.0538578033447266,
+ "learning_rate": 3.4323232323232326e-06,
+ "loss": 1.1947424411773682,
+ "mean_token_accuracy": 0.6969956159591675,
+ "num_tokens": 12730368.0,
+ "step": 777
+ },
+ {
+ "entropy": 1.3332773447036743,
+ "epoch": 1.5717171717171716,
+ "grad_norm": 1.9941202402114868,
+ "learning_rate": 3.4303030303030305e-06,
+ "loss": 1.3345956802368164,
+ "mean_token_accuracy": 0.6761724352836609,
+ "num_tokens": 12746752.0,
+ "step": 778
+ },
+ {
+ "entropy": 1.1724364757537842,
+ "epoch": 1.5737373737373739,
+ "grad_norm": 2.213365316390991,
+ "learning_rate": 3.4282828282828284e-06,
+ "loss": 1.1479787826538086,
+ "mean_token_accuracy": 0.7132999300956726,
+ "num_tokens": 12763136.0,
+ "step": 779
+ },
+ {
+ "entropy": 1.4718358516693115,
+ "epoch": 1.5757575757575757,
+ "grad_norm": 1.9743510484695435,
+ "learning_rate": 3.4262626262626262e-06,
+ "loss": 1.4844720363616943,
+ "mean_token_accuracy": 0.656448483467102,
+ "num_tokens": 12779520.0,
+ "step": 780
+ },
+ {
+ "entropy": 1.362882375717163,
+ "epoch": 1.5777777777777777,
+ "grad_norm": 2.370483875274658,
+ "learning_rate": 3.4242424242424246e-06,
+ "loss": 1.350801706314087,
+ "mean_token_accuracy": 0.6707376837730408,
+ "num_tokens": 12795904.0,
+ "step": 781
+ },
+ {
+ "entropy": 1.8302308320999146,
+ "epoch": 1.5797979797979798,
+ "grad_norm": 2.19045090675354,
+ "learning_rate": 3.4222222222222224e-06,
+ "loss": 1.8371320962905884,
+ "mean_token_accuracy": 0.5929408669471741,
+ "num_tokens": 12812288.0,
+ "step": 782
+ },
+ {
+ "entropy": 1.3435529470443726,
+ "epoch": 1.5818181818181818,
+ "grad_norm": 2.1254351139068604,
+ "learning_rate": 3.4202020202020203e-06,
+ "loss": 1.3621928691864014,
+ "mean_token_accuracy": 0.6867367029190063,
+ "num_tokens": 12828672.0,
+ "step": 783
+ },
+ {
+ "entropy": 1.8402602672576904,
+ "epoch": 1.5838383838383838,
+ "grad_norm": 2.123666524887085,
+ "learning_rate": 3.4181818181818182e-06,
+ "loss": 1.8564848899841309,
+ "mean_token_accuracy": 0.5871397256851196,
+ "num_tokens": 12845056.0,
+ "step": 784
+ },
+ {
+ "entropy": 1.441917896270752,
+ "epoch": 1.5858585858585859,
+ "grad_norm": 2.3286895751953125,
+ "learning_rate": 3.416161616161616e-06,
+ "loss": 1.4724184274673462,
+ "mean_token_accuracy": 0.6438080072402954,
+ "num_tokens": 12861440.0,
+ "step": 785
+ },
+ {
+ "entropy": 1.3071452379226685,
+ "epoch": 1.587878787878788,
+ "grad_norm": 2.149143934249878,
+ "learning_rate": 3.414141414141414e-06,
+ "loss": 1.331390142440796,
+ "mean_token_accuracy": 0.679347813129425,
+ "num_tokens": 12877824.0,
+ "step": 786
+ },
+ {
+ "entropy": 1.3547145128250122,
+ "epoch": 1.58989898989899,
+ "grad_norm": 2.2531180381774902,
+ "learning_rate": 3.4121212121212123e-06,
+ "loss": 1.361851453781128,
+ "mean_token_accuracy": 0.6767830848693848,
+ "num_tokens": 12894208.0,
+ "step": 787
+ },
+ {
+ "entropy": 0.9811291098594666,
+ "epoch": 1.591919191919192,
+ "grad_norm": 2.1053760051727295,
+ "learning_rate": 3.41010101010101e-06,
+ "loss": 0.9885507822036743,
+ "mean_token_accuracy": 0.7480459213256836,
+ "num_tokens": 12910592.0,
+ "step": 788
+ },
+ {
+ "entropy": 1.4940402507781982,
+ "epoch": 1.593939393939394,
+ "grad_norm": 2.240074634552002,
+ "learning_rate": 3.408080808080808e-06,
+ "loss": 1.5014076232910156,
+ "mean_token_accuracy": 0.6441743969917297,
+ "num_tokens": 12926976.0,
+ "step": 789
+ },
+ {
+ "entropy": 1.8134219646453857,
+ "epoch": 1.595959595959596,
+ "grad_norm": 2.040239095687866,
+ "learning_rate": 3.406060606060606e-06,
+ "loss": 1.8437519073486328,
+ "mean_token_accuracy": 0.5904372334480286,
+ "num_tokens": 12943360.0,
+ "step": 790
+ },
+ {
+ "entropy": 1.7233681678771973,
+ "epoch": 1.5979797979797978,
+ "grad_norm": 2.381786823272705,
+ "learning_rate": 3.4040404040404047e-06,
+ "loss": 1.7328217029571533,
+ "mean_token_accuracy": 0.6004518866539001,
+ "num_tokens": 12959744.0,
+ "step": 791
+ },
+ {
+ "entropy": 1.3966271877288818,
+ "epoch": 1.6,
+ "grad_norm": 2.17006778717041,
+ "learning_rate": 3.4020202020202026e-06,
+ "loss": 1.4065337181091309,
+ "mean_token_accuracy": 0.6629824042320251,
+ "num_tokens": 12976128.0,
+ "step": 792
+ },
+ {
+ "entropy": 1.5002557039260864,
+ "epoch": 1.602020202020202,
+ "grad_norm": 2.558037281036377,
+ "learning_rate": 3.4000000000000005e-06,
+ "loss": 1.4841307401657104,
+ "mean_token_accuracy": 0.647838294506073,
+ "num_tokens": 12992512.0,
+ "step": 793
+ },
+ {
+ "entropy": 1.7432687282562256,
+ "epoch": 1.6040404040404042,
+ "grad_norm": 2.134734630584717,
+ "learning_rate": 3.3979797979797984e-06,
+ "loss": 1.7539207935333252,
+ "mean_token_accuracy": 0.6028333902359009,
+ "num_tokens": 13008896.0,
+ "step": 794
+ },
+ {
+ "entropy": 1.4490517377853394,
+ "epoch": 1.606060606060606,
+ "grad_norm": 2.1590232849121094,
+ "learning_rate": 3.3959595959595963e-06,
+ "loss": 1.468411922454834,
+ "mean_token_accuracy": 0.656509518623352,
+ "num_tokens": 13025280.0,
+ "step": 795
+ },
+ {
+ "entropy": 1.3823497295379639,
+ "epoch": 1.6080808080808082,
+ "grad_norm": 2.1508285999298096,
+ "learning_rate": 3.3939393939393946e-06,
+ "loss": 1.4011330604553223,
+ "mean_token_accuracy": 0.6530898809432983,
+ "num_tokens": 13041664.0,
+ "step": 796
+ },
+ {
+ "entropy": 1.6982436180114746,
+ "epoch": 1.61010101010101,
+ "grad_norm": 2.077131986618042,
+ "learning_rate": 3.3919191919191925e-06,
+ "loss": 1.727889060974121,
+ "mean_token_accuracy": 0.6232290863990784,
+ "num_tokens": 13058048.0,
+ "step": 797
+ },
+ {
+ "entropy": 1.7268915176391602,
+ "epoch": 1.612121212121212,
+ "grad_norm": 2.0703542232513428,
+ "learning_rate": 3.3898989898989903e-06,
+ "loss": 1.7333730459213257,
+ "mean_token_accuracy": 0.6110160946846008,
+ "num_tokens": 13074432.0,
+ "step": 798
+ },
+ {
+ "entropy": 1.6253595352172852,
+ "epoch": 1.614141414141414,
+ "grad_norm": 2.241154432296753,
+ "learning_rate": 3.3878787878787882e-06,
+ "loss": 1.6275582313537598,
+ "mean_token_accuracy": 0.6198094487190247,
+ "num_tokens": 13090816.0,
+ "step": 799
+ },
+ {
+ "entropy": 1.8784692287445068,
+ "epoch": 1.6161616161616161,
+ "grad_norm": 2.505871057510376,
+ "learning_rate": 3.385858585858586e-06,
+ "loss": 1.8784008026123047,
+ "mean_token_accuracy": 0.5944064259529114,
+ "num_tokens": 13107200.0,
+ "step": 800
+ },
+ {
+ "epoch": 1.6161616161616161,
+ "eval_entropy": 1.4455043437980837,
+ "eval_loss": 1.5330992937088013,
+ "eval_mean_token_accuracy": 0.6448845310557273,
+ "eval_num_tokens": 13107200.0,
+ "eval_runtime": 43.766,
+ "eval_samples_per_second": 22.62,
+ "eval_steps_per_second": 2.833,
+ "step": 800
+ },
+ {
+ "entropy": 1.921440601348877,
+ "epoch": 1.6181818181818182,
+ "grad_norm": 2.1760783195495605,
+ "learning_rate": 3.3838383838383844e-06,
+ "loss": 1.9410110712051392,
+ "mean_token_accuracy": 0.5776746273040771,
+ "num_tokens": 13123584.0,
+ "step": 801
+ },
+ {
+ "entropy": 1.4233160018920898,
+ "epoch": 1.6202020202020202,
+ "grad_norm": 2.2687666416168213,
+ "learning_rate": 3.3818181818181823e-06,
+ "loss": 1.4465446472167969,
+ "mean_token_accuracy": 0.6488153338432312,
+ "num_tokens": 13139968.0,
+ "step": 802
+ },
+ {
+ "entropy": 1.697847843170166,
+ "epoch": 1.6222222222222222,
+ "grad_norm": 2.0649187564849854,
+ "learning_rate": 3.37979797979798e-06,
+ "loss": 1.7235385179519653,
+ "mean_token_accuracy": 0.6196262836456299,
+ "num_tokens": 13156352.0,
+ "step": 803
+ },
+ {
+ "entropy": 1.2027701139450073,
+ "epoch": 1.6242424242424243,
+ "grad_norm": 2.0115890502929688,
+ "learning_rate": 3.377777777777778e-06,
+ "loss": 1.1890676021575928,
+ "mean_token_accuracy": 0.6928431987762451,
+ "num_tokens": 13172736.0,
+ "step": 804
+ },
+ {
+ "entropy": 1.4847105741500854,
+ "epoch": 1.6262626262626263,
+ "grad_norm": 2.145098924636841,
+ "learning_rate": 3.375757575757576e-06,
+ "loss": 1.4817116260528564,
+ "mean_token_accuracy": 0.6415486335754395,
+ "num_tokens": 13189120.0,
+ "step": 805
+ },
+ {
+ "entropy": 1.3548893928527832,
+ "epoch": 1.6282828282828283,
+ "grad_norm": 2.3037521839141846,
+ "learning_rate": 3.3737373737373743e-06,
+ "loss": 1.3433952331542969,
+ "mean_token_accuracy": 0.6681729555130005,
+ "num_tokens": 13205504.0,
+ "step": 806
+ },
+ {
+ "entropy": 1.375793218612671,
+ "epoch": 1.6303030303030304,
+ "grad_norm": 2.0814852714538574,
+ "learning_rate": 3.371717171717172e-06,
+ "loss": 1.3506405353546143,
+ "mean_token_accuracy": 0.6634098887443542,
+ "num_tokens": 13221888.0,
+ "step": 807
+ },
+ {
+ "entropy": 1.1417323350906372,
+ "epoch": 1.6323232323232322,
+ "grad_norm": 2.0166866779327393,
+ "learning_rate": 3.36969696969697e-06,
+ "loss": 1.1731860637664795,
+ "mean_token_accuracy": 0.7123839855194092,
+ "num_tokens": 13238272.0,
+ "step": 808
+ },
+ {
+ "entropy": 1.3159022331237793,
+ "epoch": 1.6343434343434344,
+ "grad_norm": 1.9626281261444092,
+ "learning_rate": 3.367676767676768e-06,
+ "loss": 1.3286499977111816,
+ "mean_token_accuracy": 0.675561785697937,
+ "num_tokens": 13254656.0,
+ "step": 809
+ },
+ {
+ "entropy": 1.325775384902954,
+ "epoch": 1.6363636363636362,
+ "grad_norm": 2.1239242553710938,
+ "learning_rate": 3.365656565656566e-06,
+ "loss": 1.345514178276062,
+ "mean_token_accuracy": 0.6780043840408325,
+ "num_tokens": 13271040.0,
+ "step": 810
+ },
+ {
+ "entropy": 1.8307870626449585,
+ "epoch": 1.6383838383838385,
+ "grad_norm": 2.208296298980713,
+ "learning_rate": 3.3636363636363637e-06,
+ "loss": 1.8402234315872192,
+ "mean_token_accuracy": 0.5958719849586487,
+ "num_tokens": 13287424.0,
+ "step": 811
+ },
+ {
+ "entropy": 1.2596150636672974,
+ "epoch": 1.6404040404040403,
+ "grad_norm": 2.0936427116394043,
+ "learning_rate": 3.361616161616162e-06,
+ "loss": 1.2552804946899414,
+ "mean_token_accuracy": 0.6874083876609802,
+ "num_tokens": 13303808.0,
+ "step": 812
+ },
+ {
+ "entropy": 1.2575111389160156,
+ "epoch": 1.6424242424242426,
+ "grad_norm": 2.0232772827148438,
+ "learning_rate": 3.35959595959596e-06,
+ "loss": 1.2805989980697632,
+ "mean_token_accuracy": 0.6916829347610474,
+ "num_tokens": 13320192.0,
+ "step": 813
+ },
+ {
+ "entropy": 1.0270155668258667,
+ "epoch": 1.6444444444444444,
+ "grad_norm": 1.9090715646743774,
+ "learning_rate": 3.357575757575758e-06,
+ "loss": 1.0277824401855469,
+ "mean_token_accuracy": 0.7379091382026672,
+ "num_tokens": 13336576.0,
+ "step": 814
+ },
+ {
+ "entropy": 1.2992243766784668,
+ "epoch": 1.6464646464646466,
+ "grad_norm": 2.1718814373016357,
+ "learning_rate": 3.3555555555555557e-06,
+ "loss": 1.3278638124465942,
+ "mean_token_accuracy": 0.6764777898788452,
+ "num_tokens": 13352960.0,
+ "step": 815
+ },
+ {
+ "entropy": 1.5365362167358398,
+ "epoch": 1.6484848484848484,
+ "grad_norm": 2.0298495292663574,
+ "learning_rate": 3.3535353535353536e-06,
+ "loss": 1.5257508754730225,
+ "mean_token_accuracy": 0.6511358022689819,
+ "num_tokens": 13369344.0,
+ "step": 816
+ },
+ {
+ "entropy": 1.4384456872940063,
+ "epoch": 1.6505050505050505,
+ "grad_norm": 2.375277042388916,
+ "learning_rate": 3.351515151515152e-06,
+ "loss": 1.4287049770355225,
+ "mean_token_accuracy": 0.645639955997467,
+ "num_tokens": 13385728.0,
+ "step": 817
+ },
+ {
+ "entropy": 1.7104265689849854,
+ "epoch": 1.6525252525252525,
+ "grad_norm": 2.052535057067871,
+ "learning_rate": 3.34949494949495e-06,
+ "loss": 1.6971676349639893,
+ "mean_token_accuracy": 0.6159623861312866,
+ "num_tokens": 13402112.0,
+ "step": 818
+ },
+ {
+ "entropy": 1.5908539295196533,
+ "epoch": 1.6545454545454545,
+ "grad_norm": 2.047088861465454,
+ "learning_rate": 3.3474747474747477e-06,
+ "loss": 1.5640246868133545,
+ "mean_token_accuracy": 0.640754759311676,
+ "num_tokens": 13418496.0,
+ "step": 819
+ },
+ {
+ "entropy": 1.3660207986831665,
+ "epoch": 1.6565656565656566,
+ "grad_norm": 2.2711079120635986,
+ "learning_rate": 3.3454545454545456e-06,
+ "loss": 1.3601036071777344,
+ "mean_token_accuracy": 0.6577919125556946,
+ "num_tokens": 13434880.0,
+ "step": 820
+ },
+ {
+ "entropy": 1.5897634029388428,
+ "epoch": 1.6585858585858586,
+ "grad_norm": 2.0869712829589844,
+ "learning_rate": 3.3434343434343435e-06,
+ "loss": 1.5801420211791992,
+ "mean_token_accuracy": 0.6312897205352783,
+ "num_tokens": 13451264.0,
+ "step": 821
+ },
+ {
+ "entropy": 1.428406834602356,
+ "epoch": 1.6606060606060606,
+ "grad_norm": 2.049370765686035,
+ "learning_rate": 3.3414141414141413e-06,
+ "loss": 1.4145596027374268,
+ "mean_token_accuracy": 0.6649975776672363,
+ "num_tokens": 13467648.0,
+ "step": 822
+ },
+ {
+ "entropy": 1.5026675462722778,
+ "epoch": 1.6626262626262627,
+ "grad_norm": 2.157031297683716,
+ "learning_rate": 3.3393939393939397e-06,
+ "loss": 1.4902830123901367,
+ "mean_token_accuracy": 0.6524181962013245,
+ "num_tokens": 13484032.0,
+ "step": 823
+ },
+ {
+ "entropy": 1.4907052516937256,
+ "epoch": 1.6646464646464647,
+ "grad_norm": 2.1476478576660156,
+ "learning_rate": 3.3373737373737375e-06,
+ "loss": 1.5102436542510986,
+ "mean_token_accuracy": 0.6485100388526917,
+ "num_tokens": 13500416.0,
+ "step": 824
+ },
+ {
+ "entropy": 1.4277782440185547,
+ "epoch": 1.6666666666666665,
+ "grad_norm": 2.2880661487579346,
+ "learning_rate": 3.3353535353535354e-06,
+ "loss": 1.4358019828796387,
+ "mean_token_accuracy": 0.6522960662841797,
+ "num_tokens": 13516800.0,
+ "step": 825
+ },
+ {
+ "entropy": 1.5560128688812256,
+ "epoch": 1.6686868686868688,
+ "grad_norm": 2.048403024673462,
+ "learning_rate": 3.3333333333333333e-06,
+ "loss": 1.5472501516342163,
+ "mean_token_accuracy": 0.6488153338432312,
+ "num_tokens": 13533184.0,
+ "step": 826
+ },
+ {
+ "entropy": 1.665900707244873,
+ "epoch": 1.6707070707070706,
+ "grad_norm": 2.0828492641448975,
+ "learning_rate": 3.331313131313131e-06,
+ "loss": 1.6905372142791748,
+ "mean_token_accuracy": 0.629885196685791,
+ "num_tokens": 13549568.0,
+ "step": 827
+ },
+ {
+ "entropy": 1.3484426736831665,
+ "epoch": 1.6727272727272728,
+ "grad_norm": 2.1873531341552734,
+ "learning_rate": 3.3292929292929295e-06,
+ "loss": 1.3510258197784424,
+ "mean_token_accuracy": 0.6721421480178833,
+ "num_tokens": 13565952.0,
+ "step": 828
+ },
+ {
+ "entropy": 1.7577860355377197,
+ "epoch": 1.6747474747474747,
+ "grad_norm": 2.0467135906219482,
+ "learning_rate": 3.3272727272727274e-06,
+ "loss": 1.7791969776153564,
+ "mean_token_accuracy": 0.6028944849967957,
+ "num_tokens": 13582336.0,
+ "step": 829
+ },
+ {
+ "entropy": 1.4549124240875244,
+ "epoch": 1.676767676767677,
+ "grad_norm": 2.2004988193511963,
+ "learning_rate": 3.3252525252525253e-06,
+ "loss": 1.4512457847595215,
+ "mean_token_accuracy": 0.6521128416061401,
+ "num_tokens": 13598720.0,
+ "step": 830
+ },
+ {
+ "entropy": 1.2226758003234863,
+ "epoch": 1.6787878787878787,
+ "grad_norm": 2.0866682529449463,
+ "learning_rate": 3.323232323232323e-06,
+ "loss": 1.226635456085205,
+ "mean_token_accuracy": 0.6969345211982727,
+ "num_tokens": 13615104.0,
+ "step": 831
+ },
+ {
+ "entropy": 0.9826563000679016,
+ "epoch": 1.680808080808081,
+ "grad_norm": 1.9865771532058716,
+ "learning_rate": 3.321212121212121e-06,
+ "loss": 1.009648323059082,
+ "mean_token_accuracy": 0.7507327795028687,
+ "num_tokens": 13631488.0,
+ "step": 832
+ },
+ {
+ "entropy": 1.620038390159607,
+ "epoch": 1.6828282828282828,
+ "grad_norm": 2.1745760440826416,
+ "learning_rate": 3.3191919191919194e-06,
+ "loss": 1.615234136581421,
+ "mean_token_accuracy": 0.6305569410324097,
+ "num_tokens": 13647872.0,
+ "step": 833
+ },
+ {
+ "entropy": 1.4803876876831055,
+ "epoch": 1.6848484848484848,
+ "grad_norm": 2.328029155731201,
+ "learning_rate": 3.3171717171717177e-06,
+ "loss": 1.509049892425537,
+ "mean_token_accuracy": 0.6353810429573059,
+ "num_tokens": 13664256.0,
+ "step": 834
+ },
+ {
+ "entropy": 1.2290453910827637,
+ "epoch": 1.6868686868686869,
+ "grad_norm": 2.1777281761169434,
+ "learning_rate": 3.3151515151515156e-06,
+ "loss": 1.222726583480835,
+ "mean_token_accuracy": 0.6842330098152161,
+ "num_tokens": 13680640.0,
+ "step": 835
+ },
+ {
+ "entropy": 1.1965430974960327,
+ "epoch": 1.6888888888888889,
+ "grad_norm": 2.211397886276245,
+ "learning_rate": 3.3131313131313135e-06,
+ "loss": 1.2027852535247803,
+ "mean_token_accuracy": 0.7028578519821167,
+ "num_tokens": 13697024.0,
+ "step": 836
+ },
+ {
+ "entropy": 1.5326080322265625,
+ "epoch": 1.690909090909091,
+ "grad_norm": 2.8396589756011963,
+ "learning_rate": 3.3111111111111118e-06,
+ "loss": 1.5819613933563232,
+ "mean_token_accuracy": 0.625,
+ "num_tokens": 13713408.0,
+ "step": 837
+ },
+ {
+ "entropy": 1.2514865398406982,
+ "epoch": 1.692929292929293,
+ "grad_norm": 1.923771858215332,
+ "learning_rate": 3.3090909090909097e-06,
+ "loss": 1.2518126964569092,
+ "mean_token_accuracy": 0.6955300569534302,
+ "num_tokens": 13729792.0,
+ "step": 838
+ },
+ {
+ "entropy": 1.3717304468154907,
+ "epoch": 1.694949494949495,
+ "grad_norm": 2.0393056869506836,
+ "learning_rate": 3.3070707070707076e-06,
+ "loss": 1.400814175605774,
+ "mean_token_accuracy": 0.6618832349777222,
+ "num_tokens": 13746176.0,
+ "step": 839
+ },
+ {
+ "entropy": 1.2871394157409668,
+ "epoch": 1.696969696969697,
+ "grad_norm": 2.114823579788208,
+ "learning_rate": 3.3050505050505054e-06,
+ "loss": 1.302154541015625,
+ "mean_token_accuracy": 0.6790425181388855,
+ "num_tokens": 13762560.0,
+ "step": 840
+ },
+ {
+ "entropy": 1.518404483795166,
+ "epoch": 1.698989898989899,
+ "grad_norm": 2.186727523803711,
+ "learning_rate": 3.3030303030303033e-06,
+ "loss": 1.5067068338394165,
+ "mean_token_accuracy": 0.638006865978241,
+ "num_tokens": 13778944.0,
+ "step": 841
+ },
+ {
+ "entropy": 1.5698254108428955,
+ "epoch": 1.7010101010101009,
+ "grad_norm": 2.159721851348877,
+ "learning_rate": 3.3010101010101016e-06,
+ "loss": 1.5568101406097412,
+ "mean_token_accuracy": 0.6398998498916626,
+ "num_tokens": 13795328.0,
+ "step": 842
+ },
+ {
+ "entropy": 1.5159670114517212,
+ "epoch": 1.7030303030303031,
+ "grad_norm": 2.0655386447906494,
+ "learning_rate": 3.2989898989898995e-06,
+ "loss": 1.5152846574783325,
+ "mean_token_accuracy": 0.6441133618354797,
+ "num_tokens": 13811712.0,
+ "step": 843
+ },
+ {
+ "entropy": 1.231048822402954,
+ "epoch": 1.705050505050505,
+ "grad_norm": 1.99764084815979,
+ "learning_rate": 3.2969696969696974e-06,
+ "loss": 1.2419885396957397,
+ "mean_token_accuracy": 0.6936370134353638,
+ "num_tokens": 13828096.0,
+ "step": 844
+ },
+ {
+ "entropy": 1.1431403160095215,
+ "epoch": 1.7070707070707072,
+ "grad_norm": 2.0046544075012207,
+ "learning_rate": 3.2949494949494953e-06,
+ "loss": 1.147315502166748,
+ "mean_token_accuracy": 0.7155593633651733,
+ "num_tokens": 13844480.0,
+ "step": 845
+ },
+ {
+ "entropy": 1.903164267539978,
+ "epoch": 1.709090909090909,
+ "grad_norm": 2.068286418914795,
+ "learning_rate": 3.292929292929293e-06,
+ "loss": 1.9086098670959473,
+ "mean_token_accuracy": 0.5785295367240906,
+ "num_tokens": 13860864.0,
+ "step": 846
+ },
+ {
+ "entropy": 1.028931736946106,
+ "epoch": 1.7111111111111112,
+ "grad_norm": 2.008197546005249,
+ "learning_rate": 3.290909090909091e-06,
+ "loss": 1.0251559019088745,
+ "mean_token_accuracy": 0.7308866381645203,
+ "num_tokens": 13877248.0,
+ "step": 847
+ },
+ {
+ "entropy": 1.8438282012939453,
+ "epoch": 1.713131313131313,
+ "grad_norm": 2.180811882019043,
+ "learning_rate": 3.2888888888888894e-06,
+ "loss": 1.8572561740875244,
+ "mean_token_accuracy": 0.5788959264755249,
+ "num_tokens": 13893632.0,
+ "step": 848
+ },
+ {
+ "entropy": 1.437190294265747,
+ "epoch": 1.7151515151515153,
+ "grad_norm": 2.1013762950897217,
+ "learning_rate": 3.2868686868686873e-06,
+ "loss": 1.4337413311004639,
+ "mean_token_accuracy": 0.6489985585212708,
+ "num_tokens": 13910016.0,
+ "step": 849
+ },
+ {
+ "entropy": 1.482985496520996,
+ "epoch": 1.7171717171717171,
+ "grad_norm": 2.290910243988037,
+ "learning_rate": 3.284848484848485e-06,
+ "loss": 1.4959741830825806,
+ "mean_token_accuracy": 0.6447850465774536,
+ "num_tokens": 13926400.0,
+ "step": 850
+ },
+ {
+ "epoch": 1.7171717171717171,
+ "eval_entropy": 1.447427170411233,
+ "eval_loss": 1.5309957265853882,
+ "eval_mean_token_accuracy": 0.6452888370521607,
+ "eval_num_tokens": 13926400.0,
+ "eval_runtime": 43.7619,
+ "eval_samples_per_second": 22.622,
+ "eval_steps_per_second": 2.834,
+ "step": 850
+ },
+ {
+ "entropy": 1.4470106363296509,
+ "epoch": 1.7191919191919192,
+ "grad_norm": 2.2015624046325684,
+ "learning_rate": 3.282828282828283e-06,
+ "loss": 1.462576150894165,
+ "mean_token_accuracy": 0.643991231918335,
+ "num_tokens": 13942784.0,
+ "step": 851
+ },
+ {
+ "entropy": 1.4063832759857178,
+ "epoch": 1.7212121212121212,
+ "grad_norm": 2.1743109226226807,
+ "learning_rate": 3.280808080808081e-06,
+ "loss": 1.402491569519043,
+ "mean_token_accuracy": 0.6533952355384827,
+ "num_tokens": 13959168.0,
+ "step": 852
+ },
+ {
+ "entropy": 1.3735251426696777,
+ "epoch": 1.7232323232323232,
+ "grad_norm": 2.1350960731506348,
+ "learning_rate": 3.2787878787878793e-06,
+ "loss": 1.383405327796936,
+ "mean_token_accuracy": 0.6767830848693848,
+ "num_tokens": 13975552.0,
+ "step": 853
+ },
+ {
+ "entropy": 1.359702229499817,
+ "epoch": 1.7252525252525253,
+ "grad_norm": 2.1547393798828125,
+ "learning_rate": 3.276767676767677e-06,
+ "loss": 1.3454554080963135,
+ "mean_token_accuracy": 0.6712262034416199,
+ "num_tokens": 13991936.0,
+ "step": 854
+ },
+ {
+ "entropy": 1.0641778707504272,
+ "epoch": 1.7272727272727273,
+ "grad_norm": 2.0301826000213623,
+ "learning_rate": 3.274747474747475e-06,
+ "loss": 1.0626115798950195,
+ "mean_token_accuracy": 0.7188568711280823,
+ "num_tokens": 14008320.0,
+ "step": 855
+ },
+ {
+ "entropy": 1.598792552947998,
+ "epoch": 1.7292929292929293,
+ "grad_norm": 2.3324217796325684,
+ "learning_rate": 3.272727272727273e-06,
+ "loss": 1.6028941869735718,
+ "mean_token_accuracy": 0.6462506055831909,
+ "num_tokens": 14024704.0,
+ "step": 856
+ },
+ {
+ "entropy": 1.6304137706756592,
+ "epoch": 1.7313131313131314,
+ "grad_norm": 1.9968358278274536,
+ "learning_rate": 3.270707070707071e-06,
+ "loss": 1.6449415683746338,
+ "mean_token_accuracy": 0.6322056651115417,
+ "num_tokens": 14041088.0,
+ "step": 857
+ },
+ {
+ "entropy": 1.3219174146652222,
+ "epoch": 1.7333333333333334,
+ "grad_norm": 2.059927225112915,
+ "learning_rate": 3.2686868686868687e-06,
+ "loss": 1.3487168550491333,
+ "mean_token_accuracy": 0.6769052147865295,
+ "num_tokens": 14057472.0,
+ "step": 858
+ },
+ {
+ "entropy": 1.4645651578903198,
+ "epoch": 1.7353535353535352,
+ "grad_norm": 2.068962812423706,
+ "learning_rate": 3.266666666666667e-06,
+ "loss": 1.4480202198028564,
+ "mean_token_accuracy": 0.6521128416061401,
+ "num_tokens": 14073856.0,
+ "step": 859
+ },
+ {
+ "entropy": 1.5005300045013428,
+ "epoch": 1.7373737373737375,
+ "grad_norm": 2.228736162185669,
+ "learning_rate": 3.264646464646465e-06,
+ "loss": 1.5110201835632324,
+ "mean_token_accuracy": 0.6482657790184021,
+ "num_tokens": 14090240.0,
+ "step": 860
+ },
+ {
+ "entropy": 1.1641852855682373,
+ "epoch": 1.7393939393939393,
+ "grad_norm": 2.0611488819122314,
+ "learning_rate": 3.262626262626263e-06,
+ "loss": 1.1905086040496826,
+ "mean_token_accuracy": 0.7104909420013428,
+ "num_tokens": 14106624.0,
+ "step": 861
+ },
+ {
+ "entropy": 1.4315950870513916,
+ "epoch": 1.7414141414141415,
+ "grad_norm": 1.9914237260818481,
+ "learning_rate": 3.2606060606060607e-06,
+ "loss": 1.467012882232666,
+ "mean_token_accuracy": 0.657608687877655,
+ "num_tokens": 14123008.0,
+ "step": 862
+ },
+ {
+ "entropy": 1.5196973085403442,
+ "epoch": 1.7434343434343433,
+ "grad_norm": 2.3258447647094727,
+ "learning_rate": 3.2585858585858586e-06,
+ "loss": 1.5382301807403564,
+ "mean_token_accuracy": 0.6370298266410828,
+ "num_tokens": 14139392.0,
+ "step": 863
+ },
+ {
+ "entropy": 1.5098228454589844,
+ "epoch": 1.7454545454545456,
+ "grad_norm": 2.146327495574951,
+ "learning_rate": 3.256565656565657e-06,
+ "loss": 1.540649652481079,
+ "mean_token_accuracy": 0.6439301371574402,
+ "num_tokens": 14155776.0,
+ "step": 864
+ },
+ {
+ "entropy": 1.128011703491211,
+ "epoch": 1.7474747474747474,
+ "grad_norm": 1.967443823814392,
+ "learning_rate": 3.2545454545454548e-06,
+ "loss": 1.1707985401153564,
+ "mean_token_accuracy": 0.7298485636711121,
+ "num_tokens": 14172160.0,
+ "step": 865
+ },
+ {
+ "entropy": 1.9117931127548218,
+ "epoch": 1.7494949494949497,
+ "grad_norm": 2.097781181335449,
+ "learning_rate": 3.2525252525252527e-06,
+ "loss": 1.9291374683380127,
+ "mean_token_accuracy": 0.580483615398407,
+ "num_tokens": 14188544.0,
+ "step": 866
+ },
+ {
+ "entropy": 1.9053187370300293,
+ "epoch": 1.7515151515151515,
+ "grad_norm": 1.9865039587020874,
+ "learning_rate": 3.2505050505050505e-06,
+ "loss": 1.932790756225586,
+ "mean_token_accuracy": 0.5770639777183533,
+ "num_tokens": 14204928.0,
+ "step": 867
+ },
+ {
+ "entropy": 1.2647870779037476,
+ "epoch": 1.7535353535353535,
+ "grad_norm": 2.099891185760498,
+ "learning_rate": 3.2484848484848484e-06,
+ "loss": 1.281562328338623,
+ "mean_token_accuracy": 0.6938812732696533,
+ "num_tokens": 14221312.0,
+ "step": 868
+ },
+ {
+ "entropy": 1.5446428060531616,
+ "epoch": 1.7555555555555555,
+ "grad_norm": 1.9044189453125,
+ "learning_rate": 3.2464646464646467e-06,
+ "loss": 1.5440425872802734,
+ "mean_token_accuracy": 0.6547996997833252,
+ "num_tokens": 14237696.0,
+ "step": 869
+ },
+ {
+ "entropy": 1.179898977279663,
+ "epoch": 1.7575757575757576,
+ "grad_norm": 2.0257933139801025,
+ "learning_rate": 3.2444444444444446e-06,
+ "loss": 1.1607933044433594,
+ "mean_token_accuracy": 0.7053614854812622,
+ "num_tokens": 14254080.0,
+ "step": 870
+ },
+ {
+ "entropy": 1.0462855100631714,
+ "epoch": 1.7595959595959596,
+ "grad_norm": 1.8995661735534668,
+ "learning_rate": 3.2424242424242425e-06,
+ "loss": 1.0395057201385498,
+ "mean_token_accuracy": 0.7405959963798523,
+ "num_tokens": 14270464.0,
+ "step": 871
+ },
+ {
+ "entropy": 0.8866081833839417,
+ "epoch": 1.7616161616161616,
+ "grad_norm": 1.7807546854019165,
+ "learning_rate": 3.2404040404040404e-06,
+ "loss": 0.8795047998428345,
+ "mean_token_accuracy": 0.7721666097640991,
+ "num_tokens": 14286848.0,
+ "step": 872
+ },
+ {
+ "entropy": 1.9576365947723389,
+ "epoch": 1.7636363636363637,
+ "grad_norm": 2.1867618560791016,
+ "learning_rate": 3.2383838383838383e-06,
+ "loss": 1.9605103731155396,
+ "mean_token_accuracy": 0.5809110999107361,
+ "num_tokens": 14303232.0,
+ "step": 873
+ },
+ {
+ "entropy": 1.010751485824585,
+ "epoch": 1.7656565656565657,
+ "grad_norm": 1.9643300771713257,
+ "learning_rate": 3.236363636363636e-06,
+ "loss": 0.9990894794464111,
+ "mean_token_accuracy": 0.7386419177055359,
+ "num_tokens": 14319616.0,
+ "step": 874
+ },
+ {
+ "entropy": 1.3587232828140259,
+ "epoch": 1.7676767676767677,
+ "grad_norm": 2.2632040977478027,
+ "learning_rate": 3.2343434343434345e-06,
+ "loss": 1.3631021976470947,
+ "mean_token_accuracy": 0.665791392326355,
+ "num_tokens": 14336000.0,
+ "step": 875
+ },
+ {
+ "entropy": 1.6577962636947632,
+ "epoch": 1.7696969696969695,
+ "grad_norm": 2.3661980628967285,
+ "learning_rate": 3.232323232323233e-06,
+ "loss": 1.6802719831466675,
+ "mean_token_accuracy": 0.6024059653282166,
+ "num_tokens": 14352384.0,
+ "step": 876
+ },
+ {
+ "entropy": 1.4332902431488037,
+ "epoch": 1.7717171717171718,
+ "grad_norm": 2.2296342849731445,
+ "learning_rate": 3.2303030303030307e-06,
+ "loss": 1.4129014015197754,
+ "mean_token_accuracy": 0.6511358022689819,
+ "num_tokens": 14368768.0,
+ "step": 877
+ },
+ {
+ "entropy": 1.2260215282440186,
+ "epoch": 1.7737373737373736,
+ "grad_norm": 2.0659983158111572,
+ "learning_rate": 3.228282828282829e-06,
+ "loss": 1.2417192459106445,
+ "mean_token_accuracy": 0.693514883518219,
+ "num_tokens": 14385152.0,
+ "step": 878
+ },
+ {
+ "entropy": 1.5402687788009644,
+ "epoch": 1.7757575757575759,
+ "grad_norm": 2.2024054527282715,
+ "learning_rate": 3.226262626262627e-06,
+ "loss": 1.5535321235656738,
+ "mean_token_accuracy": 0.6331827044487,
+ "num_tokens": 14401536.0,
+ "step": 879
+ },
+ {
+ "entropy": 1.2205268144607544,
+ "epoch": 1.7777777777777777,
+ "grad_norm": 2.0056662559509277,
+ "learning_rate": 3.2242424242424248e-06,
+ "loss": 1.2244641780853271,
+ "mean_token_accuracy": 0.7037127614021301,
+ "num_tokens": 14417920.0,
+ "step": 880
+ },
+ {
+ "entropy": 1.542244791984558,
+ "epoch": 1.77979797979798,
+ "grad_norm": 2.0935254096984863,
+ "learning_rate": 3.2222222222222227e-06,
+ "loss": 1.5638453960418701,
+ "mean_token_accuracy": 0.648937463760376,
+ "num_tokens": 14434304.0,
+ "step": 881
+ },
+ {
+ "entropy": 1.4249905347824097,
+ "epoch": 1.7818181818181817,
+ "grad_norm": 1.9853826761245728,
+ "learning_rate": 3.2202020202020206e-06,
+ "loss": 1.4453399181365967,
+ "mean_token_accuracy": 0.6869809627532959,
+ "num_tokens": 14450688.0,
+ "step": 882
+ },
+ {
+ "entropy": 1.4777165651321411,
+ "epoch": 1.783838383838384,
+ "grad_norm": 2.064542055130005,
+ "learning_rate": 3.2181818181818184e-06,
+ "loss": 1.4756921529769897,
+ "mean_token_accuracy": 0.6640205383300781,
+ "num_tokens": 14467072.0,
+ "step": 883
+ },
+ {
+ "entropy": 1.3442282676696777,
+ "epoch": 1.7858585858585858,
+ "grad_norm": 2.043712854385376,
+ "learning_rate": 3.2161616161616168e-06,
+ "loss": 1.3412827253341675,
+ "mean_token_accuracy": 0.6821568012237549,
+ "num_tokens": 14483456.0,
+ "step": 884
+ },
+ {
+ "entropy": 1.5201579332351685,
+ "epoch": 1.7878787878787878,
+ "grad_norm": 2.060661792755127,
+ "learning_rate": 3.2141414141414146e-06,
+ "loss": 1.5367834568023682,
+ "mean_token_accuracy": 0.6442354917526245,
+ "num_tokens": 14499840.0,
+ "step": 885
+ },
+ {
+ "entropy": 1.1536649465560913,
+ "epoch": 1.7898989898989899,
+ "grad_norm": 2.1190037727355957,
+ "learning_rate": 3.2121212121212125e-06,
+ "loss": 1.1588165760040283,
+ "mean_token_accuracy": 0.7039570212364197,
+ "num_tokens": 14516224.0,
+ "step": 886
+ },
+ {
+ "entropy": 1.109966516494751,
+ "epoch": 1.791919191919192,
+ "grad_norm": 2.0320141315460205,
+ "learning_rate": 3.2101010101010104e-06,
+ "loss": 1.1113003492355347,
+ "mean_token_accuracy": 0.7123228907585144,
+ "num_tokens": 14532608.0,
+ "step": 887
+ },
+ {
+ "entropy": 1.5709099769592285,
+ "epoch": 1.793939393939394,
+ "grad_norm": 2.0467336177825928,
+ "learning_rate": 3.2080808080808083e-06,
+ "loss": 1.5823577642440796,
+ "mean_token_accuracy": 0.6404494643211365,
+ "num_tokens": 14548992.0,
+ "step": 888
+ },
+ {
+ "entropy": 1.4305545091629028,
+ "epoch": 1.795959595959596,
+ "grad_norm": 2.0764269828796387,
+ "learning_rate": 3.2060606060606066e-06,
+ "loss": 1.438415765762329,
+ "mean_token_accuracy": 0.6574255228042603,
+ "num_tokens": 14565376.0,
+ "step": 889
+ },
+ {
+ "entropy": 1.3523615598678589,
+ "epoch": 1.797979797979798,
+ "grad_norm": 2.167036294937134,
+ "learning_rate": 3.2040404040404045e-06,
+ "loss": 1.3782477378845215,
+ "mean_token_accuracy": 0.6645090579986572,
+ "num_tokens": 14581760.0,
+ "step": 890
+ },
+ {
+ "entropy": 1.491297721862793,
+ "epoch": 1.8,
+ "grad_norm": 2.3090412616729736,
+ "learning_rate": 3.2020202020202024e-06,
+ "loss": 1.4965641498565674,
+ "mean_token_accuracy": 0.636907696723938,
+ "num_tokens": 14598144.0,
+ "step": 891
+ },
+ {
+ "entropy": 1.2584501504898071,
+ "epoch": 1.802020202020202,
+ "grad_norm": 2.1699554920196533,
+ "learning_rate": 3.2000000000000003e-06,
+ "loss": 1.2622835636138916,
+ "mean_token_accuracy": 0.6901563405990601,
+ "num_tokens": 14614528.0,
+ "step": 892
+ },
+ {
+ "entropy": 2.149350643157959,
+ "epoch": 1.8040404040404039,
+ "grad_norm": 2.2669718265533447,
+ "learning_rate": 3.197979797979798e-06,
+ "loss": 2.167891263961792,
+ "mean_token_accuracy": 0.5497679710388184,
+ "num_tokens": 14630912.0,
+ "step": 893
+ },
+ {
+ "entropy": 1.1999561786651611,
+ "epoch": 1.8060606060606061,
+ "grad_norm": 2.163228988647461,
+ "learning_rate": 3.195959595959596e-06,
+ "loss": 1.2024329900741577,
+ "mean_token_accuracy": 0.7035906314849854,
+ "num_tokens": 14647296.0,
+ "step": 894
+ },
+ {
+ "entropy": 1.770652413368225,
+ "epoch": 1.808080808080808,
+ "grad_norm": 2.1631994247436523,
+ "learning_rate": 3.1939393939393944e-06,
+ "loss": 1.7568567991256714,
+ "mean_token_accuracy": 0.6030776500701904,
+ "num_tokens": 14663680.0,
+ "step": 895
+ },
+ {
+ "entropy": 1.2487529516220093,
+ "epoch": 1.8101010101010102,
+ "grad_norm": 1.987622857093811,
+ "learning_rate": 3.1919191919191923e-06,
+ "loss": 1.255242109298706,
+ "mean_token_accuracy": 0.6929042339324951,
+ "num_tokens": 14680064.0,
+ "step": 896
+ },
+ {
+ "entropy": 1.138524055480957,
+ "epoch": 1.812121212121212,
+ "grad_norm": 2.0367984771728516,
+ "learning_rate": 3.18989898989899e-06,
+ "loss": 1.1514946222305298,
+ "mean_token_accuracy": 0.7093917727470398,
+ "num_tokens": 14696448.0,
+ "step": 897
+ },
+ {
+ "entropy": 1.3337515592575073,
+ "epoch": 1.8141414141414143,
+ "grad_norm": 1.9483140707015991,
+ "learning_rate": 3.187878787878788e-06,
+ "loss": 1.3251439332962036,
+ "mean_token_accuracy": 0.6861260533332825,
+ "num_tokens": 14712832.0,
+ "step": 898
+ },
+ {
+ "entropy": 1.047120451927185,
+ "epoch": 1.816161616161616,
+ "grad_norm": 2.0481390953063965,
+ "learning_rate": 3.185858585858586e-06,
+ "loss": 1.071772575378418,
+ "mean_token_accuracy": 0.7275891304016113,
+ "num_tokens": 14729216.0,
+ "step": 899
+ },
+ {
+ "entropy": 1.3311363458633423,
+ "epoch": 1.8181818181818183,
+ "grad_norm": 2.3203413486480713,
+ "learning_rate": 3.1838383838383842e-06,
+ "loss": 1.334134817123413,
+ "mean_token_accuracy": 0.6643869280815125,
+ "num_tokens": 14745600.0,
+ "step": 900
+ },
+ {
+ "epoch": 1.8181818181818183,
+ "eval_entropy": 1.4476436400605786,
+ "eval_loss": 1.5290467739105225,
+ "eval_mean_token_accuracy": 0.6456143543604882,
+ "eval_num_tokens": 14745600.0,
+ "eval_runtime": 43.8113,
+ "eval_samples_per_second": 22.597,
+ "eval_steps_per_second": 2.83,
+ "step": 900
+ },
+ {
+ "entropy": 1.6842114925384521,
+ "epoch": 1.8202020202020202,
+ "grad_norm": 2.1709823608398438,
+ "learning_rate": 3.181818181818182e-06,
+ "loss": 1.6816682815551758,
+ "mean_token_accuracy": 0.6033830046653748,
+ "num_tokens": 14761984.0,
+ "step": 901
+ },
+ {
+ "entropy": 1.2661798000335693,
+ "epoch": 1.8222222222222222,
+ "grad_norm": 2.113783836364746,
+ "learning_rate": 3.17979797979798e-06,
+ "loss": 1.249183177947998,
+ "mean_token_accuracy": 0.6865534782409668,
+ "num_tokens": 14778368.0,
+ "step": 902
+ },
+ {
+ "entropy": 1.2666339874267578,
+ "epoch": 1.8242424242424242,
+ "grad_norm": 2.0210978984832764,
+ "learning_rate": 3.177777777777778e-06,
+ "loss": 1.2548515796661377,
+ "mean_token_accuracy": 0.686431348323822,
+ "num_tokens": 14794752.0,
+ "step": 903
+ },
+ {
+ "entropy": 1.1660066843032837,
+ "epoch": 1.8262626262626263,
+ "grad_norm": 2.0998034477233887,
+ "learning_rate": 3.1757575757575758e-06,
+ "loss": 1.1654324531555176,
+ "mean_token_accuracy": 0.695713222026825,
+ "num_tokens": 14811136.0,
+ "step": 904
+ },
+ {
+ "entropy": 1.2902735471725464,
+ "epoch": 1.8282828282828283,
+ "grad_norm": 2.1510303020477295,
+ "learning_rate": 3.173737373737374e-06,
+ "loss": 1.2879812717437744,
+ "mean_token_accuracy": 0.6693942546844482,
+ "num_tokens": 14827520.0,
+ "step": 905
+ },
+ {
+ "entropy": 1.1441799402236938,
+ "epoch": 1.8303030303030303,
+ "grad_norm": 1.989449143409729,
+ "learning_rate": 3.171717171717172e-06,
+ "loss": 1.1552281379699707,
+ "mean_token_accuracy": 0.7127503752708435,
+ "num_tokens": 14843904.0,
+ "step": 906
+ },
+ {
+ "entropy": 1.08464515209198,
+ "epoch": 1.8323232323232324,
+ "grad_norm": 2.094696044921875,
+ "learning_rate": 3.16969696969697e-06,
+ "loss": 1.0758531093597412,
+ "mean_token_accuracy": 0.7164142727851868,
+ "num_tokens": 14860288.0,
+ "step": 907
+ },
+ {
+ "entropy": 1.3847618103027344,
+ "epoch": 1.8343434343434344,
+ "grad_norm": 2.207976818084717,
+ "learning_rate": 3.1676767676767678e-06,
+ "loss": 1.3722931146621704,
+ "mean_token_accuracy": 0.6741573214530945,
+ "num_tokens": 14876672.0,
+ "step": 908
+ },
+ {
+ "entropy": 1.3558534383773804,
+ "epoch": 1.8363636363636364,
+ "grad_norm": 2.166674852371216,
+ "learning_rate": 3.1656565656565656e-06,
+ "loss": 1.3612618446350098,
+ "mean_token_accuracy": 0.6669516563415527,
+ "num_tokens": 14893056.0,
+ "step": 909
+ },
+ {
+ "entropy": 1.4160822629928589,
+ "epoch": 1.8383838383838382,
+ "grad_norm": 2.2242727279663086,
+ "learning_rate": 3.1636363636363635e-06,
+ "loss": 1.408278226852417,
+ "mean_token_accuracy": 0.6634098887443542,
+ "num_tokens": 14909440.0,
+ "step": 910
+ },
+ {
+ "entropy": 1.1337573528289795,
+ "epoch": 1.8404040404040405,
+ "grad_norm": 2.1882550716400146,
+ "learning_rate": 3.161616161616162e-06,
+ "loss": 1.1503205299377441,
+ "mean_token_accuracy": 0.7059721350669861,
+ "num_tokens": 14925824.0,
+ "step": 911
+ },
+ {
+ "entropy": 1.4373403787612915,
+ "epoch": 1.8424242424242423,
+ "grad_norm": 2.1595654487609863,
+ "learning_rate": 3.1595959595959597e-06,
+ "loss": 1.4344165325164795,
+ "mean_token_accuracy": 0.6599902510643005,
+ "num_tokens": 14942208.0,
+ "step": 912
+ },
+ {
+ "entropy": 1.0798827409744263,
+ "epoch": 1.8444444444444446,
+ "grad_norm": 2.101088047027588,
+ "learning_rate": 3.1575757575757576e-06,
+ "loss": 1.0873513221740723,
+ "mean_token_accuracy": 0.7169027924537659,
+ "num_tokens": 14958592.0,
+ "step": 913
+ },
+ {
+ "entropy": 1.7024599313735962,
+ "epoch": 1.8464646464646464,
+ "grad_norm": 2.291907548904419,
+ "learning_rate": 3.1555555555555555e-06,
+ "loss": 1.7014999389648438,
+ "mean_token_accuracy": 0.6105275750160217,
+ "num_tokens": 14974976.0,
+ "step": 914
+ },
+ {
+ "entropy": 1.7765953540802002,
+ "epoch": 1.8484848484848486,
+ "grad_norm": 2.1465439796447754,
+ "learning_rate": 3.1535353535353534e-06,
+ "loss": 1.788269281387329,
+ "mean_token_accuracy": 0.6066194176673889,
+ "num_tokens": 14991360.0,
+ "step": 915
+ },
+ {
+ "entropy": 1.5223884582519531,
+ "epoch": 1.8505050505050504,
+ "grad_norm": 2.313291549682617,
+ "learning_rate": 3.1515151515151517e-06,
+ "loss": 1.55397367477417,
+ "mean_token_accuracy": 0.6377626061439514,
+ "num_tokens": 15007744.0,
+ "step": 916
+ },
+ {
+ "entropy": 1.5820955038070679,
+ "epoch": 1.8525252525252527,
+ "grad_norm": 2.094886541366577,
+ "learning_rate": 3.1494949494949496e-06,
+ "loss": 1.58372962474823,
+ "mean_token_accuracy": 0.6339154839515686,
+ "num_tokens": 15024128.0,
+ "step": 917
+ },
+ {
+ "entropy": 1.7399076223373413,
+ "epoch": 1.8545454545454545,
+ "grad_norm": 2.0311105251312256,
+ "learning_rate": 3.1474747474747475e-06,
+ "loss": 1.7362079620361328,
+ "mean_token_accuracy": 0.604421079158783,
+ "num_tokens": 15040512.0,
+ "step": 918
+ },
+ {
+ "entropy": 1.4311537742614746,
+ "epoch": 1.8565656565656565,
+ "grad_norm": 2.13740873336792,
+ "learning_rate": 3.145454545454546e-06,
+ "loss": 1.4574000835418701,
+ "mean_token_accuracy": 0.648876428604126,
+ "num_tokens": 15056896.0,
+ "step": 919
+ },
+ {
+ "entropy": 1.6997709274291992,
+ "epoch": 1.8585858585858586,
+ "grad_norm": 2.1604959964752197,
+ "learning_rate": 3.143434343434344e-06,
+ "loss": 1.694093942642212,
+ "mean_token_accuracy": 0.6144357323646545,
+ "num_tokens": 15073280.0,
+ "step": 920
+ },
+ {
+ "entropy": 1.5796921253204346,
+ "epoch": 1.8606060606060606,
+ "grad_norm": 2.3852932453155518,
+ "learning_rate": 3.141414141414142e-06,
+ "loss": 1.5799579620361328,
+ "mean_token_accuracy": 0.6322667598724365,
+ "num_tokens": 15089664.0,
+ "step": 921
+ },
+ {
+ "entropy": 1.6185977458953857,
+ "epoch": 1.8626262626262626,
+ "grad_norm": 2.0631208419799805,
+ "learning_rate": 3.13939393939394e-06,
+ "loss": 1.6541715860366821,
+ "mean_token_accuracy": 0.632083535194397,
+ "num_tokens": 15106048.0,
+ "step": 922
+ },
+ {
+ "entropy": 1.5872372388839722,
+ "epoch": 1.8646464646464647,
+ "grad_norm": 2.154554843902588,
+ "learning_rate": 3.1373737373737378e-06,
+ "loss": 1.5946767330169678,
+ "mean_token_accuracy": 0.6294577717781067,
+ "num_tokens": 15122432.0,
+ "step": 923
+ },
+ {
+ "entropy": 1.6059986352920532,
+ "epoch": 1.8666666666666667,
+ "grad_norm": 1.9893742799758911,
+ "learning_rate": 3.1353535353535357e-06,
+ "loss": 1.6153039932250977,
+ "mean_token_accuracy": 0.6373962163925171,
+ "num_tokens": 15138816.0,
+ "step": 924
+ },
+ {
+ "entropy": 1.3143881559371948,
+ "epoch": 1.8686868686868687,
+ "grad_norm": 1.8989832401275635,
+ "learning_rate": 3.133333333333334e-06,
+ "loss": 1.3332319259643555,
+ "mean_token_accuracy": 0.6842330098152161,
+ "num_tokens": 15155200.0,
+ "step": 925
+ },
+ {
+ "entropy": 1.0474469661712646,
+ "epoch": 1.8707070707070708,
+ "grad_norm": 1.9129916429519653,
+ "learning_rate": 3.131313131313132e-06,
+ "loss": 1.0238264799118042,
+ "mean_token_accuracy": 0.7459697127342224,
+ "num_tokens": 15171584.0,
+ "step": 926
+ },
+ {
+ "entropy": 1.581508994102478,
+ "epoch": 1.8727272727272726,
+ "grad_norm": 2.0047285556793213,
+ "learning_rate": 3.1292929292929297e-06,
+ "loss": 1.6087100505828857,
+ "mean_token_accuracy": 0.6282364726066589,
+ "num_tokens": 15187968.0,
+ "step": 927
+ },
+ {
+ "entropy": 1.0985661745071411,
+ "epoch": 1.8747474747474748,
+ "grad_norm": 2.1207540035247803,
+ "learning_rate": 3.1272727272727276e-06,
+ "loss": 1.1158297061920166,
+ "mean_token_accuracy": 0.7111626863479614,
+ "num_tokens": 15204352.0,
+ "step": 928
+ },
+ {
+ "entropy": 1.4591490030288696,
+ "epoch": 1.8767676767676766,
+ "grad_norm": 2.2879691123962402,
+ "learning_rate": 3.1252525252525255e-06,
+ "loss": 1.4775316715240479,
+ "mean_token_accuracy": 0.6477161645889282,
+ "num_tokens": 15220736.0,
+ "step": 929
+ },
+ {
+ "entropy": 1.5469937324523926,
+ "epoch": 1.878787878787879,
+ "grad_norm": 2.0972726345062256,
+ "learning_rate": 3.1232323232323234e-06,
+ "loss": 1.5949021577835083,
+ "mean_token_accuracy": 0.6547996997833252,
+ "num_tokens": 15237120.0,
+ "step": 930
+ },
+ {
+ "entropy": 1.363472819328308,
+ "epoch": 1.8808080808080807,
+ "grad_norm": 2.149606943130493,
+ "learning_rate": 3.1212121212121217e-06,
+ "loss": 1.358612298965454,
+ "mean_token_accuracy": 0.6780654788017273,
+ "num_tokens": 15253504.0,
+ "step": 931
+ },
+ {
+ "entropy": 1.189648985862732,
+ "epoch": 1.882828282828283,
+ "grad_norm": 2.2575035095214844,
+ "learning_rate": 3.1191919191919196e-06,
+ "loss": 1.209303379058838,
+ "mean_token_accuracy": 0.7002320289611816,
+ "num_tokens": 15269888.0,
+ "step": 932
+ },
+ {
+ "entropy": 1.3054684400558472,
+ "epoch": 1.8848484848484848,
+ "grad_norm": 2.243915319442749,
+ "learning_rate": 3.1171717171717175e-06,
+ "loss": 1.333367109298706,
+ "mean_token_accuracy": 0.6703712940216064,
+ "num_tokens": 15286272.0,
+ "step": 933
+ },
+ {
+ "entropy": 1.6298167705535889,
+ "epoch": 1.886868686868687,
+ "grad_norm": 1.9654396772384644,
+ "learning_rate": 3.1151515151515154e-06,
+ "loss": 1.6496453285217285,
+ "mean_token_accuracy": 0.6248778700828552,
+ "num_tokens": 15302656.0,
+ "step": 934
+ },
+ {
+ "entropy": 1.1228219270706177,
+ "epoch": 1.8888888888888888,
+ "grad_norm": 2.2492377758026123,
+ "learning_rate": 3.1131313131313133e-06,
+ "loss": 1.1560063362121582,
+ "mean_token_accuracy": 0.699499249458313,
+ "num_tokens": 15319040.0,
+ "step": 935
+ },
+ {
+ "entropy": 1.7481759786605835,
+ "epoch": 1.8909090909090909,
+ "grad_norm": 2.1635665893554688,
+ "learning_rate": 3.1111111111111116e-06,
+ "loss": 1.737868070602417,
+ "mean_token_accuracy": 0.6030776500701904,
+ "num_tokens": 15335424.0,
+ "step": 936
+ },
+ {
+ "entropy": 1.3964993953704834,
+ "epoch": 1.892929292929293,
+ "grad_norm": 2.262946367263794,
+ "learning_rate": 3.1090909090909095e-06,
+ "loss": 1.3927817344665527,
+ "mean_token_accuracy": 0.6546775698661804,
+ "num_tokens": 15351808.0,
+ "step": 937
+ },
+ {
+ "entropy": 1.2503907680511475,
+ "epoch": 1.894949494949495,
+ "grad_norm": 1.9837188720703125,
+ "learning_rate": 3.1070707070707074e-06,
+ "loss": 1.2529189586639404,
+ "mean_token_accuracy": 0.6965681314468384,
+ "num_tokens": 15368192.0,
+ "step": 938
+ },
+ {
+ "entropy": 0.9774144291877747,
+ "epoch": 1.896969696969697,
+ "grad_norm": 1.9201257228851318,
+ "learning_rate": 3.1050505050505052e-06,
+ "loss": 0.9864459037780762,
+ "mean_token_accuracy": 0.7534807324409485,
+ "num_tokens": 15384576.0,
+ "step": 939
+ },
+ {
+ "entropy": 1.4065781831741333,
+ "epoch": 1.898989898989899,
+ "grad_norm": 2.357865571975708,
+ "learning_rate": 3.103030303030303e-06,
+ "loss": 1.428115725517273,
+ "mean_token_accuracy": 0.6493649482727051,
+ "num_tokens": 15400960.0,
+ "step": 940
+ },
+ {
+ "entropy": 1.1632903814315796,
+ "epoch": 1.901010101010101,
+ "grad_norm": 1.8824918270111084,
+ "learning_rate": 3.1010101010101014e-06,
+ "loss": 1.1478252410888672,
+ "mean_token_accuracy": 0.7158036231994629,
+ "num_tokens": 15417344.0,
+ "step": 941
+ },
+ {
+ "entropy": 1.7470908164978027,
+ "epoch": 1.903030303030303,
+ "grad_norm": 2.2958669662475586,
+ "learning_rate": 3.0989898989898993e-06,
+ "loss": 1.7697877883911133,
+ "mean_token_accuracy": 0.5892769694328308,
+ "num_tokens": 15433728.0,
+ "step": 942
+ },
+ {
+ "entropy": 1.2897151708602905,
+ "epoch": 1.905050505050505,
+ "grad_norm": 2.155731439590454,
+ "learning_rate": 3.0969696969696972e-06,
+ "loss": 1.2971919775009155,
+ "mean_token_accuracy": 0.6731802821159363,
+ "num_tokens": 15450112.0,
+ "step": 943
+ },
+ {
+ "entropy": 1.2687026262283325,
+ "epoch": 1.907070707070707,
+ "grad_norm": 2.0896284580230713,
+ "learning_rate": 3.094949494949495e-06,
+ "loss": 1.2423906326293945,
+ "mean_token_accuracy": 0.6988885998725891,
+ "num_tokens": 15466496.0,
+ "step": 944
+ },
+ {
+ "entropy": 1.5710495710372925,
+ "epoch": 1.9090909090909092,
+ "grad_norm": 2.200758457183838,
+ "learning_rate": 3.092929292929293e-06,
+ "loss": 1.5329574346542358,
+ "mean_token_accuracy": 0.6560820937156677,
+ "num_tokens": 15482880.0,
+ "step": 945
+ },
+ {
+ "entropy": 1.1555253267288208,
+ "epoch": 1.911111111111111,
+ "grad_norm": 2.278230905532837,
+ "learning_rate": 3.090909090909091e-06,
+ "loss": 1.1469731330871582,
+ "mean_token_accuracy": 0.7220932841300964,
+ "num_tokens": 15499264.0,
+ "step": 946
+ },
+ {
+ "entropy": 1.2870018482208252,
+ "epoch": 1.9131313131313132,
+ "grad_norm": 1.8973854780197144,
+ "learning_rate": 3.088888888888889e-06,
+ "loss": 1.2787907123565674,
+ "mean_token_accuracy": 0.6885075569152832,
+ "num_tokens": 15515648.0,
+ "step": 947
+ },
+ {
+ "entropy": 1.324872612953186,
+ "epoch": 1.915151515151515,
+ "grad_norm": 2.1889214515686035,
+ "learning_rate": 3.086868686868687e-06,
+ "loss": 1.3286981582641602,
+ "mean_token_accuracy": 0.6743404865264893,
+ "num_tokens": 15532032.0,
+ "step": 948
+ },
+ {
+ "entropy": 1.5601656436920166,
+ "epoch": 1.9171717171717173,
+ "grad_norm": 2.304993152618408,
+ "learning_rate": 3.084848484848485e-06,
+ "loss": 1.5766493082046509,
+ "mean_token_accuracy": 0.6172447204589844,
+ "num_tokens": 15548416.0,
+ "step": 949
+ },
+ {
+ "entropy": 1.4633477926254272,
+ "epoch": 1.9191919191919191,
+ "grad_norm": 1.9083963632583618,
+ "learning_rate": 3.082828282828283e-06,
+ "loss": 1.4554777145385742,
+ "mean_token_accuracy": 0.6667073965072632,
+ "num_tokens": 15564800.0,
+ "step": 950
+ },
+ {
+ "epoch": 1.9191919191919191,
+ "eval_entropy": 1.4443931824737979,
+ "eval_loss": 1.5272752046585083,
+ "eval_mean_token_accuracy": 0.6459201723337173,
+ "eval_num_tokens": 15564800.0,
+ "eval_runtime": 43.7243,
+ "eval_samples_per_second": 22.642,
+ "eval_steps_per_second": 2.836,
+ "step": 950
+ },
+ {
+ "entropy": 1.3350679874420166,
+ "epoch": 1.9212121212121214,
+ "grad_norm": 2.139172315597534,
+ "learning_rate": 3.0808080808080807e-06,
+ "loss": 1.3285927772521973,
+ "mean_token_accuracy": 0.6833781003952026,
+ "num_tokens": 15581184.0,
+ "step": 951
+ },
+ {
+ "entropy": 1.2373814582824707,
+ "epoch": 1.9232323232323232,
+ "grad_norm": 2.1068115234375,
+ "learning_rate": 3.078787878787879e-06,
+ "loss": 1.2527921199798584,
+ "mean_token_accuracy": 0.6969345211982727,
+ "num_tokens": 15597568.0,
+ "step": 952
+ },
+ {
+ "entropy": 1.2182631492614746,
+ "epoch": 1.9252525252525252,
+ "grad_norm": 2.1326634883880615,
+ "learning_rate": 3.076767676767677e-06,
+ "loss": 1.2174904346466064,
+ "mean_token_accuracy": 0.6905227303504944,
+ "num_tokens": 15613952.0,
+ "step": 953
+ },
+ {
+ "entropy": 0.9768888354301453,
+ "epoch": 1.9272727272727272,
+ "grad_norm": 1.9833085536956787,
+ "learning_rate": 3.074747474747475e-06,
+ "loss": 0.9956569671630859,
+ "mean_token_accuracy": 0.7401685118675232,
+ "num_tokens": 15630336.0,
+ "step": 954
+ },
+ {
+ "entropy": 0.7818430066108704,
+ "epoch": 1.9292929292929293,
+ "grad_norm": 1.9219205379486084,
+ "learning_rate": 3.0727272727272727e-06,
+ "loss": 0.7922182083129883,
+ "mean_token_accuracy": 0.7827919125556946,
+ "num_tokens": 15646720.0,
+ "step": 955
+ },
+ {
+ "entropy": 1.6172020435333252,
+ "epoch": 1.9313131313131313,
+ "grad_norm": 1.9390695095062256,
+ "learning_rate": 3.0707070707070706e-06,
+ "loss": 1.6629729270935059,
+ "mean_token_accuracy": 0.6192598938941956,
+ "num_tokens": 15663104.0,
+ "step": 956
+ },
+ {
+ "entropy": 1.291207194328308,
+ "epoch": 1.9333333333333333,
+ "grad_norm": 1.9242857694625854,
+ "learning_rate": 3.068686868686869e-06,
+ "loss": 1.2721552848815918,
+ "mean_token_accuracy": 0.6867367029190063,
+ "num_tokens": 15679488.0,
+ "step": 957
+ },
+ {
+ "entropy": 1.3730298280715942,
+ "epoch": 1.9353535353535354,
+ "grad_norm": 1.9483041763305664,
+ "learning_rate": 3.066666666666667e-06,
+ "loss": 1.3835055828094482,
+ "mean_token_accuracy": 0.6745237112045288,
+ "num_tokens": 15695872.0,
+ "step": 958
+ },
+ {
+ "entropy": 1.1266423463821411,
+ "epoch": 1.9373737373737374,
+ "grad_norm": 2.1663992404937744,
+ "learning_rate": 3.0646464646464647e-06,
+ "loss": 1.1091173887252808,
+ "mean_token_accuracy": 0.7151318788528442,
+ "num_tokens": 15712256.0,
+ "step": 959
+ },
+ {
+ "entropy": 1.298812747001648,
+ "epoch": 1.9393939393939394,
+ "grad_norm": 2.0564448833465576,
+ "learning_rate": 3.0626262626262626e-06,
+ "loss": 1.3167932033538818,
+ "mean_token_accuracy": 0.6812408566474915,
+ "num_tokens": 15728640.0,
+ "step": 960
+ },
+ {
+ "entropy": 1.7007938623428345,
+ "epoch": 1.9414141414141413,
+ "grad_norm": 2.1043670177459717,
+ "learning_rate": 3.0606060606060605e-06,
+ "loss": 1.6956250667572021,
+ "mean_token_accuracy": 0.6194430589675903,
+ "num_tokens": 15745024.0,
+ "step": 961
+ },
+ {
+ "entropy": 1.408955693244934,
+ "epoch": 1.9434343434343435,
+ "grad_norm": 2.0432121753692627,
+ "learning_rate": 3.058585858585859e-06,
+ "loss": 1.3905062675476074,
+ "mean_token_accuracy": 0.6662799119949341,
+ "num_tokens": 15761408.0,
+ "step": 962
+ },
+ {
+ "entropy": 1.938867211341858,
+ "epoch": 1.9454545454545453,
+ "grad_norm": 2.3359744548797607,
+ "learning_rate": 3.056565656565657e-06,
+ "loss": 1.9670305252075195,
+ "mean_token_accuracy": 0.5845139026641846,
+ "num_tokens": 15777792.0,
+ "step": 963
+ },
+ {
+ "entropy": 1.5049448013305664,
+ "epoch": 1.9474747474747476,
+ "grad_norm": 2.1907997131347656,
+ "learning_rate": 3.054545454545455e-06,
+ "loss": 1.534651756286621,
+ "mean_token_accuracy": 0.652723491191864,
+ "num_tokens": 15794176.0,
+ "step": 964
+ },
+ {
+ "entropy": 1.1916700601577759,
+ "epoch": 1.9494949494949494,
+ "grad_norm": 2.098696231842041,
+ "learning_rate": 3.052525252525253e-06,
+ "loss": 1.191361904144287,
+ "mean_token_accuracy": 0.715192973613739,
+ "num_tokens": 15810560.0,
+ "step": 965
+ },
+ {
+ "entropy": 1.6707743406295776,
+ "epoch": 1.9515151515151516,
+ "grad_norm": 2.106846570968628,
+ "learning_rate": 3.0505050505050508e-06,
+ "loss": 1.6789159774780273,
+ "mean_token_accuracy": 0.6133365631103516,
+ "num_tokens": 15826944.0,
+ "step": 966
+ },
+ {
+ "entropy": 1.4679991006851196,
+ "epoch": 1.9535353535353535,
+ "grad_norm": 2.1568050384521484,
+ "learning_rate": 3.048484848484849e-06,
+ "loss": 1.4427556991577148,
+ "mean_token_accuracy": 0.6526013612747192,
+ "num_tokens": 15843328.0,
+ "step": 967
+ },
+ {
+ "entropy": 0.9881705641746521,
+ "epoch": 1.9555555555555557,
+ "grad_norm": 2.029680013656616,
+ "learning_rate": 3.046464646464647e-06,
+ "loss": 0.9961811304092407,
+ "mean_token_accuracy": 0.7408402562141418,
+ "num_tokens": 15859712.0,
+ "step": 968
+ },
+ {
+ "entropy": 1.0721628665924072,
+ "epoch": 1.9575757575757575,
+ "grad_norm": 1.9763140678405762,
+ "learning_rate": 3.044444444444445e-06,
+ "loss": 1.067029356956482,
+ "mean_token_accuracy": 0.733146071434021,
+ "num_tokens": 15876096.0,
+ "step": 969
+ },
+ {
+ "entropy": 1.7053834199905396,
+ "epoch": 1.9595959595959596,
+ "grad_norm": 2.3659849166870117,
+ "learning_rate": 3.0424242424242427e-06,
+ "loss": 1.7629203796386719,
+ "mean_token_accuracy": 0.6105275750160217,
+ "num_tokens": 15892480.0,
+ "step": 970
+ },
+ {
+ "entropy": 1.8083471059799194,
+ "epoch": 1.9616161616161616,
+ "grad_norm": 2.081869602203369,
+ "learning_rate": 3.0404040404040406e-06,
+ "loss": 1.8430367708206177,
+ "mean_token_accuracy": 0.6009404063224792,
+ "num_tokens": 15908864.0,
+ "step": 971
+ },
+ {
+ "entropy": 1.2576755285263062,
+ "epoch": 1.9636363636363636,
+ "grad_norm": 2.128230571746826,
+ "learning_rate": 3.038383838383839e-06,
+ "loss": 1.2493822574615479,
+ "mean_token_accuracy": 0.6913776397705078,
+ "num_tokens": 15925248.0,
+ "step": 972
+ },
+ {
+ "entropy": 0.8995128273963928,
+ "epoch": 1.9656565656565657,
+ "grad_norm": 2.0216031074523926,
+ "learning_rate": 3.036363636363637e-06,
+ "loss": 0.9069615602493286,
+ "mean_token_accuracy": 0.7620908617973328,
+ "num_tokens": 15941632.0,
+ "step": 973
+ },
+ {
+ "entropy": 1.615832805633545,
+ "epoch": 1.9676767676767677,
+ "grad_norm": 2.26552152633667,
+ "learning_rate": 3.0343434343434347e-06,
+ "loss": 1.6284873485565186,
+ "mean_token_accuracy": 0.6193209290504456,
+ "num_tokens": 15958016.0,
+ "step": 974
+ },
+ {
+ "entropy": 1.404674768447876,
+ "epoch": 1.9696969696969697,
+ "grad_norm": 2.1790690422058105,
+ "learning_rate": 3.0323232323232326e-06,
+ "loss": 1.4130914211273193,
+ "mean_token_accuracy": 0.6594406366348267,
+ "num_tokens": 15974400.0,
+ "step": 975
+ },
+ {
+ "entropy": 1.3440049886703491,
+ "epoch": 1.9717171717171718,
+ "grad_norm": 2.121338367462158,
+ "learning_rate": 3.0303030303030305e-06,
+ "loss": 1.3652238845825195,
+ "mean_token_accuracy": 0.6794699430465698,
+ "num_tokens": 15990784.0,
+ "step": 976
+ },
+ {
+ "entropy": 1.0894689559936523,
+ "epoch": 1.9737373737373738,
+ "grad_norm": 2.0155327320098877,
+ "learning_rate": 3.028282828282829e-06,
+ "loss": 1.0642526149749756,
+ "mean_token_accuracy": 0.7223986387252808,
+ "num_tokens": 16007168.0,
+ "step": 977
+ },
+ {
+ "entropy": 1.469588041305542,
+ "epoch": 1.9757575757575756,
+ "grad_norm": 2.2047550678253174,
+ "learning_rate": 3.0262626262626267e-06,
+ "loss": 1.498295545578003,
+ "mean_token_accuracy": 0.6441133618354797,
+ "num_tokens": 16023552.0,
+ "step": 978
+ },
+ {
+ "entropy": 1.6617093086242676,
+ "epoch": 1.9777777777777779,
+ "grad_norm": 1.9770722389221191,
+ "learning_rate": 3.0242424242424246e-06,
+ "loss": 1.6753556728363037,
+ "mean_token_accuracy": 0.6284807324409485,
+ "num_tokens": 16039936.0,
+ "step": 979
+ },
+ {
+ "entropy": 1.8029206991195679,
+ "epoch": 1.9797979797979797,
+ "grad_norm": 2.2365968227386475,
+ "learning_rate": 3.0222222222222225e-06,
+ "loss": 1.7957369089126587,
+ "mean_token_accuracy": 0.601062536239624,
+ "num_tokens": 16056320.0,
+ "step": 980
+ },
+ {
+ "entropy": 1.6331384181976318,
+ "epoch": 1.981818181818182,
+ "grad_norm": 2.0565884113311768,
+ "learning_rate": 3.0202020202020203e-06,
+ "loss": 1.673048973083496,
+ "mean_token_accuracy": 0.6203590631484985,
+ "num_tokens": 16072704.0,
+ "step": 981
+ },
+ {
+ "entropy": 1.1730273962020874,
+ "epoch": 1.9838383838383837,
+ "grad_norm": 2.0347228050231934,
+ "learning_rate": 3.0181818181818182e-06,
+ "loss": 1.1709872484207153,
+ "mean_token_accuracy": 0.7086589932441711,
+ "num_tokens": 16089088.0,
+ "step": 982
+ },
+ {
+ "entropy": 1.1109111309051514,
+ "epoch": 1.985858585858586,
+ "grad_norm": 1.9992051124572754,
+ "learning_rate": 3.0161616161616165e-06,
+ "loss": 1.097399353981018,
+ "mean_token_accuracy": 0.7245969772338867,
+ "num_tokens": 16105472.0,
+ "step": 983
+ },
+ {
+ "entropy": 1.2580686807632446,
+ "epoch": 1.9878787878787878,
+ "grad_norm": 2.1748640537261963,
+ "learning_rate": 3.0141414141414144e-06,
+ "loss": 1.2503113746643066,
+ "mean_token_accuracy": 0.6883854269981384,
+ "num_tokens": 16121856.0,
+ "step": 984
+ },
+ {
+ "entropy": 1.4884002208709717,
+ "epoch": 1.98989898989899,
+ "grad_norm": 2.3283042907714844,
+ "learning_rate": 3.0121212121212123e-06,
+ "loss": 1.4856922626495361,
+ "mean_token_accuracy": 0.6461895704269409,
+ "num_tokens": 16138240.0,
+ "step": 985
+ },
+ {
+ "entropy": 1.6224603652954102,
+ "epoch": 1.9919191919191919,
+ "grad_norm": 2.0596823692321777,
+ "learning_rate": 3.0101010101010102e-06,
+ "loss": 1.6202584505081177,
+ "mean_token_accuracy": 0.6388617753982544,
+ "num_tokens": 16154624.0,
+ "step": 986
+ },
+ {
+ "entropy": 1.559553623199463,
+ "epoch": 1.993939393939394,
+ "grad_norm": 2.100667953491211,
+ "learning_rate": 3.008080808080808e-06,
+ "loss": 1.5781179666519165,
+ "mean_token_accuracy": 0.6409379839897156,
+ "num_tokens": 16171008.0,
+ "step": 987
+ },
+ {
+ "entropy": 1.299710750579834,
+ "epoch": 1.995959595959596,
+ "grad_norm": 2.2846767902374268,
+ "learning_rate": 3.0060606060606064e-06,
+ "loss": 1.2758262157440186,
+ "mean_token_accuracy": 0.6882632970809937,
+ "num_tokens": 16187392.0,
+ "step": 988
+ },
+ {
+ "entropy": 1.3966935873031616,
+ "epoch": 1.997979797979798,
+ "grad_norm": 2.1244044303894043,
+ "learning_rate": 3.0040404040404043e-06,
+ "loss": 1.420769214630127,
+ "mean_token_accuracy": 0.6670737862586975,
+ "num_tokens": 16203776.0,
+ "step": 989
+ },
+ {
+ "entropy": 1.5851658582687378,
+ "epoch": 2.0,
+ "grad_norm": 2.277179002761841,
+ "learning_rate": 3.002020202020202e-06,
+ "loss": 1.5963382720947266,
+ "mean_token_accuracy": 0.6276868581771851,
+ "num_tokens": 16220160.0,
+ "step": 990
+ },
+ {
+ "entropy": 1.9146819114685059,
+ "epoch": 2.002020202020202,
+ "grad_norm": 2.0008981227874756,
+ "learning_rate": 3e-06,
+ "loss": 1.8820562362670898,
+ "mean_token_accuracy": 0.5895212292671204,
+ "num_tokens": 16236544.0,
+ "step": 991
+ },
+ {
+ "entropy": 1.3451478481292725,
+ "epoch": 2.004040404040404,
+ "grad_norm": 2.236388921737671,
+ "learning_rate": 2.997979797979798e-06,
+ "loss": 1.2709381580352783,
+ "mean_token_accuracy": 0.6848436594009399,
+ "num_tokens": 16252928.0,
+ "step": 992
+ },
+ {
+ "entropy": 1.4454528093338013,
+ "epoch": 2.006060606060606,
+ "grad_norm": 2.220324754714966,
+ "learning_rate": 2.9959595959595963e-06,
+ "loss": 1.387961506843567,
+ "mean_token_accuracy": 0.6642647981643677,
+ "num_tokens": 16269312.0,
+ "step": 993
+ },
+ {
+ "entropy": 1.2083219289779663,
+ "epoch": 2.008080808080808,
+ "grad_norm": 2.092430830001831,
+ "learning_rate": 2.993939393939394e-06,
+ "loss": 1.1450953483581543,
+ "mean_token_accuracy": 0.7159257531166077,
+ "num_tokens": 16285696.0,
+ "step": 994
+ },
+ {
+ "entropy": 1.3030426502227783,
+ "epoch": 2.01010101010101,
+ "grad_norm": 2.0722665786743164,
+ "learning_rate": 2.991919191919192e-06,
+ "loss": 1.2463462352752686,
+ "mean_token_accuracy": 0.6780654788017273,
+ "num_tokens": 16302080.0,
+ "step": 995
+ },
+ {
+ "entropy": 1.3418742418289185,
+ "epoch": 2.012121212121212,
+ "grad_norm": 1.9004424810409546,
+ "learning_rate": 2.98989898989899e-06,
+ "loss": 1.3043954372406006,
+ "mean_token_accuracy": 0.6850268840789795,
+ "num_tokens": 16318464.0,
+ "step": 996
+ },
+ {
+ "entropy": 1.6736825704574585,
+ "epoch": 2.014141414141414,
+ "grad_norm": 2.132983446121216,
+ "learning_rate": 2.987878787878788e-06,
+ "loss": 1.631580114364624,
+ "mean_token_accuracy": 0.619076669216156,
+ "num_tokens": 16334848.0,
+ "step": 997
+ },
+ {
+ "entropy": 1.435375690460205,
+ "epoch": 2.0161616161616163,
+ "grad_norm": 2.2508206367492676,
+ "learning_rate": 2.9858585858585857e-06,
+ "loss": 1.3867809772491455,
+ "mean_token_accuracy": 0.662432849407196,
+ "num_tokens": 16351232.0,
+ "step": 998
+ },
+ {
+ "entropy": 1.050812005996704,
+ "epoch": 2.018181818181818,
+ "grad_norm": 1.878668189048767,
+ "learning_rate": 2.983838383838384e-06,
+ "loss": 1.0428903102874756,
+ "mean_token_accuracy": 0.7329018115997314,
+ "num_tokens": 16367616.0,
+ "step": 999
+ },
+ {
+ "entropy": 1.325859546661377,
+ "epoch": 2.0202020202020203,
+ "grad_norm": 1.8450349569320679,
+ "learning_rate": 2.981818181818182e-06,
+ "loss": 1.302099347114563,
+ "mean_token_accuracy": 0.6926599740982056,
+ "num_tokens": 16384000.0,
+ "step": 1000
+ },
+ {
+ "epoch": 2.0202020202020203,
+ "eval_entropy": 1.3899660999736478,
+ "eval_loss": 1.5310550928115845,
+ "eval_mean_token_accuracy": 0.6458596015168775,
+ "eval_num_tokens": 16384000.0,
+ "eval_runtime": 43.8845,
+ "eval_samples_per_second": 22.559,
+ "eval_steps_per_second": 2.826,
+ "step": 1000
+ }
+ ],
+ "logging_steps": 1,
+ "max_steps": 2475,
+ "num_input_tokens_seen": 0,
+ "num_train_epochs": 5,
+ "save_steps": 500,
+ "stateful_callbacks": {
+ "TrainerControl": {
+ "args": {
+ "should_epoch_stop": false,
+ "should_evaluate": false,
+ "should_log": false,
+ "should_save": true,
+ "should_training_stop": false
+ },
+ "attributes": {}
+ }
+ },
+ "total_flos": 1.38550645358592e+17,
+ "train_batch_size": 4,
+ "trial_name": null,
+ "trial_params": null
+}
diff --git a/qwen3-1.7b-teutonic-5e6/checkpoint-1000/training_args.bin b/qwen3-1.7b-teutonic-5e6/checkpoint-1000/training_args.bin
new file mode 100644
index 0000000000000000000000000000000000000000..37649d4bae96a1df88666daf20b4e5b6e092d976
--- /dev/null
+++ b/qwen3-1.7b-teutonic-5e6/checkpoint-1000/training_args.bin
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:58bbcabfe99a1cf24f2e76aab66b507c2b720d9271dc9f17c8208d741a9c3a65
+size 7121
diff --git a/qwen3-1.7b-teutonic-5e6/checkpoint-1000/zero_to_fp32.py b/qwen3-1.7b-teutonic-5e6/checkpoint-1000/zero_to_fp32.py
new file mode 100644
index 0000000000000000000000000000000000000000..3970548c400fd4361bd923b763db90e963ddaf8c
--- /dev/null
+++ b/qwen3-1.7b-teutonic-5e6/checkpoint-1000/zero_to_fp32.py
@@ -0,0 +1,854 @@
+#!/usr/bin/env python
+
+# Copyright (c) Microsoft Corporation.
+# SPDX-License-Identifier: Apache-2.0
+
+# DeepSpeed Team
+
+# This script extracts fp32 consolidated weights from a zero 1, 2 and 3 DeepSpeed checkpoints. It gets
+# copied into the top level checkpoint dir, so the user can easily do the conversion at any point in
+# the future. Once extracted, the weights don't require DeepSpeed and can be used in any
+# application.
+#
+# example:
+# python zero_to_fp32.py . output_dir/
+# or
+# python zero_to_fp32.py . output_dir/ --safe_serialization
+
+import argparse
+import torch
+import glob
+import math
+import os
+import re
+import gc
+import json
+import numpy as np
+from tqdm import tqdm
+from collections import OrderedDict
+from dataclasses import dataclass
+
+# while this script doesn't use deepspeed to recover data, since the checkpoints are pickled with
+# DeepSpeed data structures it has to be available in the current python environment.
+from deepspeed.utils import logger
+from deepspeed.checkpoint.constants import (DS_VERSION, OPTIMIZER_STATE_DICT, SINGLE_PARTITION_OF_FP32_GROUPS,
+ FP32_FLAT_GROUPS, ZERO_STAGE, PARTITION_COUNT, PARAM_SHAPES, BUFFER_NAMES,
+ FROZEN_PARAM_SHAPES, FROZEN_PARAM_FRAGMENTS, AUTOEP_LAYERS_KEY,
+ AUTOEP_LAYERS_KEY_LEGACY, AUTOEP_ZERO3_EXPERT_STATE_FORMAT_KEY,
+ AUTOEP_ZERO3_PARTITIONED_EXPERT_STATE_FORMAT, PARAM_ALIGNMENT_PADDINGS)
+
+
+@dataclass
+class zero_model_state:
+ buffers: dict()
+ param_shapes: dict()
+ shared_params: list
+ ds_version: int
+ frozen_param_shapes: dict()
+ frozen_param_fragments: dict()
+
+
+debug = 0
+
+# load to cpu
+device = torch.device('cpu')
+
+OUTPUT_DTYPE_NAMES = {
+ 'float32': torch.float32,
+ 'fp32': torch.float32,
+ 'float16': torch.float16,
+ 'fp16': torch.float16,
+ 'bfloat16': torch.bfloat16,
+ 'bf16': torch.bfloat16,
+}
+
+
+def _resolve_output_dtype(dtype):
+ requested_dtype = dtype
+ if isinstance(dtype, str):
+ dtype_name = dtype[6:] if dtype.startswith('torch.') else dtype
+ dtype = OUTPUT_DTYPE_NAMES.get(dtype_name.lower())
+ if dtype not in set(OUTPUT_DTYPE_NAMES.values()):
+ supported = ', '.join(sorted(OUTPUT_DTYPE_NAMES))
+ raise ValueError(f"Unsupported output dtype {requested_dtype!r}. Choose one of: {supported}")
+ return dtype
+
+
+def atoi(text):
+ return int(text) if text.isdigit() else text
+
+
+def natural_keys(text):
+ '''
+ alist.sort(key=natural_keys) sorts in human order
+ http://nedbatchelder.com/blog/200712/human_sorting.html
+ (See Toothy's implementation in the comments)
+ '''
+ return [atoi(c) for c in re.split(r'(\d+)', text)]
+
+
+def get_model_state_file(checkpoint_dir, zero_stage):
+ if not os.path.isdir(checkpoint_dir):
+ raise FileNotFoundError(f"Directory '{checkpoint_dir}' doesn't exist")
+
+ # there should be only one file
+ if zero_stage <= 2:
+ file = os.path.join(checkpoint_dir, "mp_rank_00_model_states.pt")
+ elif zero_stage == 3:
+ file = os.path.join(checkpoint_dir, "zero_pp_rank_0_mp_rank_00_model_states.pt")
+
+ if not os.path.exists(file):
+ raise FileNotFoundError(f"can't find model states file at '{file}'")
+
+ return file
+
+
+def get_checkpoint_files(checkpoint_dir, glob_pattern):
+ # XXX: need to test that this simple glob rule works for multi-node setup too
+ ckpt_files = sorted(glob.glob(os.path.join(checkpoint_dir, glob_pattern)), key=natural_keys)
+
+ if len(ckpt_files) == 0:
+ raise FileNotFoundError(f"can't find {glob_pattern} files in directory '{checkpoint_dir}'")
+
+ return ckpt_files
+
+
+def get_optim_files(checkpoint_dir):
+ return get_checkpoint_files(checkpoint_dir, "*_optim_states.pt")
+
+
+def get_model_state_files(checkpoint_dir):
+ return get_checkpoint_files(checkpoint_dir, "*_model_states.pt")
+
+
+def _has_autoep_zero3_partitioned_metadata(state_dict):
+ autoep_layers = state_dict.get(AUTOEP_LAYERS_KEY)
+ if autoep_layers is None:
+ autoep_layers = state_dict.get(AUTOEP_LAYERS_KEY_LEGACY)
+ if not isinstance(autoep_layers, list):
+ return False
+ return any(
+ isinstance(entry, dict)
+ and entry.get(AUTOEP_ZERO3_EXPERT_STATE_FORMAT_KEY) == AUTOEP_ZERO3_PARTITIONED_EXPERT_STATE_FORMAT
+ for entry in autoep_layers)
+
+
+def _raise_if_autoep_zero3_partitioned_state(state_dict):
+ if _has_autoep_zero3_partitioned_metadata(state_dict):
+ raise NotImplementedError("zero_to_fp32 does not support AutoEP ZeRO-3 partition-native checkpoints. "
+ "AutoEP expert parameters are partitioned over expert replica groups, so "
+ "global data-parallel consolidation would produce incomplete expert tensors. "
+ "Use ds_to_universal.py for expert-aware conversion.")
+
+
+def _raise_if_autoep_zero3_partitioned_checkpoint(model_files):
+ for file in model_files:
+ state_dict = torch.load(file, map_location=device, weights_only=False)
+ _raise_if_autoep_zero3_partitioned_state(state_dict)
+
+
+def parse_model_states(files):
+ zero_model_states = []
+ for file in files:
+ state_dict = torch.load(file, map_location=device, weights_only=False)
+ _raise_if_autoep_zero3_partitioned_state(state_dict)
+
+ if BUFFER_NAMES not in state_dict:
+ raise ValueError(f"{file} is not a model state checkpoint")
+ buffer_names = state_dict[BUFFER_NAMES]
+ if debug:
+ print("Found buffers:", buffer_names)
+
+ # recover just the buffers while restoring them to fp32 if they were saved in fp16
+ buffers = {k: v.float() for k, v in state_dict["module"].items() if k in buffer_names}
+ param_shapes = state_dict[PARAM_SHAPES]
+
+ # collect parameters that are included in param_shapes
+ param_names = []
+ for s in param_shapes:
+ for name in s.keys():
+ param_names.append(name)
+
+ # update with frozen parameters
+ frozen_param_shapes = state_dict.get(FROZEN_PARAM_SHAPES, None)
+ if frozen_param_shapes is not None:
+ if debug:
+ print(f"Found frozen_param_shapes: {frozen_param_shapes}")
+ param_names += list(frozen_param_shapes.keys())
+
+ # handle shared params
+ shared_params = [[k, v] for k, v in state_dict["shared_params"].items()]
+
+ ds_version = state_dict.get(DS_VERSION, None)
+
+ frozen_param_fragments = state_dict.get(FROZEN_PARAM_FRAGMENTS, None)
+
+ z_model_state = zero_model_state(buffers=buffers,
+ param_shapes=param_shapes,
+ shared_params=shared_params,
+ ds_version=ds_version,
+ frozen_param_shapes=frozen_param_shapes,
+ frozen_param_fragments=frozen_param_fragments)
+ zero_model_states.append(z_model_state)
+
+ return zero_model_states
+
+
+def parse_optim_states(files, ds_checkpoint_dir):
+ total_files = len(files)
+ state_dicts = []
+ for f in tqdm(files, desc='Loading checkpoint shards'):
+ state_dict = torch.load(f, map_location=device, mmap=True, weights_only=False)
+ # immediately discard the potentially huge 2 optimizer states as we only care for fp32 master weights
+ # and also handle the case where it was already removed by another helper script
+ state_dict["optimizer_state_dict"].pop("optimizer_state_dict", None)
+ state_dicts.append(state_dict)
+
+ if ZERO_STAGE not in state_dicts[0][OPTIMIZER_STATE_DICT]:
+ raise ValueError(f"{files[0]} is not a zero checkpoint")
+ zero_stage = state_dicts[0][OPTIMIZER_STATE_DICT][ZERO_STAGE]
+ world_size = state_dicts[0][OPTIMIZER_STATE_DICT][PARTITION_COUNT]
+
+ # For ZeRO-2 each param group can have different partition_count as data parallelism for expert
+ # parameters can be different from data parallelism for non-expert parameters. So we can just
+ # use the max of the partition_count to get the dp world_size.
+
+ if type(world_size) is list:
+ world_size = max(world_size)
+
+ if world_size != total_files:
+ raise ValueError(
+ f"Expected {world_size} of '*_optim_states.pt' under '{ds_checkpoint_dir}' but found {total_files} files. "
+ "Possibly due to an overwrite of an old checkpoint, or a checkpoint didn't get saved by one or more processes."
+ )
+
+ # the groups are named differently in each stage
+ if zero_stage <= 2:
+ fp32_groups_key = SINGLE_PARTITION_OF_FP32_GROUPS
+ elif zero_stage == 3:
+ fp32_groups_key = FP32_FLAT_GROUPS
+ else:
+ raise ValueError(f"unknown zero stage {zero_stage}")
+
+ fp32_flat_groups = [state_dicts[i][OPTIMIZER_STATE_DICT][fp32_groups_key] for i in range(len(state_dicts))]
+ param_alignment_paddings = state_dicts[0][OPTIMIZER_STATE_DICT].get(PARAM_ALIGNMENT_PADDINGS)
+ return zero_stage, world_size, fp32_flat_groups, param_alignment_paddings
+
+
+def _get_fp32_state_dict_from_zero_checkpoint(ds_checkpoint_dir, exclude_frozen_parameters):
+ """
+ Returns fp32 state_dict reconstructed from ds checkpoint
+
+ Args:
+ - ``ds_checkpoint_dir``: path to the deepspeed checkpoint folder (where the optimizer files are)
+
+ """
+ print(f"Processing zero checkpoint '{ds_checkpoint_dir}'")
+
+ # parse_model_states rejects AutoEP ZeRO-3 partition-native checkpoints
+ # before the expensive optimizer-shard load below.
+ model_files = get_model_state_files(ds_checkpoint_dir)
+ zero_model_states = parse_model_states(model_files)
+ print(f'Parsing checkpoint created by deepspeed=={zero_model_states[0].ds_version}')
+
+ optim_files = get_optim_files(ds_checkpoint_dir)
+ zero_stage, world_size, fp32_flat_groups, param_alignment_paddings = parse_optim_states(
+ optim_files, ds_checkpoint_dir)
+ print(f"Detected checkpoint of type zero stage {zero_stage}, world_size: {world_size}")
+
+ if zero_stage <= 2:
+ return _get_fp32_state_dict_from_zero2_checkpoint(world_size, fp32_flat_groups, zero_model_states,
+ exclude_frozen_parameters, param_alignment_paddings)
+ elif zero_stage == 3:
+ return _get_fp32_state_dict_from_zero3_checkpoint(world_size, fp32_flat_groups, zero_model_states,
+ exclude_frozen_parameters)
+
+
+def _zero2_merge_frozen_params(state_dict, zero_model_states):
+ if zero_model_states[0].frozen_param_shapes is None or len(zero_model_states[0].frozen_param_shapes) == 0:
+ return
+
+ frozen_param_shapes = zero_model_states[0].frozen_param_shapes
+ frozen_param_fragments = zero_model_states[0].frozen_param_fragments
+
+ if debug:
+ num_elem = sum(s.numel() for s in frozen_param_shapes.values())
+ print(f'rank 0: {FROZEN_PARAM_SHAPES}.numel = {num_elem}')
+
+ wanted_params = len(frozen_param_shapes)
+ wanted_numel = sum(s.numel() for s in frozen_param_shapes.values())
+ avail_numel = sum([p.numel() for p in frozen_param_fragments.values()])
+ print(f'Frozen params: Have {avail_numel} numels to process.')
+ print(f'Frozen params: Need {wanted_numel} numels in {wanted_params} params')
+
+ total_params = 0
+ total_numel = 0
+ for name, shape in frozen_param_shapes.items():
+ total_params += 1
+ unpartitioned_numel = shape.numel()
+ total_numel += unpartitioned_numel
+
+ state_dict[name] = frozen_param_fragments[name]
+
+ if debug:
+ print(f"{name} full shape: {shape} unpartitioned numel {unpartitioned_numel} ")
+
+ print(f"Reconstructed Frozen fp32 state dict with {total_params} params {total_numel} elements")
+
+
+def _has_callable(obj, fn):
+ attr = getattr(obj, fn, None)
+ return callable(attr)
+
+
+def _zero2_merge_trainable_params(state_dict,
+ world_size,
+ fp32_flat_groups,
+ zero_model_states,
+ param_alignment_paddings=None):
+ param_shapes = zero_model_states[0].param_shapes
+
+ # Reconstruction protocol:
+ #
+ # XXX: document this
+
+ if debug:
+ for i in range(world_size):
+ for j in range(len(fp32_flat_groups[0])):
+ print(f"{FP32_FLAT_GROUPS}[{i}][{j}].shape={fp32_flat_groups[i][j].shape}")
+
+ # XXX: memory usage doubles here (zero2)
+ num_param_groups = len(fp32_flat_groups[0])
+ merged_single_partition_of_fp32_groups = []
+ for i in range(num_param_groups):
+ merged_partitions = [sd[i] for sd in fp32_flat_groups]
+ full_single_fp32_vector = torch.cat(merged_partitions, 0)
+ merged_single_partition_of_fp32_groups.append(full_single_fp32_vector)
+ avail_numel = sum(
+ [full_single_fp32_vector.numel() for full_single_fp32_vector in merged_single_partition_of_fp32_groups])
+
+ if debug:
+ wanted_params = sum([len(shapes) for shapes in param_shapes])
+ wanted_numel = sum([sum(shape.numel() for shape in shapes.values()) for shapes in param_shapes])
+ # not asserting if there is a mismatch due to possible padding
+ print(f"Have {avail_numel} numels to process.")
+ print(f"Need {wanted_numel} numels in {wanted_params} params.")
+
+ # params
+ # XXX: for huge models that can't fit into the host's RAM we will have to recode this to support
+ # out-of-core computing solution
+ total_numel = 0
+ total_params = 0
+ for group_idx, (shapes,
+ full_single_fp32_vector) in enumerate(zip(param_shapes, merged_single_partition_of_fp32_groups)):
+ offset = 0
+ avail_numel = full_single_fp32_vector.numel()
+ group_alignment_paddings = None
+ if param_alignment_paddings is not None:
+ group_alignment_paddings = param_alignment_paddings[group_idx]
+ if len(group_alignment_paddings) != len(shapes):
+ raise ValueError(f"Expected {len(shapes)} parameter alignment paddings for group {group_idx}, "
+ f"but found {len(group_alignment_paddings)}")
+
+ for param_idx, (name, shape) in enumerate(shapes.items()):
+
+ unpartitioned_numel = shape.numel() if _has_callable(shape, 'numel') else math.prod(shape)
+ total_numel += unpartitioned_numel
+ total_params += 1
+
+ if debug:
+ print(f"{name} full shape: {shape} unpartitioned numel {unpartitioned_numel} ")
+ state_dict[name] = full_single_fp32_vector.narrow(0, offset, unpartitioned_numel).view(shape)
+ offset += unpartitioned_numel
+ if group_alignment_paddings is not None:
+ offset += group_alignment_paddings[param_idx]
+
+ # Z2 started to align to 2*world_size to improve nccl performance. Therefore both offset and
+ # avail_numel can differ by anywhere between 0..2*world_size. Due to two unrelated complex
+ # paddings performed in the code it's almost impossible to predict the exact numbers w/o the
+ # live optimizer object, so we are checking that the numbers are within the right range
+ align_to = 2 * world_size
+
+ def zero2_align(x):
+ return align_to * math.ceil(x / align_to)
+
+ if debug:
+ print(f"original offset={offset}, avail_numel={avail_numel}")
+
+ offset = zero2_align(offset)
+ avail_numel = zero2_align(avail_numel)
+
+ if debug:
+ print(f"aligned offset={offset}, avail_numel={avail_numel}")
+
+ # Sanity check
+ if offset != avail_numel:
+ raise ValueError(f"consumed {offset} numels out of {avail_numel} - something is wrong")
+
+ print(f"Reconstructed fp32 state dict with {total_params} params {total_numel} elements")
+
+
+def _get_fp32_state_dict_from_zero2_checkpoint(world_size,
+ fp32_flat_groups,
+ zero_model_states,
+ exclude_frozen_parameters,
+ param_alignment_paddings=None):
+ state_dict = OrderedDict()
+
+ # buffers
+ buffers = zero_model_states[0].buffers
+ state_dict.update(buffers)
+ if debug:
+ print(f"added {len(buffers)} buffers")
+
+ if not exclude_frozen_parameters:
+ _zero2_merge_frozen_params(state_dict, zero_model_states)
+
+ _zero2_merge_trainable_params(state_dict, world_size, fp32_flat_groups, zero_model_states,
+ param_alignment_paddings)
+
+ # recover shared parameters
+ for pair in zero_model_states[0].shared_params:
+ if pair[1] in state_dict:
+ state_dict[pair[0]] = state_dict[pair[1]]
+
+ return state_dict
+
+
+def zero3_partitioned_param_info(unpartitioned_numel, world_size):
+ remainder = unpartitioned_numel % world_size
+ padding_numel = (world_size - remainder) if remainder else 0
+ partitioned_numel = math.ceil(unpartitioned_numel / world_size)
+ return partitioned_numel, padding_numel
+
+
+def _zero3_merge_frozen_params(state_dict, world_size, zero_model_states):
+ if zero_model_states[0].frozen_param_shapes is None or len(zero_model_states[0].frozen_param_shapes) == 0:
+ return
+
+ if debug:
+ for i in range(world_size):
+ num_elem = sum(s.numel() for s in zero_model_states[i].frozen_param_fragments.values())
+ print(f'rank {i}: {FROZEN_PARAM_SHAPES}.numel = {num_elem}')
+
+ frozen_param_shapes = zero_model_states[0].frozen_param_shapes
+ wanted_params = len(frozen_param_shapes)
+ wanted_numel = sum(s.numel() for s in frozen_param_shapes.values())
+ avail_numel = sum([p.numel() for p in zero_model_states[0].frozen_param_fragments.values()]) * world_size
+ print(f'Frozen params: Have {avail_numel} numels to process.')
+ print(f'Frozen params: Need {wanted_numel} numels in {wanted_params} params')
+
+ total_params = 0
+ total_numel = 0
+ for name, shape in zero_model_states[0].frozen_param_shapes.items():
+ total_params += 1
+ unpartitioned_numel = shape.numel()
+ total_numel += unpartitioned_numel
+
+ param_frags = tuple(model_state.frozen_param_fragments[name] for model_state in zero_model_states)
+ state_dict[name] = torch.cat(param_frags, 0).narrow(0, 0, unpartitioned_numel).view(shape)
+
+ partitioned_numel, partitioned_padding_numel = zero3_partitioned_param_info(unpartitioned_numel, world_size)
+
+ if debug:
+ print(
+ f"Frozen params: {total_params} {name} full shape: {shape} partition0 numel={partitioned_numel} partitioned_padding_numel={partitioned_padding_numel}"
+ )
+
+ print(f"Reconstructed Frozen fp32 state dict with {total_params} params {total_numel} elements")
+
+
+class GatheredTensor:
+ """
+ A pseudo tensor that collects partitioned weights.
+ It is more memory efficient when there are multiple groups.
+ """
+
+ def __init__(self, flat_groups, flat_groups_offset, offset, partitioned_numel, shape):
+ self.flat_groups = flat_groups
+ self.flat_groups_offset = flat_groups_offset
+ self.offset = offset
+ self.partitioned_numel = partitioned_numel
+ self.shape = shape
+ self.dtype = self.flat_groups[0][0].dtype
+
+ def contiguous(self):
+ """
+ Merge partitioned weights from flat_groups into a single tensor.
+ """
+ end_idx = self.offset + self.partitioned_numel
+ world_size = len(self.flat_groups)
+ pad_flat_param_chunks = []
+
+ for rank_i in range(world_size):
+ # for each rank, we need to collect weights from related group/groups
+ flat_groups_at_rank_i = self.flat_groups[rank_i]
+ start_group_id = None
+ end_group_id = None
+ for group_id in range(len(self.flat_groups_offset)):
+ if self.flat_groups_offset[group_id] <= self.offset < self.flat_groups_offset[group_id + 1]:
+ start_group_id = group_id
+ if self.flat_groups_offset[group_id] < end_idx <= self.flat_groups_offset[group_id + 1]:
+ end_group_id = group_id
+ break
+ # collect weights from related group/groups
+ for group_id in range(start_group_id, end_group_id + 1):
+ flat_tensor = flat_groups_at_rank_i[group_id]
+ start_offset = self.offset - self.flat_groups_offset[group_id]
+ end_offset = min(end_idx, self.flat_groups_offset[group_id + 1]) - self.flat_groups_offset[group_id]
+ pad_flat_param_chunks.append(flat_tensor[start_offset:end_offset])
+
+ # collect weights from all ranks
+ pad_flat_param = torch.cat(pad_flat_param_chunks, dim=0)
+ param = pad_flat_param[:self.shape.numel()].view(self.shape).contiguous()
+ return param
+
+
+def _zero3_merge_trainable_params(state_dict, world_size, fp32_flat_groups, zero_model_states):
+ param_shapes = zero_model_states[0].param_shapes
+ avail_numel = sum([flat_group.numel() for flat_group in fp32_flat_groups[0]]) * world_size
+
+ # Reconstruction protocol: For zero3 we need to zip the partitions together at boundary of each
+ # param, re-consolidating each param, while dealing with padding if any
+
+ # merge list of dicts, preserving order
+ param_shapes = {k: v for d in param_shapes for k, v in d.items()}
+
+ if debug:
+ for i in range(world_size):
+ print(f"{FP32_FLAT_GROUPS}[{i}].shape={fp32_flat_groups[i].shape}")
+
+ wanted_params = len(param_shapes)
+ wanted_numel = sum(shape.numel() for shape in param_shapes.values())
+ # not asserting if there is a mismatch due to possible padding
+ avail_numel = fp32_flat_groups[0].numel() * world_size
+ print(f"Trainable params: Have {avail_numel} numels to process.")
+ print(f"Trainable params: Need {wanted_numel} numels in {wanted_params} params.")
+
+ # params
+ # XXX: for huge models that can't fit into the host's RAM we will have to recode this to support
+ # out-of-core computing solution
+ offset = 0
+ total_numel = 0
+ total_params = 0
+ flat_groups_offset = [0] + list(np.cumsum([flat_tensor.numel() for flat_tensor in fp32_flat_groups[0]]))
+ for name, shape in tqdm(param_shapes.items(), desc='Gathering sharded weights'):
+ unpartitioned_numel = shape.numel()
+ total_numel += unpartitioned_numel
+ total_params += 1
+ partitioned_numel, partitioned_padding_numel = zero3_partitioned_param_info(unpartitioned_numel, world_size)
+
+ if debug:
+ print(
+ f"Trainable params: {total_params} {name} full shape: {shape} partition0 numel={partitioned_numel} partitioned_padding_numel={partitioned_padding_numel}"
+ )
+
+ # memory efficient tensor
+ tensor = GatheredTensor(fp32_flat_groups, flat_groups_offset, offset, partitioned_numel, shape)
+ state_dict[name] = tensor
+ offset += partitioned_numel
+
+ offset *= world_size
+
+ # Sanity check
+ if offset != avail_numel:
+ raise ValueError(f"consumed {offset} numels out of {avail_numel} - something is wrong")
+
+ print(f"Reconstructed Trainable fp32 state dict with {total_params} params {total_numel} elements")
+
+
+def _get_fp32_state_dict_from_zero3_checkpoint(world_size, fp32_flat_groups, zero_model_states,
+ exclude_frozen_parameters):
+ state_dict = OrderedDict()
+
+ # buffers
+ buffers = zero_model_states[0].buffers
+ state_dict.update(buffers)
+ if debug:
+ print(f"added {len(buffers)} buffers")
+
+ if not exclude_frozen_parameters:
+ _zero3_merge_frozen_params(state_dict, world_size, zero_model_states)
+
+ _zero3_merge_trainable_params(state_dict, world_size, fp32_flat_groups, zero_model_states)
+
+ # recover shared parameters
+ for pair in zero_model_states[0].shared_params:
+ if pair[1] in state_dict:
+ state_dict[pair[0]] = state_dict[pair[1]]
+
+ return state_dict
+
+
+def to_torch_tensor(state_dict, return_empty_tensor=False, dtype=None):
+ """
+ Convert state_dict of GatheredTensor to torch tensor
+ """
+ if dtype is not None:
+ dtype = _resolve_output_dtype(dtype)
+
+ torch_state_dict = {}
+ converted_tensors = {}
+ for name, tensor in state_dict.items():
+ tensor_id = id(tensor)
+ if tensor_id in converted_tensors: # shared tensors
+ shared_tensor = torch_state_dict[converted_tensors[tensor_id]]
+ torch_state_dict[name] = shared_tensor
+ else:
+ converted_tensors[tensor_id] = name
+ if return_empty_tensor:
+ torch_state_dict[name] = torch.empty(tensor.shape, dtype=dtype or tensor.dtype)
+ else:
+ contiguous_tensor = tensor.contiguous()
+ torch_state_dict[name] = contiguous_tensor.to(dtype=dtype) if dtype else contiguous_tensor
+ return torch_state_dict
+
+
+def get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir,
+ tag=None,
+ exclude_frozen_parameters=False,
+ lazy_mode=False):
+ """
+ Convert ZeRO 2 or 3 checkpoint into a single fp32 consolidated state_dict that can be loaded with
+ ``load_state_dict()`` and used for training without DeepSpeed or shared with others, for example
+ via a model hub.
+
+ Args:
+ - ``checkpoint_dir``: path to the desired checkpoint folder
+ - ``tag``: checkpoint tag used as a unique identifier for checkpoint. If not provided will attempt to load tag in 'latest' file. e.g., ``global_step14``
+ - ``exclude_frozen_parameters``: exclude frozen parameters
+ - ``lazy_mode``: get state_dict in lazy mode. It returns a dict of pesduo tensor instead of torch tensor, which is more memory efficient.
+ Convert the pesduo tensor to torch tensor by ``.contiguous()``
+
+ Returns:
+ - pytorch ``state_dict``
+
+ A typical usage might be ::
+
+ from deepspeed.utils.zero_to_fp32 import get_fp32_state_dict_from_zero_checkpoint
+ # do the training and checkpoint saving
+ state_dict = get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir) # already on cpu
+ model = model.cpu() # move to cpu
+ model.load_state_dict(state_dict)
+ # submit to model hub or save the model to share with others
+
+ In this example the ``model`` will no longer be usable in the deepspeed context of the same
+ application. i.e. you will need to re-initialize the deepspeed engine, since
+ ``model.load_state_dict(state_dict)`` will remove all the deepspeed magic from it.
+
+ If you want it all done for you, use ``load_state_dict_from_zero_checkpoint`` instead.
+
+ Note: the above usage may not work if your application doesn't have sufficient free CPU memory.
+ You may need to use the offline approach using the ``zero_to_fp32.py`` script that is saved with
+ the checkpoint. Or you can load state_dict in lazy mode ::
+
+ from deepspeed.utils.zero_to_fp32 import get_fp32_state_dict_from_zero_checkpoint
+ state_dict = get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir, lazy_mode=True) # not on cpu
+ for name, lazy_tensor in state_dict.item():
+ tensor = lazy_tensor.contiguous() # to cpu
+ print(name, tensor)
+ # del tensor to release memory if it no longer in use
+ """
+ if tag is None:
+ latest_path = os.path.join(checkpoint_dir, 'latest')
+ if os.path.isfile(latest_path):
+ with open(latest_path, 'r') as fd:
+ tag = fd.read().strip()
+ else:
+ raise ValueError(f"Unable to find 'latest' file at {latest_path}")
+
+ ds_checkpoint_dir = os.path.join(checkpoint_dir, tag)
+
+ if not os.path.isdir(ds_checkpoint_dir):
+ raise FileNotFoundError(f"Directory '{ds_checkpoint_dir}' doesn't exist")
+
+ state_dict = _get_fp32_state_dict_from_zero_checkpoint(ds_checkpoint_dir, exclude_frozen_parameters)
+ if lazy_mode:
+ return state_dict
+ else:
+ return to_torch_tensor(state_dict)
+
+
+def convert_zero_checkpoint_to_state_dict(checkpoint_dir,
+ output_dir,
+ dtype=torch.float32,
+ max_shard_size="5GB",
+ safe_serialization=False,
+ tag=None,
+ exclude_frozen_parameters=False):
+ """
+ Convert ZeRO 2 or 3 checkpoint into a consolidated ``state_dict`` file that can be
+ loaded with ``torch.load(file)`` + ``load_state_dict()`` and used for training without DeepSpeed.
+
+ Args:
+ - ``checkpoint_dir``: path to the desired checkpoint folder. (one that contains the tag-folder, like ``global_step14``)
+ - ``output_dir``: directory for the PyTorch state_dict output files
+ - ``dtype``: output tensor dtype. Supports float32, float16, and bfloat16 as strings or torch dtypes.
+ - ``max_shard_size``: the maximum size for a checkpoint before being sharded, default value is 5GB
+ - ``safe_serialization``: whether to save the model using `safetensors` or the traditional PyTorch way (that uses `pickle`).
+ - ``tag``: checkpoint tag used as a unique identifier for checkpoint. If not provided will attempt to load tag in the file named ``latest`` in the checkpoint folder, e.g., ``global_step14``
+ - ``exclude_frozen_parameters``: exclude frozen parameters
+ """
+
+ dtype = _resolve_output_dtype(dtype)
+
+ # Dependency pre-check
+ if safe_serialization:
+ try:
+ from safetensors.torch import save_file
+ except ImportError:
+ print('If you want to use `safe_serialization`, please `pip install safetensors`')
+ raise
+ if max_shard_size is not None:
+ try:
+ from huggingface_hub import split_torch_state_dict_into_shards
+ except ImportError:
+ print('If you want to use `max_shard_size`, please `pip install huggingface_hub`')
+ raise
+
+ # Convert zero checkpoint to state_dict
+ state_dict = get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir,
+ tag,
+ exclude_frozen_parameters,
+ lazy_mode=True)
+
+ # Shard the model if it is too big.
+ weights_name = "model.safetensors" if safe_serialization else "pytorch_model.bin"
+ if max_shard_size is not None:
+ filename_pattern = weights_name.replace(".bin", "{suffix}.bin").replace(".safetensors", "{suffix}.safetensors")
+ # an memory-efficient approach for sharding
+ empty_state_dict = to_torch_tensor(state_dict, return_empty_tensor=True, dtype=dtype)
+ state_dict_split = split_torch_state_dict_into_shards(empty_state_dict,
+ filename_pattern=filename_pattern,
+ max_shard_size=max_shard_size)
+ else:
+ from collections import namedtuple
+ StateDictSplit = namedtuple("StateDictSplit", ["is_sharded", "filename_to_tensors"])
+ state_dict_split = StateDictSplit(is_sharded=False,
+ filename_to_tensors={weights_name: list(state_dict.keys())})
+
+ # Save the model by shard
+ os.makedirs(output_dir, exist_ok=True)
+ filename_to_tensors = state_dict_split.filename_to_tensors.items()
+ for shard_file, tensors in tqdm(filename_to_tensors, desc="Saving checkpoint shards"):
+ shard_state_dict = {tensor_name: state_dict[tensor_name] for tensor_name in tensors}
+ shard_state_dict = to_torch_tensor(shard_state_dict, dtype=dtype)
+ output_path = os.path.join(output_dir, shard_file)
+ if safe_serialization:
+ save_file(shard_state_dict, output_path, metadata={"format": "pt"})
+ else:
+ torch.save(shard_state_dict, output_path)
+ # release the memory of current shard
+ for tensor_name in list(shard_state_dict.keys()):
+ del state_dict[tensor_name]
+ del shard_state_dict[tensor_name]
+ del shard_state_dict
+ gc.collect()
+
+ # Save index if sharded
+ if state_dict_split.is_sharded:
+ index = {
+ "metadata": state_dict_split.metadata,
+ "weight_map": state_dict_split.tensor_to_filename,
+ }
+ save_index_file = "model.safetensors.index.json" if safe_serialization else "pytorch_model.bin.index.json"
+ save_index_file = os.path.join(output_dir, save_index_file)
+ with open(save_index_file, "w", encoding="utf-8") as f:
+ content = json.dumps(index, indent=2, sort_keys=True) + "\n"
+ f.write(content)
+
+
+def convert_zero_checkpoint_to_fp32_state_dict(checkpoint_dir,
+ output_dir,
+ max_shard_size="5GB",
+ safe_serialization=False,
+ tag=None,
+ exclude_frozen_parameters=False):
+ """Backward-compatible fp32 checkpoint conversion."""
+ return convert_zero_checkpoint_to_state_dict(checkpoint_dir,
+ output_dir,
+ dtype=torch.float32,
+ max_shard_size=max_shard_size,
+ safe_serialization=safe_serialization,
+ tag=tag,
+ exclude_frozen_parameters=exclude_frozen_parameters)
+
+
+def load_state_dict_from_zero_checkpoint(model, checkpoint_dir, tag=None):
+ """
+ 1. Put the provided model to cpu
+ 2. Convert ZeRO 2 or 3 checkpoint into a single fp32 consolidated ``state_dict``
+ 3. Load it into the provided model
+
+ Args:
+ - ``model``: the model object to update
+ - ``checkpoint_dir``: path to the desired checkpoint folder. (one that contains the tag-folder, like ``global_step14``)
+ - ``tag``: checkpoint tag used as a unique identifier for checkpoint. If not provided will attempt to load tag in the file named ``latest`` in the checkpoint folder, e.g., ``global_step14``
+
+ Returns:
+ - ``model`: modified model
+
+ Make sure you have plenty of CPU memory available before you call this function. If you don't
+ have enough use the ``zero_to_fp32.py`` utility to do the conversion. You will find it
+ conveniently placed for you in the checkpoint folder.
+
+ A typical usage might be ::
+
+ from deepspeed.utils.zero_to_fp32 import load_state_dict_from_zero_checkpoint
+ model = load_state_dict_from_zero_checkpoint(trainer.model, checkpoint_dir)
+ # submit to model hub or save the model to share with others
+
+ Note, that once this was run, the ``model`` will no longer be usable in the deepspeed context
+ of the same application. i.e. you will need to re-initialize the deepspeed engine, since
+ ``model.load_state_dict(state_dict)`` will remove all the deepspeed magic from it.
+
+ """
+ logger.info("Extracting fp32 weights")
+ state_dict = get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir, tag)
+
+ logger.info("Overwriting model with fp32 weights")
+ model = model.cpu()
+ model.load_state_dict(state_dict, strict=False)
+
+ return model
+
+
+if __name__ == "__main__":
+ parser = argparse.ArgumentParser()
+ parser.add_argument("checkpoint_dir",
+ type=str,
+ help="path to the desired checkpoint folder, e.g., path/checkpoint-12")
+ parser.add_argument("output_dir",
+ type=str,
+ help="directory to the pytorch fp32 state_dict output files"
+ "(e.g. path/checkpoint-12-output/)")
+ parser.add_argument(
+ "--max_shard_size",
+ type=str,
+ default="5GB",
+ help="The maximum size for a checkpoint before being sharded. Checkpoints shard will then be each of size"
+ "lower than this size. If expressed as a string, needs to be digits followed by a unit (like `5MB`"
+ "We default it to 5GB in order for models to be able to run easily on free-tier google colab instances"
+ "without CPU OOM issues.")
+ parser.add_argument(
+ "--safe_serialization",
+ default=False,
+ action='store_true',
+ help="Whether to save the model using `safetensors` or the traditional PyTorch way (that uses `pickle`).")
+ parser.add_argument("-t",
+ "--tag",
+ type=str,
+ default=None,
+ help="checkpoint tag used as a unique identifier for checkpoint. e.g., global_step1")
+ parser.add_argument("--exclude_frozen_parameters", action='store_true', help="exclude frozen parameters")
+ parser.add_argument("-d", "--debug", action='store_true', help="enable debug")
+ args = parser.parse_args()
+
+ debug = args.debug
+
+ convert_zero_checkpoint_to_fp32_state_dict(args.checkpoint_dir,
+ args.output_dir,
+ max_shard_size=args.max_shard_size,
+ safe_serialization=args.safe_serialization,
+ tag=args.tag,
+ exclude_frozen_parameters=args.exclude_frozen_parameters)
diff --git a/qwen3-1.7b-teutonic-5e6/checkpoint-1000/zero_to_torch.py b/qwen3-1.7b-teutonic-5e6/checkpoint-1000/zero_to_torch.py
new file mode 100644
index 0000000000000000000000000000000000000000..81312e252400d77f03cc1a88522f8f18ebe70ee7
--- /dev/null
+++ b/qwen3-1.7b-teutonic-5e6/checkpoint-1000/zero_to_torch.py
@@ -0,0 +1,54 @@
+#!/usr/bin/env python
+
+# SPDX-License-Identifier: Apache-2.0
+# DeepSpeed Team
+
+import argparse
+
+if __package__:
+ from . import zero_to_fp32
+else:
+ import zero_to_fp32
+
+
+def main(args=None):
+ parser = argparse.ArgumentParser(
+ description="Convert a DeepSpeed ZeRO checkpoint to float32, float16, or bfloat16 PyTorch weights.")
+ parser.add_argument("checkpoint_dir",
+ type=str,
+ help="path to the desired checkpoint folder, e.g., path/checkpoint-12")
+ parser.add_argument("output_dir", type=str, help="directory for the converted PyTorch state_dict files")
+ parser.add_argument("--dtype",
+ type=str,
+ choices=sorted(zero_to_fp32.OUTPUT_DTYPE_NAMES),
+ required=True,
+ help="output tensor dtype")
+ parser.add_argument("--max_shard_size",
+ type=str,
+ default="5GB",
+ help="maximum size of each checkpoint shard, such as 5GB or 500MB")
+ parser.add_argument("--safe_serialization",
+ default=False,
+ action='store_true',
+ help="save with safetensors instead of PyTorch pickle serialization")
+ parser.add_argument("-t",
+ "--tag",
+ type=str,
+ default=None,
+ help="checkpoint tag used as a unique identifier, e.g., global_step1")
+ parser.add_argument("--exclude_frozen_parameters", action='store_true', help="exclude frozen parameters")
+ parser.add_argument("-d", "--debug", action='store_true', help="enable debug output")
+ parsed_args = parser.parse_args(args)
+
+ zero_to_fp32.debug = parsed_args.debug
+ zero_to_fp32.convert_zero_checkpoint_to_state_dict(parsed_args.checkpoint_dir,
+ parsed_args.output_dir,
+ dtype=parsed_args.dtype,
+ max_shard_size=parsed_args.max_shard_size,
+ safe_serialization=parsed_args.safe_serialization,
+ tag=parsed_args.tag,
+ exclude_frozen_parameters=parsed_args.exclude_frozen_parameters)
+
+
+if __name__ == "__main__":
+ main()
diff --git a/qwen3-1.7b-teutonic-5e6/checkpoint-1500/chat_template.jinja b/qwen3-1.7b-teutonic-5e6/checkpoint-1500/chat_template.jinja
new file mode 100644
index 0000000000000000000000000000000000000000..01be9b307daa2d425f7c168c9fb145a286e0afb4
--- /dev/null
+++ b/qwen3-1.7b-teutonic-5e6/checkpoint-1500/chat_template.jinja
@@ -0,0 +1,89 @@
+{%- if tools %}
+ {{- '<|im_start|>system\n' }}
+ {%- if messages[0].role == 'system' %}
+ {{- messages[0].content + '\n\n' }}
+ {%- endif %}
+ {{- "# Tools\n\nYou may call one or more functions to assist with the user query.\n\nYou are provided with function signatures within XML tags:\n" }}
+ {%- for tool in tools %}
+ {{- "\n" }}
+ {{- tool | tojson }}
+ {%- endfor %}
+ {{- "\n\n\nFor each function call, return a json object with function name and arguments within XML tags:\n\n{\"name\": , \"arguments\": }\n<|im_end|>\n" }}
+{%- else %}
+ {%- if messages[0].role == 'system' %}
+ {{- '<|im_start|>system\n' + messages[0].content + '<|im_end|>\n' }}
+ {%- endif %}
+{%- endif %}
+{%- set ns = namespace(multi_step_tool=true, last_query_index=messages|length - 1) %}
+{%- for message in messages[::-1] %}
+ {%- set index = (messages|length - 1) - loop.index0 %}
+ {%- if ns.multi_step_tool and message.role == "user" and message.content is string and not(message.content.startswith('') and message.content.endswith('')) %}
+ {%- set ns.multi_step_tool = false %}
+ {%- set ns.last_query_index = index %}
+ {%- endif %}
+{%- endfor %}
+{%- for message in messages %}
+ {%- if message.content is string %}
+ {%- set content = message.content %}
+ {%- else %}
+ {%- set content = '' %}
+ {%- endif %}
+ {%- if (message.role == "user") or (message.role == "system" and not loop.first) %}
+ {{- '<|im_start|>' + message.role + '\n' + content + '<|im_end|>' + '\n' }}
+ {%- elif message.role == "assistant" %}
+ {%- set reasoning_content = '' %}
+ {%- if message.reasoning_content is string %}
+ {%- set reasoning_content = message.reasoning_content %}
+ {%- else %}
+ {%- if '' in content %}
+ {%- set reasoning_content = content.split('')[0].rstrip('\n').split('')[-1].lstrip('\n') %}
+ {%- set content = content.split('')[-1].lstrip('\n') %}
+ {%- endif %}
+ {%- endif %}
+ {%- if loop.index0 > ns.last_query_index %}
+ {%- if loop.last or (not loop.last and reasoning_content) %}
+ {{- '<|im_start|>' + message.role + '\n\n' + reasoning_content.strip('\n') + '\n\n\n' + content.lstrip('\n') }}
+ {%- else %}
+ {{- '<|im_start|>' + message.role + '\n' + content }}
+ {%- endif %}
+ {%- else %}
+ {{- '<|im_start|>' + message.role + '\n' + content }}
+ {%- endif %}
+ {%- if message.tool_calls %}
+ {%- for tool_call in message.tool_calls %}
+ {%- if (loop.first and content) or (not loop.first) %}
+ {{- '\n' }}
+ {%- endif %}
+ {%- if tool_call.function %}
+ {%- set tool_call = tool_call.function %}
+ {%- endif %}
+ {{- '\n{"name": "' }}
+ {{- tool_call.name }}
+ {{- '", "arguments": ' }}
+ {%- if tool_call.arguments is string %}
+ {{- tool_call.arguments }}
+ {%- else %}
+ {{- tool_call.arguments | tojson }}
+ {%- endif %}
+ {{- '}\n' }}
+ {%- endfor %}
+ {%- endif %}
+ {{- '<|im_end|>\n' }}
+ {%- elif message.role == "tool" %}
+ {%- if loop.first or (messages[loop.index0 - 1].role != "tool") %}
+ {{- '<|im_start|>user' }}
+ {%- endif %}
+ {{- '\n\n' }}
+ {{- content }}
+ {{- '\n' }}
+ {%- if loop.last or (messages[loop.index0 + 1].role != "tool") %}
+ {{- '<|im_end|>\n' }}
+ {%- endif %}
+ {%- endif %}
+{%- endfor %}
+{%- if add_generation_prompt %}
+ {{- '<|im_start|>assistant\n' }}
+ {%- if enable_thinking is defined and enable_thinking is false %}
+ {{- '\n\n\n\n' }}
+ {%- endif %}
+{%- endif %}
\ No newline at end of file
diff --git a/qwen3-1.7b-teutonic-5e6/checkpoint-1500/config.json b/qwen3-1.7b-teutonic-5e6/checkpoint-1500/config.json
new file mode 100644
index 0000000000000000000000000000000000000000..5d9cef07396baadff5390e4508eb33025967a029
--- /dev/null
+++ b/qwen3-1.7b-teutonic-5e6/checkpoint-1500/config.json
@@ -0,0 +1,63 @@
+{
+ "architectures": [
+ "Qwen3ForCausalLM"
+ ],
+ "attention_bias": false,
+ "attention_dropout": 0.0,
+ "bos_token_id": null,
+ "dtype": "bfloat16",
+ "eos_token_id": 151645,
+ "head_dim": 128,
+ "hidden_act": "silu",
+ "hidden_size": 2048,
+ "initializer_range": 0.02,
+ "intermediate_size": 6144,
+ "layer_types": [
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention"
+ ],
+ "max_position_embeddings": 40960,
+ "max_window_layers": 28,
+ "model_type": "qwen3",
+ "num_attention_heads": 16,
+ "num_hidden_layers": 28,
+ "num_key_value_heads": 8,
+ "pad_token_id": 151643,
+ "rms_norm_eps": 1e-06,
+ "rope_parameters": {
+ "rope_theta": 1000000,
+ "rope_type": "default"
+ },
+ "sliding_window": null,
+ "tie_word_embeddings": true,
+ "transformers_version": "5.17.0",
+ "use_cache": false,
+ "use_sliding_window": false,
+ "vocab_size": 151936
+}
diff --git a/qwen3-1.7b-teutonic-5e6/checkpoint-1500/generation_config.json b/qwen3-1.7b-teutonic-5e6/checkpoint-1500/generation_config.json
new file mode 100644
index 0000000000000000000000000000000000000000..e6941fe4b04f26113d6eef6255d005c4d7090bda
--- /dev/null
+++ b/qwen3-1.7b-teutonic-5e6/checkpoint-1500/generation_config.json
@@ -0,0 +1,12 @@
+{
+ "do_sample": true,
+ "eos_token_id": [
+ 151645,
+ 151643
+ ],
+ "pad_token_id": 151643,
+ "temperature": 0.6,
+ "top_k": 20,
+ "top_p": 0.95,
+ "transformers_version": "5.17.0"
+}
diff --git a/qwen3-1.7b-teutonic-5e6/checkpoint-1500/global_step1500/bf16_zero_pp_rank_0_mp_rank_00_optim_states.pt b/qwen3-1.7b-teutonic-5e6/checkpoint-1500/global_step1500/bf16_zero_pp_rank_0_mp_rank_00_optim_states.pt
new file mode 100644
index 0000000000000000000000000000000000000000..45a0eb383aad786eb5d7a821b9001b27077695e2
--- /dev/null
+++ b/qwen3-1.7b-teutonic-5e6/checkpoint-1500/global_step1500/bf16_zero_pp_rank_0_mp_rank_00_optim_states.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d5077930dfb821ac5bc488ed9247c90a00d51842213c669f29520242479685bc
+size 10323466465
diff --git a/qwen3-1.7b-teutonic-5e6/checkpoint-1500/global_step1500/bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt b/qwen3-1.7b-teutonic-5e6/checkpoint-1500/global_step1500/bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt
new file mode 100644
index 0000000000000000000000000000000000000000..178177a1a20fd51b6eee1986ac67dacb7d761e96
--- /dev/null
+++ b/qwen3-1.7b-teutonic-5e6/checkpoint-1500/global_step1500/bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6f90b6cb6f7fdcf8772c092842627541743e4bc83174c27ba4c87c65cac672b9
+size 10323469601
diff --git a/qwen3-1.7b-teutonic-5e6/checkpoint-1500/global_step1500/mp_rank_00_model_states.pt b/qwen3-1.7b-teutonic-5e6/checkpoint-1500/global_step1500/mp_rank_00_model_states.pt
new file mode 100644
index 0000000000000000000000000000000000000000..51f16515475f795806e72bcec24f54fdbce8222e
--- /dev/null
+++ b/qwen3-1.7b-teutonic-5e6/checkpoint-1500/global_step1500/mp_rank_00_model_states.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:523ec813c393fd3f84897681df01c102a4d3a66fc72527bf1d35c3a922771077
+size 3441239653
diff --git a/qwen3-1.7b-teutonic-5e6/checkpoint-1500/latest b/qwen3-1.7b-teutonic-5e6/checkpoint-1500/latest
new file mode 100644
index 0000000000000000000000000000000000000000..c56ff7708f44fb7928fea2f70d6d7342ce0d5b67
--- /dev/null
+++ b/qwen3-1.7b-teutonic-5e6/checkpoint-1500/latest
@@ -0,0 +1 @@
+global_step1500
\ No newline at end of file
diff --git a/qwen3-1.7b-teutonic-5e6/checkpoint-1500/model.safetensors b/qwen3-1.7b-teutonic-5e6/checkpoint-1500/model.safetensors
new file mode 100644
index 0000000000000000000000000000000000000000..a2f04bfe78a5dc92beab6864ba55a934bf32971c
--- /dev/null
+++ b/qwen3-1.7b-teutonic-5e6/checkpoint-1500/model.safetensors
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a9175b75d3713c74280504d975bccb5374f789063c1ddf46bea2f3c0543006f0
+size 4063515640
diff --git a/qwen3-1.7b-teutonic-5e6/checkpoint-1500/rng_state_0.pth b/qwen3-1.7b-teutonic-5e6/checkpoint-1500/rng_state_0.pth
new file mode 100644
index 0000000000000000000000000000000000000000..d16a29846bf1c6a082460f85cb978e35b206c0de
--- /dev/null
+++ b/qwen3-1.7b-teutonic-5e6/checkpoint-1500/rng_state_0.pth
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:27a469d99b2f16f69a43f163d81133ab2eca2b9bce1eb579887f4eaf6efaa9da
+size 14917
diff --git a/qwen3-1.7b-teutonic-5e6/checkpoint-1500/rng_state_1.pth b/qwen3-1.7b-teutonic-5e6/checkpoint-1500/rng_state_1.pth
new file mode 100644
index 0000000000000000000000000000000000000000..5571d4000197098fd8379a6cd2fdd0eab372c18f
--- /dev/null
+++ b/qwen3-1.7b-teutonic-5e6/checkpoint-1500/rng_state_1.pth
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:fdf02b0060e448a3cb44dfd5a4ab2653449e45bbc811b8f72f63d96c164d8054
+size 14917
diff --git a/qwen3-1.7b-teutonic-5e6/checkpoint-1500/scheduler.pt b/qwen3-1.7b-teutonic-5e6/checkpoint-1500/scheduler.pt
new file mode 100644
index 0000000000000000000000000000000000000000..e0a6820cffa14a33c2a0689e24a0fca9059341c2
--- /dev/null
+++ b/qwen3-1.7b-teutonic-5e6/checkpoint-1500/scheduler.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:335ffb4d9b33a883d23d5c0d0aaee61492fba1956a2a4256ce15e6ef24d2782f
+size 1465
diff --git a/qwen3-1.7b-teutonic-5e6/checkpoint-1500/tokenizer.json b/qwen3-1.7b-teutonic-5e6/checkpoint-1500/tokenizer.json
new file mode 100644
index 0000000000000000000000000000000000000000..c7afbed2efcdf019f88ab0572ec29d3bf595dfe2
--- /dev/null
+++ b/qwen3-1.7b-teutonic-5e6/checkpoint-1500/tokenizer.json
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:be75606093db2094d7cd20f3c2f385c212750648bd6ea4fb2bf507a6a4c55506
+size 11422650
diff --git a/qwen3-1.7b-teutonic-5e6/checkpoint-1500/tokenizer_config.json b/qwen3-1.7b-teutonic-5e6/checkpoint-1500/tokenizer_config.json
new file mode 100644
index 0000000000000000000000000000000000000000..770e41d6c92519d525eede4cbcf3ba27f6425311
--- /dev/null
+++ b/qwen3-1.7b-teutonic-5e6/checkpoint-1500/tokenizer_config.json
@@ -0,0 +1,30 @@
+{
+ "add_prefix_space": false,
+ "backend": "tokenizers",
+ "bos_token": null,
+ "clean_up_tokenization_spaces": false,
+ "eos_token": "<|im_end|>",
+ "errors": "replace",
+ "extra_special_tokens": [
+ "<|im_start|>",
+ "<|im_end|>",
+ "<|object_ref_start|>",
+ "<|object_ref_end|>",
+ "<|box_start|>",
+ "<|box_end|>",
+ "<|quad_start|>",
+ "<|quad_end|>",
+ "<|vision_start|>",
+ "<|vision_end|>",
+ "<|vision_pad|>",
+ "<|image_pad|>",
+ "<|video_pad|>"
+ ],
+ "is_local": false,
+ "local_files_only": false,
+ "model_max_length": 131072,
+ "pad_token": "<|endoftext|>",
+ "split_special_tokens": false,
+ "tokenizer_class": "Qwen2Tokenizer",
+ "unk_token": null
+}
diff --git a/qwen3-1.7b-teutonic-5e6/checkpoint-1500/trainer_state.json b/qwen3-1.7b-teutonic-5e6/checkpoint-1500/trainer_state.json
new file mode 100644
index 0000000000000000000000000000000000000000..795344881c0c7b55cb0af2012bc236022d4dd27e
--- /dev/null
+++ b/qwen3-1.7b-teutonic-5e6/checkpoint-1500/trainer_state.json
@@ -0,0 +1,15364 @@
+{
+ "best_global_step": 950,
+ "best_metric": 1.5272752046585083,
+ "best_model_checkpoint": "./checkpoints/qwen3-1.7b-teutonic-5e6/checkpoint-500",
+ "epoch": 3.0303030303030303,
+ "eval_steps": 50,
+ "global_step": 1500,
+ "is_hyper_param_search": false,
+ "is_local_process_zero": true,
+ "is_world_process_zero": true,
+ "log_history": [
+ {
+ "entropy": 0.8295683860778809,
+ "epoch": 0.00202020202020202,
+ "grad_norm": 10.886772155761719,
+ "learning_rate": 5e-06,
+ "loss": 1.5185801982879639,
+ "mean_token_accuracy": 0.672874927520752,
+ "num_tokens": 16384.0,
+ "step": 1
+ },
+ {
+ "entropy": 1.1781222820281982,
+ "epoch": 0.00404040404040404,
+ "grad_norm": 13.80163288116455,
+ "learning_rate": 4.997979797979798e-06,
+ "loss": 2.18377685546875,
+ "mean_token_accuracy": 0.5700415372848511,
+ "num_tokens": 32768.0,
+ "step": 2
+ },
+ {
+ "entropy": 0.7972303032875061,
+ "epoch": 0.006060606060606061,
+ "grad_norm": 13.71261215209961,
+ "learning_rate": 4.995959595959596e-06,
+ "loss": 1.5810277462005615,
+ "mean_token_accuracy": 0.6782486438751221,
+ "num_tokens": 49152.0,
+ "step": 3
+ },
+ {
+ "entropy": 0.9709298610687256,
+ "epoch": 0.00808080808080808,
+ "grad_norm": 9.18797779083252,
+ "learning_rate": 4.993939393939394e-06,
+ "loss": 1.6923656463623047,
+ "mean_token_accuracy": 0.6447850465774536,
+ "num_tokens": 65536.0,
+ "step": 4
+ },
+ {
+ "entropy": 1.2161898612976074,
+ "epoch": 0.010101010101010102,
+ "grad_norm": 8.97508430480957,
+ "learning_rate": 4.991919191919192e-06,
+ "loss": 1.9361608028411865,
+ "mean_token_accuracy": 0.5922691822052002,
+ "num_tokens": 81920.0,
+ "step": 5
+ },
+ {
+ "entropy": 0.9950039982795715,
+ "epoch": 0.012121212121212121,
+ "grad_norm": 6.453213214874268,
+ "learning_rate": 4.98989898989899e-06,
+ "loss": 1.5012480020523071,
+ "mean_token_accuracy": 0.6671959161758423,
+ "num_tokens": 98304.0,
+ "step": 6
+ },
+ {
+ "entropy": 1.452682614326477,
+ "epoch": 0.014141414141414142,
+ "grad_norm": 8.50411605834961,
+ "learning_rate": 4.987878787878789e-06,
+ "loss": 2.1297783851623535,
+ "mean_token_accuracy": 0.5807889699935913,
+ "num_tokens": 114688.0,
+ "step": 7
+ },
+ {
+ "entropy": 1.4661014080047607,
+ "epoch": 0.01616161616161616,
+ "grad_norm": 4.290146827697754,
+ "learning_rate": 4.9858585858585865e-06,
+ "loss": 1.7391963005065918,
+ "mean_token_accuracy": 0.611993134021759,
+ "num_tokens": 131072.0,
+ "step": 8
+ },
+ {
+ "entropy": 1.9030903577804565,
+ "epoch": 0.01818181818181818,
+ "grad_norm": 4.503584384918213,
+ "learning_rate": 4.983838383838384e-06,
+ "loss": 2.283596992492676,
+ "mean_token_accuracy": 0.5261968970298767,
+ "num_tokens": 147456.0,
+ "step": 9
+ },
+ {
+ "entropy": 1.2784137725830078,
+ "epoch": 0.020202020202020204,
+ "grad_norm": 3.427410840988159,
+ "learning_rate": 4.981818181818182e-06,
+ "loss": 1.5032392740249634,
+ "mean_token_accuracy": 0.6578529477119446,
+ "num_tokens": 163840.0,
+ "step": 10
+ },
+ {
+ "entropy": 1.7487188577651978,
+ "epoch": 0.022222222222222223,
+ "grad_norm": 3.20536470413208,
+ "learning_rate": 4.97979797979798e-06,
+ "loss": 1.9184643030166626,
+ "mean_token_accuracy": 0.6093673706054688,
+ "num_tokens": 180224.0,
+ "step": 11
+ },
+ {
+ "entropy": 1.6585394144058228,
+ "epoch": 0.024242424242424242,
+ "grad_norm": 3.2463598251342773,
+ "learning_rate": 4.977777777777778e-06,
+ "loss": 1.8364639282226562,
+ "mean_token_accuracy": 0.6014899611473083,
+ "num_tokens": 196608.0,
+ "step": 12
+ },
+ {
+ "entropy": 1.4869519472122192,
+ "epoch": 0.026262626262626262,
+ "grad_norm": 2.8681719303131104,
+ "learning_rate": 4.975757575757576e-06,
+ "loss": 1.6356265544891357,
+ "mean_token_accuracy": 0.6361138224601746,
+ "num_tokens": 212992.0,
+ "step": 13
+ },
+ {
+ "entropy": 1.5179094076156616,
+ "epoch": 0.028282828282828285,
+ "grad_norm": 3.0470707416534424,
+ "learning_rate": 4.973737373737374e-06,
+ "loss": 1.6317660808563232,
+ "mean_token_accuracy": 0.6334269642829895,
+ "num_tokens": 229376.0,
+ "step": 14
+ },
+ {
+ "entropy": 1.7384330034255981,
+ "epoch": 0.030303030303030304,
+ "grad_norm": 2.6149742603302,
+ "learning_rate": 4.971717171717172e-06,
+ "loss": 1.67923903465271,
+ "mean_token_accuracy": 0.6469223499298096,
+ "num_tokens": 245760.0,
+ "step": 15
+ },
+ {
+ "entropy": 2.108799457550049,
+ "epoch": 0.03232323232323232,
+ "grad_norm": 3.694795608520508,
+ "learning_rate": 4.9696969696969696e-06,
+ "loss": 1.9052371978759766,
+ "mean_token_accuracy": 0.5927577018737793,
+ "num_tokens": 262144.0,
+ "step": 16
+ },
+ {
+ "entropy": 1.7532848119735718,
+ "epoch": 0.03434343434343434,
+ "grad_norm": 3.1510567665100098,
+ "learning_rate": 4.9676767676767675e-06,
+ "loss": 1.6382691860198975,
+ "mean_token_accuracy": 0.6311675906181335,
+ "num_tokens": 278528.0,
+ "step": 17
+ },
+ {
+ "entropy": 1.723548412322998,
+ "epoch": 0.03636363636363636,
+ "grad_norm": 3.093825340270996,
+ "learning_rate": 4.965656565656566e-06,
+ "loss": 1.6039624214172363,
+ "mean_token_accuracy": 0.6313507556915283,
+ "num_tokens": 294912.0,
+ "step": 18
+ },
+ {
+ "entropy": 1.4790765047073364,
+ "epoch": 0.03838383838383838,
+ "grad_norm": 2.7150535583496094,
+ "learning_rate": 4.963636363636364e-06,
+ "loss": 1.4173667430877686,
+ "mean_token_accuracy": 0.6627381443977356,
+ "num_tokens": 311296.0,
+ "step": 19
+ },
+ {
+ "entropy": 1.5827536582946777,
+ "epoch": 0.04040404040404041,
+ "grad_norm": 2.610966444015503,
+ "learning_rate": 4.961616161616162e-06,
+ "loss": 1.4604644775390625,
+ "mean_token_accuracy": 0.656509518623352,
+ "num_tokens": 327680.0,
+ "step": 20
+ },
+ {
+ "entropy": 2.054673194885254,
+ "epoch": 0.04242424242424243,
+ "grad_norm": 2.612940788269043,
+ "learning_rate": 4.95959595959596e-06,
+ "loss": 1.962029218673706,
+ "mean_token_accuracy": 0.5719956159591675,
+ "num_tokens": 344064.0,
+ "step": 21
+ },
+ {
+ "entropy": 1.6684198379516602,
+ "epoch": 0.044444444444444446,
+ "grad_norm": 2.5426013469696045,
+ "learning_rate": 4.957575757575758e-06,
+ "loss": 1.591176986694336,
+ "mean_token_accuracy": 0.6519907116889954,
+ "num_tokens": 360448.0,
+ "step": 22
+ },
+ {
+ "entropy": 1.9070855379104614,
+ "epoch": 0.046464646464646465,
+ "grad_norm": 2.5853357315063477,
+ "learning_rate": 4.9555555555555565e-06,
+ "loss": 1.930276870727539,
+ "mean_token_accuracy": 0.5895823240280151,
+ "num_tokens": 376832.0,
+ "step": 23
+ },
+ {
+ "entropy": 1.716689109802246,
+ "epoch": 0.048484848484848485,
+ "grad_norm": 2.7608494758605957,
+ "learning_rate": 4.953535353535354e-06,
+ "loss": 1.739159345626831,
+ "mean_token_accuracy": 0.6028944849967957,
+ "num_tokens": 393216.0,
+ "step": 24
+ },
+ {
+ "entropy": 1.5666695833206177,
+ "epoch": 0.050505050505050504,
+ "grad_norm": 2.5973074436187744,
+ "learning_rate": 4.951515151515152e-06,
+ "loss": 1.6016302108764648,
+ "mean_token_accuracy": 0.626099169254303,
+ "num_tokens": 409600.0,
+ "step": 25
+ },
+ {
+ "entropy": 1.4267652034759521,
+ "epoch": 0.052525252525252523,
+ "grad_norm": 2.163451910018921,
+ "learning_rate": 4.94949494949495e-06,
+ "loss": 1.5093598365783691,
+ "mean_token_accuracy": 0.6530288457870483,
+ "num_tokens": 425984.0,
+ "step": 26
+ },
+ {
+ "entropy": 1.8537150621414185,
+ "epoch": 0.05454545454545454,
+ "grad_norm": 2.6983258724212646,
+ "learning_rate": 4.947474747474748e-06,
+ "loss": 1.8831868171691895,
+ "mean_token_accuracy": 0.5802393555641174,
+ "num_tokens": 442368.0,
+ "step": 27
+ },
+ {
+ "entropy": 1.410015344619751,
+ "epoch": 0.05656565656565657,
+ "grad_norm": 2.4777796268463135,
+ "learning_rate": 4.945454545454546e-06,
+ "loss": 1.4835734367370605,
+ "mean_token_accuracy": 0.6519907116889954,
+ "num_tokens": 458752.0,
+ "step": 28
+ },
+ {
+ "entropy": 1.2399017810821533,
+ "epoch": 0.05858585858585859,
+ "grad_norm": 2.1676478385925293,
+ "learning_rate": 4.943434343434344e-06,
+ "loss": 1.330633282661438,
+ "mean_token_accuracy": 0.6802638173103333,
+ "num_tokens": 475136.0,
+ "step": 29
+ },
+ {
+ "entropy": 1.6478898525238037,
+ "epoch": 0.06060606060606061,
+ "grad_norm": 2.5325536727905273,
+ "learning_rate": 4.941414141414142e-06,
+ "loss": 1.771777868270874,
+ "mean_token_accuracy": 0.6128480434417725,
+ "num_tokens": 491520.0,
+ "step": 30
+ },
+ {
+ "entropy": 1.7915360927581787,
+ "epoch": 0.06262626262626263,
+ "grad_norm": 2.2419557571411133,
+ "learning_rate": 4.93939393939394e-06,
+ "loss": 1.870645523071289,
+ "mean_token_accuracy": 0.6074743270874023,
+ "num_tokens": 507904.0,
+ "step": 31
+ },
+ {
+ "entropy": 1.7638899087905884,
+ "epoch": 0.06464646464646465,
+ "grad_norm": 2.8063058853149414,
+ "learning_rate": 4.937373737373738e-06,
+ "loss": 1.83987557888031,
+ "mean_token_accuracy": 0.5836589932441711,
+ "num_tokens": 524288.0,
+ "step": 32
+ },
+ {
+ "entropy": 1.4097516536712646,
+ "epoch": 0.06666666666666667,
+ "grad_norm": 2.1857714653015137,
+ "learning_rate": 4.935353535353536e-06,
+ "loss": 1.477668285369873,
+ "mean_token_accuracy": 0.6507083773612976,
+ "num_tokens": 540672.0,
+ "step": 33
+ },
+ {
+ "entropy": 1.4447426795959473,
+ "epoch": 0.06868686868686869,
+ "grad_norm": 2.3771870136260986,
+ "learning_rate": 4.933333333333334e-06,
+ "loss": 1.536318063735962,
+ "mean_token_accuracy": 0.6383732557296753,
+ "num_tokens": 557056.0,
+ "step": 34
+ },
+ {
+ "entropy": 1.9658164978027344,
+ "epoch": 0.0707070707070707,
+ "grad_norm": 2.4784936904907227,
+ "learning_rate": 4.931313131313132e-06,
+ "loss": 2.0565035343170166,
+ "mean_token_accuracy": 0.5581949353218079,
+ "num_tokens": 573440.0,
+ "step": 35
+ },
+ {
+ "entropy": 1.6045317649841309,
+ "epoch": 0.07272727272727272,
+ "grad_norm": 2.1128504276275635,
+ "learning_rate": 4.92929292929293e-06,
+ "loss": 1.6234952211380005,
+ "mean_token_accuracy": 0.6293356418609619,
+ "num_tokens": 589824.0,
+ "step": 36
+ },
+ {
+ "entropy": 1.5491269826889038,
+ "epoch": 0.07474747474747474,
+ "grad_norm": 2.1672937870025635,
+ "learning_rate": 4.927272727272728e-06,
+ "loss": 1.5809822082519531,
+ "mean_token_accuracy": 0.6419760584831238,
+ "num_tokens": 606208.0,
+ "step": 37
+ },
+ {
+ "entropy": 1.545534610748291,
+ "epoch": 0.07676767676767676,
+ "grad_norm": 2.2526934146881104,
+ "learning_rate": 4.925252525252526e-06,
+ "loss": 1.5707473754882812,
+ "mean_token_accuracy": 0.6344650983810425,
+ "num_tokens": 622592.0,
+ "step": 38
+ },
+ {
+ "entropy": 1.579596757888794,
+ "epoch": 0.07878787878787878,
+ "grad_norm": 2.3988492488861084,
+ "learning_rate": 4.9232323232323235e-06,
+ "loss": 1.6725983619689941,
+ "mean_token_accuracy": 0.6243282556533813,
+ "num_tokens": 638976.0,
+ "step": 39
+ },
+ {
+ "entropy": 2.0438785552978516,
+ "epoch": 0.08080808080808081,
+ "grad_norm": 2.3314359188079834,
+ "learning_rate": 4.9212121212121214e-06,
+ "loss": 2.0424888134002686,
+ "mean_token_accuracy": 0.570713222026825,
+ "num_tokens": 655360.0,
+ "step": 40
+ },
+ {
+ "entropy": 1.8690773248672485,
+ "epoch": 0.08282828282828283,
+ "grad_norm": 2.130401134490967,
+ "learning_rate": 4.919191919191919e-06,
+ "loss": 1.8796970844268799,
+ "mean_token_accuracy": 0.5882999300956726,
+ "num_tokens": 671744.0,
+ "step": 41
+ },
+ {
+ "entropy": 1.756626009941101,
+ "epoch": 0.08484848484848485,
+ "grad_norm": 2.342318534851074,
+ "learning_rate": 4.917171717171717e-06,
+ "loss": 1.71901535987854,
+ "mean_token_accuracy": 0.6173668503761292,
+ "num_tokens": 688128.0,
+ "step": 42
+ },
+ {
+ "entropy": 1.4642508029937744,
+ "epoch": 0.08686868686868687,
+ "grad_norm": 1.993338704109192,
+ "learning_rate": 4.915151515151516e-06,
+ "loss": 1.4660165309906006,
+ "mean_token_accuracy": 0.6687225103378296,
+ "num_tokens": 704512.0,
+ "step": 43
+ },
+ {
+ "entropy": 1.790807843208313,
+ "epoch": 0.08888888888888889,
+ "grad_norm": 2.3186943531036377,
+ "learning_rate": 4.913131313131314e-06,
+ "loss": 1.7102060317993164,
+ "mean_token_accuracy": 0.6238397359848022,
+ "num_tokens": 720896.0,
+ "step": 44
+ },
+ {
+ "entropy": 1.5970485210418701,
+ "epoch": 0.09090909090909091,
+ "grad_norm": 2.299307346343994,
+ "learning_rate": 4.911111111111112e-06,
+ "loss": 1.5170013904571533,
+ "mean_token_accuracy": 0.652662456035614,
+ "num_tokens": 737280.0,
+ "step": 45
+ },
+ {
+ "entropy": 1.3955466747283936,
+ "epoch": 0.09292929292929293,
+ "grad_norm": 2.171952962875366,
+ "learning_rate": 4.90909090909091e-06,
+ "loss": 1.4059661626815796,
+ "mean_token_accuracy": 0.6654250025749207,
+ "num_tokens": 753664.0,
+ "step": 46
+ },
+ {
+ "entropy": 1.7198575735092163,
+ "epoch": 0.09494949494949495,
+ "grad_norm": 2.385092258453369,
+ "learning_rate": 4.9070707070707075e-06,
+ "loss": 1.730346918106079,
+ "mean_token_accuracy": 0.6191987991333008,
+ "num_tokens": 770048.0,
+ "step": 47
+ },
+ {
+ "entropy": 1.3542555570602417,
+ "epoch": 0.09696969696969697,
+ "grad_norm": 2.1463189125061035,
+ "learning_rate": 4.905050505050505e-06,
+ "loss": 1.346085786819458,
+ "mean_token_accuracy": 0.680446982383728,
+ "num_tokens": 786432.0,
+ "step": 48
+ },
+ {
+ "entropy": 1.8084443807601929,
+ "epoch": 0.09898989898989899,
+ "grad_norm": 2.1505849361419678,
+ "learning_rate": 4.903030303030303e-06,
+ "loss": 1.847151756286621,
+ "mean_token_accuracy": 0.5926355719566345,
+ "num_tokens": 802816.0,
+ "step": 49
+ },
+ {
+ "entropy": 1.751160979270935,
+ "epoch": 0.10101010101010101,
+ "grad_norm": 2.1436259746551514,
+ "learning_rate": 4.901010101010101e-06,
+ "loss": 1.7802404165267944,
+ "mean_token_accuracy": 0.5996580123901367,
+ "num_tokens": 819200.0,
+ "step": 50
+ },
+ {
+ "epoch": 0.10101010101010101,
+ "eval_entropy": 1.6292865045609013,
+ "eval_loss": 1.6725393533706665,
+ "eval_mean_token_accuracy": 0.6230050469598463,
+ "eval_num_tokens": 819200.0,
+ "eval_runtime": 43.9148,
+ "eval_samples_per_second": 22.544,
+ "eval_steps_per_second": 2.824,
+ "step": 50
+ },
+ {
+ "entropy": 1.4428319931030273,
+ "epoch": 0.10303030303030303,
+ "grad_norm": 2.0954954624176025,
+ "learning_rate": 4.898989898989899e-06,
+ "loss": 1.4927232265472412,
+ "mean_token_accuracy": 0.6522349715232849,
+ "num_tokens": 835584.0,
+ "step": 51
+ },
+ {
+ "entropy": 1.7493115663528442,
+ "epoch": 0.10505050505050505,
+ "grad_norm": 2.300030469894409,
+ "learning_rate": 4.896969696969697e-06,
+ "loss": 1.737417221069336,
+ "mean_token_accuracy": 0.6213361024856567,
+ "num_tokens": 851968.0,
+ "step": 52
+ },
+ {
+ "entropy": 1.7801647186279297,
+ "epoch": 0.10707070707070707,
+ "grad_norm": 2.3947081565856934,
+ "learning_rate": 4.894949494949495e-06,
+ "loss": 1.8372564315795898,
+ "mean_token_accuracy": 0.5931240916252136,
+ "num_tokens": 868352.0,
+ "step": 53
+ },
+ {
+ "entropy": 1.6934887170791626,
+ "epoch": 0.10909090909090909,
+ "grad_norm": 2.1981089115142822,
+ "learning_rate": 4.8929292929292936e-06,
+ "loss": 1.734646201133728,
+ "mean_token_accuracy": 0.611932098865509,
+ "num_tokens": 884736.0,
+ "step": 54
+ },
+ {
+ "entropy": 1.3401941061019897,
+ "epoch": 0.1111111111111111,
+ "grad_norm": 2.193511724472046,
+ "learning_rate": 4.8909090909090914e-06,
+ "loss": 1.399888038635254,
+ "mean_token_accuracy": 0.6745847463607788,
+ "num_tokens": 901120.0,
+ "step": 55
+ },
+ {
+ "entropy": 1.8244661092758179,
+ "epoch": 0.11313131313131314,
+ "grad_norm": 2.4358489513397217,
+ "learning_rate": 4.888888888888889e-06,
+ "loss": 1.8716282844543457,
+ "mean_token_accuracy": 0.5867122411727905,
+ "num_tokens": 917504.0,
+ "step": 56
+ },
+ {
+ "entropy": 1.5019619464874268,
+ "epoch": 0.11515151515151516,
+ "grad_norm": 2.1135261058807373,
+ "learning_rate": 4.886868686868687e-06,
+ "loss": 1.523103952407837,
+ "mean_token_accuracy": 0.6414265036582947,
+ "num_tokens": 933888.0,
+ "step": 57
+ },
+ {
+ "entropy": 1.6132855415344238,
+ "epoch": 0.11717171717171718,
+ "grad_norm": 2.026301145553589,
+ "learning_rate": 4.884848484848485e-06,
+ "loss": 1.6233609914779663,
+ "mean_token_accuracy": 0.6279311180114746,
+ "num_tokens": 950272.0,
+ "step": 58
+ },
+ {
+ "entropy": 1.738538384437561,
+ "epoch": 0.1191919191919192,
+ "grad_norm": 2.1539394855499268,
+ "learning_rate": 4.882828282828283e-06,
+ "loss": 1.8147332668304443,
+ "mean_token_accuracy": 0.6124816536903381,
+ "num_tokens": 966656.0,
+ "step": 59
+ },
+ {
+ "entropy": 1.5533764362335205,
+ "epoch": 0.12121212121212122,
+ "grad_norm": 2.2603628635406494,
+ "learning_rate": 4.880808080808081e-06,
+ "loss": 1.562873125076294,
+ "mean_token_accuracy": 0.6398388147354126,
+ "num_tokens": 983040.0,
+ "step": 60
+ },
+ {
+ "entropy": 1.722406268119812,
+ "epoch": 0.12323232323232323,
+ "grad_norm": 2.3630757331848145,
+ "learning_rate": 4.878787878787879e-06,
+ "loss": 1.7461689710617065,
+ "mean_token_accuracy": 0.6023448705673218,
+ "num_tokens": 999424.0,
+ "step": 61
+ },
+ {
+ "entropy": 1.6533517837524414,
+ "epoch": 0.12525252525252525,
+ "grad_norm": 2.2165415287017822,
+ "learning_rate": 4.876767676767677e-06,
+ "loss": 1.676560640335083,
+ "mean_token_accuracy": 0.6437469720840454,
+ "num_tokens": 1015808.0,
+ "step": 62
+ },
+ {
+ "entropy": 1.749995470046997,
+ "epoch": 0.12727272727272726,
+ "grad_norm": 2.1572678089141846,
+ "learning_rate": 4.8747474747474745e-06,
+ "loss": 1.784087896347046,
+ "mean_token_accuracy": 0.5884831547737122,
+ "num_tokens": 1032192.0,
+ "step": 63
+ },
+ {
+ "entropy": 1.442152738571167,
+ "epoch": 0.1292929292929293,
+ "grad_norm": 2.163490056991577,
+ "learning_rate": 4.872727272727273e-06,
+ "loss": 1.4307503700256348,
+ "mean_token_accuracy": 0.6618832349777222,
+ "num_tokens": 1048576.0,
+ "step": 64
+ },
+ {
+ "entropy": 1.2657029628753662,
+ "epoch": 0.13131313131313133,
+ "grad_norm": 2.1225337982177734,
+ "learning_rate": 4.870707070707071e-06,
+ "loss": 1.2731966972351074,
+ "mean_token_accuracy": 0.688568651676178,
+ "num_tokens": 1064960.0,
+ "step": 65
+ },
+ {
+ "entropy": 1.1613880395889282,
+ "epoch": 0.13333333333333333,
+ "grad_norm": 1.9527196884155273,
+ "learning_rate": 4.868686868686869e-06,
+ "loss": 1.13922119140625,
+ "mean_token_accuracy": 0.7389472126960754,
+ "num_tokens": 1081344.0,
+ "step": 66
+ },
+ {
+ "entropy": 1.6936380863189697,
+ "epoch": 0.13535353535353536,
+ "grad_norm": 2.004284620285034,
+ "learning_rate": 4.866666666666667e-06,
+ "loss": 1.6982238292694092,
+ "mean_token_accuracy": 0.6191987991333008,
+ "num_tokens": 1097728.0,
+ "step": 67
+ },
+ {
+ "entropy": 1.750565528869629,
+ "epoch": 0.13737373737373737,
+ "grad_norm": 2.225955009460449,
+ "learning_rate": 4.864646464646466e-06,
+ "loss": 1.796521782875061,
+ "mean_token_accuracy": 0.5934293866157532,
+ "num_tokens": 1114112.0,
+ "step": 68
+ },
+ {
+ "entropy": 1.6608220338821411,
+ "epoch": 0.1393939393939394,
+ "grad_norm": 2.127035617828369,
+ "learning_rate": 4.8626262626262636e-06,
+ "loss": 1.6633095741271973,
+ "mean_token_accuracy": 0.6356253027915955,
+ "num_tokens": 1130496.0,
+ "step": 69
+ },
+ {
+ "entropy": 1.4848661422729492,
+ "epoch": 0.1414141414141414,
+ "grad_norm": 2.0338215827941895,
+ "learning_rate": 4.8606060606060615e-06,
+ "loss": 1.4789674282073975,
+ "mean_token_accuracy": 0.6680508255958557,
+ "num_tokens": 1146880.0,
+ "step": 70
+ },
+ {
+ "entropy": 1.2310466766357422,
+ "epoch": 0.14343434343434344,
+ "grad_norm": 2.105898141860962,
+ "learning_rate": 4.858585858585859e-06,
+ "loss": 1.2301937341690063,
+ "mean_token_accuracy": 0.7040791511535645,
+ "num_tokens": 1163264.0,
+ "step": 71
+ },
+ {
+ "entropy": 1.6310514211654663,
+ "epoch": 0.14545454545454545,
+ "grad_norm": 2.152125358581543,
+ "learning_rate": 4.856565656565657e-06,
+ "loss": 1.643636703491211,
+ "mean_token_accuracy": 0.6221299171447754,
+ "num_tokens": 1179648.0,
+ "step": 72
+ },
+ {
+ "entropy": 1.4747940301895142,
+ "epoch": 0.14747474747474748,
+ "grad_norm": 2.096461296081543,
+ "learning_rate": 4.854545454545455e-06,
+ "loss": 1.42953360080719,
+ "mean_token_accuracy": 0.6651807427406311,
+ "num_tokens": 1196032.0,
+ "step": 73
+ },
+ {
+ "entropy": 1.819259524345398,
+ "epoch": 0.1494949494949495,
+ "grad_norm": 2.2852063179016113,
+ "learning_rate": 4.852525252525253e-06,
+ "loss": 1.845325231552124,
+ "mean_token_accuracy": 0.6036272644996643,
+ "num_tokens": 1212416.0,
+ "step": 74
+ },
+ {
+ "entropy": 1.4233598709106445,
+ "epoch": 0.15151515151515152,
+ "grad_norm": 2.1157381534576416,
+ "learning_rate": 4.850505050505051e-06,
+ "loss": 1.4565438032150269,
+ "mean_token_accuracy": 0.6607230305671692,
+ "num_tokens": 1228800.0,
+ "step": 75
+ },
+ {
+ "entropy": 1.6441459655761719,
+ "epoch": 0.15353535353535352,
+ "grad_norm": 2.2092180252075195,
+ "learning_rate": 4.848484848484849e-06,
+ "loss": 1.6467639207839966,
+ "mean_token_accuracy": 0.6285417675971985,
+ "num_tokens": 1245184.0,
+ "step": 76
+ },
+ {
+ "entropy": 1.6124308109283447,
+ "epoch": 0.15555555555555556,
+ "grad_norm": 2.231876850128174,
+ "learning_rate": 4.846464646464647e-06,
+ "loss": 1.617384672164917,
+ "mean_token_accuracy": 0.6340987086296082,
+ "num_tokens": 1261568.0,
+ "step": 77
+ },
+ {
+ "entropy": 1.828405499458313,
+ "epoch": 0.15757575757575756,
+ "grad_norm": 2.1372313499450684,
+ "learning_rate": 4.8444444444444446e-06,
+ "loss": 1.8333203792572021,
+ "mean_token_accuracy": 0.5979481935501099,
+ "num_tokens": 1277952.0,
+ "step": 78
+ },
+ {
+ "entropy": 1.855564832687378,
+ "epoch": 0.1595959595959596,
+ "grad_norm": 2.1422762870788574,
+ "learning_rate": 4.842424242424243e-06,
+ "loss": 1.9133609533309937,
+ "mean_token_accuracy": 0.570652186870575,
+ "num_tokens": 1294336.0,
+ "step": 79
+ },
+ {
+ "entropy": 1.7908183336257935,
+ "epoch": 0.16161616161616163,
+ "grad_norm": 2.172368049621582,
+ "learning_rate": 4.840404040404041e-06,
+ "loss": 1.8109419345855713,
+ "mean_token_accuracy": 0.60332190990448,
+ "num_tokens": 1310720.0,
+ "step": 80
+ },
+ {
+ "entropy": 1.9537721872329712,
+ "epoch": 0.16363636363636364,
+ "grad_norm": 2.199505090713501,
+ "learning_rate": 4.838383838383839e-06,
+ "loss": 2.0001401901245117,
+ "mean_token_accuracy": 0.5585613250732422,
+ "num_tokens": 1327104.0,
+ "step": 81
+ },
+ {
+ "entropy": 1.5365772247314453,
+ "epoch": 0.16565656565656567,
+ "grad_norm": 2.0778913497924805,
+ "learning_rate": 4.836363636363637e-06,
+ "loss": 1.575089931488037,
+ "mean_token_accuracy": 0.6375183463096619,
+ "num_tokens": 1343488.0,
+ "step": 82
+ },
+ {
+ "entropy": 1.7177143096923828,
+ "epoch": 0.16767676767676767,
+ "grad_norm": 2.1010894775390625,
+ "learning_rate": 4.834343434343435e-06,
+ "loss": 1.7428388595581055,
+ "mean_token_accuracy": 0.6113824844360352,
+ "num_tokens": 1359872.0,
+ "step": 83
+ },
+ {
+ "entropy": 1.5080527067184448,
+ "epoch": 0.1696969696969697,
+ "grad_norm": 2.021969795227051,
+ "learning_rate": 4.832323232323233e-06,
+ "loss": 1.5361201763153076,
+ "mean_token_accuracy": 0.656509518623352,
+ "num_tokens": 1376256.0,
+ "step": 84
+ },
+ {
+ "entropy": 1.5042120218276978,
+ "epoch": 0.1717171717171717,
+ "grad_norm": 2.121314287185669,
+ "learning_rate": 4.830303030303031e-06,
+ "loss": 1.4912033081054688,
+ "mean_token_accuracy": 0.6621274948120117,
+ "num_tokens": 1392640.0,
+ "step": 85
+ },
+ {
+ "entropy": 1.9523948431015015,
+ "epoch": 0.17373737373737375,
+ "grad_norm": 2.1702609062194824,
+ "learning_rate": 4.8282828282828285e-06,
+ "loss": 1.9369449615478516,
+ "mean_token_accuracy": 0.5727283954620361,
+ "num_tokens": 1409024.0,
+ "step": 86
+ },
+ {
+ "entropy": 1.6174770593643188,
+ "epoch": 0.17575757575757575,
+ "grad_norm": 2.222412109375,
+ "learning_rate": 4.826262626262626e-06,
+ "loss": 1.6794973611831665,
+ "mean_token_accuracy": 0.6151685118675232,
+ "num_tokens": 1425408.0,
+ "step": 87
+ },
+ {
+ "entropy": 1.1480307579040527,
+ "epoch": 0.17777777777777778,
+ "grad_norm": 1.98936128616333,
+ "learning_rate": 4.824242424242424e-06,
+ "loss": 1.1324063539505005,
+ "mean_token_accuracy": 0.7193453907966614,
+ "num_tokens": 1441792.0,
+ "step": 88
+ },
+ {
+ "entropy": 1.5290263891220093,
+ "epoch": 0.1797979797979798,
+ "grad_norm": 2.098860263824463,
+ "learning_rate": 4.822222222222222e-06,
+ "loss": 1.556044340133667,
+ "mean_token_accuracy": 0.6415486335754395,
+ "num_tokens": 1458176.0,
+ "step": 89
+ },
+ {
+ "entropy": 1.1366127729415894,
+ "epoch": 0.18181818181818182,
+ "grad_norm": 1.9387420415878296,
+ "learning_rate": 4.820202020202021e-06,
+ "loss": 1.1635141372680664,
+ "mean_token_accuracy": 0.7168416976928711,
+ "num_tokens": 1474560.0,
+ "step": 90
+ },
+ {
+ "entropy": 1.5142875909805298,
+ "epoch": 0.18383838383838383,
+ "grad_norm": 2.259131908416748,
+ "learning_rate": 4.818181818181819e-06,
+ "loss": 1.5329954624176025,
+ "mean_token_accuracy": 0.6471666097640991,
+ "num_tokens": 1490944.0,
+ "step": 91
+ },
+ {
+ "entropy": 1.5197135210037231,
+ "epoch": 0.18585858585858586,
+ "grad_norm": 2.1230807304382324,
+ "learning_rate": 4.816161616161617e-06,
+ "loss": 1.541062593460083,
+ "mean_token_accuracy": 0.6398388147354126,
+ "num_tokens": 1507328.0,
+ "step": 92
+ },
+ {
+ "entropy": 1.5469954013824463,
+ "epoch": 0.18787878787878787,
+ "grad_norm": 2.0568583011627197,
+ "learning_rate": 4.8141414141414146e-06,
+ "loss": 1.5564078092575073,
+ "mean_token_accuracy": 0.6384953856468201,
+ "num_tokens": 1523712.0,
+ "step": 93
+ },
+ {
+ "entropy": 1.5429692268371582,
+ "epoch": 0.1898989898989899,
+ "grad_norm": 2.200144052505493,
+ "learning_rate": 4.8121212121212125e-06,
+ "loss": 1.5292989015579224,
+ "mean_token_accuracy": 0.6373351216316223,
+ "num_tokens": 1540096.0,
+ "step": 94
+ },
+ {
+ "entropy": 1.4556076526641846,
+ "epoch": 0.1919191919191919,
+ "grad_norm": 2.250291585922241,
+ "learning_rate": 4.81010101010101e-06,
+ "loss": 1.426419734954834,
+ "mean_token_accuracy": 0.6604176759719849,
+ "num_tokens": 1556480.0,
+ "step": 95
+ },
+ {
+ "entropy": 1.4021440744400024,
+ "epoch": 0.19393939393939394,
+ "grad_norm": 2.287038803100586,
+ "learning_rate": 4.808080808080808e-06,
+ "loss": 1.4377000331878662,
+ "mean_token_accuracy": 0.6572422981262207,
+ "num_tokens": 1572864.0,
+ "step": 96
+ },
+ {
+ "entropy": 1.658683180809021,
+ "epoch": 0.19595959595959597,
+ "grad_norm": 2.1817963123321533,
+ "learning_rate": 4.806060606060606e-06,
+ "loss": 1.6514620780944824,
+ "mean_token_accuracy": 0.6278700828552246,
+ "num_tokens": 1589248.0,
+ "step": 97
+ },
+ {
+ "entropy": 1.5182844400405884,
+ "epoch": 0.19797979797979798,
+ "grad_norm": 2.144071340560913,
+ "learning_rate": 4.804040404040404e-06,
+ "loss": 1.5554628372192383,
+ "mean_token_accuracy": 0.6311064958572388,
+ "num_tokens": 1605632.0,
+ "step": 98
+ },
+ {
+ "entropy": 1.5997719764709473,
+ "epoch": 0.2,
+ "grad_norm": 2.0921664237976074,
+ "learning_rate": 4.802020202020202e-06,
+ "loss": 1.6185517311096191,
+ "mean_token_accuracy": 0.6286028623580933,
+ "num_tokens": 1622016.0,
+ "step": 99
+ },
+ {
+ "entropy": 1.4011279344558716,
+ "epoch": 0.20202020202020202,
+ "grad_norm": 2.2240700721740723,
+ "learning_rate": 4.800000000000001e-06,
+ "loss": 1.4112927913665771,
+ "mean_token_accuracy": 0.6696995496749878,
+ "num_tokens": 1638400.0,
+ "step": 100
+ },
+ {
+ "epoch": 0.20202020202020202,
+ "eval_entropy": 1.606662715634992,
+ "eval_loss": 1.6283859014511108,
+ "eval_mean_token_accuracy": 0.6293055717983553,
+ "eval_num_tokens": 1638400.0,
+ "eval_runtime": 43.7782,
+ "eval_samples_per_second": 22.614,
+ "eval_steps_per_second": 2.832,
+ "step": 100
+ },
+ {
+ "entropy": 1.6743865013122559,
+ "epoch": 0.20404040404040405,
+ "grad_norm": 2.025153398513794,
+ "learning_rate": 4.7979797979797985e-06,
+ "loss": 1.6404725313186646,
+ "mean_token_accuracy": 0.656509518623352,
+ "num_tokens": 1654784.0,
+ "step": 101
+ },
+ {
+ "entropy": 1.6196308135986328,
+ "epoch": 0.20606060606060606,
+ "grad_norm": 2.037229299545288,
+ "learning_rate": 4.795959595959596e-06,
+ "loss": 1.647303581237793,
+ "mean_token_accuracy": 0.6203590631484985,
+ "num_tokens": 1671168.0,
+ "step": 102
+ },
+ {
+ "entropy": 1.5600125789642334,
+ "epoch": 0.2080808080808081,
+ "grad_norm": 2.144221782684326,
+ "learning_rate": 4.793939393939394e-06,
+ "loss": 1.6199731826782227,
+ "mean_token_accuracy": 0.6249389052391052,
+ "num_tokens": 1687552.0,
+ "step": 103
+ },
+ {
+ "entropy": 2.064497947692871,
+ "epoch": 0.2101010101010101,
+ "grad_norm": 2.3956470489501953,
+ "learning_rate": 4.791919191919192e-06,
+ "loss": 2.048987627029419,
+ "mean_token_accuracy": 0.5507450103759766,
+ "num_tokens": 1703936.0,
+ "step": 104
+ },
+ {
+ "entropy": 1.7934812307357788,
+ "epoch": 0.21212121212121213,
+ "grad_norm": 2.487302780151367,
+ "learning_rate": 4.78989898989899e-06,
+ "loss": 1.8285956382751465,
+ "mean_token_accuracy": 0.5749877691268921,
+ "num_tokens": 1720320.0,
+ "step": 105
+ },
+ {
+ "entropy": 1.365216851234436,
+ "epoch": 0.21414141414141413,
+ "grad_norm": 2.0714964866638184,
+ "learning_rate": 4.787878787878788e-06,
+ "loss": 1.3870220184326172,
+ "mean_token_accuracy": 0.6692721247673035,
+ "num_tokens": 1736704.0,
+ "step": 106
+ },
+ {
+ "entropy": 1.4983874559402466,
+ "epoch": 0.21616161616161617,
+ "grad_norm": 1.9995853900909424,
+ "learning_rate": 4.785858585858586e-06,
+ "loss": 1.4949266910552979,
+ "mean_token_accuracy": 0.6554103493690491,
+ "num_tokens": 1753088.0,
+ "step": 107
+ },
+ {
+ "entropy": 2.0503509044647217,
+ "epoch": 0.21818181818181817,
+ "grad_norm": 2.190884590148926,
+ "learning_rate": 4.783838383838385e-06,
+ "loss": 2.079286813735962,
+ "mean_token_accuracy": 0.5657669901847839,
+ "num_tokens": 1769472.0,
+ "step": 108
+ },
+ {
+ "entropy": 1.5557374954223633,
+ "epoch": 0.2202020202020202,
+ "grad_norm": 2.044100761413574,
+ "learning_rate": 4.7818181818181825e-06,
+ "loss": 1.5735318660736084,
+ "mean_token_accuracy": 0.6359916925430298,
+ "num_tokens": 1785856.0,
+ "step": 109
+ },
+ {
+ "entropy": 1.5567635297775269,
+ "epoch": 0.2222222222222222,
+ "grad_norm": 2.3714160919189453,
+ "learning_rate": 4.77979797979798e-06,
+ "loss": 1.6026921272277832,
+ "mean_token_accuracy": 0.6290302872657776,
+ "num_tokens": 1802240.0,
+ "step": 110
+ },
+ {
+ "entropy": 1.8457536697387695,
+ "epoch": 0.22424242424242424,
+ "grad_norm": 2.202939987182617,
+ "learning_rate": 4.777777777777778e-06,
+ "loss": 1.7986081838607788,
+ "mean_token_accuracy": 0.5952613353729248,
+ "num_tokens": 1818624.0,
+ "step": 111
+ },
+ {
+ "entropy": 1.8822020292282104,
+ "epoch": 0.22626262626262628,
+ "grad_norm": 2.2674992084503174,
+ "learning_rate": 4.775757575757576e-06,
+ "loss": 1.9095954895019531,
+ "mean_token_accuracy": 0.5769418478012085,
+ "num_tokens": 1835008.0,
+ "step": 112
+ },
+ {
+ "entropy": 1.5445998907089233,
+ "epoch": 0.22828282828282828,
+ "grad_norm": 2.2848100662231445,
+ "learning_rate": 4.773737373737374e-06,
+ "loss": 1.5832195281982422,
+ "mean_token_accuracy": 0.6337933540344238,
+ "num_tokens": 1851392.0,
+ "step": 113
+ },
+ {
+ "entropy": 1.934509515762329,
+ "epoch": 0.23030303030303031,
+ "grad_norm": 2.3585174083709717,
+ "learning_rate": 4.771717171717172e-06,
+ "loss": 1.9809319972991943,
+ "mean_token_accuracy": 0.5789570212364197,
+ "num_tokens": 1867776.0,
+ "step": 114
+ },
+ {
+ "entropy": 1.7354214191436768,
+ "epoch": 0.23232323232323232,
+ "grad_norm": 2.13913631439209,
+ "learning_rate": 4.769696969696971e-06,
+ "loss": 1.7527806758880615,
+ "mean_token_accuracy": 0.6061308979988098,
+ "num_tokens": 1884160.0,
+ "step": 115
+ },
+ {
+ "entropy": 1.7714784145355225,
+ "epoch": 0.23434343434343435,
+ "grad_norm": 1.9915403127670288,
+ "learning_rate": 4.7676767676767685e-06,
+ "loss": 1.8065431118011475,
+ "mean_token_accuracy": 0.5986199378967285,
+ "num_tokens": 1900544.0,
+ "step": 116
+ },
+ {
+ "entropy": 1.5272125005722046,
+ "epoch": 0.23636363636363636,
+ "grad_norm": 2.004565715789795,
+ "learning_rate": 4.765656565656566e-06,
+ "loss": 1.5074517726898193,
+ "mean_token_accuracy": 0.6601123809814453,
+ "num_tokens": 1916928.0,
+ "step": 117
+ },
+ {
+ "entropy": 1.5393218994140625,
+ "epoch": 0.2383838383838384,
+ "grad_norm": 2.018089532852173,
+ "learning_rate": 4.763636363636364e-06,
+ "loss": 1.5607573986053467,
+ "mean_token_accuracy": 0.6540058851242065,
+ "num_tokens": 1933312.0,
+ "step": 118
+ },
+ {
+ "entropy": 1.7827534675598145,
+ "epoch": 0.2404040404040404,
+ "grad_norm": 2.062368392944336,
+ "learning_rate": 4.761616161616162e-06,
+ "loss": 1.7844001054763794,
+ "mean_token_accuracy": 0.6187102794647217,
+ "num_tokens": 1949696.0,
+ "step": 119
+ },
+ {
+ "entropy": 1.416417121887207,
+ "epoch": 0.24242424242424243,
+ "grad_norm": 2.0980024337768555,
+ "learning_rate": 4.75959595959596e-06,
+ "loss": 1.4041050672531128,
+ "mean_token_accuracy": 0.6561431288719177,
+ "num_tokens": 1966080.0,
+ "step": 120
+ },
+ {
+ "entropy": 1.3325153589248657,
+ "epoch": 0.24444444444444444,
+ "grad_norm": 1.9985002279281616,
+ "learning_rate": 4.757575757575758e-06,
+ "loss": 1.3197274208068848,
+ "mean_token_accuracy": 0.6806912422180176,
+ "num_tokens": 1982464.0,
+ "step": 121
+ },
+ {
+ "entropy": 1.7306798696517944,
+ "epoch": 0.24646464646464647,
+ "grad_norm": 2.2098441123962402,
+ "learning_rate": 4.755555555555556e-06,
+ "loss": 1.7595295906066895,
+ "mean_token_accuracy": 0.6110160946846008,
+ "num_tokens": 1998848.0,
+ "step": 122
+ },
+ {
+ "entropy": 1.3733264207839966,
+ "epoch": 0.24848484848484848,
+ "grad_norm": 1.9827327728271484,
+ "learning_rate": 4.753535353535354e-06,
+ "loss": 1.4026854038238525,
+ "mean_token_accuracy": 0.65486079454422,
+ "num_tokens": 2015232.0,
+ "step": 123
+ },
+ {
+ "entropy": 1.5910528898239136,
+ "epoch": 0.2505050505050505,
+ "grad_norm": 1.9615319967269897,
+ "learning_rate": 4.751515151515152e-06,
+ "loss": 1.6071114540100098,
+ "mean_token_accuracy": 0.648937463760376,
+ "num_tokens": 2031616.0,
+ "step": 124
+ },
+ {
+ "entropy": 1.3300938606262207,
+ "epoch": 0.25252525252525254,
+ "grad_norm": 1.9878504276275635,
+ "learning_rate": 4.7494949494949495e-06,
+ "loss": 1.3623383045196533,
+ "mean_token_accuracy": 0.6873473525047302,
+ "num_tokens": 2048000.0,
+ "step": 125
+ },
+ {
+ "entropy": 1.6087368726730347,
+ "epoch": 0.2545454545454545,
+ "grad_norm": 2.264647960662842,
+ "learning_rate": 4.747474747474748e-06,
+ "loss": 1.634058952331543,
+ "mean_token_accuracy": 0.6211528778076172,
+ "num_tokens": 2064384.0,
+ "step": 126
+ },
+ {
+ "entropy": 1.536401391029358,
+ "epoch": 0.25656565656565655,
+ "grad_norm": 1.9561539888381958,
+ "learning_rate": 4.745454545454546e-06,
+ "loss": 1.5699501037597656,
+ "mean_token_accuracy": 0.6248167753219604,
+ "num_tokens": 2080768.0,
+ "step": 127
+ },
+ {
+ "entropy": 1.656266212463379,
+ "epoch": 0.2585858585858586,
+ "grad_norm": 2.1124353408813477,
+ "learning_rate": 4.743434343434344e-06,
+ "loss": 1.6612167358398438,
+ "mean_token_accuracy": 0.623900830745697,
+ "num_tokens": 2097152.0,
+ "step": 128
+ },
+ {
+ "entropy": 1.170629620552063,
+ "epoch": 0.2606060606060606,
+ "grad_norm": 1.917840600013733,
+ "learning_rate": 4.741414141414142e-06,
+ "loss": 1.1992300748825073,
+ "mean_token_accuracy": 0.6995603442192078,
+ "num_tokens": 2113536.0,
+ "step": 129
+ },
+ {
+ "entropy": 1.4126694202423096,
+ "epoch": 0.26262626262626265,
+ "grad_norm": 1.9041539430618286,
+ "learning_rate": 4.73939393939394e-06,
+ "loss": 1.4334447383880615,
+ "mean_token_accuracy": 0.6637151837348938,
+ "num_tokens": 2129920.0,
+ "step": 130
+ },
+ {
+ "entropy": 1.8490277528762817,
+ "epoch": 0.26464646464646463,
+ "grad_norm": 2.1440138816833496,
+ "learning_rate": 4.737373737373738e-06,
+ "loss": 1.9226353168487549,
+ "mean_token_accuracy": 0.5805447101593018,
+ "num_tokens": 2146304.0,
+ "step": 131
+ },
+ {
+ "entropy": 1.5000964403152466,
+ "epoch": 0.26666666666666666,
+ "grad_norm": 1.9615508317947388,
+ "learning_rate": 4.735353535353536e-06,
+ "loss": 1.495596170425415,
+ "mean_token_accuracy": 0.6542501449584961,
+ "num_tokens": 2162688.0,
+ "step": 132
+ },
+ {
+ "entropy": 1.9371142387390137,
+ "epoch": 0.2686868686868687,
+ "grad_norm": 2.202200412750244,
+ "learning_rate": 4.7333333333333335e-06,
+ "loss": 1.94151771068573,
+ "mean_token_accuracy": 0.5810332298278809,
+ "num_tokens": 2179072.0,
+ "step": 133
+ },
+ {
+ "entropy": 1.4535126686096191,
+ "epoch": 0.27070707070707073,
+ "grad_norm": 1.9804027080535889,
+ "learning_rate": 4.731313131313131e-06,
+ "loss": 1.4401545524597168,
+ "mean_token_accuracy": 0.6585246920585632,
+ "num_tokens": 2195456.0,
+ "step": 134
+ },
+ {
+ "entropy": 2.0871424674987793,
+ "epoch": 0.2727272727272727,
+ "grad_norm": 2.1417081356048584,
+ "learning_rate": 4.729292929292929e-06,
+ "loss": 2.117375373840332,
+ "mean_token_accuracy": 0.5600268840789795,
+ "num_tokens": 2211840.0,
+ "step": 135
+ },
+ {
+ "entropy": 1.2986469268798828,
+ "epoch": 0.27474747474747474,
+ "grad_norm": 2.0299136638641357,
+ "learning_rate": 4.727272727272728e-06,
+ "loss": 1.3631991147994995,
+ "mean_token_accuracy": 0.6750732660293579,
+ "num_tokens": 2228224.0,
+ "step": 136
+ },
+ {
+ "entropy": 1.163429617881775,
+ "epoch": 0.2767676767676768,
+ "grad_norm": 1.8368210792541504,
+ "learning_rate": 4.725252525252526e-06,
+ "loss": 1.1603795289993286,
+ "mean_token_accuracy": 0.7150708436965942,
+ "num_tokens": 2244608.0,
+ "step": 137
+ },
+ {
+ "entropy": 1.4301519393920898,
+ "epoch": 0.2787878787878788,
+ "grad_norm": 1.996767520904541,
+ "learning_rate": 4.723232323232324e-06,
+ "loss": 1.422170639038086,
+ "mean_token_accuracy": 0.6816682815551758,
+ "num_tokens": 2260992.0,
+ "step": 138
+ },
+ {
+ "entropy": 1.5160106420516968,
+ "epoch": 0.2808080808080808,
+ "grad_norm": 2.004770278930664,
+ "learning_rate": 4.721212121212122e-06,
+ "loss": 1.5385751724243164,
+ "mean_token_accuracy": 0.640693724155426,
+ "num_tokens": 2277376.0,
+ "step": 139
+ },
+ {
+ "entropy": 1.2483867406845093,
+ "epoch": 0.2828282828282828,
+ "grad_norm": 1.8488364219665527,
+ "learning_rate": 4.7191919191919195e-06,
+ "loss": 1.2563843727111816,
+ "mean_token_accuracy": 0.6998046040534973,
+ "num_tokens": 2293760.0,
+ "step": 140
+ },
+ {
+ "entropy": 1.3214129209518433,
+ "epoch": 0.28484848484848485,
+ "grad_norm": 2.1334660053253174,
+ "learning_rate": 4.717171717171717e-06,
+ "loss": 1.3392664194107056,
+ "mean_token_accuracy": 0.6822789311408997,
+ "num_tokens": 2310144.0,
+ "step": 141
+ },
+ {
+ "entropy": 1.611849069595337,
+ "epoch": 0.2868686868686869,
+ "grad_norm": 2.0539803504943848,
+ "learning_rate": 4.715151515151515e-06,
+ "loss": 1.6335396766662598,
+ "mean_token_accuracy": 0.6235954761505127,
+ "num_tokens": 2326528.0,
+ "step": 142
+ },
+ {
+ "entropy": 1.5847896337509155,
+ "epoch": 0.28888888888888886,
+ "grad_norm": 2.1658759117126465,
+ "learning_rate": 4.713131313131313e-06,
+ "loss": 1.603764533996582,
+ "mean_token_accuracy": 0.633671224117279,
+ "num_tokens": 2342912.0,
+ "step": 143
+ },
+ {
+ "entropy": 1.5731406211853027,
+ "epoch": 0.2909090909090909,
+ "grad_norm": 2.0830390453338623,
+ "learning_rate": 4.711111111111111e-06,
+ "loss": 1.558933973312378,
+ "mean_token_accuracy": 0.6392892003059387,
+ "num_tokens": 2359296.0,
+ "step": 144
+ },
+ {
+ "entropy": 1.8901628255844116,
+ "epoch": 0.29292929292929293,
+ "grad_norm": 2.1436922550201416,
+ "learning_rate": 4.709090909090909e-06,
+ "loss": 1.878631353378296,
+ "mean_token_accuracy": 0.57333904504776,
+ "num_tokens": 2375680.0,
+ "step": 145
+ },
+ {
+ "entropy": 1.9344228506088257,
+ "epoch": 0.29494949494949496,
+ "grad_norm": 2.1245176792144775,
+ "learning_rate": 4.707070707070707e-06,
+ "loss": 1.9546704292297363,
+ "mean_token_accuracy": 0.5987420678138733,
+ "num_tokens": 2392064.0,
+ "step": 146
+ },
+ {
+ "entropy": 1.2455096244812012,
+ "epoch": 0.296969696969697,
+ "grad_norm": 1.896581768989563,
+ "learning_rate": 4.705050505050506e-06,
+ "loss": 1.2738728523254395,
+ "mean_token_accuracy": 0.7025524973869324,
+ "num_tokens": 2408448.0,
+ "step": 147
+ },
+ {
+ "entropy": 1.4235948324203491,
+ "epoch": 0.298989898989899,
+ "grad_norm": 2.522636890411377,
+ "learning_rate": 4.7030303030303035e-06,
+ "loss": 1.4524480104446411,
+ "mean_token_accuracy": 0.6441133618354797,
+ "num_tokens": 2424832.0,
+ "step": 148
+ },
+ {
+ "entropy": 1.764552116394043,
+ "epoch": 0.301010101010101,
+ "grad_norm": 2.0232255458831787,
+ "learning_rate": 4.701010101010101e-06,
+ "loss": 1.7811740636825562,
+ "mean_token_accuracy": 0.6229848265647888,
+ "num_tokens": 2441216.0,
+ "step": 149
+ },
+ {
+ "entropy": 1.74843168258667,
+ "epoch": 0.30303030303030304,
+ "grad_norm": 2.1589369773864746,
+ "learning_rate": 4.698989898989899e-06,
+ "loss": 1.7337679862976074,
+ "mean_token_accuracy": 0.5975207686424255,
+ "num_tokens": 2457600.0,
+ "step": 150
+ },
+ {
+ "epoch": 0.30303030303030304,
+ "eval_entropy": 1.6033287221385586,
+ "eval_loss": 1.6036633253097534,
+ "eval_mean_token_accuracy": 0.6329842450157288,
+ "eval_num_tokens": 2457600.0,
+ "eval_runtime": 43.7655,
+ "eval_samples_per_second": 22.621,
+ "eval_steps_per_second": 2.833,
+ "step": 150
+ },
+ {
+ "entropy": 2.0572290420532227,
+ "epoch": 0.30505050505050507,
+ "grad_norm": 2.0511579513549805,
+ "learning_rate": 4.696969696969698e-06,
+ "loss": 2.0196030139923096,
+ "mean_token_accuracy": 0.5519052147865295,
+ "num_tokens": 2473984.0,
+ "step": 151
+ },
+ {
+ "entropy": 1.5792397260665894,
+ "epoch": 0.30707070707070705,
+ "grad_norm": 2.048396587371826,
+ "learning_rate": 4.694949494949496e-06,
+ "loss": 1.575985074043274,
+ "mean_token_accuracy": 0.623961865901947,
+ "num_tokens": 2490368.0,
+ "step": 152
+ },
+ {
+ "entropy": 1.487528681755066,
+ "epoch": 0.3090909090909091,
+ "grad_norm": 2.1903791427612305,
+ "learning_rate": 4.692929292929294e-06,
+ "loss": 1.479973316192627,
+ "mean_token_accuracy": 0.6595017313957214,
+ "num_tokens": 2506752.0,
+ "step": 153
+ },
+ {
+ "entropy": 1.766797423362732,
+ "epoch": 0.3111111111111111,
+ "grad_norm": 2.1843011379241943,
+ "learning_rate": 4.690909090909092e-06,
+ "loss": 1.7993412017822266,
+ "mean_token_accuracy": 0.5965437293052673,
+ "num_tokens": 2523136.0,
+ "step": 154
+ },
+ {
+ "entropy": 1.7058254480361938,
+ "epoch": 0.31313131313131315,
+ "grad_norm": 2.204461097717285,
+ "learning_rate": 4.6888888888888895e-06,
+ "loss": 1.7346007823944092,
+ "mean_token_accuracy": 0.6077185869216919,
+ "num_tokens": 2539520.0,
+ "step": 155
+ },
+ {
+ "entropy": 1.4929591417312622,
+ "epoch": 0.3151515151515151,
+ "grad_norm": 2.1739237308502197,
+ "learning_rate": 4.6868686868686874e-06,
+ "loss": 1.4708621501922607,
+ "mean_token_accuracy": 0.6427088379859924,
+ "num_tokens": 2555904.0,
+ "step": 156
+ },
+ {
+ "entropy": 1.5797587633132935,
+ "epoch": 0.31717171717171716,
+ "grad_norm": 2.150561571121216,
+ "learning_rate": 4.684848484848485e-06,
+ "loss": 1.5921857357025146,
+ "mean_token_accuracy": 0.6262823939323425,
+ "num_tokens": 2572288.0,
+ "step": 157
+ },
+ {
+ "entropy": 1.6198230981826782,
+ "epoch": 0.3191919191919192,
+ "grad_norm": 2.061169385910034,
+ "learning_rate": 4.682828282828283e-06,
+ "loss": 1.6601009368896484,
+ "mean_token_accuracy": 0.619015634059906,
+ "num_tokens": 2588672.0,
+ "step": 158
+ },
+ {
+ "entropy": 1.4026126861572266,
+ "epoch": 0.3212121212121212,
+ "grad_norm": 2.066208839416504,
+ "learning_rate": 4.680808080808081e-06,
+ "loss": 1.4062483310699463,
+ "mean_token_accuracy": 0.6681729555130005,
+ "num_tokens": 2605056.0,
+ "step": 159
+ },
+ {
+ "entropy": 1.4608066082000732,
+ "epoch": 0.32323232323232326,
+ "grad_norm": 2.022627353668213,
+ "learning_rate": 4.678787878787879e-06,
+ "loss": 1.4644970893859863,
+ "mean_token_accuracy": 0.6533341407775879,
+ "num_tokens": 2621440.0,
+ "step": 160
+ },
+ {
+ "entropy": 1.222448468208313,
+ "epoch": 0.32525252525252524,
+ "grad_norm": 2.0399329662323,
+ "learning_rate": 4.676767676767677e-06,
+ "loss": 1.2547852993011475,
+ "mean_token_accuracy": 0.6925378441810608,
+ "num_tokens": 2637824.0,
+ "step": 161
+ },
+ {
+ "entropy": 1.8702255487442017,
+ "epoch": 0.32727272727272727,
+ "grad_norm": 2.177307367324829,
+ "learning_rate": 4.674747474747476e-06,
+ "loss": 1.9041712284088135,
+ "mean_token_accuracy": 0.5754152536392212,
+ "num_tokens": 2654208.0,
+ "step": 162
+ },
+ {
+ "entropy": 1.6861947774887085,
+ "epoch": 0.3292929292929293,
+ "grad_norm": 2.009544610977173,
+ "learning_rate": 4.6727272727272735e-06,
+ "loss": 1.7079355716705322,
+ "mean_token_accuracy": 0.615290641784668,
+ "num_tokens": 2670592.0,
+ "step": 163
+ },
+ {
+ "entropy": 1.3880327939987183,
+ "epoch": 0.33131313131313134,
+ "grad_norm": 2.16359543800354,
+ "learning_rate": 4.670707070707071e-06,
+ "loss": 1.4220571517944336,
+ "mean_token_accuracy": 0.6604787707328796,
+ "num_tokens": 2686976.0,
+ "step": 164
+ },
+ {
+ "entropy": 1.5760643482208252,
+ "epoch": 0.3333333333333333,
+ "grad_norm": 2.09773325920105,
+ "learning_rate": 4.668686868686869e-06,
+ "loss": 1.6036784648895264,
+ "mean_token_accuracy": 0.6267098188400269,
+ "num_tokens": 2703360.0,
+ "step": 165
+ },
+ {
+ "entropy": 1.078303575515747,
+ "epoch": 0.33535353535353535,
+ "grad_norm": 1.7760599851608276,
+ "learning_rate": 4.666666666666667e-06,
+ "loss": 1.0612695217132568,
+ "mean_token_accuracy": 0.7349169254302979,
+ "num_tokens": 2719744.0,
+ "step": 166
+ },
+ {
+ "entropy": 1.3722130060195923,
+ "epoch": 0.3373737373737374,
+ "grad_norm": 2.0816409587860107,
+ "learning_rate": 4.664646464646465e-06,
+ "loss": 1.3693504333496094,
+ "mean_token_accuracy": 0.6682950854301453,
+ "num_tokens": 2736128.0,
+ "step": 167
+ },
+ {
+ "entropy": 1.8752760887145996,
+ "epoch": 0.3393939393939394,
+ "grad_norm": 2.1670358180999756,
+ "learning_rate": 4.662626262626263e-06,
+ "loss": 1.8874578475952148,
+ "mean_token_accuracy": 0.5881778001785278,
+ "num_tokens": 2752512.0,
+ "step": 168
+ },
+ {
+ "entropy": 1.7384581565856934,
+ "epoch": 0.3414141414141414,
+ "grad_norm": 2.0072429180145264,
+ "learning_rate": 4.660606060606061e-06,
+ "loss": 1.7491583824157715,
+ "mean_token_accuracy": 0.6041157841682434,
+ "num_tokens": 2768896.0,
+ "step": 169
+ },
+ {
+ "entropy": 1.6651691198349,
+ "epoch": 0.3434343434343434,
+ "grad_norm": 2.0812578201293945,
+ "learning_rate": 4.658585858585859e-06,
+ "loss": 1.6808438301086426,
+ "mean_token_accuracy": 0.6273204684257507,
+ "num_tokens": 2785280.0,
+ "step": 170
+ },
+ {
+ "entropy": 1.3426616191864014,
+ "epoch": 0.34545454545454546,
+ "grad_norm": 1.9108822345733643,
+ "learning_rate": 4.656565656565657e-06,
+ "loss": 1.3404264450073242,
+ "mean_token_accuracy": 0.6703101992607117,
+ "num_tokens": 2801664.0,
+ "step": 171
+ },
+ {
+ "entropy": 1.631722092628479,
+ "epoch": 0.3474747474747475,
+ "grad_norm": 2.07317852973938,
+ "learning_rate": 4.654545454545455e-06,
+ "loss": 1.6545813083648682,
+ "mean_token_accuracy": 0.6143136024475098,
+ "num_tokens": 2818048.0,
+ "step": 172
+ },
+ {
+ "entropy": 1.3678289651870728,
+ "epoch": 0.34949494949494947,
+ "grad_norm": 1.8517454862594604,
+ "learning_rate": 4.652525252525253e-06,
+ "loss": 1.3759769201278687,
+ "mean_token_accuracy": 0.6693942546844482,
+ "num_tokens": 2834432.0,
+ "step": 173
+ },
+ {
+ "entropy": 1.3927773237228394,
+ "epoch": 0.3515151515151515,
+ "grad_norm": 2.1304867267608643,
+ "learning_rate": 4.650505050505051e-06,
+ "loss": 1.4104211330413818,
+ "mean_token_accuracy": 0.6612725853919983,
+ "num_tokens": 2850816.0,
+ "step": 174
+ },
+ {
+ "entropy": 1.3352863788604736,
+ "epoch": 0.35353535353535354,
+ "grad_norm": 2.2788777351379395,
+ "learning_rate": 4.648484848484849e-06,
+ "loss": 1.3665719032287598,
+ "mean_token_accuracy": 0.6640205383300781,
+ "num_tokens": 2867200.0,
+ "step": 175
+ },
+ {
+ "entropy": 1.6800754070281982,
+ "epoch": 0.35555555555555557,
+ "grad_norm": 2.030787944793701,
+ "learning_rate": 4.646464646464647e-06,
+ "loss": 1.7239181995391846,
+ "mean_token_accuracy": 0.6193209290504456,
+ "num_tokens": 2883584.0,
+ "step": 176
+ },
+ {
+ "entropy": 1.2852731943130493,
+ "epoch": 0.3575757575757576,
+ "grad_norm": 1.9148248434066772,
+ "learning_rate": 4.644444444444445e-06,
+ "loss": 1.2934880256652832,
+ "mean_token_accuracy": 0.6798363327980042,
+ "num_tokens": 2899968.0,
+ "step": 177
+ },
+ {
+ "entropy": 1.6366593837738037,
+ "epoch": 0.3595959595959596,
+ "grad_norm": 2.2264015674591064,
+ "learning_rate": 4.642424242424243e-06,
+ "loss": 1.632002592086792,
+ "mean_token_accuracy": 0.6180385947227478,
+ "num_tokens": 2916352.0,
+ "step": 178
+ },
+ {
+ "entropy": 1.3935565948486328,
+ "epoch": 0.3616161616161616,
+ "grad_norm": 2.0410475730895996,
+ "learning_rate": 4.6404040404040406e-06,
+ "loss": 1.407370686531067,
+ "mean_token_accuracy": 0.6722032427787781,
+ "num_tokens": 2932736.0,
+ "step": 179
+ },
+ {
+ "entropy": 1.0952510833740234,
+ "epoch": 0.36363636363636365,
+ "grad_norm": 1.937270164489746,
+ "learning_rate": 4.6383838383838384e-06,
+ "loss": 1.1101973056793213,
+ "mean_token_accuracy": 0.730947732925415,
+ "num_tokens": 2949120.0,
+ "step": 180
+ },
+ {
+ "entropy": 1.184926152229309,
+ "epoch": 0.3656565656565657,
+ "grad_norm": 1.9867528676986694,
+ "learning_rate": 4.636363636363636e-06,
+ "loss": 1.1932196617126465,
+ "mean_token_accuracy": 0.7085368633270264,
+ "num_tokens": 2965504.0,
+ "step": 181
+ },
+ {
+ "entropy": 1.4552072286605835,
+ "epoch": 0.36767676767676766,
+ "grad_norm": 1.9395766258239746,
+ "learning_rate": 4.634343434343434e-06,
+ "loss": 1.4559956789016724,
+ "mean_token_accuracy": 0.6534562706947327,
+ "num_tokens": 2981888.0,
+ "step": 182
+ },
+ {
+ "entropy": 1.6437443494796753,
+ "epoch": 0.3696969696969697,
+ "grad_norm": 2.1257567405700684,
+ "learning_rate": 4.632323232323233e-06,
+ "loss": 1.6260980367660522,
+ "mean_token_accuracy": 0.625,
+ "num_tokens": 2998272.0,
+ "step": 183
+ },
+ {
+ "entropy": 1.8264633417129517,
+ "epoch": 0.3717171717171717,
+ "grad_norm": 2.187041759490967,
+ "learning_rate": 4.630303030303031e-06,
+ "loss": 1.8431676626205444,
+ "mean_token_accuracy": 0.60332190990448,
+ "num_tokens": 3014656.0,
+ "step": 184
+ },
+ {
+ "entropy": 1.1121351718902588,
+ "epoch": 0.37373737373737376,
+ "grad_norm": 1.8460402488708496,
+ "learning_rate": 4.628282828282829e-06,
+ "loss": 1.1054425239562988,
+ "mean_token_accuracy": 0.7254518866539001,
+ "num_tokens": 3031040.0,
+ "step": 185
+ },
+ {
+ "entropy": 1.3531955480575562,
+ "epoch": 0.37575757575757573,
+ "grad_norm": 2.3463470935821533,
+ "learning_rate": 4.626262626262627e-06,
+ "loss": 1.359381914138794,
+ "mean_token_accuracy": 0.6610894203186035,
+ "num_tokens": 3047424.0,
+ "step": 186
+ },
+ {
+ "entropy": 1.2131577730178833,
+ "epoch": 0.37777777777777777,
+ "grad_norm": 1.9595171213150024,
+ "learning_rate": 4.6242424242424245e-06,
+ "loss": 1.2312005758285522,
+ "mean_token_accuracy": 0.6979115605354309,
+ "num_tokens": 3063808.0,
+ "step": 187
+ },
+ {
+ "entropy": 1.5441125631332397,
+ "epoch": 0.3797979797979798,
+ "grad_norm": 2.2971343994140625,
+ "learning_rate": 4.622222222222222e-06,
+ "loss": 1.5923339128494263,
+ "mean_token_accuracy": 0.6297020316123962,
+ "num_tokens": 3080192.0,
+ "step": 188
+ },
+ {
+ "entropy": 1.7998323440551758,
+ "epoch": 0.38181818181818183,
+ "grad_norm": 2.010552167892456,
+ "learning_rate": 4.62020202020202e-06,
+ "loss": 1.8257204294204712,
+ "mean_token_accuracy": 0.5997191071510315,
+ "num_tokens": 3096576.0,
+ "step": 189
+ },
+ {
+ "entropy": 1.389051914215088,
+ "epoch": 0.3838383838383838,
+ "grad_norm": 2.1183855533599854,
+ "learning_rate": 4.618181818181818e-06,
+ "loss": 1.431575059890747,
+ "mean_token_accuracy": 0.6537005305290222,
+ "num_tokens": 3112960.0,
+ "step": 190
+ },
+ {
+ "entropy": 2.0066142082214355,
+ "epoch": 0.38585858585858585,
+ "grad_norm": 2.061399221420288,
+ "learning_rate": 4.616161616161616e-06,
+ "loss": 2.0477406978607178,
+ "mean_token_accuracy": 0.5454934239387512,
+ "num_tokens": 3129344.0,
+ "step": 191
+ },
+ {
+ "entropy": 1.3063198328018188,
+ "epoch": 0.3878787878787879,
+ "grad_norm": 1.9490301609039307,
+ "learning_rate": 4.614141414141414e-06,
+ "loss": 1.3293564319610596,
+ "mean_token_accuracy": 0.6797142028808594,
+ "num_tokens": 3145728.0,
+ "step": 192
+ },
+ {
+ "entropy": 1.3636538982391357,
+ "epoch": 0.3898989898989899,
+ "grad_norm": 2.025458574295044,
+ "learning_rate": 4.612121212121212e-06,
+ "loss": 1.3736209869384766,
+ "mean_token_accuracy": 0.679286777973175,
+ "num_tokens": 3162112.0,
+ "step": 193
+ },
+ {
+ "entropy": 1.5669925212860107,
+ "epoch": 0.39191919191919194,
+ "grad_norm": 2.064033269882202,
+ "learning_rate": 4.6101010101010106e-06,
+ "loss": 1.5587083101272583,
+ "mean_token_accuracy": 0.6267709136009216,
+ "num_tokens": 3178496.0,
+ "step": 194
+ },
+ {
+ "entropy": 1.4489716291427612,
+ "epoch": 0.3939393939393939,
+ "grad_norm": 2.0174880027770996,
+ "learning_rate": 4.6080808080808085e-06,
+ "loss": 1.4331417083740234,
+ "mean_token_accuracy": 0.6571812629699707,
+ "num_tokens": 3194880.0,
+ "step": 195
+ },
+ {
+ "entropy": 2.026416778564453,
+ "epoch": 0.39595959595959596,
+ "grad_norm": 2.1820878982543945,
+ "learning_rate": 4.606060606060606e-06,
+ "loss": 2.029003143310547,
+ "mean_token_accuracy": 0.563568651676178,
+ "num_tokens": 3211264.0,
+ "step": 196
+ },
+ {
+ "entropy": 2.1844022274017334,
+ "epoch": 0.397979797979798,
+ "grad_norm": 2.175349473953247,
+ "learning_rate": 4.604040404040404e-06,
+ "loss": 2.2053825855255127,
+ "mean_token_accuracy": 0.5553248524665833,
+ "num_tokens": 3227648.0,
+ "step": 197
+ },
+ {
+ "entropy": 1.6750906705856323,
+ "epoch": 0.4,
+ "grad_norm": 1.8300689458847046,
+ "learning_rate": 4.602020202020203e-06,
+ "loss": 1.692289113998413,
+ "mean_token_accuracy": 0.6345261335372925,
+ "num_tokens": 3244032.0,
+ "step": 198
+ },
+ {
+ "entropy": 1.6192772388458252,
+ "epoch": 0.402020202020202,
+ "grad_norm": 1.9788341522216797,
+ "learning_rate": 4.600000000000001e-06,
+ "loss": 1.628842830657959,
+ "mean_token_accuracy": 0.615229606628418,
+ "num_tokens": 3260416.0,
+ "step": 199
+ },
+ {
+ "entropy": 1.444870948791504,
+ "epoch": 0.40404040404040403,
+ "grad_norm": 2.05141544342041,
+ "learning_rate": 4.597979797979799e-06,
+ "loss": 1.4365839958190918,
+ "mean_token_accuracy": 0.652723491191864,
+ "num_tokens": 3276800.0,
+ "step": 200
+ },
+ {
+ "epoch": 0.40404040404040403,
+ "eval_entropy": 1.5766179907706477,
+ "eval_loss": 1.5878442525863647,
+ "eval_mean_token_accuracy": 0.6355829551335304,
+ "eval_num_tokens": 3276800.0,
+ "eval_runtime": 43.7272,
+ "eval_samples_per_second": 22.64,
+ "eval_steps_per_second": 2.836,
+ "step": 200
+ },
+ {
+ "entropy": 1.3354029655456543,
+ "epoch": 0.40606060606060607,
+ "grad_norm": 2.012895345687866,
+ "learning_rate": 4.595959595959597e-06,
+ "loss": 1.3289990425109863,
+ "mean_token_accuracy": 0.685271143913269,
+ "num_tokens": 3293184.0,
+ "step": 201
+ },
+ {
+ "entropy": 1.1990011930465698,
+ "epoch": 0.4080808080808081,
+ "grad_norm": 1.9462409019470215,
+ "learning_rate": 4.5939393939393945e-06,
+ "loss": 1.224869728088379,
+ "mean_token_accuracy": 0.6976673007011414,
+ "num_tokens": 3309568.0,
+ "step": 202
+ },
+ {
+ "entropy": 1.6442710161209106,
+ "epoch": 0.4101010101010101,
+ "grad_norm": 2.064532995223999,
+ "learning_rate": 4.591919191919192e-06,
+ "loss": 1.6800963878631592,
+ "mean_token_accuracy": 0.6111993193626404,
+ "num_tokens": 3325952.0,
+ "step": 203
+ },
+ {
+ "entropy": 1.8336306810379028,
+ "epoch": 0.4121212121212121,
+ "grad_norm": 2.478731393814087,
+ "learning_rate": 4.58989898989899e-06,
+ "loss": 1.8518702983856201,
+ "mean_token_accuracy": 0.6021006107330322,
+ "num_tokens": 3342336.0,
+ "step": 204
+ },
+ {
+ "entropy": 1.6767220497131348,
+ "epoch": 0.41414141414141414,
+ "grad_norm": 2.0676512718200684,
+ "learning_rate": 4.587878787878788e-06,
+ "loss": 1.674522042274475,
+ "mean_token_accuracy": 0.6178553700447083,
+ "num_tokens": 3358720.0,
+ "step": 205
+ },
+ {
+ "entropy": 1.7236497402191162,
+ "epoch": 0.4161616161616162,
+ "grad_norm": 2.057074785232544,
+ "learning_rate": 4.585858585858586e-06,
+ "loss": 1.725832462310791,
+ "mean_token_accuracy": 0.620114803314209,
+ "num_tokens": 3375104.0,
+ "step": 206
+ },
+ {
+ "entropy": 1.6493065357208252,
+ "epoch": 0.41818181818181815,
+ "grad_norm": 2.0946099758148193,
+ "learning_rate": 4.583838383838384e-06,
+ "loss": 1.6826295852661133,
+ "mean_token_accuracy": 0.6251221299171448,
+ "num_tokens": 3391488.0,
+ "step": 207
+ },
+ {
+ "entropy": 1.3565926551818848,
+ "epoch": 0.4202020202020202,
+ "grad_norm": 1.9611284732818604,
+ "learning_rate": 4.581818181818183e-06,
+ "loss": 1.346787452697754,
+ "mean_token_accuracy": 0.685332179069519,
+ "num_tokens": 3407872.0,
+ "step": 208
+ },
+ {
+ "entropy": 1.4401211738586426,
+ "epoch": 0.4222222222222222,
+ "grad_norm": 1.8614766597747803,
+ "learning_rate": 4.579797979797981e-06,
+ "loss": 1.4717891216278076,
+ "mean_token_accuracy": 0.650036633014679,
+ "num_tokens": 3424256.0,
+ "step": 209
+ },
+ {
+ "entropy": 1.194276213645935,
+ "epoch": 0.42424242424242425,
+ "grad_norm": 2.0625810623168945,
+ "learning_rate": 4.5777777777777785e-06,
+ "loss": 1.2106804847717285,
+ "mean_token_accuracy": 0.700537383556366,
+ "num_tokens": 3440640.0,
+ "step": 210
+ },
+ {
+ "entropy": 1.7622313499450684,
+ "epoch": 0.4262626262626263,
+ "grad_norm": 2.330610752105713,
+ "learning_rate": 4.575757575757576e-06,
+ "loss": 1.7872710227966309,
+ "mean_token_accuracy": 0.598436713218689,
+ "num_tokens": 3457024.0,
+ "step": 211
+ },
+ {
+ "entropy": 1.409427523612976,
+ "epoch": 0.42828282828282827,
+ "grad_norm": 2.0988259315490723,
+ "learning_rate": 4.573737373737374e-06,
+ "loss": 1.416553020477295,
+ "mean_token_accuracy": 0.6682950854301453,
+ "num_tokens": 3473408.0,
+ "step": 212
+ },
+ {
+ "entropy": 1.6856107711791992,
+ "epoch": 0.4303030303030303,
+ "grad_norm": 2.341475009918213,
+ "learning_rate": 4.571717171717172e-06,
+ "loss": 1.692287802696228,
+ "mean_token_accuracy": 0.6024059653282166,
+ "num_tokens": 3489792.0,
+ "step": 213
+ },
+ {
+ "entropy": 1.997343897819519,
+ "epoch": 0.43232323232323233,
+ "grad_norm": 2.200498104095459,
+ "learning_rate": 4.56969696969697e-06,
+ "loss": 2.026289939880371,
+ "mean_token_accuracy": 0.5781631469726562,
+ "num_tokens": 3506176.0,
+ "step": 214
+ },
+ {
+ "entropy": 1.5175039768218994,
+ "epoch": 0.43434343434343436,
+ "grad_norm": 2.105257272720337,
+ "learning_rate": 4.567676767676768e-06,
+ "loss": 1.521841049194336,
+ "mean_token_accuracy": 0.6404494643211365,
+ "num_tokens": 3522560.0,
+ "step": 215
+ },
+ {
+ "entropy": 1.3616528511047363,
+ "epoch": 0.43636363636363634,
+ "grad_norm": 2.0035297870635986,
+ "learning_rate": 4.565656565656566e-06,
+ "loss": 1.3674360513687134,
+ "mean_token_accuracy": 0.6771494746208191,
+ "num_tokens": 3538944.0,
+ "step": 216
+ },
+ {
+ "entropy": 1.42499840259552,
+ "epoch": 0.4383838383838384,
+ "grad_norm": 1.9115629196166992,
+ "learning_rate": 4.563636363636364e-06,
+ "loss": 1.4129266738891602,
+ "mean_token_accuracy": 0.6686614751815796,
+ "num_tokens": 3555328.0,
+ "step": 217
+ },
+ {
+ "entropy": 1.2844709157943726,
+ "epoch": 0.4404040404040404,
+ "grad_norm": 2.3313121795654297,
+ "learning_rate": 4.5616161616161616e-06,
+ "loss": 1.315006971359253,
+ "mean_token_accuracy": 0.681546151638031,
+ "num_tokens": 3571712.0,
+ "step": 218
+ },
+ {
+ "entropy": 1.5227075815200806,
+ "epoch": 0.44242424242424244,
+ "grad_norm": 2.1849124431610107,
+ "learning_rate": 4.55959595959596e-06,
+ "loss": 1.517989158630371,
+ "mean_token_accuracy": 0.6466169953346252,
+ "num_tokens": 3588096.0,
+ "step": 219
+ },
+ {
+ "entropy": 1.494642734527588,
+ "epoch": 0.4444444444444444,
+ "grad_norm": 2.3109116554260254,
+ "learning_rate": 4.557575757575758e-06,
+ "loss": 1.525421142578125,
+ "mean_token_accuracy": 0.6392281651496887,
+ "num_tokens": 3604480.0,
+ "step": 220
+ },
+ {
+ "entropy": 1.6428948640823364,
+ "epoch": 0.44646464646464645,
+ "grad_norm": 2.1182680130004883,
+ "learning_rate": 4.555555555555556e-06,
+ "loss": 1.6634926795959473,
+ "mean_token_accuracy": 0.6207864880561829,
+ "num_tokens": 3620864.0,
+ "step": 221
+ },
+ {
+ "entropy": 1.7323675155639648,
+ "epoch": 0.4484848484848485,
+ "grad_norm": 2.2620837688446045,
+ "learning_rate": 4.553535353535354e-06,
+ "loss": 1.7312631607055664,
+ "mean_token_accuracy": 0.6138250827789307,
+ "num_tokens": 3637248.0,
+ "step": 222
+ },
+ {
+ "entropy": 1.7863894701004028,
+ "epoch": 0.4505050505050505,
+ "grad_norm": 2.1416971683502197,
+ "learning_rate": 4.551515151515152e-06,
+ "loss": 1.799318552017212,
+ "mean_token_accuracy": 0.5906203985214233,
+ "num_tokens": 3653632.0,
+ "step": 223
+ },
+ {
+ "entropy": 1.7404330968856812,
+ "epoch": 0.45252525252525255,
+ "grad_norm": 2.0814504623413086,
+ "learning_rate": 4.54949494949495e-06,
+ "loss": 1.742197036743164,
+ "mean_token_accuracy": 0.6050317287445068,
+ "num_tokens": 3670016.0,
+ "step": 224
+ },
+ {
+ "entropy": 1.4387869834899902,
+ "epoch": 0.45454545454545453,
+ "grad_norm": 2.1386022567749023,
+ "learning_rate": 4.547474747474748e-06,
+ "loss": 1.4602317810058594,
+ "mean_token_accuracy": 0.6502198576927185,
+ "num_tokens": 3686400.0,
+ "step": 225
+ },
+ {
+ "entropy": 1.2733348608016968,
+ "epoch": 0.45656565656565656,
+ "grad_norm": 2.02687668800354,
+ "learning_rate": 4.5454545454545455e-06,
+ "loss": 1.2683714628219604,
+ "mean_token_accuracy": 0.6966902613639832,
+ "num_tokens": 3702784.0,
+ "step": 226
+ },
+ {
+ "entropy": 1.4554595947265625,
+ "epoch": 0.4585858585858586,
+ "grad_norm": 2.169268846511841,
+ "learning_rate": 4.543434343434343e-06,
+ "loss": 1.4541833400726318,
+ "mean_token_accuracy": 0.6522960662841797,
+ "num_tokens": 3719168.0,
+ "step": 227
+ },
+ {
+ "entropy": 1.487573266029358,
+ "epoch": 0.46060606060606063,
+ "grad_norm": 1.9726165533065796,
+ "learning_rate": 4.541414141414141e-06,
+ "loss": 1.5024409294128418,
+ "mean_token_accuracy": 0.6509526371955872,
+ "num_tokens": 3735552.0,
+ "step": 228
+ },
+ {
+ "entropy": 1.3991138935089111,
+ "epoch": 0.4626262626262626,
+ "grad_norm": 2.0992283821105957,
+ "learning_rate": 4.539393939393939e-06,
+ "loss": 1.4016860723495483,
+ "mean_token_accuracy": 0.6639594435691833,
+ "num_tokens": 3751936.0,
+ "step": 229
+ },
+ {
+ "entropy": 1.6564134359359741,
+ "epoch": 0.46464646464646464,
+ "grad_norm": 2.047656297683716,
+ "learning_rate": 4.537373737373738e-06,
+ "loss": 1.6598721742630005,
+ "mean_token_accuracy": 0.6166951656341553,
+ "num_tokens": 3768320.0,
+ "step": 230
+ },
+ {
+ "entropy": 1.5165014266967773,
+ "epoch": 0.4666666666666667,
+ "grad_norm": 2.079996109008789,
+ "learning_rate": 4.535353535353536e-06,
+ "loss": 1.4980010986328125,
+ "mean_token_accuracy": 0.6370908617973328,
+ "num_tokens": 3784704.0,
+ "step": 231
+ },
+ {
+ "entropy": 1.5409225225448608,
+ "epoch": 0.4686868686868687,
+ "grad_norm": 2.0996923446655273,
+ "learning_rate": 4.533333333333334e-06,
+ "loss": 1.5504257678985596,
+ "mean_token_accuracy": 0.6465559601783752,
+ "num_tokens": 3801088.0,
+ "step": 232
+ },
+ {
+ "entropy": 1.3590043783187866,
+ "epoch": 0.4707070707070707,
+ "grad_norm": 2.14617919921875,
+ "learning_rate": 4.531313131313132e-06,
+ "loss": 1.3581812381744385,
+ "mean_token_accuracy": 0.6683561205863953,
+ "num_tokens": 3817472.0,
+ "step": 233
+ },
+ {
+ "entropy": 1.641785740852356,
+ "epoch": 0.4727272727272727,
+ "grad_norm": 1.9499926567077637,
+ "learning_rate": 4.5292929292929295e-06,
+ "loss": 1.6653470993041992,
+ "mean_token_accuracy": 0.6319614052772522,
+ "num_tokens": 3833856.0,
+ "step": 234
+ },
+ {
+ "entropy": 1.4700758457183838,
+ "epoch": 0.47474747474747475,
+ "grad_norm": 1.9411437511444092,
+ "learning_rate": 4.527272727272727e-06,
+ "loss": 1.4762463569641113,
+ "mean_token_accuracy": 0.6631656289100647,
+ "num_tokens": 3850240.0,
+ "step": 235
+ },
+ {
+ "entropy": 1.5418941974639893,
+ "epoch": 0.4767676767676768,
+ "grad_norm": 2.3191940784454346,
+ "learning_rate": 4.525252525252526e-06,
+ "loss": 1.5151214599609375,
+ "mean_token_accuracy": 0.6326331496238708,
+ "num_tokens": 3866624.0,
+ "step": 236
+ },
+ {
+ "entropy": 2.1102776527404785,
+ "epoch": 0.47878787878787876,
+ "grad_norm": 2.2287707328796387,
+ "learning_rate": 4.523232323232324e-06,
+ "loss": 2.106567859649658,
+ "mean_token_accuracy": 0.5506839156150818,
+ "num_tokens": 3883008.0,
+ "step": 237
+ },
+ {
+ "entropy": 1.7448827028274536,
+ "epoch": 0.4808080808080808,
+ "grad_norm": 2.2352893352508545,
+ "learning_rate": 4.521212121212122e-06,
+ "loss": 1.7495017051696777,
+ "mean_token_accuracy": 0.60326087474823,
+ "num_tokens": 3899392.0,
+ "step": 238
+ },
+ {
+ "entropy": 1.315240740776062,
+ "epoch": 0.48282828282828283,
+ "grad_norm": 1.9933631420135498,
+ "learning_rate": 4.51919191919192e-06,
+ "loss": 1.315302848815918,
+ "mean_token_accuracy": 0.686431348323822,
+ "num_tokens": 3915776.0,
+ "step": 239
+ },
+ {
+ "entropy": 1.5851637125015259,
+ "epoch": 0.48484848484848486,
+ "grad_norm": 2.153303623199463,
+ "learning_rate": 4.517171717171718e-06,
+ "loss": 1.5947662591934204,
+ "mean_token_accuracy": 0.6297020316123962,
+ "num_tokens": 3932160.0,
+ "step": 240
+ },
+ {
+ "entropy": 1.8050944805145264,
+ "epoch": 0.4868686868686869,
+ "grad_norm": 2.025022506713867,
+ "learning_rate": 4.5151515151515155e-06,
+ "loss": 1.8051010370254517,
+ "mean_token_accuracy": 0.6055202484130859,
+ "num_tokens": 3948544.0,
+ "step": 241
+ },
+ {
+ "entropy": 1.518296718597412,
+ "epoch": 0.4888888888888889,
+ "grad_norm": 2.1021833419799805,
+ "learning_rate": 4.513131313131313e-06,
+ "loss": 1.5396809577941895,
+ "mean_token_accuracy": 0.6359916925430298,
+ "num_tokens": 3964928.0,
+ "step": 242
+ },
+ {
+ "entropy": 1.518903136253357,
+ "epoch": 0.4909090909090909,
+ "grad_norm": 2.0960140228271484,
+ "learning_rate": 4.511111111111111e-06,
+ "loss": 1.5528055429458618,
+ "mean_token_accuracy": 0.6224963068962097,
+ "num_tokens": 3981312.0,
+ "step": 243
+ },
+ {
+ "entropy": 1.6960705518722534,
+ "epoch": 0.49292929292929294,
+ "grad_norm": 1.8580718040466309,
+ "learning_rate": 4.50909090909091e-06,
+ "loss": 1.7192862033843994,
+ "mean_token_accuracy": 0.6259770393371582,
+ "num_tokens": 3997696.0,
+ "step": 244
+ },
+ {
+ "entropy": 1.4633033275604248,
+ "epoch": 0.494949494949495,
+ "grad_norm": 1.8983049392700195,
+ "learning_rate": 4.507070707070708e-06,
+ "loss": 1.451701283454895,
+ "mean_token_accuracy": 0.6669516563415527,
+ "num_tokens": 4014080.0,
+ "step": 245
+ },
+ {
+ "entropy": 1.6508095264434814,
+ "epoch": 0.49696969696969695,
+ "grad_norm": 2.298679828643799,
+ "learning_rate": 4.505050505050506e-06,
+ "loss": 1.6596897840499878,
+ "mean_token_accuracy": 0.6115046143531799,
+ "num_tokens": 4030464.0,
+ "step": 246
+ },
+ {
+ "entropy": 1.313779354095459,
+ "epoch": 0.498989898989899,
+ "grad_norm": 1.9894335269927979,
+ "learning_rate": 4.503030303030304e-06,
+ "loss": 1.3285409212112427,
+ "mean_token_accuracy": 0.6802027225494385,
+ "num_tokens": 4046848.0,
+ "step": 247
+ },
+ {
+ "entropy": 1.5139521360397339,
+ "epoch": 0.501010101010101,
+ "grad_norm": 2.028320789337158,
+ "learning_rate": 4.501010101010102e-06,
+ "loss": 1.5366487503051758,
+ "mean_token_accuracy": 0.6416707634925842,
+ "num_tokens": 4063232.0,
+ "step": 248
+ },
+ {
+ "entropy": 1.5277045965194702,
+ "epoch": 0.503030303030303,
+ "grad_norm": 2.3160979747772217,
+ "learning_rate": 4.4989898989898995e-06,
+ "loss": 1.5571036338806152,
+ "mean_token_accuracy": 0.6436858773231506,
+ "num_tokens": 4079616.0,
+ "step": 249
+ },
+ {
+ "entropy": 1.6084188222885132,
+ "epoch": 0.5050505050505051,
+ "grad_norm": 2.20550274848938,
+ "learning_rate": 4.496969696969697e-06,
+ "loss": 1.622127652168274,
+ "mean_token_accuracy": 0.6284196376800537,
+ "num_tokens": 4096000.0,
+ "step": 250
+ },
+ {
+ "epoch": 0.5050505050505051,
+ "eval_entropy": 1.5547234262189558,
+ "eval_loss": 1.5764786005020142,
+ "eval_mean_token_accuracy": 0.6375306306346771,
+ "eval_num_tokens": 4096000.0,
+ "eval_runtime": 43.7607,
+ "eval_samples_per_second": 22.623,
+ "eval_steps_per_second": 2.834,
+ "step": 250
+ },
+ {
+ "entropy": 1.4309135675430298,
+ "epoch": 0.5070707070707071,
+ "grad_norm": 2.080864906311035,
+ "learning_rate": 4.494949494949495e-06,
+ "loss": 1.4481091499328613,
+ "mean_token_accuracy": 0.6694552898406982,
+ "num_tokens": 4112384.0,
+ "step": 251
+ },
+ {
+ "entropy": 1.4067270755767822,
+ "epoch": 0.509090909090909,
+ "grad_norm": 2.036475419998169,
+ "learning_rate": 4.492929292929293e-06,
+ "loss": 1.4035298824310303,
+ "mean_token_accuracy": 0.6547996997833252,
+ "num_tokens": 4128768.0,
+ "step": 252
+ },
+ {
+ "entropy": 2.1751608848571777,
+ "epoch": 0.5111111111111111,
+ "grad_norm": 1.953995943069458,
+ "learning_rate": 4.490909090909091e-06,
+ "loss": 2.1616692543029785,
+ "mean_token_accuracy": 0.5592941045761108,
+ "num_tokens": 4145152.0,
+ "step": 253
+ },
+ {
+ "entropy": 1.45353102684021,
+ "epoch": 0.5131313131313131,
+ "grad_norm": 2.084437370300293,
+ "learning_rate": 4.488888888888889e-06,
+ "loss": 1.4633586406707764,
+ "mean_token_accuracy": 0.6620664596557617,
+ "num_tokens": 4161536.0,
+ "step": 254
+ },
+ {
+ "entropy": 1.64310884475708,
+ "epoch": 0.5151515151515151,
+ "grad_norm": 2.0700111389160156,
+ "learning_rate": 4.486868686868688e-06,
+ "loss": 1.6868078708648682,
+ "mean_token_accuracy": 0.616328775882721,
+ "num_tokens": 4177920.0,
+ "step": 255
+ },
+ {
+ "entropy": 1.5113472938537598,
+ "epoch": 0.5171717171717172,
+ "grad_norm": 2.102180242538452,
+ "learning_rate": 4.4848484848484855e-06,
+ "loss": 1.5240120887756348,
+ "mean_token_accuracy": 0.638067901134491,
+ "num_tokens": 4194304.0,
+ "step": 256
+ },
+ {
+ "entropy": 1.5752851963043213,
+ "epoch": 0.5191919191919192,
+ "grad_norm": 2.026737689971924,
+ "learning_rate": 4.4828282828282834e-06,
+ "loss": 1.6041791439056396,
+ "mean_token_accuracy": 0.6275647282600403,
+ "num_tokens": 4210688.0,
+ "step": 257
+ },
+ {
+ "entropy": 1.548423171043396,
+ "epoch": 0.5212121212121212,
+ "grad_norm": 2.0578935146331787,
+ "learning_rate": 4.480808080808081e-06,
+ "loss": 1.563529372215271,
+ "mean_token_accuracy": 0.6348925232887268,
+ "num_tokens": 4227072.0,
+ "step": 258
+ },
+ {
+ "entropy": 1.5324457883834839,
+ "epoch": 0.5232323232323233,
+ "grad_norm": 2.216235637664795,
+ "learning_rate": 4.478787878787879e-06,
+ "loss": 1.5559678077697754,
+ "mean_token_accuracy": 0.6300073266029358,
+ "num_tokens": 4243456.0,
+ "step": 259
+ },
+ {
+ "entropy": 1.5275540351867676,
+ "epoch": 0.5252525252525253,
+ "grad_norm": 2.037306070327759,
+ "learning_rate": 4.476767676767677e-06,
+ "loss": 1.541567087173462,
+ "mean_token_accuracy": 0.6424035429954529,
+ "num_tokens": 4259840.0,
+ "step": 260
+ },
+ {
+ "entropy": 1.6842185258865356,
+ "epoch": 0.5272727272727272,
+ "grad_norm": 2.0241005420684814,
+ "learning_rate": 4.474747474747475e-06,
+ "loss": 1.6819056272506714,
+ "mean_token_accuracy": 0.6102222800254822,
+ "num_tokens": 4276224.0,
+ "step": 261
+ },
+ {
+ "entropy": 1.5656248331069946,
+ "epoch": 0.5292929292929293,
+ "grad_norm": 1.993054986000061,
+ "learning_rate": 4.472727272727273e-06,
+ "loss": 1.6132277250289917,
+ "mean_token_accuracy": 0.634709358215332,
+ "num_tokens": 4292608.0,
+ "step": 262
+ },
+ {
+ "entropy": 1.6354466676712036,
+ "epoch": 0.5313131313131313,
+ "grad_norm": 2.06508731842041,
+ "learning_rate": 4.470707070707071e-06,
+ "loss": 1.6475149393081665,
+ "mean_token_accuracy": 0.6121763586997986,
+ "num_tokens": 4308992.0,
+ "step": 263
+ },
+ {
+ "entropy": 1.5267232656478882,
+ "epoch": 0.5333333333333333,
+ "grad_norm": 2.1227569580078125,
+ "learning_rate": 4.468686868686869e-06,
+ "loss": 1.5468671321868896,
+ "mean_token_accuracy": 0.6365413069725037,
+ "num_tokens": 4325376.0,
+ "step": 264
+ },
+ {
+ "entropy": 1.2252761125564575,
+ "epoch": 0.5353535353535354,
+ "grad_norm": 2.0596134662628174,
+ "learning_rate": 4.4666666666666665e-06,
+ "loss": 1.2242389917373657,
+ "mean_token_accuracy": 0.6973620057106018,
+ "num_tokens": 4341760.0,
+ "step": 265
+ },
+ {
+ "entropy": 1.6007431745529175,
+ "epoch": 0.5373737373737374,
+ "grad_norm": 1.9341918230056763,
+ "learning_rate": 4.464646464646465e-06,
+ "loss": 1.5989094972610474,
+ "mean_token_accuracy": 0.6284196376800537,
+ "num_tokens": 4358144.0,
+ "step": 266
+ },
+ {
+ "entropy": 1.2708780765533447,
+ "epoch": 0.5393939393939394,
+ "grad_norm": 4.594091415405273,
+ "learning_rate": 4.462626262626263e-06,
+ "loss": 1.3484312295913696,
+ "mean_token_accuracy": 0.6790425181388855,
+ "num_tokens": 4374528.0,
+ "step": 267
+ },
+ {
+ "entropy": 1.5423723459243774,
+ "epoch": 0.5414141414141415,
+ "grad_norm": 1.933602213859558,
+ "learning_rate": 4.460606060606061e-06,
+ "loss": 1.594527244567871,
+ "mean_token_accuracy": 0.6386785507202148,
+ "num_tokens": 4390912.0,
+ "step": 268
+ },
+ {
+ "entropy": 1.7213952541351318,
+ "epoch": 0.5434343434343434,
+ "grad_norm": 2.195904016494751,
+ "learning_rate": 4.458585858585859e-06,
+ "loss": 1.724353551864624,
+ "mean_token_accuracy": 0.6008182764053345,
+ "num_tokens": 4407296.0,
+ "step": 269
+ },
+ {
+ "entropy": 1.4772557020187378,
+ "epoch": 0.5454545454545454,
+ "grad_norm": 2.0990796089172363,
+ "learning_rate": 4.456565656565657e-06,
+ "loss": 1.486825942993164,
+ "mean_token_accuracy": 0.6507083773612976,
+ "num_tokens": 4423680.0,
+ "step": 270
+ },
+ {
+ "entropy": 1.6876367330551147,
+ "epoch": 0.5474747474747474,
+ "grad_norm": 2.1944479942321777,
+ "learning_rate": 4.454545454545455e-06,
+ "loss": 1.71107816696167,
+ "mean_token_accuracy": 0.6027112603187561,
+ "num_tokens": 4440064.0,
+ "step": 271
+ },
+ {
+ "entropy": 1.5480726957321167,
+ "epoch": 0.5494949494949495,
+ "grad_norm": 2.1579341888427734,
+ "learning_rate": 4.452525252525253e-06,
+ "loss": 1.544647216796875,
+ "mean_token_accuracy": 0.6345261335372925,
+ "num_tokens": 4456448.0,
+ "step": 272
+ },
+ {
+ "entropy": 1.6161645650863647,
+ "epoch": 0.5515151515151515,
+ "grad_norm": 1.981154203414917,
+ "learning_rate": 4.4505050505050505e-06,
+ "loss": 1.6109068393707275,
+ "mean_token_accuracy": 0.6318392753601074,
+ "num_tokens": 4472832.0,
+ "step": 273
+ },
+ {
+ "entropy": 1.2576326131820679,
+ "epoch": 0.5535353535353535,
+ "grad_norm": 1.95668625831604,
+ "learning_rate": 4.448484848484848e-06,
+ "loss": 1.2614656686782837,
+ "mean_token_accuracy": 0.7025524973869324,
+ "num_tokens": 4489216.0,
+ "step": 274
+ },
+ {
+ "entropy": 1.405206561088562,
+ "epoch": 0.5555555555555556,
+ "grad_norm": 2.0302884578704834,
+ "learning_rate": 4.446464646464646e-06,
+ "loss": 1.416546106338501,
+ "mean_token_accuracy": 0.6753786206245422,
+ "num_tokens": 4505600.0,
+ "step": 275
+ },
+ {
+ "entropy": 1.9038625955581665,
+ "epoch": 0.5575757575757576,
+ "grad_norm": 2.06475567817688,
+ "learning_rate": 4.444444444444444e-06,
+ "loss": 1.9273614883422852,
+ "mean_token_accuracy": 0.5813996195793152,
+ "num_tokens": 4521984.0,
+ "step": 276
+ },
+ {
+ "entropy": 1.7208465337753296,
+ "epoch": 0.5595959595959596,
+ "grad_norm": 2.0136189460754395,
+ "learning_rate": 4.442424242424243e-06,
+ "loss": 1.7260158061981201,
+ "mean_token_accuracy": 0.6006350517272949,
+ "num_tokens": 4538368.0,
+ "step": 277
+ },
+ {
+ "entropy": 1.7570570707321167,
+ "epoch": 0.5616161616161616,
+ "grad_norm": 2.148594379425049,
+ "learning_rate": 4.440404040404041e-06,
+ "loss": 1.7799286842346191,
+ "mean_token_accuracy": 0.5943453907966614,
+ "num_tokens": 4554752.0,
+ "step": 278
+ },
+ {
+ "entropy": 1.6388157606124878,
+ "epoch": 0.5636363636363636,
+ "grad_norm": 2.1301987171173096,
+ "learning_rate": 4.438383838383839e-06,
+ "loss": 1.63419508934021,
+ "mean_token_accuracy": 0.6405715942382812,
+ "num_tokens": 4571136.0,
+ "step": 279
+ },
+ {
+ "entropy": 1.5902295112609863,
+ "epoch": 0.5656565656565656,
+ "grad_norm": 2.0526790618896484,
+ "learning_rate": 4.436363636363637e-06,
+ "loss": 1.6278276443481445,
+ "mean_token_accuracy": 0.615229606628418,
+ "num_tokens": 4587520.0,
+ "step": 280
+ },
+ {
+ "entropy": 1.5494027137756348,
+ "epoch": 0.5676767676767677,
+ "grad_norm": 1.9863859415054321,
+ "learning_rate": 4.434343434343435e-06,
+ "loss": 1.5622975826263428,
+ "mean_token_accuracy": 0.6261602640151978,
+ "num_tokens": 4603904.0,
+ "step": 281
+ },
+ {
+ "entropy": 1.2140685319900513,
+ "epoch": 0.5696969696969697,
+ "grad_norm": 2.4206902980804443,
+ "learning_rate": 4.432323232323233e-06,
+ "loss": 1.2389612197875977,
+ "mean_token_accuracy": 0.6943697929382324,
+ "num_tokens": 4620288.0,
+ "step": 282
+ },
+ {
+ "entropy": 1.4347270727157593,
+ "epoch": 0.5717171717171717,
+ "grad_norm": 2.0198237895965576,
+ "learning_rate": 4.430303030303031e-06,
+ "loss": 1.4648659229278564,
+ "mean_token_accuracy": 0.6573033928871155,
+ "num_tokens": 4636672.0,
+ "step": 283
+ },
+ {
+ "entropy": 1.3239331245422363,
+ "epoch": 0.5737373737373738,
+ "grad_norm": 1.9862704277038574,
+ "learning_rate": 4.428282828282829e-06,
+ "loss": 1.3520365953445435,
+ "mean_token_accuracy": 0.6852100491523743,
+ "num_tokens": 4653056.0,
+ "step": 284
+ },
+ {
+ "entropy": 1.7534631490707397,
+ "epoch": 0.5757575757575758,
+ "grad_norm": 2.0964176654815674,
+ "learning_rate": 4.426262626262627e-06,
+ "loss": 1.7659757137298584,
+ "mean_token_accuracy": 0.5975207686424255,
+ "num_tokens": 4669440.0,
+ "step": 285
+ },
+ {
+ "entropy": 1.1573433876037598,
+ "epoch": 0.5777777777777777,
+ "grad_norm": 2.0671567916870117,
+ "learning_rate": 4.424242424242425e-06,
+ "loss": 1.1479461193084717,
+ "mean_token_accuracy": 0.7143380641937256,
+ "num_tokens": 4685824.0,
+ "step": 286
+ },
+ {
+ "entropy": 1.2439504861831665,
+ "epoch": 0.5797979797979798,
+ "grad_norm": 2.0317327976226807,
+ "learning_rate": 4.422222222222223e-06,
+ "loss": 1.255782961845398,
+ "mean_token_accuracy": 0.691255509853363,
+ "num_tokens": 4702208.0,
+ "step": 287
+ },
+ {
+ "entropy": 1.5569473505020142,
+ "epoch": 0.5818181818181818,
+ "grad_norm": 2.0759670734405518,
+ "learning_rate": 4.4202020202020205e-06,
+ "loss": 1.5564974546432495,
+ "mean_token_accuracy": 0.6284807324409485,
+ "num_tokens": 4718592.0,
+ "step": 288
+ },
+ {
+ "entropy": 1.478676199913025,
+ "epoch": 0.5838383838383838,
+ "grad_norm": 1.9528205394744873,
+ "learning_rate": 4.418181818181818e-06,
+ "loss": 1.491654396057129,
+ "mean_token_accuracy": 0.6520518064498901,
+ "num_tokens": 4734976.0,
+ "step": 289
+ },
+ {
+ "entropy": 1.2454503774642944,
+ "epoch": 0.5858585858585859,
+ "grad_norm": 1.9008079767227173,
+ "learning_rate": 4.416161616161616e-06,
+ "loss": 1.253904938697815,
+ "mean_token_accuracy": 0.6971787810325623,
+ "num_tokens": 4751360.0,
+ "step": 290
+ },
+ {
+ "entropy": 1.1740810871124268,
+ "epoch": 0.5878787878787879,
+ "grad_norm": 1.8888787031173706,
+ "learning_rate": 4.414141414141415e-06,
+ "loss": 1.195070505142212,
+ "mean_token_accuracy": 0.7048119306564331,
+ "num_tokens": 4767744.0,
+ "step": 291
+ },
+ {
+ "entropy": 1.6064486503601074,
+ "epoch": 0.5898989898989899,
+ "grad_norm": 2.073559284210205,
+ "learning_rate": 4.412121212121213e-06,
+ "loss": 1.6180689334869385,
+ "mean_token_accuracy": 0.6242061257362366,
+ "num_tokens": 4784128.0,
+ "step": 292
+ },
+ {
+ "entropy": 1.6119383573532104,
+ "epoch": 0.591919191919192,
+ "grad_norm": 1.8773425817489624,
+ "learning_rate": 4.410101010101011e-06,
+ "loss": 1.5875662565231323,
+ "mean_token_accuracy": 0.647838294506073,
+ "num_tokens": 4800512.0,
+ "step": 293
+ },
+ {
+ "entropy": 1.392905592918396,
+ "epoch": 0.593939393939394,
+ "grad_norm": 2.1699368953704834,
+ "learning_rate": 4.408080808080809e-06,
+ "loss": 1.3949127197265625,
+ "mean_token_accuracy": 0.6538837552070618,
+ "num_tokens": 4816896.0,
+ "step": 294
+ },
+ {
+ "entropy": 1.5057002305984497,
+ "epoch": 0.5959595959595959,
+ "grad_norm": 2.2730712890625,
+ "learning_rate": 4.4060606060606066e-06,
+ "loss": 1.4755051136016846,
+ "mean_token_accuracy": 0.6498534679412842,
+ "num_tokens": 4833280.0,
+ "step": 295
+ },
+ {
+ "entropy": 1.5468902587890625,
+ "epoch": 0.597979797979798,
+ "grad_norm": 1.9911075830459595,
+ "learning_rate": 4.4040404040404044e-06,
+ "loss": 1.539963722229004,
+ "mean_token_accuracy": 0.6520518064498901,
+ "num_tokens": 4849664.0,
+ "step": 296
+ },
+ {
+ "entropy": 1.5196901559829712,
+ "epoch": 0.6,
+ "grad_norm": 1.9859540462493896,
+ "learning_rate": 4.402020202020202e-06,
+ "loss": 1.523442029953003,
+ "mean_token_accuracy": 0.6417317986488342,
+ "num_tokens": 4866048.0,
+ "step": 297
+ },
+ {
+ "entropy": 1.8251866102218628,
+ "epoch": 0.602020202020202,
+ "grad_norm": 2.2566516399383545,
+ "learning_rate": 4.4e-06,
+ "loss": 1.8667771816253662,
+ "mean_token_accuracy": 0.5872007608413696,
+ "num_tokens": 4882432.0,
+ "step": 298
+ },
+ {
+ "entropy": 1.6816744804382324,
+ "epoch": 0.604040404040404,
+ "grad_norm": 2.0107715129852295,
+ "learning_rate": 4.397979797979798e-06,
+ "loss": 1.6907548904418945,
+ "mean_token_accuracy": 0.6176722049713135,
+ "num_tokens": 4898816.0,
+ "step": 299
+ },
+ {
+ "entropy": 1.4992223978042603,
+ "epoch": 0.6060606060606061,
+ "grad_norm": 2.1236329078674316,
+ "learning_rate": 4.395959595959596e-06,
+ "loss": 1.5015790462493896,
+ "mean_token_accuracy": 0.6372129917144775,
+ "num_tokens": 4915200.0,
+ "step": 300
+ },
+ {
+ "epoch": 0.6060606060606061,
+ "eval_entropy": 1.5544315297757425,
+ "eval_loss": 1.5676765441894531,
+ "eval_mean_token_accuracy": 0.6389380799185845,
+ "eval_num_tokens": 4915200.0,
+ "eval_runtime": 43.7857,
+ "eval_samples_per_second": 22.61,
+ "eval_steps_per_second": 2.832,
+ "step": 300
+ },
+ {
+ "entropy": 1.518556833267212,
+ "epoch": 0.6080808080808081,
+ "grad_norm": 2.88371205329895,
+ "learning_rate": 4.393939393939394e-06,
+ "loss": 1.5310916900634766,
+ "mean_token_accuracy": 0.6551660895347595,
+ "num_tokens": 4931584.0,
+ "step": 301
+ },
+ {
+ "entropy": 1.4650386571884155,
+ "epoch": 0.6101010101010101,
+ "grad_norm": 2.1849780082702637,
+ "learning_rate": 4.391919191919193e-06,
+ "loss": 1.4405598640441895,
+ "mean_token_accuracy": 0.6531509757041931,
+ "num_tokens": 4947968.0,
+ "step": 302
+ },
+ {
+ "entropy": 1.5209505558013916,
+ "epoch": 0.6121212121212121,
+ "grad_norm": 2.1135010719299316,
+ "learning_rate": 4.3898989898989905e-06,
+ "loss": 1.5324647426605225,
+ "mean_token_accuracy": 0.6561431288719177,
+ "num_tokens": 4964352.0,
+ "step": 303
+ },
+ {
+ "entropy": 1.448391318321228,
+ "epoch": 0.6141414141414141,
+ "grad_norm": 1.9959306716918945,
+ "learning_rate": 4.387878787878788e-06,
+ "loss": 1.4721965789794922,
+ "mean_token_accuracy": 0.6578529477119446,
+ "num_tokens": 4980736.0,
+ "step": 304
+ },
+ {
+ "entropy": 1.4609358310699463,
+ "epoch": 0.6161616161616161,
+ "grad_norm": 2.1308915615081787,
+ "learning_rate": 4.385858585858586e-06,
+ "loss": 1.4707520008087158,
+ "mean_token_accuracy": 0.6482046842575073,
+ "num_tokens": 4997120.0,
+ "step": 305
+ },
+ {
+ "entropy": 1.7125155925750732,
+ "epoch": 0.6181818181818182,
+ "grad_norm": 2.2170841693878174,
+ "learning_rate": 4.383838383838384e-06,
+ "loss": 1.7630269527435303,
+ "mean_token_accuracy": 0.5999022722244263,
+ "num_tokens": 5013504.0,
+ "step": 306
+ },
+ {
+ "entropy": 1.6740268468856812,
+ "epoch": 0.6202020202020202,
+ "grad_norm": 1.983644723892212,
+ "learning_rate": 4.381818181818182e-06,
+ "loss": 1.6849563121795654,
+ "mean_token_accuracy": 0.6237176060676575,
+ "num_tokens": 5029888.0,
+ "step": 307
+ },
+ {
+ "entropy": 1.3754971027374268,
+ "epoch": 0.6222222222222222,
+ "grad_norm": 2.005277395248413,
+ "learning_rate": 4.37979797979798e-06,
+ "loss": 1.385213851928711,
+ "mean_token_accuracy": 0.6638984084129333,
+ "num_tokens": 5046272.0,
+ "step": 308
+ },
+ {
+ "entropy": 1.5476733446121216,
+ "epoch": 0.6242424242424243,
+ "grad_norm": 2.4317610263824463,
+ "learning_rate": 4.377777777777778e-06,
+ "loss": 1.5417041778564453,
+ "mean_token_accuracy": 0.6323888897895813,
+ "num_tokens": 5062656.0,
+ "step": 309
+ },
+ {
+ "entropy": 1.4437031745910645,
+ "epoch": 0.6262626262626263,
+ "grad_norm": 2.1464109420776367,
+ "learning_rate": 4.375757575757576e-06,
+ "loss": 1.4704627990722656,
+ "mean_token_accuracy": 0.6772105693817139,
+ "num_tokens": 5079040.0,
+ "step": 310
+ },
+ {
+ "entropy": 1.4859641790390015,
+ "epoch": 0.6282828282828283,
+ "grad_norm": 2.0800492763519287,
+ "learning_rate": 4.373737373737374e-06,
+ "loss": 1.5072834491729736,
+ "mean_token_accuracy": 0.658707857131958,
+ "num_tokens": 5095424.0,
+ "step": 311
+ },
+ {
+ "entropy": 1.8733419179916382,
+ "epoch": 0.6303030303030303,
+ "grad_norm": 2.120965003967285,
+ "learning_rate": 4.3717171717171715e-06,
+ "loss": 1.8567254543304443,
+ "mean_token_accuracy": 0.5847581624984741,
+ "num_tokens": 5111808.0,
+ "step": 312
+ },
+ {
+ "entropy": 1.4578243494033813,
+ "epoch": 0.6323232323232323,
+ "grad_norm": 1.9577823877334595,
+ "learning_rate": 4.36969696969697e-06,
+ "loss": 1.4758052825927734,
+ "mean_token_accuracy": 0.6590742468833923,
+ "num_tokens": 5128192.0,
+ "step": 313
+ },
+ {
+ "entropy": 1.8680084943771362,
+ "epoch": 0.6343434343434343,
+ "grad_norm": 2.170994520187378,
+ "learning_rate": 4.367676767676768e-06,
+ "loss": 1.904400110244751,
+ "mean_token_accuracy": 0.5857962965965271,
+ "num_tokens": 5144576.0,
+ "step": 314
+ },
+ {
+ "entropy": 1.6488304138183594,
+ "epoch": 0.6363636363636364,
+ "grad_norm": 1.959490418434143,
+ "learning_rate": 4.365656565656566e-06,
+ "loss": 1.6570477485656738,
+ "mean_token_accuracy": 0.6257327795028687,
+ "num_tokens": 5160960.0,
+ "step": 315
+ },
+ {
+ "entropy": 1.5038025379180908,
+ "epoch": 0.6383838383838384,
+ "grad_norm": 2.072697401046753,
+ "learning_rate": 4.363636363636364e-06,
+ "loss": 1.5234860181808472,
+ "mean_token_accuracy": 0.6494259834289551,
+ "num_tokens": 5177344.0,
+ "step": 316
+ },
+ {
+ "entropy": 1.4154654741287231,
+ "epoch": 0.6404040404040404,
+ "grad_norm": 1.894587755203247,
+ "learning_rate": 4.361616161616162e-06,
+ "loss": 1.4131592512130737,
+ "mean_token_accuracy": 0.6667073965072632,
+ "num_tokens": 5193728.0,
+ "step": 317
+ },
+ {
+ "entropy": 1.5813250541687012,
+ "epoch": 0.6424242424242425,
+ "grad_norm": 2.4088737964630127,
+ "learning_rate": 4.35959595959596e-06,
+ "loss": 1.6236622333526611,
+ "mean_token_accuracy": 0.6313507556915283,
+ "num_tokens": 5210112.0,
+ "step": 318
+ },
+ {
+ "entropy": 1.4140045642852783,
+ "epoch": 0.6444444444444445,
+ "grad_norm": 1.991557240486145,
+ "learning_rate": 4.3575757575757576e-06,
+ "loss": 1.4242517948150635,
+ "mean_token_accuracy": 0.6569980382919312,
+ "num_tokens": 5226496.0,
+ "step": 319
+ },
+ {
+ "entropy": 1.615628957748413,
+ "epoch": 0.6464646464646465,
+ "grad_norm": 1.8819077014923096,
+ "learning_rate": 4.3555555555555555e-06,
+ "loss": 1.6331532001495361,
+ "mean_token_accuracy": 0.6299462914466858,
+ "num_tokens": 5242880.0,
+ "step": 320
+ },
+ {
+ "entropy": 1.3118394613265991,
+ "epoch": 0.6484848484848484,
+ "grad_norm": 2.0589284896850586,
+ "learning_rate": 4.353535353535353e-06,
+ "loss": 1.2880034446716309,
+ "mean_token_accuracy": 0.6842941045761108,
+ "num_tokens": 5259264.0,
+ "step": 321
+ },
+ {
+ "entropy": 1.2786612510681152,
+ "epoch": 0.6505050505050505,
+ "grad_norm": 2.033839225769043,
+ "learning_rate": 4.351515151515152e-06,
+ "loss": 1.2829055786132812,
+ "mean_token_accuracy": 0.6910112500190735,
+ "num_tokens": 5275648.0,
+ "step": 322
+ },
+ {
+ "entropy": 1.5632988214492798,
+ "epoch": 0.6525252525252525,
+ "grad_norm": 2.1970419883728027,
+ "learning_rate": 4.34949494949495e-06,
+ "loss": 1.5865097045898438,
+ "mean_token_accuracy": 0.642892062664032,
+ "num_tokens": 5292032.0,
+ "step": 323
+ },
+ {
+ "entropy": 1.1542725563049316,
+ "epoch": 0.6545454545454545,
+ "grad_norm": 1.9750654697418213,
+ "learning_rate": 4.347474747474748e-06,
+ "loss": 1.1401035785675049,
+ "mean_token_accuracy": 0.7061553597450256,
+ "num_tokens": 5308416.0,
+ "step": 324
+ },
+ {
+ "entropy": 1.6879942417144775,
+ "epoch": 0.6565656565656566,
+ "grad_norm": 2.0022354125976562,
+ "learning_rate": 4.345454545454546e-06,
+ "loss": 1.70950448513031,
+ "mean_token_accuracy": 0.6089398860931396,
+ "num_tokens": 5324800.0,
+ "step": 325
+ },
+ {
+ "entropy": 1.7589699029922485,
+ "epoch": 0.6585858585858586,
+ "grad_norm": 1.925520420074463,
+ "learning_rate": 4.343434343434344e-06,
+ "loss": 1.7749860286712646,
+ "mean_token_accuracy": 0.6015510559082031,
+ "num_tokens": 5341184.0,
+ "step": 326
+ },
+ {
+ "entropy": 1.4720325469970703,
+ "epoch": 0.6606060606060606,
+ "grad_norm": 1.9487124681472778,
+ "learning_rate": 4.341414141414142e-06,
+ "loss": 1.4983797073364258,
+ "mean_token_accuracy": 0.6537005305290222,
+ "num_tokens": 5357568.0,
+ "step": 327
+ },
+ {
+ "entropy": 1.676164150238037,
+ "epoch": 0.6626262626262627,
+ "grad_norm": 2.13053035736084,
+ "learning_rate": 4.33939393939394e-06,
+ "loss": 1.699425458908081,
+ "mean_token_accuracy": 0.6100390553474426,
+ "num_tokens": 5373952.0,
+ "step": 328
+ },
+ {
+ "entropy": 1.3424437046051025,
+ "epoch": 0.6646464646464646,
+ "grad_norm": 2.0245954990386963,
+ "learning_rate": 4.337373737373738e-06,
+ "loss": 1.3393046855926514,
+ "mean_token_accuracy": 0.677760124206543,
+ "num_tokens": 5390336.0,
+ "step": 329
+ },
+ {
+ "entropy": 1.4926583766937256,
+ "epoch": 0.6666666666666666,
+ "grad_norm": 1.893343448638916,
+ "learning_rate": 4.335353535353536e-06,
+ "loss": 1.4779269695281982,
+ "mean_token_accuracy": 0.6713483333587646,
+ "num_tokens": 5406720.0,
+ "step": 330
+ },
+ {
+ "entropy": 1.5753449201583862,
+ "epoch": 0.6686868686868687,
+ "grad_norm": 2.051647424697876,
+ "learning_rate": 4.333333333333334e-06,
+ "loss": 1.622597336769104,
+ "mean_token_accuracy": 0.6436248421669006,
+ "num_tokens": 5423104.0,
+ "step": 331
+ },
+ {
+ "entropy": 1.4573988914489746,
+ "epoch": 0.6707070707070707,
+ "grad_norm": 1.8709567785263062,
+ "learning_rate": 4.331313131313132e-06,
+ "loss": 1.4902422428131104,
+ "mean_token_accuracy": 0.6682950854301453,
+ "num_tokens": 5439488.0,
+ "step": 332
+ },
+ {
+ "entropy": 1.378867745399475,
+ "epoch": 0.6727272727272727,
+ "grad_norm": 2.235928773880005,
+ "learning_rate": 4.32929292929293e-06,
+ "loss": 1.4067103862762451,
+ "mean_token_accuracy": 0.662432849407196,
+ "num_tokens": 5455872.0,
+ "step": 333
+ },
+ {
+ "entropy": 1.7553350925445557,
+ "epoch": 0.6747474747474748,
+ "grad_norm": 2.0335066318511963,
+ "learning_rate": 4.327272727272728e-06,
+ "loss": 1.772943139076233,
+ "mean_token_accuracy": 0.5953834652900696,
+ "num_tokens": 5472256.0,
+ "step": 334
+ },
+ {
+ "entropy": 1.1587097644805908,
+ "epoch": 0.6767676767676768,
+ "grad_norm": 1.8764615058898926,
+ "learning_rate": 4.3252525252525255e-06,
+ "loss": 1.1396210193634033,
+ "mean_token_accuracy": 0.7197117805480957,
+ "num_tokens": 5488640.0,
+ "step": 335
+ },
+ {
+ "entropy": 1.4819872379302979,
+ "epoch": 0.6787878787878788,
+ "grad_norm": 2.0907511711120605,
+ "learning_rate": 4.323232323232323e-06,
+ "loss": 1.460550308227539,
+ "mean_token_accuracy": 0.6520518064498901,
+ "num_tokens": 5505024.0,
+ "step": 336
+ },
+ {
+ "entropy": 1.0744472742080688,
+ "epoch": 0.6808080808080809,
+ "grad_norm": 2.227606773376465,
+ "learning_rate": 4.321212121212121e-06,
+ "loss": 1.0909301042556763,
+ "mean_token_accuracy": 0.7258182764053345,
+ "num_tokens": 5521408.0,
+ "step": 337
+ },
+ {
+ "entropy": 1.507999300956726,
+ "epoch": 0.6828282828282828,
+ "grad_norm": 2.0332415103912354,
+ "learning_rate": 4.31919191919192e-06,
+ "loss": 1.5173046588897705,
+ "mean_token_accuracy": 0.6374572515487671,
+ "num_tokens": 5537792.0,
+ "step": 338
+ },
+ {
+ "entropy": 1.6792720556259155,
+ "epoch": 0.6848484848484848,
+ "grad_norm": 2.0194997787475586,
+ "learning_rate": 4.317171717171718e-06,
+ "loss": 1.679445743560791,
+ "mean_token_accuracy": 0.6138250827789307,
+ "num_tokens": 5554176.0,
+ "step": 339
+ },
+ {
+ "entropy": 1.6470589637756348,
+ "epoch": 0.6868686868686869,
+ "grad_norm": 2.09116268157959,
+ "learning_rate": 4.315151515151516e-06,
+ "loss": 1.6386632919311523,
+ "mean_token_accuracy": 0.6146799921989441,
+ "num_tokens": 5570560.0,
+ "step": 340
+ },
+ {
+ "entropy": 1.3612488508224487,
+ "epoch": 0.6888888888888889,
+ "grad_norm": 2.1586217880249023,
+ "learning_rate": 4.313131313131314e-06,
+ "loss": 1.3698725700378418,
+ "mean_token_accuracy": 0.6696995496749878,
+ "num_tokens": 5586944.0,
+ "step": 341
+ },
+ {
+ "entropy": 1.400327205657959,
+ "epoch": 0.6909090909090909,
+ "grad_norm": 2.082094192504883,
+ "learning_rate": 4.3111111111111115e-06,
+ "loss": 1.396535873413086,
+ "mean_token_accuracy": 0.6780654788017273,
+ "num_tokens": 5603328.0,
+ "step": 342
+ },
+ {
+ "entropy": 1.389290690422058,
+ "epoch": 0.692929292929293,
+ "grad_norm": 2.0780303478240967,
+ "learning_rate": 4.309090909090909e-06,
+ "loss": 1.4275782108306885,
+ "mean_token_accuracy": 0.6656082272529602,
+ "num_tokens": 5619712.0,
+ "step": 343
+ },
+ {
+ "entropy": 1.5133870840072632,
+ "epoch": 0.694949494949495,
+ "grad_norm": 1.9819531440734863,
+ "learning_rate": 4.307070707070707e-06,
+ "loss": 1.5309596061706543,
+ "mean_token_accuracy": 0.6373351216316223,
+ "num_tokens": 5636096.0,
+ "step": 344
+ },
+ {
+ "entropy": 1.483219861984253,
+ "epoch": 0.696969696969697,
+ "grad_norm": 1.9794325828552246,
+ "learning_rate": 4.305050505050505e-06,
+ "loss": 1.4693087339401245,
+ "mean_token_accuracy": 0.6550439596176147,
+ "num_tokens": 5652480.0,
+ "step": 345
+ },
+ {
+ "entropy": 1.381489634513855,
+ "epoch": 0.6989898989898989,
+ "grad_norm": 2.0637686252593994,
+ "learning_rate": 4.303030303030303e-06,
+ "loss": 1.3862476348876953,
+ "mean_token_accuracy": 0.6631045341491699,
+ "num_tokens": 5668864.0,
+ "step": 346
+ },
+ {
+ "entropy": 1.9860024452209473,
+ "epoch": 0.701010101010101,
+ "grad_norm": 2.1626803874969482,
+ "learning_rate": 4.301010101010101e-06,
+ "loss": 1.9815950393676758,
+ "mean_token_accuracy": 0.5698583126068115,
+ "num_tokens": 5685248.0,
+ "step": 347
+ },
+ {
+ "entropy": 1.5044004917144775,
+ "epoch": 0.703030303030303,
+ "grad_norm": 2.060976266860962,
+ "learning_rate": 4.298989898989899e-06,
+ "loss": 1.4937127828598022,
+ "mean_token_accuracy": 0.6524792313575745,
+ "num_tokens": 5701632.0,
+ "step": 348
+ },
+ {
+ "entropy": 1.3397772312164307,
+ "epoch": 0.705050505050505,
+ "grad_norm": 2.0162901878356934,
+ "learning_rate": 4.296969696969698e-06,
+ "loss": 1.3358572721481323,
+ "mean_token_accuracy": 0.6949804425239563,
+ "num_tokens": 5718016.0,
+ "step": 349
+ },
+ {
+ "entropy": 1.145772099494934,
+ "epoch": 0.7070707070707071,
+ "grad_norm": 2.097634792327881,
+ "learning_rate": 4.2949494949494955e-06,
+ "loss": 1.1833416223526,
+ "mean_token_accuracy": 0.7104909420013428,
+ "num_tokens": 5734400.0,
+ "step": 350
+ },
+ {
+ "epoch": 0.7070707070707071,
+ "eval_entropy": 1.5476290602837839,
+ "eval_loss": 1.5603480339050293,
+ "eval_mean_token_accuracy": 0.640111118555069,
+ "eval_num_tokens": 5734400.0,
+ "eval_runtime": 43.7844,
+ "eval_samples_per_second": 22.611,
+ "eval_steps_per_second": 2.832,
+ "step": 350
+ },
+ {
+ "entropy": 1.4311926364898682,
+ "epoch": 0.7090909090909091,
+ "grad_norm": 2.002321720123291,
+ "learning_rate": 4.292929292929293e-06,
+ "loss": 1.4534825086593628,
+ "mean_token_accuracy": 0.6614558100700378,
+ "num_tokens": 5750784.0,
+ "step": 351
+ },
+ {
+ "entropy": 1.3983922004699707,
+ "epoch": 0.7111111111111111,
+ "grad_norm": 2.1724867820739746,
+ "learning_rate": 4.290909090909091e-06,
+ "loss": 1.3969402313232422,
+ "mean_token_accuracy": 0.6652418375015259,
+ "num_tokens": 5767168.0,
+ "step": 352
+ },
+ {
+ "entropy": 1.8218920230865479,
+ "epoch": 0.7131313131313132,
+ "grad_norm": 2.1629281044006348,
+ "learning_rate": 4.288888888888889e-06,
+ "loss": 1.8196980953216553,
+ "mean_token_accuracy": 0.6027112603187561,
+ "num_tokens": 5783552.0,
+ "step": 353
+ },
+ {
+ "entropy": 1.5322320461273193,
+ "epoch": 0.7151515151515152,
+ "grad_norm": 1.8486647605895996,
+ "learning_rate": 4.286868686868687e-06,
+ "loss": 1.5504610538482666,
+ "mean_token_accuracy": 0.6533341407775879,
+ "num_tokens": 5799936.0,
+ "step": 354
+ },
+ {
+ "entropy": 1.935966968536377,
+ "epoch": 0.7171717171717171,
+ "grad_norm": 2.252814292907715,
+ "learning_rate": 4.284848484848485e-06,
+ "loss": 1.9300384521484375,
+ "mean_token_accuracy": 0.5656448602676392,
+ "num_tokens": 5816320.0,
+ "step": 355
+ },
+ {
+ "entropy": 1.6711398363113403,
+ "epoch": 0.7191919191919192,
+ "grad_norm": 2.023379325866699,
+ "learning_rate": 4.282828282828283e-06,
+ "loss": 1.6584455966949463,
+ "mean_token_accuracy": 0.6235954761505127,
+ "num_tokens": 5832704.0,
+ "step": 356
+ },
+ {
+ "entropy": 1.3922803401947021,
+ "epoch": 0.7212121212121212,
+ "grad_norm": 2.0487611293792725,
+ "learning_rate": 4.280808080808081e-06,
+ "loss": 1.4220250844955444,
+ "mean_token_accuracy": 0.6591963768005371,
+ "num_tokens": 5849088.0,
+ "step": 357
+ },
+ {
+ "entropy": 1.59521484375,
+ "epoch": 0.7232323232323232,
+ "grad_norm": 1.8598614931106567,
+ "learning_rate": 4.278787878787879e-06,
+ "loss": 1.5979329347610474,
+ "mean_token_accuracy": 0.6237176060676575,
+ "num_tokens": 5865472.0,
+ "step": 358
+ },
+ {
+ "entropy": 1.5810115337371826,
+ "epoch": 0.7252525252525253,
+ "grad_norm": 2.140115737915039,
+ "learning_rate": 4.276767676767677e-06,
+ "loss": 1.5774705410003662,
+ "mean_token_accuracy": 0.6274425983428955,
+ "num_tokens": 5881856.0,
+ "step": 359
+ },
+ {
+ "entropy": 1.5839109420776367,
+ "epoch": 0.7272727272727273,
+ "grad_norm": 2.0947749614715576,
+ "learning_rate": 4.274747474747475e-06,
+ "loss": 1.601511001586914,
+ "mean_token_accuracy": 0.630984365940094,
+ "num_tokens": 5898240.0,
+ "step": 360
+ },
+ {
+ "entropy": 1.195770263671875,
+ "epoch": 0.7292929292929293,
+ "grad_norm": 1.8740363121032715,
+ "learning_rate": 4.272727272727273e-06,
+ "loss": 1.195652961730957,
+ "mean_token_accuracy": 0.7077430486679077,
+ "num_tokens": 5914624.0,
+ "step": 361
+ },
+ {
+ "entropy": 1.375698208808899,
+ "epoch": 0.7313131313131314,
+ "grad_norm": 1.9580703973770142,
+ "learning_rate": 4.270707070707071e-06,
+ "loss": 1.3746864795684814,
+ "mean_token_accuracy": 0.6775158643722534,
+ "num_tokens": 5931008.0,
+ "step": 362
+ },
+ {
+ "entropy": 1.7259451150894165,
+ "epoch": 0.7333333333333333,
+ "grad_norm": 2.3127365112304688,
+ "learning_rate": 4.268686868686869e-06,
+ "loss": 1.745998501777649,
+ "mean_token_accuracy": 0.6040546894073486,
+ "num_tokens": 5947392.0,
+ "step": 363
+ },
+ {
+ "entropy": 1.209228754043579,
+ "epoch": 0.7353535353535353,
+ "grad_norm": 1.9004793167114258,
+ "learning_rate": 4.266666666666668e-06,
+ "loss": 1.232427716255188,
+ "mean_token_accuracy": 0.6925989389419556,
+ "num_tokens": 5963776.0,
+ "step": 364
+ },
+ {
+ "entropy": 1.2811263799667358,
+ "epoch": 0.7373737373737373,
+ "grad_norm": 1.9568246603012085,
+ "learning_rate": 4.2646464646464655e-06,
+ "loss": 1.291853427886963,
+ "mean_token_accuracy": 0.6884465217590332,
+ "num_tokens": 5980160.0,
+ "step": 365
+ },
+ {
+ "entropy": 1.843044638633728,
+ "epoch": 0.7393939393939394,
+ "grad_norm": 2.132735252380371,
+ "learning_rate": 4.262626262626263e-06,
+ "loss": 1.8818857669830322,
+ "mean_token_accuracy": 0.5785295367240906,
+ "num_tokens": 5996544.0,
+ "step": 366
+ },
+ {
+ "entropy": 1.3353440761566162,
+ "epoch": 0.7414141414141414,
+ "grad_norm": 1.8893013000488281,
+ "learning_rate": 4.260606060606061e-06,
+ "loss": 1.354011058807373,
+ "mean_token_accuracy": 0.6797752976417542,
+ "num_tokens": 6012928.0,
+ "step": 367
+ },
+ {
+ "entropy": 1.3889728784561157,
+ "epoch": 0.7434343434343434,
+ "grad_norm": 1.980757236480713,
+ "learning_rate": 4.258585858585859e-06,
+ "loss": 1.3671875,
+ "mean_token_accuracy": 0.6627381443977356,
+ "num_tokens": 6029312.0,
+ "step": 368
+ },
+ {
+ "entropy": 1.4605348110198975,
+ "epoch": 0.7454545454545455,
+ "grad_norm": 2.1033780574798584,
+ "learning_rate": 4.256565656565657e-06,
+ "loss": 1.455024003982544,
+ "mean_token_accuracy": 0.6451514363288879,
+ "num_tokens": 6045696.0,
+ "step": 369
+ },
+ {
+ "entropy": 1.4382058382034302,
+ "epoch": 0.7474747474747475,
+ "grad_norm": 2.1068074703216553,
+ "learning_rate": 4.254545454545455e-06,
+ "loss": 1.4422768354415894,
+ "mean_token_accuracy": 0.6630434989929199,
+ "num_tokens": 6062080.0,
+ "step": 370
+ },
+ {
+ "entropy": 1.8046759366989136,
+ "epoch": 0.7494949494949495,
+ "grad_norm": 2.214466094970703,
+ "learning_rate": 4.252525252525253e-06,
+ "loss": 1.8247106075286865,
+ "mean_token_accuracy": 0.6003297567367554,
+ "num_tokens": 6078464.0,
+ "step": 371
+ },
+ {
+ "entropy": 1.671690583229065,
+ "epoch": 0.7515151515151515,
+ "grad_norm": 1.9790785312652588,
+ "learning_rate": 4.250505050505051e-06,
+ "loss": 1.6907753944396973,
+ "mean_token_accuracy": 0.6232290863990784,
+ "num_tokens": 6094848.0,
+ "step": 372
+ },
+ {
+ "entropy": 1.6881897449493408,
+ "epoch": 0.7535353535353535,
+ "grad_norm": 1.8638463020324707,
+ "learning_rate": 4.248484848484849e-06,
+ "loss": 1.7132716178894043,
+ "mean_token_accuracy": 0.6357474327087402,
+ "num_tokens": 6111232.0,
+ "step": 373
+ },
+ {
+ "entropy": 1.2555122375488281,
+ "epoch": 0.7555555555555555,
+ "grad_norm": 2.082378387451172,
+ "learning_rate": 4.246464646464647e-06,
+ "loss": 1.240688681602478,
+ "mean_token_accuracy": 0.6965070962905884,
+ "num_tokens": 6127616.0,
+ "step": 374
+ },
+ {
+ "entropy": 1.847135305404663,
+ "epoch": 0.7575757575757576,
+ "grad_norm": 2.142962694168091,
+ "learning_rate": 4.244444444444445e-06,
+ "loss": 1.8840911388397217,
+ "mean_token_accuracy": 0.5822545289993286,
+ "num_tokens": 6144000.0,
+ "step": 375
+ },
+ {
+ "entropy": 1.239328384399414,
+ "epoch": 0.7595959595959596,
+ "grad_norm": 2.0589468479156494,
+ "learning_rate": 4.242424242424243e-06,
+ "loss": 1.256324052810669,
+ "mean_token_accuracy": 0.6845383644104004,
+ "num_tokens": 6160384.0,
+ "step": 376
+ },
+ {
+ "entropy": 1.5047202110290527,
+ "epoch": 0.7616161616161616,
+ "grad_norm": 1.7951308488845825,
+ "learning_rate": 4.240404040404041e-06,
+ "loss": 1.5063304901123047,
+ "mean_token_accuracy": 0.6583414673805237,
+ "num_tokens": 6176768.0,
+ "step": 377
+ },
+ {
+ "entropy": 1.4072566032409668,
+ "epoch": 0.7636363636363637,
+ "grad_norm": 2.1670594215393066,
+ "learning_rate": 4.238383838383839e-06,
+ "loss": 1.4205389022827148,
+ "mean_token_accuracy": 0.658707857131958,
+ "num_tokens": 6193152.0,
+ "step": 378
+ },
+ {
+ "entropy": 1.3207885026931763,
+ "epoch": 0.7656565656565657,
+ "grad_norm": 1.9017083644866943,
+ "learning_rate": 4.236363636363637e-06,
+ "loss": 1.3169896602630615,
+ "mean_token_accuracy": 0.6954079270362854,
+ "num_tokens": 6209536.0,
+ "step": 379
+ },
+ {
+ "entropy": 1.4762436151504517,
+ "epoch": 0.7676767676767676,
+ "grad_norm": 2.0023069381713867,
+ "learning_rate": 4.234343434343435e-06,
+ "loss": 1.4725041389465332,
+ "mean_token_accuracy": 0.6620053648948669,
+ "num_tokens": 6225920.0,
+ "step": 380
+ },
+ {
+ "entropy": 1.4895304441452026,
+ "epoch": 0.7696969696969697,
+ "grad_norm": 1.9528017044067383,
+ "learning_rate": 4.2323232323232325e-06,
+ "loss": 1.49650239944458,
+ "mean_token_accuracy": 0.6508915424346924,
+ "num_tokens": 6242304.0,
+ "step": 381
+ },
+ {
+ "entropy": 1.3024516105651855,
+ "epoch": 0.7717171717171717,
+ "grad_norm": 1.9855588674545288,
+ "learning_rate": 4.2303030303030304e-06,
+ "loss": 1.34218168258667,
+ "mean_token_accuracy": 0.6838055849075317,
+ "num_tokens": 6258688.0,
+ "step": 382
+ },
+ {
+ "entropy": 1.6005626916885376,
+ "epoch": 0.7737373737373737,
+ "grad_norm": 2.028286933898926,
+ "learning_rate": 4.228282828282828e-06,
+ "loss": 1.6299283504486084,
+ "mean_token_accuracy": 0.6315950155258179,
+ "num_tokens": 6275072.0,
+ "step": 383
+ },
+ {
+ "entropy": 1.7050484418869019,
+ "epoch": 0.7757575757575758,
+ "grad_norm": 2.474047899246216,
+ "learning_rate": 4.226262626262626e-06,
+ "loss": 1.777151346206665,
+ "mean_token_accuracy": 0.5919027924537659,
+ "num_tokens": 6291456.0,
+ "step": 384
+ },
+ {
+ "entropy": 1.5014543533325195,
+ "epoch": 0.7777777777777778,
+ "grad_norm": 1.9866594076156616,
+ "learning_rate": 4.224242424242425e-06,
+ "loss": 1.5308949947357178,
+ "mean_token_accuracy": 0.639106035232544,
+ "num_tokens": 6307840.0,
+ "step": 385
+ },
+ {
+ "entropy": 1.6957359313964844,
+ "epoch": 0.7797979797979798,
+ "grad_norm": 2.229820966720581,
+ "learning_rate": 4.222222222222223e-06,
+ "loss": 1.728761911392212,
+ "mean_token_accuracy": 0.6242061257362366,
+ "num_tokens": 6324224.0,
+ "step": 386
+ },
+ {
+ "entropy": 1.6823521852493286,
+ "epoch": 0.7818181818181819,
+ "grad_norm": 2.033383369445801,
+ "learning_rate": 4.220202020202021e-06,
+ "loss": 1.7310154438018799,
+ "mean_token_accuracy": 0.6257938742637634,
+ "num_tokens": 6340608.0,
+ "step": 387
+ },
+ {
+ "entropy": 1.710307240486145,
+ "epoch": 0.7838383838383839,
+ "grad_norm": 2.061861038208008,
+ "learning_rate": 4.218181818181819e-06,
+ "loss": 1.7066943645477295,
+ "mean_token_accuracy": 0.6123595237731934,
+ "num_tokens": 6356992.0,
+ "step": 388
+ },
+ {
+ "entropy": 1.241638422012329,
+ "epoch": 0.7858585858585858,
+ "grad_norm": 1.9743852615356445,
+ "learning_rate": 4.2161616161616165e-06,
+ "loss": 1.2520272731781006,
+ "mean_token_accuracy": 0.692415714263916,
+ "num_tokens": 6373376.0,
+ "step": 389
+ },
+ {
+ "entropy": 1.5918776988983154,
+ "epoch": 0.7878787878787878,
+ "grad_norm": 2.156675100326538,
+ "learning_rate": 4.214141414141414e-06,
+ "loss": 1.6060255765914917,
+ "mean_token_accuracy": 0.6178553700447083,
+ "num_tokens": 6389760.0,
+ "step": 390
+ },
+ {
+ "entropy": 1.7307862043380737,
+ "epoch": 0.7898989898989899,
+ "grad_norm": 2.171247959136963,
+ "learning_rate": 4.212121212121212e-06,
+ "loss": 1.7327581644058228,
+ "mean_token_accuracy": 0.6502808928489685,
+ "num_tokens": 6406144.0,
+ "step": 391
+ },
+ {
+ "entropy": 1.6725553274154663,
+ "epoch": 0.7919191919191919,
+ "grad_norm": 2.020228624343872,
+ "learning_rate": 4.21010101010101e-06,
+ "loss": 1.6803646087646484,
+ "mean_token_accuracy": 0.6383732557296753,
+ "num_tokens": 6422528.0,
+ "step": 392
+ },
+ {
+ "entropy": 1.4120928049087524,
+ "epoch": 0.793939393939394,
+ "grad_norm": 1.9896639585494995,
+ "learning_rate": 4.208080808080808e-06,
+ "loss": 1.3974279165267944,
+ "mean_token_accuracy": 0.6661577820777893,
+ "num_tokens": 6438912.0,
+ "step": 393
+ },
+ {
+ "entropy": 0.933545708656311,
+ "epoch": 0.795959595959596,
+ "grad_norm": 1.6884223222732544,
+ "learning_rate": 4.206060606060606e-06,
+ "loss": 0.9287986755371094,
+ "mean_token_accuracy": 0.7636175155639648,
+ "num_tokens": 6455296.0,
+ "step": 394
+ },
+ {
+ "entropy": 1.419004201889038,
+ "epoch": 0.797979797979798,
+ "grad_norm": 2.2590551376342773,
+ "learning_rate": 4.204040404040405e-06,
+ "loss": 1.4652538299560547,
+ "mean_token_accuracy": 0.6589521169662476,
+ "num_tokens": 6471680.0,
+ "step": 395
+ },
+ {
+ "entropy": 1.750221610069275,
+ "epoch": 0.8,
+ "grad_norm": 2.195030450820923,
+ "learning_rate": 4.2020202020202026e-06,
+ "loss": 1.7328863143920898,
+ "mean_token_accuracy": 0.6008182764053345,
+ "num_tokens": 6488064.0,
+ "step": 396
+ },
+ {
+ "entropy": 1.7440015077590942,
+ "epoch": 0.802020202020202,
+ "grad_norm": 1.9833927154541016,
+ "learning_rate": 4.2000000000000004e-06,
+ "loss": 1.7617835998535156,
+ "mean_token_accuracy": 0.5992916226387024,
+ "num_tokens": 6504448.0,
+ "step": 397
+ },
+ {
+ "entropy": 1.1636452674865723,
+ "epoch": 0.804040404040404,
+ "grad_norm": 1.9506802558898926,
+ "learning_rate": 4.197979797979798e-06,
+ "loss": 1.1344032287597656,
+ "mean_token_accuracy": 0.7122618556022644,
+ "num_tokens": 6520832.0,
+ "step": 398
+ },
+ {
+ "entropy": 1.6173542737960815,
+ "epoch": 0.806060606060606,
+ "grad_norm": 2.0997231006622314,
+ "learning_rate": 4.195959595959596e-06,
+ "loss": 1.6303956508636475,
+ "mean_token_accuracy": 0.6215192675590515,
+ "num_tokens": 6537216.0,
+ "step": 399
+ },
+ {
+ "entropy": 1.4391542673110962,
+ "epoch": 0.8080808080808081,
+ "grad_norm": 2.356828212738037,
+ "learning_rate": 4.193939393939394e-06,
+ "loss": 1.4465923309326172,
+ "mean_token_accuracy": 0.6594406366348267,
+ "num_tokens": 6553600.0,
+ "step": 400
+ },
+ {
+ "epoch": 0.8080808080808081,
+ "eval_entropy": 1.5504949785047961,
+ "eval_loss": 1.5544542074203491,
+ "eval_mean_token_accuracy": 0.6409837569921247,
+ "eval_num_tokens": 6553600.0,
+ "eval_runtime": 43.7986,
+ "eval_samples_per_second": 22.603,
+ "eval_steps_per_second": 2.831,
+ "step": 400
+ },
+ {
+ "entropy": 1.60548996925354,
+ "epoch": 0.8101010101010101,
+ "grad_norm": 2.1894404888153076,
+ "learning_rate": 4.191919191919192e-06,
+ "loss": 1.6116085052490234,
+ "mean_token_accuracy": 0.6276868581771851,
+ "num_tokens": 6569984.0,
+ "step": 401
+ },
+ {
+ "entropy": 1.436041235923767,
+ "epoch": 0.8121212121212121,
+ "grad_norm": 2.1180264949798584,
+ "learning_rate": 4.18989898989899e-06,
+ "loss": 1.4203698635101318,
+ "mean_token_accuracy": 0.6554714441299438,
+ "num_tokens": 6586368.0,
+ "step": 402
+ },
+ {
+ "entropy": 1.1348106861114502,
+ "epoch": 0.8141414141414142,
+ "grad_norm": 2.0420851707458496,
+ "learning_rate": 4.187878787878788e-06,
+ "loss": 1.1424527168273926,
+ "mean_token_accuracy": 0.7154372334480286,
+ "num_tokens": 6602752.0,
+ "step": 403
+ },
+ {
+ "entropy": 1.4039727449417114,
+ "epoch": 0.8161616161616162,
+ "grad_norm": 2.148340940475464,
+ "learning_rate": 4.185858585858586e-06,
+ "loss": 1.4246132373809814,
+ "mean_token_accuracy": 0.6723864078521729,
+ "num_tokens": 6619136.0,
+ "step": 404
+ },
+ {
+ "entropy": 1.4222626686096191,
+ "epoch": 0.8181818181818182,
+ "grad_norm": 1.9436827898025513,
+ "learning_rate": 4.1838383838383835e-06,
+ "loss": 1.4235990047454834,
+ "mean_token_accuracy": 0.6759281754493713,
+ "num_tokens": 6635520.0,
+ "step": 405
+ },
+ {
+ "entropy": 1.4771310091018677,
+ "epoch": 0.8202020202020202,
+ "grad_norm": 2.0953726768493652,
+ "learning_rate": 4.181818181818182e-06,
+ "loss": 1.501934289932251,
+ "mean_token_accuracy": 0.6535173654556274,
+ "num_tokens": 6651904.0,
+ "step": 406
+ },
+ {
+ "entropy": 1.526256799697876,
+ "epoch": 0.8222222222222222,
+ "grad_norm": 2.245994806289673,
+ "learning_rate": 4.17979797979798e-06,
+ "loss": 1.539383888244629,
+ "mean_token_accuracy": 0.6417317986488342,
+ "num_tokens": 6668288.0,
+ "step": 407
+ },
+ {
+ "entropy": 1.1716097593307495,
+ "epoch": 0.8242424242424242,
+ "grad_norm": 1.8283652067184448,
+ "learning_rate": 4.177777777777778e-06,
+ "loss": 1.1798359155654907,
+ "mean_token_accuracy": 0.7123839855194092,
+ "num_tokens": 6684672.0,
+ "step": 408
+ },
+ {
+ "entropy": 1.4581540822982788,
+ "epoch": 0.8262626262626263,
+ "grad_norm": 1.9325168132781982,
+ "learning_rate": 4.175757575757576e-06,
+ "loss": 1.4526585340499878,
+ "mean_token_accuracy": 0.662432849407196,
+ "num_tokens": 6701056.0,
+ "step": 409
+ },
+ {
+ "entropy": 1.4073102474212646,
+ "epoch": 0.8282828282828283,
+ "grad_norm": 2.1543467044830322,
+ "learning_rate": 4.173737373737375e-06,
+ "loss": 1.437713861465454,
+ "mean_token_accuracy": 0.6725696325302124,
+ "num_tokens": 6717440.0,
+ "step": 410
+ },
+ {
+ "entropy": 1.5970062017440796,
+ "epoch": 0.8303030303030303,
+ "grad_norm": 2.1714792251586914,
+ "learning_rate": 4.1717171717171726e-06,
+ "loss": 1.613295078277588,
+ "mean_token_accuracy": 0.6229848265647888,
+ "num_tokens": 6733824.0,
+ "step": 411
+ },
+ {
+ "entropy": 1.449837327003479,
+ "epoch": 0.8323232323232324,
+ "grad_norm": 2.2499871253967285,
+ "learning_rate": 4.1696969696969705e-06,
+ "loss": 1.489758014678955,
+ "mean_token_accuracy": 0.645639955997467,
+ "num_tokens": 6750208.0,
+ "step": 412
+ },
+ {
+ "entropy": 1.7696173191070557,
+ "epoch": 0.8343434343434344,
+ "grad_norm": 2.3637073040008545,
+ "learning_rate": 4.167676767676768e-06,
+ "loss": 1.8147599697113037,
+ "mean_token_accuracy": 0.5821323990821838,
+ "num_tokens": 6766592.0,
+ "step": 413
+ },
+ {
+ "entropy": 1.99376380443573,
+ "epoch": 0.8363636363636363,
+ "grad_norm": 2.265683174133301,
+ "learning_rate": 4.165656565656566e-06,
+ "loss": 2.009024143218994,
+ "mean_token_accuracy": 0.5591108798980713,
+ "num_tokens": 6782976.0,
+ "step": 414
+ },
+ {
+ "entropy": 1.7276980876922607,
+ "epoch": 0.8383838383838383,
+ "grad_norm": 1.9407477378845215,
+ "learning_rate": 4.163636363636364e-06,
+ "loss": 1.7130229473114014,
+ "mean_token_accuracy": 0.6202979683876038,
+ "num_tokens": 6799360.0,
+ "step": 415
+ },
+ {
+ "entropy": 1.5418223142623901,
+ "epoch": 0.8404040404040404,
+ "grad_norm": 1.9765743017196655,
+ "learning_rate": 4.161616161616162e-06,
+ "loss": 1.5627632141113281,
+ "mean_token_accuracy": 0.6433194875717163,
+ "num_tokens": 6815744.0,
+ "step": 416
+ },
+ {
+ "entropy": 1.6040345430374146,
+ "epoch": 0.8424242424242424,
+ "grad_norm": 2.1942877769470215,
+ "learning_rate": 4.15959595959596e-06,
+ "loss": 1.6044622659683228,
+ "mean_token_accuracy": 0.6278700828552246,
+ "num_tokens": 6832128.0,
+ "step": 417
+ },
+ {
+ "entropy": 1.8461577892303467,
+ "epoch": 0.8444444444444444,
+ "grad_norm": 2.289196729660034,
+ "learning_rate": 4.157575757575758e-06,
+ "loss": 1.8679372072219849,
+ "mean_token_accuracy": 0.5887884497642517,
+ "num_tokens": 6848512.0,
+ "step": 418
+ },
+ {
+ "entropy": 1.4079350233078003,
+ "epoch": 0.8464646464646465,
+ "grad_norm": 1.9526047706604004,
+ "learning_rate": 4.155555555555556e-06,
+ "loss": 1.4145114421844482,
+ "mean_token_accuracy": 0.6731802821159363,
+ "num_tokens": 6864896.0,
+ "step": 419
+ },
+ {
+ "entropy": 1.1973973512649536,
+ "epoch": 0.8484848484848485,
+ "grad_norm": 2.0716679096221924,
+ "learning_rate": 4.1535353535353536e-06,
+ "loss": 1.260810136795044,
+ "mean_token_accuracy": 0.701697587966919,
+ "num_tokens": 6881280.0,
+ "step": 420
+ },
+ {
+ "entropy": 1.507702350616455,
+ "epoch": 0.8505050505050505,
+ "grad_norm": 2.0233314037323,
+ "learning_rate": 4.151515151515152e-06,
+ "loss": 1.513720154762268,
+ "mean_token_accuracy": 0.6497313380241394,
+ "num_tokens": 6897664.0,
+ "step": 421
+ },
+ {
+ "entropy": 1.54402756690979,
+ "epoch": 0.8525252525252526,
+ "grad_norm": 2.23366379737854,
+ "learning_rate": 4.14949494949495e-06,
+ "loss": 1.5434964895248413,
+ "mean_token_accuracy": 0.6502198576927185,
+ "num_tokens": 6914048.0,
+ "step": 422
+ },
+ {
+ "entropy": 1.820296049118042,
+ "epoch": 0.8545454545454545,
+ "grad_norm": 2.066905975341797,
+ "learning_rate": 4.147474747474748e-06,
+ "loss": 1.8356924057006836,
+ "mean_token_accuracy": 0.5886663198471069,
+ "num_tokens": 6930432.0,
+ "step": 423
+ },
+ {
+ "entropy": 1.4747720956802368,
+ "epoch": 0.8565656565656565,
+ "grad_norm": 2.040022850036621,
+ "learning_rate": 4.145454545454546e-06,
+ "loss": 1.4495999813079834,
+ "mean_token_accuracy": 0.6532731056213379,
+ "num_tokens": 6946816.0,
+ "step": 424
+ },
+ {
+ "entropy": 1.7536696195602417,
+ "epoch": 0.8585858585858586,
+ "grad_norm": 2.0884320735931396,
+ "learning_rate": 4.143434343434344e-06,
+ "loss": 1.7520158290863037,
+ "mean_token_accuracy": 0.6083903312683105,
+ "num_tokens": 6963200.0,
+ "step": 425
+ },
+ {
+ "entropy": 1.715582251548767,
+ "epoch": 0.8606060606060606,
+ "grad_norm": 2.2991514205932617,
+ "learning_rate": 4.141414141414142e-06,
+ "loss": 1.7138090133666992,
+ "mean_token_accuracy": 0.607107937335968,
+ "num_tokens": 6979584.0,
+ "step": 426
+ },
+ {
+ "entropy": 1.8116382360458374,
+ "epoch": 0.8626262626262626,
+ "grad_norm": 2.289909839630127,
+ "learning_rate": 4.13939393939394e-06,
+ "loss": 1.7755894660949707,
+ "mean_token_accuracy": 0.5776746273040771,
+ "num_tokens": 6995968.0,
+ "step": 427
+ },
+ {
+ "entropy": 1.487213373184204,
+ "epoch": 0.8646464646464647,
+ "grad_norm": 1.8834803104400635,
+ "learning_rate": 4.1373737373737375e-06,
+ "loss": 1.511157751083374,
+ "mean_token_accuracy": 0.6546165347099304,
+ "num_tokens": 7012352.0,
+ "step": 428
+ },
+ {
+ "entropy": 1.2769891023635864,
+ "epoch": 0.8666666666666667,
+ "grad_norm": 1.892616629600525,
+ "learning_rate": 4.135353535353535e-06,
+ "loss": 1.2772598266601562,
+ "mean_token_accuracy": 0.6984611749649048,
+ "num_tokens": 7028736.0,
+ "step": 429
+ },
+ {
+ "entropy": 1.579519271850586,
+ "epoch": 0.8686868686868687,
+ "grad_norm": 1.9801563024520874,
+ "learning_rate": 4.133333333333333e-06,
+ "loss": 1.5647528171539307,
+ "mean_token_accuracy": 0.6482046842575073,
+ "num_tokens": 7045120.0,
+ "step": 430
+ },
+ {
+ "entropy": 1.407600998878479,
+ "epoch": 0.8707070707070707,
+ "grad_norm": 2.1737446784973145,
+ "learning_rate": 4.131313131313132e-06,
+ "loss": 1.4184494018554688,
+ "mean_token_accuracy": 0.6640815734863281,
+ "num_tokens": 7061504.0,
+ "step": 431
+ },
+ {
+ "entropy": 1.9118441343307495,
+ "epoch": 0.8727272727272727,
+ "grad_norm": 1.9541205167770386,
+ "learning_rate": 4.12929292929293e-06,
+ "loss": 1.9581422805786133,
+ "mean_token_accuracy": 0.564667820930481,
+ "num_tokens": 7077888.0,
+ "step": 432
+ },
+ {
+ "entropy": 1.925604224205017,
+ "epoch": 0.8747474747474747,
+ "grad_norm": 1.9696223735809326,
+ "learning_rate": 4.127272727272728e-06,
+ "loss": 1.9226163625717163,
+ "mean_token_accuracy": 0.5870786309242249,
+ "num_tokens": 7094272.0,
+ "step": 433
+ },
+ {
+ "entropy": 1.3994735479354858,
+ "epoch": 0.8767676767676768,
+ "grad_norm": 2.003532886505127,
+ "learning_rate": 4.125252525252526e-06,
+ "loss": 1.4291752576828003,
+ "mean_token_accuracy": 0.669516384601593,
+ "num_tokens": 7110656.0,
+ "step": 434
+ },
+ {
+ "entropy": 1.5550929307937622,
+ "epoch": 0.8787878787878788,
+ "grad_norm": 2.2658586502075195,
+ "learning_rate": 4.1232323232323236e-06,
+ "loss": 1.5533335208892822,
+ "mean_token_accuracy": 0.6242672204971313,
+ "num_tokens": 7127040.0,
+ "step": 435
+ },
+ {
+ "entropy": 1.6622785329818726,
+ "epoch": 0.8808080808080808,
+ "grad_norm": 2.0748393535614014,
+ "learning_rate": 4.1212121212121215e-06,
+ "loss": 1.700000524520874,
+ "mean_token_accuracy": 0.6221299171447754,
+ "num_tokens": 7143424.0,
+ "step": 436
+ },
+ {
+ "entropy": 1.1223996877670288,
+ "epoch": 0.8828282828282829,
+ "grad_norm": 2.0348756313323975,
+ "learning_rate": 4.119191919191919e-06,
+ "loss": 1.143293857574463,
+ "mean_token_accuracy": 0.7045676708221436,
+ "num_tokens": 7159808.0,
+ "step": 437
+ },
+ {
+ "entropy": 1.7156380414962769,
+ "epoch": 0.8848484848484849,
+ "grad_norm": 2.306549549102783,
+ "learning_rate": 4.117171717171717e-06,
+ "loss": 1.7625794410705566,
+ "mean_token_accuracy": 0.6049706935882568,
+ "num_tokens": 7176192.0,
+ "step": 438
+ },
+ {
+ "entropy": 1.8507202863693237,
+ "epoch": 0.8868686868686869,
+ "grad_norm": 2.2955853939056396,
+ "learning_rate": 4.115151515151515e-06,
+ "loss": 1.8923052549362183,
+ "mean_token_accuracy": 0.5938568711280823,
+ "num_tokens": 7192576.0,
+ "step": 439
+ },
+ {
+ "entropy": 1.813754916191101,
+ "epoch": 0.8888888888888888,
+ "grad_norm": 2.095700263977051,
+ "learning_rate": 4.113131313131313e-06,
+ "loss": 1.8375307321548462,
+ "mean_token_accuracy": 0.5848192572593689,
+ "num_tokens": 7208960.0,
+ "step": 440
+ },
+ {
+ "entropy": 1.4185007810592651,
+ "epoch": 0.8909090909090909,
+ "grad_norm": 2.118213176727295,
+ "learning_rate": 4.111111111111111e-06,
+ "loss": 1.4548171758651733,
+ "mean_token_accuracy": 0.657608687877655,
+ "num_tokens": 7225344.0,
+ "step": 441
+ },
+ {
+ "entropy": 1.1041690111160278,
+ "epoch": 0.8929292929292929,
+ "grad_norm": 1.9638988971710205,
+ "learning_rate": 4.10909090909091e-06,
+ "loss": 1.1232322454452515,
+ "mean_token_accuracy": 0.7068881392478943,
+ "num_tokens": 7241728.0,
+ "step": 442
+ },
+ {
+ "entropy": 1.6423180103302002,
+ "epoch": 0.8949494949494949,
+ "grad_norm": 2.32987117767334,
+ "learning_rate": 4.1070707070707075e-06,
+ "loss": 1.665462613105774,
+ "mean_token_accuracy": 0.6133365631103516,
+ "num_tokens": 7258112.0,
+ "step": 443
+ },
+ {
+ "entropy": 1.4778310060501099,
+ "epoch": 0.896969696969697,
+ "grad_norm": 2.1247661113739014,
+ "learning_rate": 4.105050505050505e-06,
+ "loss": 1.4554922580718994,
+ "mean_token_accuracy": 0.6474108695983887,
+ "num_tokens": 7274496.0,
+ "step": 444
+ },
+ {
+ "entropy": 1.3777755498886108,
+ "epoch": 0.898989898989899,
+ "grad_norm": 1.9243263006210327,
+ "learning_rate": 4.103030303030303e-06,
+ "loss": 1.3942883014678955,
+ "mean_token_accuracy": 0.6656692624092102,
+ "num_tokens": 7290880.0,
+ "step": 445
+ },
+ {
+ "entropy": 1.1036415100097656,
+ "epoch": 0.901010101010101,
+ "grad_norm": 1.9711178541183472,
+ "learning_rate": 4.101010101010101e-06,
+ "loss": 1.1070351600646973,
+ "mean_token_accuracy": 0.7213605046272278,
+ "num_tokens": 7307264.0,
+ "step": 446
+ },
+ {
+ "entropy": 1.695487380027771,
+ "epoch": 0.9030303030303031,
+ "grad_norm": 2.1113150119781494,
+ "learning_rate": 4.098989898989899e-06,
+ "loss": 1.6993653774261475,
+ "mean_token_accuracy": 0.5992305874824524,
+ "num_tokens": 7323648.0,
+ "step": 447
+ },
+ {
+ "entropy": 1.496254801750183,
+ "epoch": 0.9050505050505051,
+ "grad_norm": 2.072986602783203,
+ "learning_rate": 4.096969696969697e-06,
+ "loss": 1.5046567916870117,
+ "mean_token_accuracy": 0.6558378338813782,
+ "num_tokens": 7340032.0,
+ "step": 448
+ },
+ {
+ "entropy": 2.1845033168792725,
+ "epoch": 0.907070707070707,
+ "grad_norm": 2.3246262073516846,
+ "learning_rate": 4.094949494949495e-06,
+ "loss": 2.1508708000183105,
+ "mean_token_accuracy": 0.5376160144805908,
+ "num_tokens": 7356416.0,
+ "step": 449
+ },
+ {
+ "entropy": 1.6329164505004883,
+ "epoch": 0.9090909090909091,
+ "grad_norm": 1.997310757637024,
+ "learning_rate": 4.092929292929294e-06,
+ "loss": 1.6422264575958252,
+ "mean_token_accuracy": 0.6232290863990784,
+ "num_tokens": 7372800.0,
+ "step": 450
+ },
+ {
+ "epoch": 0.9090909090909091,
+ "eval_entropy": 1.5515337480652718,
+ "eval_loss": 1.5485161542892456,
+ "eval_mean_token_accuracy": 0.6417716929028111,
+ "eval_num_tokens": 7372800.0,
+ "eval_runtime": 43.7901,
+ "eval_samples_per_second": 22.608,
+ "eval_steps_per_second": 2.832,
+ "step": 450
+ },
+ {
+ "entropy": 1.7854291200637817,
+ "epoch": 0.9111111111111111,
+ "grad_norm": 2.0212173461914062,
+ "learning_rate": 4.0909090909090915e-06,
+ "loss": 1.7920666933059692,
+ "mean_token_accuracy": 0.5950170755386353,
+ "num_tokens": 7389184.0,
+ "step": 451
+ },
+ {
+ "entropy": 1.216304898262024,
+ "epoch": 0.9131313131313131,
+ "grad_norm": 1.8800705671310425,
+ "learning_rate": 4.088888888888889e-06,
+ "loss": 1.2014267444610596,
+ "mean_token_accuracy": 0.7076819539070129,
+ "num_tokens": 7405568.0,
+ "step": 452
+ },
+ {
+ "entropy": 1.5593703985214233,
+ "epoch": 0.9151515151515152,
+ "grad_norm": 2.101381301879883,
+ "learning_rate": 4.086868686868687e-06,
+ "loss": 1.559610366821289,
+ "mean_token_accuracy": 0.6337933540344238,
+ "num_tokens": 7421952.0,
+ "step": 453
+ },
+ {
+ "entropy": 1.4199841022491455,
+ "epoch": 0.9171717171717172,
+ "grad_norm": 2.0038692951202393,
+ "learning_rate": 4.084848484848485e-06,
+ "loss": 1.4040653705596924,
+ "mean_token_accuracy": 0.660845160484314,
+ "num_tokens": 7438336.0,
+ "step": 454
+ },
+ {
+ "entropy": 1.2010453939437866,
+ "epoch": 0.9191919191919192,
+ "grad_norm": 2.004110336303711,
+ "learning_rate": 4.082828282828283e-06,
+ "loss": 1.194115400314331,
+ "mean_token_accuracy": 0.6955910921096802,
+ "num_tokens": 7454720.0,
+ "step": 455
+ },
+ {
+ "entropy": 1.2295804023742676,
+ "epoch": 0.9212121212121213,
+ "grad_norm": 2.130387544631958,
+ "learning_rate": 4.080808080808081e-06,
+ "loss": 1.2125787734985352,
+ "mean_token_accuracy": 0.6993771195411682,
+ "num_tokens": 7471104.0,
+ "step": 456
+ },
+ {
+ "entropy": 1.3706485033035278,
+ "epoch": 0.9232323232323232,
+ "grad_norm": 2.1667706966400146,
+ "learning_rate": 4.07878787878788e-06,
+ "loss": 1.3886957168579102,
+ "mean_token_accuracy": 0.6686614751815796,
+ "num_tokens": 7487488.0,
+ "step": 457
+ },
+ {
+ "entropy": 1.714059591293335,
+ "epoch": 0.9252525252525252,
+ "grad_norm": 2.0084264278411865,
+ "learning_rate": 4.0767676767676775e-06,
+ "loss": 1.724153757095337,
+ "mean_token_accuracy": 0.602161705493927,
+ "num_tokens": 7503872.0,
+ "step": 458
+ },
+ {
+ "entropy": 1.3170617818832397,
+ "epoch": 0.9272727272727272,
+ "grad_norm": 1.9923994541168213,
+ "learning_rate": 4.0747474747474754e-06,
+ "loss": 1.3518096208572388,
+ "mean_token_accuracy": 0.6801416873931885,
+ "num_tokens": 7520256.0,
+ "step": 459
+ },
+ {
+ "entropy": 1.7946380376815796,
+ "epoch": 0.9292929292929293,
+ "grad_norm": 2.3475260734558105,
+ "learning_rate": 4.072727272727273e-06,
+ "loss": 1.824514389038086,
+ "mean_token_accuracy": 0.6005740165710449,
+ "num_tokens": 7536640.0,
+ "step": 460
+ },
+ {
+ "entropy": 1.3695449829101562,
+ "epoch": 0.9313131313131313,
+ "grad_norm": 2.0983943939208984,
+ "learning_rate": 4.070707070707071e-06,
+ "loss": 1.3934273719787598,
+ "mean_token_accuracy": 0.6723253726959229,
+ "num_tokens": 7553024.0,
+ "step": 461
+ },
+ {
+ "entropy": 1.5890663862228394,
+ "epoch": 0.9333333333333333,
+ "grad_norm": 2.1465413570404053,
+ "learning_rate": 4.068686868686869e-06,
+ "loss": 1.611652135848999,
+ "mean_token_accuracy": 0.6408158540725708,
+ "num_tokens": 7569408.0,
+ "step": 462
+ },
+ {
+ "entropy": 1.2593727111816406,
+ "epoch": 0.9353535353535354,
+ "grad_norm": 1.7885241508483887,
+ "learning_rate": 4.066666666666667e-06,
+ "loss": 1.2781705856323242,
+ "mean_token_accuracy": 0.6946751475334167,
+ "num_tokens": 7585792.0,
+ "step": 463
+ },
+ {
+ "entropy": 1.5680429935455322,
+ "epoch": 0.9373737373737374,
+ "grad_norm": 2.1466636657714844,
+ "learning_rate": 4.064646464646465e-06,
+ "loss": 1.60218346118927,
+ "mean_token_accuracy": 0.6334269642829895,
+ "num_tokens": 7602176.0,
+ "step": 464
+ },
+ {
+ "entropy": 1.681670069694519,
+ "epoch": 0.9393939393939394,
+ "grad_norm": 2.2211875915527344,
+ "learning_rate": 4.062626262626263e-06,
+ "loss": 1.6819908618927002,
+ "mean_token_accuracy": 0.6177943348884583,
+ "num_tokens": 7618560.0,
+ "step": 465
+ },
+ {
+ "entropy": 1.2239924669265747,
+ "epoch": 0.9414141414141414,
+ "grad_norm": 1.9997022151947021,
+ "learning_rate": 4.060606060606061e-06,
+ "loss": 1.2383348941802979,
+ "mean_token_accuracy": 0.6922325491905212,
+ "num_tokens": 7634944.0,
+ "step": 466
+ },
+ {
+ "entropy": 1.291772723197937,
+ "epoch": 0.9434343434343434,
+ "grad_norm": 2.0868117809295654,
+ "learning_rate": 4.058585858585859e-06,
+ "loss": 1.2879221439361572,
+ "mean_token_accuracy": 0.6883854269981384,
+ "num_tokens": 7651328.0,
+ "step": 467
+ },
+ {
+ "entropy": 1.2860186100006104,
+ "epoch": 0.9454545454545454,
+ "grad_norm": 1.8586393594741821,
+ "learning_rate": 4.056565656565657e-06,
+ "loss": 1.2820203304290771,
+ "mean_token_accuracy": 0.6802638173103333,
+ "num_tokens": 7667712.0,
+ "step": 468
+ },
+ {
+ "entropy": 1.860795259475708,
+ "epoch": 0.9474747474747475,
+ "grad_norm": 2.364405393600464,
+ "learning_rate": 4.054545454545455e-06,
+ "loss": 1.8880727291107178,
+ "mean_token_accuracy": 0.5821323990821838,
+ "num_tokens": 7684096.0,
+ "step": 469
+ },
+ {
+ "entropy": 1.6017589569091797,
+ "epoch": 0.9494949494949495,
+ "grad_norm": 2.023054599761963,
+ "learning_rate": 4.052525252525253e-06,
+ "loss": 1.5993852615356445,
+ "mean_token_accuracy": 0.6366634368896484,
+ "num_tokens": 7700480.0,
+ "step": 470
+ },
+ {
+ "entropy": 1.8498003482818604,
+ "epoch": 0.9515151515151515,
+ "grad_norm": 2.4474003314971924,
+ "learning_rate": 4.050505050505051e-06,
+ "loss": 1.868700623512268,
+ "mean_token_accuracy": 0.5826209187507629,
+ "num_tokens": 7716864.0,
+ "step": 471
+ },
+ {
+ "entropy": 1.5625540018081665,
+ "epoch": 0.9535353535353536,
+ "grad_norm": 2.1018362045288086,
+ "learning_rate": 4.048484848484849e-06,
+ "loss": 1.571077585220337,
+ "mean_token_accuracy": 0.6433805823326111,
+ "num_tokens": 7733248.0,
+ "step": 472
+ },
+ {
+ "entropy": 1.8534579277038574,
+ "epoch": 0.9555555555555556,
+ "grad_norm": 2.407588243484497,
+ "learning_rate": 4.046464646464647e-06,
+ "loss": 1.8789153099060059,
+ "mean_token_accuracy": 0.5859794616699219,
+ "num_tokens": 7749632.0,
+ "step": 473
+ },
+ {
+ "entropy": 1.3264559507369995,
+ "epoch": 0.9575757575757575,
+ "grad_norm": 2.007199764251709,
+ "learning_rate": 4.044444444444445e-06,
+ "loss": 1.3489338159561157,
+ "mean_token_accuracy": 0.6650586128234863,
+ "num_tokens": 7766016.0,
+ "step": 474
+ },
+ {
+ "entropy": 1.1361761093139648,
+ "epoch": 0.9595959595959596,
+ "grad_norm": 1.9226998090744019,
+ "learning_rate": 4.0424242424242425e-06,
+ "loss": 1.156738519668579,
+ "mean_token_accuracy": 0.7226428985595703,
+ "num_tokens": 7782400.0,
+ "step": 475
+ },
+ {
+ "entropy": 1.550872802734375,
+ "epoch": 0.9616161616161616,
+ "grad_norm": 2.0082473754882812,
+ "learning_rate": 4.04040404040404e-06,
+ "loss": 1.5657379627227783,
+ "mean_token_accuracy": 0.642953097820282,
+ "num_tokens": 7798784.0,
+ "step": 476
+ },
+ {
+ "entropy": 1.0122721195220947,
+ "epoch": 0.9636363636363636,
+ "grad_norm": 1.9946776628494263,
+ "learning_rate": 4.038383838383838e-06,
+ "loss": 1.0301119089126587,
+ "mean_token_accuracy": 0.7361993193626404,
+ "num_tokens": 7815168.0,
+ "step": 477
+ },
+ {
+ "entropy": 1.61614990234375,
+ "epoch": 0.9656565656565657,
+ "grad_norm": 2.5009427070617676,
+ "learning_rate": 4.036363636363637e-06,
+ "loss": 1.6194994449615479,
+ "mean_token_accuracy": 0.620175838470459,
+ "num_tokens": 7831552.0,
+ "step": 478
+ },
+ {
+ "entropy": 1.5723443031311035,
+ "epoch": 0.9676767676767677,
+ "grad_norm": 1.9101917743682861,
+ "learning_rate": 4.034343434343435e-06,
+ "loss": 1.5586073398590088,
+ "mean_token_accuracy": 0.6223741769790649,
+ "num_tokens": 7847936.0,
+ "step": 479
+ },
+ {
+ "entropy": 1.271834373474121,
+ "epoch": 0.9696969696969697,
+ "grad_norm": 1.768438458442688,
+ "learning_rate": 4.032323232323233e-06,
+ "loss": 1.2466087341308594,
+ "mean_token_accuracy": 0.692354679107666,
+ "num_tokens": 7864320.0,
+ "step": 480
+ },
+ {
+ "entropy": 1.4550020694732666,
+ "epoch": 0.9717171717171718,
+ "grad_norm": 1.9753917455673218,
+ "learning_rate": 4.030303030303031e-06,
+ "loss": 1.4689980745315552,
+ "mean_token_accuracy": 0.6530288457870483,
+ "num_tokens": 7880704.0,
+ "step": 481
+ },
+ {
+ "entropy": 2.023750066757202,
+ "epoch": 0.9737373737373738,
+ "grad_norm": 2.3014674186706543,
+ "learning_rate": 4.0282828282828285e-06,
+ "loss": 2.0601019859313965,
+ "mean_token_accuracy": 0.5528212189674377,
+ "num_tokens": 7897088.0,
+ "step": 482
+ },
+ {
+ "entropy": 1.3225218057632446,
+ "epoch": 0.9757575757575757,
+ "grad_norm": 1.9937688112258911,
+ "learning_rate": 4.0262626262626264e-06,
+ "loss": 1.3186583518981934,
+ "mean_token_accuracy": 0.6751343607902527,
+ "num_tokens": 7913472.0,
+ "step": 483
+ },
+ {
+ "entropy": 1.4357911348342896,
+ "epoch": 0.9777777777777777,
+ "grad_norm": 1.9591492414474487,
+ "learning_rate": 4.024242424242424e-06,
+ "loss": 1.4157384634017944,
+ "mean_token_accuracy": 0.6618832349777222,
+ "num_tokens": 7929856.0,
+ "step": 484
+ },
+ {
+ "entropy": 1.470274567604065,
+ "epoch": 0.9797979797979798,
+ "grad_norm": 1.9318779706954956,
+ "learning_rate": 4.022222222222222e-06,
+ "loss": 1.4754853248596191,
+ "mean_token_accuracy": 0.6497923731803894,
+ "num_tokens": 7946240.0,
+ "step": 485
+ },
+ {
+ "entropy": 1.345750331878662,
+ "epoch": 0.9818181818181818,
+ "grad_norm": 1.9079619646072388,
+ "learning_rate": 4.02020202020202e-06,
+ "loss": 1.336526870727539,
+ "mean_token_accuracy": 0.6800805926322937,
+ "num_tokens": 7962624.0,
+ "step": 486
+ },
+ {
+ "entropy": 1.4641839265823364,
+ "epoch": 0.9838383838383838,
+ "grad_norm": 1.8413265943527222,
+ "learning_rate": 4.018181818181818e-06,
+ "loss": 1.4556326866149902,
+ "mean_token_accuracy": 0.6591963768005371,
+ "num_tokens": 7979008.0,
+ "step": 487
+ },
+ {
+ "entropy": 1.7966911792755127,
+ "epoch": 0.9858585858585859,
+ "grad_norm": 2.138899564743042,
+ "learning_rate": 4.016161616161616e-06,
+ "loss": 1.822898507118225,
+ "mean_token_accuracy": 0.5959941148757935,
+ "num_tokens": 7995392.0,
+ "step": 488
+ },
+ {
+ "entropy": 1.4334503412246704,
+ "epoch": 0.9878787878787879,
+ "grad_norm": 2.206122636795044,
+ "learning_rate": 4.014141414141415e-06,
+ "loss": 1.4127681255340576,
+ "mean_token_accuracy": 0.6609672904014587,
+ "num_tokens": 8011776.0,
+ "step": 489
+ },
+ {
+ "entropy": 1.684112548828125,
+ "epoch": 0.98989898989899,
+ "grad_norm": 2.0637400150299072,
+ "learning_rate": 4.0121212121212125e-06,
+ "loss": 1.6790317296981812,
+ "mean_token_accuracy": 0.6159012913703918,
+ "num_tokens": 8028160.0,
+ "step": 490
+ },
+ {
+ "entropy": 2.0716898441314697,
+ "epoch": 0.9919191919191919,
+ "grad_norm": 3.3912627696990967,
+ "learning_rate": 4.01010101010101e-06,
+ "loss": 2.0814826488494873,
+ "mean_token_accuracy": 0.5522105693817139,
+ "num_tokens": 8044544.0,
+ "step": 491
+ },
+ {
+ "entropy": 1.8764609098434448,
+ "epoch": 0.9939393939393939,
+ "grad_norm": 2.00569748878479,
+ "learning_rate": 4.008080808080808e-06,
+ "loss": 1.9150068759918213,
+ "mean_token_accuracy": 0.5823766589164734,
+ "num_tokens": 8060928.0,
+ "step": 492
+ },
+ {
+ "entropy": 1.630242943763733,
+ "epoch": 0.9959595959595959,
+ "grad_norm": 2.0648858547210693,
+ "learning_rate": 4.006060606060607e-06,
+ "loss": 1.6354784965515137,
+ "mean_token_accuracy": 0.621213972568512,
+ "num_tokens": 8077312.0,
+ "step": 493
+ },
+ {
+ "entropy": 1.7174078226089478,
+ "epoch": 0.997979797979798,
+ "grad_norm": 2.1376583576202393,
+ "learning_rate": 4.004040404040405e-06,
+ "loss": 1.7092773914337158,
+ "mean_token_accuracy": 0.6129701733589172,
+ "num_tokens": 8093696.0,
+ "step": 494
+ },
+ {
+ "entropy": 1.2112717628479004,
+ "epoch": 1.0,
+ "grad_norm": 1.9164371490478516,
+ "learning_rate": 4.002020202020203e-06,
+ "loss": 1.2106354236602783,
+ "mean_token_accuracy": 0.7040180563926697,
+ "num_tokens": 8110080.0,
+ "step": 495
+ },
+ {
+ "entropy": 1.2168891429901123,
+ "epoch": 1.002020202020202,
+ "grad_norm": 1.9845075607299805,
+ "learning_rate": 4.000000000000001e-06,
+ "loss": 1.1398870944976807,
+ "mean_token_accuracy": 0.7053004503250122,
+ "num_tokens": 8126464.0,
+ "step": 496
+ },
+ {
+ "entropy": 1.4670923948287964,
+ "epoch": 1.004040404040404,
+ "grad_norm": 2.0864014625549316,
+ "learning_rate": 3.9979797979797986e-06,
+ "loss": 1.4241242408752441,
+ "mean_token_accuracy": 0.6605398058891296,
+ "num_tokens": 8142848.0,
+ "step": 497
+ },
+ {
+ "entropy": 1.560593843460083,
+ "epoch": 1.006060606060606,
+ "grad_norm": 1.8882137537002563,
+ "learning_rate": 3.9959595959595964e-06,
+ "loss": 1.5164835453033447,
+ "mean_token_accuracy": 0.6610894203186035,
+ "num_tokens": 8159232.0,
+ "step": 498
+ },
+ {
+ "entropy": 1.2573609352111816,
+ "epoch": 1.0080808080808081,
+ "grad_norm": 2.011486530303955,
+ "learning_rate": 3.993939393939394e-06,
+ "loss": 1.2119636535644531,
+ "mean_token_accuracy": 0.6832559704780579,
+ "num_tokens": 8175616.0,
+ "step": 499
+ },
+ {
+ "entropy": 1.6119122505187988,
+ "epoch": 1.0101010101010102,
+ "grad_norm": 2.0089032649993896,
+ "learning_rate": 3.991919191919192e-06,
+ "loss": 1.5869622230529785,
+ "mean_token_accuracy": 0.6301905512809753,
+ "num_tokens": 8192000.0,
+ "step": 500
+ },
+ {
+ "epoch": 1.0101010101010102,
+ "eval_entropy": 1.4907484121860997,
+ "eval_loss": 1.5444872379302979,
+ "eval_mean_token_accuracy": 0.6426531960887294,
+ "eval_num_tokens": 8192000.0,
+ "eval_runtime": 43.729,
+ "eval_samples_per_second": 22.639,
+ "eval_steps_per_second": 2.836,
+ "step": 500
+ },
+ {
+ "entropy": 1.5645418167114258,
+ "epoch": 1.0121212121212122,
+ "grad_norm": 2.171133279800415,
+ "learning_rate": 3.98989898989899e-06,
+ "loss": 1.5252666473388672,
+ "mean_token_accuracy": 0.639594554901123,
+ "num_tokens": 8208384.0,
+ "step": 501
+ },
+ {
+ "entropy": 1.2985379695892334,
+ "epoch": 1.0141414141414142,
+ "grad_norm": 1.9075100421905518,
+ "learning_rate": 3.987878787878788e-06,
+ "loss": 1.2863168716430664,
+ "mean_token_accuracy": 0.6797142028808594,
+ "num_tokens": 8224768.0,
+ "step": 502
+ },
+ {
+ "entropy": 1.4055086374282837,
+ "epoch": 1.0161616161616163,
+ "grad_norm": 1.8511557579040527,
+ "learning_rate": 3.985858585858587e-06,
+ "loss": 1.440205454826355,
+ "mean_token_accuracy": 0.6671348214149475,
+ "num_tokens": 8241152.0,
+ "step": 503
+ },
+ {
+ "entropy": 1.0818580389022827,
+ "epoch": 1.018181818181818,
+ "grad_norm": 1.8912067413330078,
+ "learning_rate": 3.983838383838385e-06,
+ "loss": 1.0614542961120605,
+ "mean_token_accuracy": 0.7345505356788635,
+ "num_tokens": 8257536.0,
+ "step": 504
+ },
+ {
+ "entropy": 0.9176992177963257,
+ "epoch": 1.02020202020202,
+ "grad_norm": 1.7737077474594116,
+ "learning_rate": 3.9818181818181825e-06,
+ "loss": 0.9281337261199951,
+ "mean_token_accuracy": 0.7594040036201477,
+ "num_tokens": 8273920.0,
+ "step": 505
+ },
+ {
+ "entropy": 1.8801840543746948,
+ "epoch": 1.0222222222222221,
+ "grad_norm": 2.191689968109131,
+ "learning_rate": 3.97979797979798e-06,
+ "loss": 1.9127342700958252,
+ "mean_token_accuracy": 0.5732169151306152,
+ "num_tokens": 8290304.0,
+ "step": 506
+ },
+ {
+ "entropy": 1.2336418628692627,
+ "epoch": 1.0242424242424242,
+ "grad_norm": 2.0369555950164795,
+ "learning_rate": 3.977777777777778e-06,
+ "loss": 1.2513363361358643,
+ "mean_token_accuracy": 0.6852100491523743,
+ "num_tokens": 8306688.0,
+ "step": 507
+ },
+ {
+ "entropy": 1.8379974365234375,
+ "epoch": 1.0262626262626262,
+ "grad_norm": 2.176361083984375,
+ "learning_rate": 3.975757575757576e-06,
+ "loss": 1.8472888469696045,
+ "mean_token_accuracy": 0.5863458514213562,
+ "num_tokens": 8323072.0,
+ "step": 508
+ },
+ {
+ "entropy": 1.4421337842941284,
+ "epoch": 1.0282828282828282,
+ "grad_norm": 2.305441379547119,
+ "learning_rate": 3.973737373737374e-06,
+ "loss": 1.4835591316223145,
+ "mean_token_accuracy": 0.6530288457870483,
+ "num_tokens": 8339456.0,
+ "step": 509
+ },
+ {
+ "entropy": 1.535184621810913,
+ "epoch": 1.0303030303030303,
+ "grad_norm": 2.0329015254974365,
+ "learning_rate": 3.971717171717172e-06,
+ "loss": 1.547795057296753,
+ "mean_token_accuracy": 0.6268319487571716,
+ "num_tokens": 8355840.0,
+ "step": 510
+ },
+ {
+ "entropy": 1.295592188835144,
+ "epoch": 1.0323232323232323,
+ "grad_norm": 2.0321359634399414,
+ "learning_rate": 3.96969696969697e-06,
+ "loss": 1.2885974645614624,
+ "mean_token_accuracy": 0.681485116481781,
+ "num_tokens": 8372224.0,
+ "step": 511
+ },
+ {
+ "entropy": 1.345553994178772,
+ "epoch": 1.0343434343434343,
+ "grad_norm": 1.9151290655136108,
+ "learning_rate": 3.967676767676768e-06,
+ "loss": 1.3364320993423462,
+ "mean_token_accuracy": 0.6849657893180847,
+ "num_tokens": 8388608.0,
+ "step": 512
+ },
+ {
+ "entropy": 1.3150815963745117,
+ "epoch": 1.0363636363636364,
+ "grad_norm": 1.9379258155822754,
+ "learning_rate": 3.965656565656566e-06,
+ "loss": 1.317258596420288,
+ "mean_token_accuracy": 0.6873473525047302,
+ "num_tokens": 8404992.0,
+ "step": 513
+ },
+ {
+ "entropy": 1.46832275390625,
+ "epoch": 1.0383838383838384,
+ "grad_norm": 2.1009161472320557,
+ "learning_rate": 3.963636363636364e-06,
+ "loss": 1.4794366359710693,
+ "mean_token_accuracy": 0.6513800621032715,
+ "num_tokens": 8421376.0,
+ "step": 514
+ },
+ {
+ "entropy": 1.389290452003479,
+ "epoch": 1.0404040404040404,
+ "grad_norm": 1.8813941478729248,
+ "learning_rate": 3.961616161616162e-06,
+ "loss": 1.3930776119232178,
+ "mean_token_accuracy": 0.6799584627151489,
+ "num_tokens": 8437760.0,
+ "step": 515
+ },
+ {
+ "entropy": 1.2919796705245972,
+ "epoch": 1.0424242424242425,
+ "grad_norm": 2.0765745639801025,
+ "learning_rate": 3.95959595959596e-06,
+ "loss": 1.2943801879882812,
+ "mean_token_accuracy": 0.6929653286933899,
+ "num_tokens": 8454144.0,
+ "step": 516
+ },
+ {
+ "entropy": 1.172440767288208,
+ "epoch": 1.0444444444444445,
+ "grad_norm": 2.126800298690796,
+ "learning_rate": 3.957575757575758e-06,
+ "loss": 1.1923961639404297,
+ "mean_token_accuracy": 0.7098192572593689,
+ "num_tokens": 8470528.0,
+ "step": 517
+ },
+ {
+ "entropy": 1.8239458799362183,
+ "epoch": 1.0464646464646465,
+ "grad_norm": 1.96744966506958,
+ "learning_rate": 3.955555555555556e-06,
+ "loss": 1.840916633605957,
+ "mean_token_accuracy": 0.6033830046653748,
+ "num_tokens": 8486912.0,
+ "step": 518
+ },
+ {
+ "entropy": 1.4117076396942139,
+ "epoch": 1.0484848484848486,
+ "grad_norm": 2.0560126304626465,
+ "learning_rate": 3.953535353535354e-06,
+ "loss": 1.421530842781067,
+ "mean_token_accuracy": 0.6581583023071289,
+ "num_tokens": 8503296.0,
+ "step": 519
+ },
+ {
+ "entropy": 1.7085175514221191,
+ "epoch": 1.0505050505050506,
+ "grad_norm": 2.1386539936065674,
+ "learning_rate": 3.951515151515152e-06,
+ "loss": 1.7334365844726562,
+ "mean_token_accuracy": 0.6282975077629089,
+ "num_tokens": 8519680.0,
+ "step": 520
+ },
+ {
+ "entropy": 1.0607928037643433,
+ "epoch": 1.0525252525252524,
+ "grad_norm": 1.981020212173462,
+ "learning_rate": 3.9494949494949496e-06,
+ "loss": 1.0639690160751343,
+ "mean_token_accuracy": 0.722337543964386,
+ "num_tokens": 8536064.0,
+ "step": 521
+ },
+ {
+ "entropy": 1.2227518558502197,
+ "epoch": 1.0545454545454545,
+ "grad_norm": 1.9116015434265137,
+ "learning_rate": 3.9474747474747474e-06,
+ "loss": 1.2139748334884644,
+ "mean_token_accuracy": 0.704384446144104,
+ "num_tokens": 8552448.0,
+ "step": 522
+ },
+ {
+ "entropy": 1.2655551433563232,
+ "epoch": 1.0565656565656565,
+ "grad_norm": 2.176706075668335,
+ "learning_rate": 3.945454545454545e-06,
+ "loss": 1.2869982719421387,
+ "mean_token_accuracy": 0.678798258304596,
+ "num_tokens": 8568832.0,
+ "step": 523
+ },
+ {
+ "entropy": 1.436476707458496,
+ "epoch": 1.0585858585858585,
+ "grad_norm": 2.034846544265747,
+ "learning_rate": 3.943434343434343e-06,
+ "loss": 1.431445837020874,
+ "mean_token_accuracy": 0.6650586128234863,
+ "num_tokens": 8585216.0,
+ "step": 524
+ },
+ {
+ "entropy": 1.4563549757003784,
+ "epoch": 1.0606060606060606,
+ "grad_norm": 2.0301241874694824,
+ "learning_rate": 3.941414141414142e-06,
+ "loss": 1.468353033065796,
+ "mean_token_accuracy": 0.6550439596176147,
+ "num_tokens": 8601600.0,
+ "step": 525
+ },
+ {
+ "entropy": 1.4636993408203125,
+ "epoch": 1.0626262626262626,
+ "grad_norm": 2.092679023742676,
+ "learning_rate": 3.93939393939394e-06,
+ "loss": 1.4488987922668457,
+ "mean_token_accuracy": 0.652723491191864,
+ "num_tokens": 8617984.0,
+ "step": 526
+ },
+ {
+ "entropy": 1.1217654943466187,
+ "epoch": 1.0646464646464646,
+ "grad_norm": 1.953627347946167,
+ "learning_rate": 3.937373737373738e-06,
+ "loss": 1.1403257846832275,
+ "mean_token_accuracy": 0.7172080874443054,
+ "num_tokens": 8634368.0,
+ "step": 527
+ },
+ {
+ "entropy": 1.7785910367965698,
+ "epoch": 1.0666666666666667,
+ "grad_norm": 1.8496789932250977,
+ "learning_rate": 3.935353535353536e-06,
+ "loss": 1.7961615324020386,
+ "mean_token_accuracy": 0.6036272644996643,
+ "num_tokens": 8650752.0,
+ "step": 528
+ },
+ {
+ "entropy": 1.0998575687408447,
+ "epoch": 1.0686868686868687,
+ "grad_norm": 1.8732962608337402,
+ "learning_rate": 3.9333333333333335e-06,
+ "loss": 1.0960031747817993,
+ "mean_token_accuracy": 0.7223986387252808,
+ "num_tokens": 8667136.0,
+ "step": 529
+ },
+ {
+ "entropy": 1.6707724332809448,
+ "epoch": 1.0707070707070707,
+ "grad_norm": 2.220453977584839,
+ "learning_rate": 3.931313131313131e-06,
+ "loss": 1.6837453842163086,
+ "mean_token_accuracy": 0.6148021221160889,
+ "num_tokens": 8683520.0,
+ "step": 530
+ },
+ {
+ "entropy": 1.6108022928237915,
+ "epoch": 1.0727272727272728,
+ "grad_norm": 2.2464118003845215,
+ "learning_rate": 3.929292929292929e-06,
+ "loss": 1.633517861366272,
+ "mean_token_accuracy": 0.6195651888847351,
+ "num_tokens": 8699904.0,
+ "step": 531
+ },
+ {
+ "entropy": 1.1888173818588257,
+ "epoch": 1.0747474747474748,
+ "grad_norm": 1.8803986310958862,
+ "learning_rate": 3.927272727272727e-06,
+ "loss": 1.1904420852661133,
+ "mean_token_accuracy": 0.7134220600128174,
+ "num_tokens": 8716288.0,
+ "step": 532
+ },
+ {
+ "entropy": 1.0656445026397705,
+ "epoch": 1.0767676767676768,
+ "grad_norm": 2.038243532180786,
+ "learning_rate": 3.925252525252525e-06,
+ "loss": 1.0505216121673584,
+ "mean_token_accuracy": 0.7284440398216248,
+ "num_tokens": 8732672.0,
+ "step": 533
+ },
+ {
+ "entropy": 1.1473655700683594,
+ "epoch": 1.0787878787878789,
+ "grad_norm": 1.981107234954834,
+ "learning_rate": 3.923232323232323e-06,
+ "loss": 1.136179804801941,
+ "mean_token_accuracy": 0.7085368633270264,
+ "num_tokens": 8749056.0,
+ "step": 534
+ },
+ {
+ "entropy": 1.340027928352356,
+ "epoch": 1.0808080808080809,
+ "grad_norm": 2.2797436714172363,
+ "learning_rate": 3.921212121212122e-06,
+ "loss": 1.334214210510254,
+ "mean_token_accuracy": 0.6749511361122131,
+ "num_tokens": 8765440.0,
+ "step": 535
+ },
+ {
+ "entropy": 1.1464864015579224,
+ "epoch": 1.082828282828283,
+ "grad_norm": 1.9428092241287231,
+ "learning_rate": 3.9191919191919196e-06,
+ "loss": 1.1649314165115356,
+ "mean_token_accuracy": 0.72013920545578,
+ "num_tokens": 8781824.0,
+ "step": 536
+ },
+ {
+ "entropy": 1.784925103187561,
+ "epoch": 1.084848484848485,
+ "grad_norm": 2.157468795776367,
+ "learning_rate": 3.9171717171717175e-06,
+ "loss": 1.7725508213043213,
+ "mean_token_accuracy": 0.6014899611473083,
+ "num_tokens": 8798208.0,
+ "step": 537
+ },
+ {
+ "entropy": 1.362166166305542,
+ "epoch": 1.0868686868686868,
+ "grad_norm": 2.109646797180176,
+ "learning_rate": 3.915151515151515e-06,
+ "loss": 1.331969976425171,
+ "mean_token_accuracy": 0.6720200181007385,
+ "num_tokens": 8814592.0,
+ "step": 538
+ },
+ {
+ "entropy": 1.5614176988601685,
+ "epoch": 1.0888888888888888,
+ "grad_norm": 2.1047098636627197,
+ "learning_rate": 3.913131313131314e-06,
+ "loss": 1.534292459487915,
+ "mean_token_accuracy": 0.6446629166603088,
+ "num_tokens": 8830976.0,
+ "step": 539
+ },
+ {
+ "entropy": 1.5551490783691406,
+ "epoch": 1.0909090909090908,
+ "grad_norm": 2.10304594039917,
+ "learning_rate": 3.911111111111112e-06,
+ "loss": 1.5706363916397095,
+ "mean_token_accuracy": 0.6322056651115417,
+ "num_tokens": 8847360.0,
+ "step": 540
+ },
+ {
+ "entropy": 1.7433172464370728,
+ "epoch": 1.0929292929292929,
+ "grad_norm": 1.8621257543563843,
+ "learning_rate": 3.90909090909091e-06,
+ "loss": 1.7367552518844604,
+ "mean_token_accuracy": 0.6162066459655762,
+ "num_tokens": 8863744.0,
+ "step": 541
+ },
+ {
+ "entropy": 1.4073128700256348,
+ "epoch": 1.094949494949495,
+ "grad_norm": 1.939588189125061,
+ "learning_rate": 3.907070707070708e-06,
+ "loss": 1.3844857215881348,
+ "mean_token_accuracy": 0.6634709239006042,
+ "num_tokens": 8880128.0,
+ "step": 542
+ },
+ {
+ "entropy": 1.479836344718933,
+ "epoch": 1.096969696969697,
+ "grad_norm": 2.06921648979187,
+ "learning_rate": 3.905050505050506e-06,
+ "loss": 1.4933744668960571,
+ "mean_token_accuracy": 0.6510136723518372,
+ "num_tokens": 8896512.0,
+ "step": 543
+ },
+ {
+ "entropy": 1.3934576511383057,
+ "epoch": 1.098989898989899,
+ "grad_norm": 2.0697920322418213,
+ "learning_rate": 3.9030303030303035e-06,
+ "loss": 1.3886666297912598,
+ "mean_token_accuracy": 0.6775158643722534,
+ "num_tokens": 8912896.0,
+ "step": 544
+ },
+ {
+ "entropy": 1.5223709344863892,
+ "epoch": 1.101010101010101,
+ "grad_norm": 2.1627066135406494,
+ "learning_rate": 3.901010101010101e-06,
+ "loss": 1.5601061582565308,
+ "mean_token_accuracy": 0.6392892003059387,
+ "num_tokens": 8929280.0,
+ "step": 545
+ },
+ {
+ "entropy": 1.4462933540344238,
+ "epoch": 1.103030303030303,
+ "grad_norm": 1.9871046543121338,
+ "learning_rate": 3.898989898989899e-06,
+ "loss": 1.463792085647583,
+ "mean_token_accuracy": 0.6518075466156006,
+ "num_tokens": 8945664.0,
+ "step": 546
+ },
+ {
+ "entropy": 1.325921654701233,
+ "epoch": 1.105050505050505,
+ "grad_norm": 1.9081087112426758,
+ "learning_rate": 3.896969696969697e-06,
+ "loss": 1.3303362131118774,
+ "mean_token_accuracy": 0.7009648084640503,
+ "num_tokens": 8962048.0,
+ "step": 547
+ },
+ {
+ "entropy": 1.415048360824585,
+ "epoch": 1.107070707070707,
+ "grad_norm": 2.3306756019592285,
+ "learning_rate": 3.894949494949495e-06,
+ "loss": 1.4139145612716675,
+ "mean_token_accuracy": 0.6849047541618347,
+ "num_tokens": 8978432.0,
+ "step": 548
+ },
+ {
+ "entropy": 1.5863295793533325,
+ "epoch": 1.1090909090909091,
+ "grad_norm": 1.993159294128418,
+ "learning_rate": 3.892929292929293e-06,
+ "loss": 1.6010534763336182,
+ "mean_token_accuracy": 0.6424035429954529,
+ "num_tokens": 8994816.0,
+ "step": 549
+ },
+ {
+ "entropy": 1.5526721477508545,
+ "epoch": 1.1111111111111112,
+ "grad_norm": 2.0528876781463623,
+ "learning_rate": 3.890909090909092e-06,
+ "loss": 1.5465538501739502,
+ "mean_token_accuracy": 0.659807026386261,
+ "num_tokens": 9011200.0,
+ "step": 550
+ },
+ {
+ "epoch": 1.1111111111111112,
+ "eval_entropy": 1.4479231247978825,
+ "eval_loss": 1.5459802150726318,
+ "eval_mean_token_accuracy": 0.6425906530311031,
+ "eval_num_tokens": 9011200.0,
+ "eval_runtime": 43.7828,
+ "eval_samples_per_second": 22.612,
+ "eval_steps_per_second": 2.832,
+ "step": 550
+ },
+ {
+ "entropy": 1.4322376251220703,
+ "epoch": 1.1131313131313132,
+ "grad_norm": 1.933549404144287,
+ "learning_rate": 3.88888888888889e-06,
+ "loss": 1.4371025562286377,
+ "mean_token_accuracy": 0.670676589012146,
+ "num_tokens": 9027584.0,
+ "step": 551
+ },
+ {
+ "entropy": 1.5050671100616455,
+ "epoch": 1.1151515151515152,
+ "grad_norm": 1.9736099243164062,
+ "learning_rate": 3.8868686868686875e-06,
+ "loss": 1.5362370014190674,
+ "mean_token_accuracy": 0.651624321937561,
+ "num_tokens": 9043968.0,
+ "step": 552
+ },
+ {
+ "entropy": 1.4335887432098389,
+ "epoch": 1.1171717171717173,
+ "grad_norm": 1.8896421194076538,
+ "learning_rate": 3.884848484848485e-06,
+ "loss": 1.4439888000488281,
+ "mean_token_accuracy": 0.670615553855896,
+ "num_tokens": 9060352.0,
+ "step": 553
+ },
+ {
+ "entropy": 1.5979355573654175,
+ "epoch": 1.1191919191919193,
+ "grad_norm": 2.1710526943206787,
+ "learning_rate": 3.882828282828283e-06,
+ "loss": 1.629098653793335,
+ "mean_token_accuracy": 0.6226184368133545,
+ "num_tokens": 9076736.0,
+ "step": 554
+ },
+ {
+ "entropy": 1.2901698350906372,
+ "epoch": 1.121212121212121,
+ "grad_norm": 2.129443407058716,
+ "learning_rate": 3.880808080808081e-06,
+ "loss": 1.2824913263320923,
+ "mean_token_accuracy": 0.6816682815551758,
+ "num_tokens": 9093120.0,
+ "step": 555
+ },
+ {
+ "entropy": 1.403212308883667,
+ "epoch": 1.1232323232323231,
+ "grad_norm": 1.883240818977356,
+ "learning_rate": 3.878787878787879e-06,
+ "loss": 1.3960213661193848,
+ "mean_token_accuracy": 0.6651197075843811,
+ "num_tokens": 9109504.0,
+ "step": 556
+ },
+ {
+ "entropy": 1.3959479331970215,
+ "epoch": 1.1252525252525252,
+ "grad_norm": 2.1145691871643066,
+ "learning_rate": 3.876767676767677e-06,
+ "loss": 1.4046590328216553,
+ "mean_token_accuracy": 0.6607230305671692,
+ "num_tokens": 9125888.0,
+ "step": 557
+ },
+ {
+ "entropy": 1.0899873971939087,
+ "epoch": 1.1272727272727272,
+ "grad_norm": 1.8862192630767822,
+ "learning_rate": 3.874747474747475e-06,
+ "loss": 1.0965509414672852,
+ "mean_token_accuracy": 0.7245969772338867,
+ "num_tokens": 9142272.0,
+ "step": 558
+ },
+ {
+ "entropy": 1.0380184650421143,
+ "epoch": 1.1292929292929292,
+ "grad_norm": 1.946702241897583,
+ "learning_rate": 3.872727272727273e-06,
+ "loss": 1.0539875030517578,
+ "mean_token_accuracy": 0.7423058152198792,
+ "num_tokens": 9158656.0,
+ "step": 559
+ },
+ {
+ "entropy": 1.1173628568649292,
+ "epoch": 1.1313131313131313,
+ "grad_norm": 2.002847909927368,
+ "learning_rate": 3.8707070707070706e-06,
+ "loss": 1.128816843032837,
+ "mean_token_accuracy": 0.7112848162651062,
+ "num_tokens": 9175040.0,
+ "step": 560
+ },
+ {
+ "entropy": 1.6536206007003784,
+ "epoch": 1.1333333333333333,
+ "grad_norm": 2.212761640548706,
+ "learning_rate": 3.868686868686869e-06,
+ "loss": 1.6286437511444092,
+ "mean_token_accuracy": 0.6277479529380798,
+ "num_tokens": 9191424.0,
+ "step": 561
+ },
+ {
+ "entropy": 1.7187970876693726,
+ "epoch": 1.1353535353535353,
+ "grad_norm": 2.207310199737549,
+ "learning_rate": 3.866666666666667e-06,
+ "loss": 1.7072829008102417,
+ "mean_token_accuracy": 0.6180996298789978,
+ "num_tokens": 9207808.0,
+ "step": 562
+ },
+ {
+ "entropy": 1.7057682275772095,
+ "epoch": 1.1373737373737374,
+ "grad_norm": 2.1582961082458496,
+ "learning_rate": 3.864646464646465e-06,
+ "loss": 1.7216498851776123,
+ "mean_token_accuracy": 0.6105886697769165,
+ "num_tokens": 9224192.0,
+ "step": 563
+ },
+ {
+ "entropy": 1.8016951084136963,
+ "epoch": 1.1393939393939394,
+ "grad_norm": 2.035249948501587,
+ "learning_rate": 3.862626262626263e-06,
+ "loss": 1.8101240396499634,
+ "mean_token_accuracy": 0.6199315786361694,
+ "num_tokens": 9240576.0,
+ "step": 564
+ },
+ {
+ "entropy": 1.094889760017395,
+ "epoch": 1.1414141414141414,
+ "grad_norm": 2.119861364364624,
+ "learning_rate": 3.860606060606061e-06,
+ "loss": 1.0990344285964966,
+ "mean_token_accuracy": 0.7071323990821838,
+ "num_tokens": 9256960.0,
+ "step": 565
+ },
+ {
+ "entropy": 1.537480115890503,
+ "epoch": 1.1434343434343435,
+ "grad_norm": 2.167386770248413,
+ "learning_rate": 3.858585858585859e-06,
+ "loss": 1.5262541770935059,
+ "mean_token_accuracy": 0.6343429684638977,
+ "num_tokens": 9273344.0,
+ "step": 566
+ },
+ {
+ "entropy": 1.2940728664398193,
+ "epoch": 1.1454545454545455,
+ "grad_norm": 1.941097617149353,
+ "learning_rate": 3.856565656565657e-06,
+ "loss": 1.3145123720169067,
+ "mean_token_accuracy": 0.6822789311408997,
+ "num_tokens": 9289728.0,
+ "step": 567
+ },
+ {
+ "entropy": 1.3154000043869019,
+ "epoch": 1.1474747474747475,
+ "grad_norm": 2.088576078414917,
+ "learning_rate": 3.8545454545454545e-06,
+ "loss": 1.3154902458190918,
+ "mean_token_accuracy": 0.6703101992607117,
+ "num_tokens": 9306112.0,
+ "step": 568
+ },
+ {
+ "entropy": 1.3576427698135376,
+ "epoch": 1.1494949494949496,
+ "grad_norm": 1.9895246028900146,
+ "learning_rate": 3.852525252525252e-06,
+ "loss": 1.3295378684997559,
+ "mean_token_accuracy": 0.6762335300445557,
+ "num_tokens": 9322496.0,
+ "step": 569
+ },
+ {
+ "entropy": 1.6067878007888794,
+ "epoch": 1.1515151515151516,
+ "grad_norm": 2.021191358566284,
+ "learning_rate": 3.85050505050505e-06,
+ "loss": 1.6273870468139648,
+ "mean_token_accuracy": 0.6329995393753052,
+ "num_tokens": 9338880.0,
+ "step": 570
+ },
+ {
+ "entropy": 1.4831246137619019,
+ "epoch": 1.1535353535353536,
+ "grad_norm": 2.114612340927124,
+ "learning_rate": 3.848484848484848e-06,
+ "loss": 1.4925904273986816,
+ "mean_token_accuracy": 0.6566316485404968,
+ "num_tokens": 9355264.0,
+ "step": 571
+ },
+ {
+ "entropy": 1.4476258754730225,
+ "epoch": 1.1555555555555554,
+ "grad_norm": 2.3435230255126953,
+ "learning_rate": 3.846464646464647e-06,
+ "loss": 1.438629388809204,
+ "mean_token_accuracy": 0.656020998954773,
+ "num_tokens": 9371648.0,
+ "step": 572
+ },
+ {
+ "entropy": 1.7930315732955933,
+ "epoch": 1.1575757575757575,
+ "grad_norm": 2.1700010299682617,
+ "learning_rate": 3.844444444444445e-06,
+ "loss": 1.786928653717041,
+ "mean_token_accuracy": 0.6028333902359009,
+ "num_tokens": 9388032.0,
+ "step": 573
+ },
+ {
+ "entropy": 1.3800512552261353,
+ "epoch": 1.1595959595959595,
+ "grad_norm": 2.2036688327789307,
+ "learning_rate": 3.842424242424243e-06,
+ "loss": 1.3767224550247192,
+ "mean_token_accuracy": 0.6692110300064087,
+ "num_tokens": 9404416.0,
+ "step": 574
+ },
+ {
+ "entropy": 0.8562329411506653,
+ "epoch": 1.1616161616161615,
+ "grad_norm": 1.765388011932373,
+ "learning_rate": 3.840404040404041e-06,
+ "loss": 0.8366047143936157,
+ "mean_token_accuracy": 0.783707857131958,
+ "num_tokens": 9420800.0,
+ "step": 575
+ },
+ {
+ "entropy": 1.4267486333847046,
+ "epoch": 1.1636363636363636,
+ "grad_norm": 2.0768239498138428,
+ "learning_rate": 3.8383838383838385e-06,
+ "loss": 1.4165449142456055,
+ "mean_token_accuracy": 0.656020998954773,
+ "num_tokens": 9437184.0,
+ "step": 576
+ },
+ {
+ "entropy": 1.4123388528823853,
+ "epoch": 1.1656565656565656,
+ "grad_norm": 2.028716802597046,
+ "learning_rate": 3.836363636363636e-06,
+ "loss": 1.420403003692627,
+ "mean_token_accuracy": 0.6729360222816467,
+ "num_tokens": 9453568.0,
+ "step": 577
+ },
+ {
+ "entropy": 1.1549286842346191,
+ "epoch": 1.1676767676767676,
+ "grad_norm": 2.1142919063568115,
+ "learning_rate": 3.834343434343435e-06,
+ "loss": 1.1487064361572266,
+ "mean_token_accuracy": 0.7040180563926697,
+ "num_tokens": 9469952.0,
+ "step": 578
+ },
+ {
+ "entropy": 1.4779953956604004,
+ "epoch": 1.1696969696969697,
+ "grad_norm": 2.166046142578125,
+ "learning_rate": 3.832323232323233e-06,
+ "loss": 1.4788683652877808,
+ "mean_token_accuracy": 0.6436858773231506,
+ "num_tokens": 9486336.0,
+ "step": 579
+ },
+ {
+ "entropy": 1.568039059638977,
+ "epoch": 1.1717171717171717,
+ "grad_norm": 2.067704200744629,
+ "learning_rate": 3.830303030303031e-06,
+ "loss": 1.5830929279327393,
+ "mean_token_accuracy": 0.6461895704269409,
+ "num_tokens": 9502720.0,
+ "step": 580
+ },
+ {
+ "entropy": 1.1785298585891724,
+ "epoch": 1.1737373737373737,
+ "grad_norm": 1.9789953231811523,
+ "learning_rate": 3.828282828282829e-06,
+ "loss": 1.2083481550216675,
+ "mean_token_accuracy": 0.7120786309242249,
+ "num_tokens": 9519104.0,
+ "step": 581
+ },
+ {
+ "entropy": 1.3783847093582153,
+ "epoch": 1.1757575757575758,
+ "grad_norm": 2.1503992080688477,
+ "learning_rate": 3.826262626262627e-06,
+ "loss": 1.3721052408218384,
+ "mean_token_accuracy": 0.6669516563415527,
+ "num_tokens": 9535488.0,
+ "step": 582
+ },
+ {
+ "entropy": 1.120093584060669,
+ "epoch": 1.1777777777777778,
+ "grad_norm": 1.9822942018508911,
+ "learning_rate": 3.8242424242424245e-06,
+ "loss": 1.1215628385543823,
+ "mean_token_accuracy": 0.7122618556022644,
+ "num_tokens": 9551872.0,
+ "step": 583
+ },
+ {
+ "entropy": 1.4621506929397583,
+ "epoch": 1.1797979797979798,
+ "grad_norm": 2.159489631652832,
+ "learning_rate": 3.8222222222222224e-06,
+ "loss": 1.4614722728729248,
+ "mean_token_accuracy": 0.6535173654556274,
+ "num_tokens": 9568256.0,
+ "step": 584
+ },
+ {
+ "entropy": 1.015797734260559,
+ "epoch": 1.1818181818181819,
+ "grad_norm": 1.846848726272583,
+ "learning_rate": 3.82020202020202e-06,
+ "loss": 1.0082862377166748,
+ "mean_token_accuracy": 0.7394968271255493,
+ "num_tokens": 9584640.0,
+ "step": 585
+ },
+ {
+ "entropy": 1.364890217781067,
+ "epoch": 1.183838383838384,
+ "grad_norm": 2.175546646118164,
+ "learning_rate": 3.818181818181819e-06,
+ "loss": 1.3496818542480469,
+ "mean_token_accuracy": 0.6665241718292236,
+ "num_tokens": 9601024.0,
+ "step": 586
+ },
+ {
+ "entropy": 1.6056870222091675,
+ "epoch": 1.185858585858586,
+ "grad_norm": 2.1069748401641846,
+ "learning_rate": 3.816161616161617e-06,
+ "loss": 1.6048622131347656,
+ "mean_token_accuracy": 0.642892062664032,
+ "num_tokens": 9617408.0,
+ "step": 587
+ },
+ {
+ "entropy": 1.415107250213623,
+ "epoch": 1.187878787878788,
+ "grad_norm": 2.184544324874878,
+ "learning_rate": 3.8141414141414144e-06,
+ "loss": 1.4305826425552368,
+ "mean_token_accuracy": 0.6619443297386169,
+ "num_tokens": 9633792.0,
+ "step": 588
+ },
+ {
+ "entropy": 1.6554296016693115,
+ "epoch": 1.1898989898989898,
+ "grad_norm": 2.109793186187744,
+ "learning_rate": 3.8121212121212127e-06,
+ "loss": 1.6850776672363281,
+ "mean_token_accuracy": 0.6116267442703247,
+ "num_tokens": 9650176.0,
+ "step": 589
+ },
+ {
+ "entropy": 1.681472659111023,
+ "epoch": 1.1919191919191918,
+ "grad_norm": 2.3179714679718018,
+ "learning_rate": 3.8101010101010106e-06,
+ "loss": 1.699328064918518,
+ "mean_token_accuracy": 0.6105275750160217,
+ "num_tokens": 9666560.0,
+ "step": 590
+ },
+ {
+ "entropy": 1.113366961479187,
+ "epoch": 1.1939393939393939,
+ "grad_norm": 1.9885324239730835,
+ "learning_rate": 3.8080808080808085e-06,
+ "loss": 1.1357836723327637,
+ "mean_token_accuracy": 0.7157425284385681,
+ "num_tokens": 9682944.0,
+ "step": 591
+ },
+ {
+ "entropy": 1.1010088920593262,
+ "epoch": 1.195959595959596,
+ "grad_norm": 1.9745922088623047,
+ "learning_rate": 3.8060606060606064e-06,
+ "loss": 1.094293475151062,
+ "mean_token_accuracy": 0.7214215993881226,
+ "num_tokens": 9699328.0,
+ "step": 592
+ },
+ {
+ "entropy": 1.562911033630371,
+ "epoch": 1.197979797979798,
+ "grad_norm": 2.0526134967803955,
+ "learning_rate": 3.8040404040404043e-06,
+ "loss": 1.5747783184051514,
+ "mean_token_accuracy": 0.6342818737030029,
+ "num_tokens": 9715712.0,
+ "step": 593
+ },
+ {
+ "entropy": 1.6541608572006226,
+ "epoch": 1.2,
+ "grad_norm": 2.1268227100372314,
+ "learning_rate": 3.8020202020202026e-06,
+ "loss": 1.696983814239502,
+ "mean_token_accuracy": 0.6127259135246277,
+ "num_tokens": 9732096.0,
+ "step": 594
+ },
+ {
+ "entropy": 1.5259482860565186,
+ "epoch": 1.202020202020202,
+ "grad_norm": 2.240692377090454,
+ "learning_rate": 3.8000000000000005e-06,
+ "loss": 1.5582221746444702,
+ "mean_token_accuracy": 0.644052267074585,
+ "num_tokens": 9748480.0,
+ "step": 595
+ },
+ {
+ "entropy": 1.455372929573059,
+ "epoch": 1.204040404040404,
+ "grad_norm": 2.0934698581695557,
+ "learning_rate": 3.7979797979797984e-06,
+ "loss": 1.464172124862671,
+ "mean_token_accuracy": 0.6555935740470886,
+ "num_tokens": 9764864.0,
+ "step": 596
+ },
+ {
+ "entropy": 1.5074836015701294,
+ "epoch": 1.206060606060606,
+ "grad_norm": 1.9779568910598755,
+ "learning_rate": 3.7959595959595962e-06,
+ "loss": 1.5280466079711914,
+ "mean_token_accuracy": 0.6460063457489014,
+ "num_tokens": 9781248.0,
+ "step": 597
+ },
+ {
+ "entropy": 1.3286548852920532,
+ "epoch": 1.208080808080808,
+ "grad_norm": 1.9357717037200928,
+ "learning_rate": 3.793939393939394e-06,
+ "loss": 1.3448848724365234,
+ "mean_token_accuracy": 0.6816072463989258,
+ "num_tokens": 9797632.0,
+ "step": 598
+ },
+ {
+ "entropy": 1.5161375999450684,
+ "epoch": 1.2101010101010101,
+ "grad_norm": 2.224217414855957,
+ "learning_rate": 3.791919191919192e-06,
+ "loss": 1.5168235301971436,
+ "mean_token_accuracy": 0.6453346610069275,
+ "num_tokens": 9814016.0,
+ "step": 599
+ },
+ {
+ "entropy": 1.3337587118148804,
+ "epoch": 1.2121212121212122,
+ "grad_norm": 1.9308290481567383,
+ "learning_rate": 3.7898989898989903e-06,
+ "loss": 1.3429791927337646,
+ "mean_token_accuracy": 0.6827064156532288,
+ "num_tokens": 9830400.0,
+ "step": 600
+ },
+ {
+ "epoch": 1.2121212121212122,
+ "eval_entropy": 1.4592116455877981,
+ "eval_loss": 1.5424447059631348,
+ "eval_mean_token_accuracy": 0.6431668299821115,
+ "eval_num_tokens": 9830400.0,
+ "eval_runtime": 43.8289,
+ "eval_samples_per_second": 22.588,
+ "eval_steps_per_second": 2.829,
+ "step": 600
+ },
+ {
+ "entropy": 1.6697322130203247,
+ "epoch": 1.2141414141414142,
+ "grad_norm": 1.9462894201278687,
+ "learning_rate": 3.7878787878787882e-06,
+ "loss": 1.6434354782104492,
+ "mean_token_accuracy": 0.635808527469635,
+ "num_tokens": 9846784.0,
+ "step": 601
+ },
+ {
+ "entropy": 1.427184820175171,
+ "epoch": 1.2161616161616162,
+ "grad_norm": 2.196017026901245,
+ "learning_rate": 3.785858585858586e-06,
+ "loss": 1.4024749994277954,
+ "mean_token_accuracy": 0.6720200181007385,
+ "num_tokens": 9863168.0,
+ "step": 602
+ },
+ {
+ "entropy": 1.4623222351074219,
+ "epoch": 1.2181818181818183,
+ "grad_norm": 2.160585403442383,
+ "learning_rate": 3.783838383838384e-06,
+ "loss": 1.4455416202545166,
+ "mean_token_accuracy": 0.6557767391204834,
+ "num_tokens": 9879552.0,
+ "step": 603
+ },
+ {
+ "entropy": 1.348331093788147,
+ "epoch": 1.2202020202020203,
+ "grad_norm": 2.079458713531494,
+ "learning_rate": 3.781818181818182e-06,
+ "loss": 1.3672473430633545,
+ "mean_token_accuracy": 0.6834391951560974,
+ "num_tokens": 9895936.0,
+ "step": 604
+ },
+ {
+ "entropy": 1.362277865409851,
+ "epoch": 1.2222222222222223,
+ "grad_norm": 2.10718035697937,
+ "learning_rate": 3.77979797979798e-06,
+ "loss": 1.3567216396331787,
+ "mean_token_accuracy": 0.6659135222434998,
+ "num_tokens": 9912320.0,
+ "step": 605
+ },
+ {
+ "entropy": 1.2743226289749146,
+ "epoch": 1.2242424242424241,
+ "grad_norm": 2.116549491882324,
+ "learning_rate": 3.777777777777778e-06,
+ "loss": 1.303470253944397,
+ "mean_token_accuracy": 0.6791035532951355,
+ "num_tokens": 9928704.0,
+ "step": 606
+ },
+ {
+ "entropy": 1.079649567604065,
+ "epoch": 1.2262626262626264,
+ "grad_norm": 1.8429855108261108,
+ "learning_rate": 3.775757575757576e-06,
+ "loss": 1.0616915225982666,
+ "mean_token_accuracy": 0.7321690320968628,
+ "num_tokens": 9945088.0,
+ "step": 607
+ },
+ {
+ "entropy": 1.363136649131775,
+ "epoch": 1.2282828282828282,
+ "grad_norm": 2.171295166015625,
+ "learning_rate": 3.773737373737374e-06,
+ "loss": 1.3525331020355225,
+ "mean_token_accuracy": 0.6732413172721863,
+ "num_tokens": 9961472.0,
+ "step": 608
+ },
+ {
+ "entropy": 1.324924349784851,
+ "epoch": 1.2303030303030302,
+ "grad_norm": 2.0872693061828613,
+ "learning_rate": 3.7717171717171717e-06,
+ "loss": 1.3282644748687744,
+ "mean_token_accuracy": 0.674462616443634,
+ "num_tokens": 9977856.0,
+ "step": 609
+ },
+ {
+ "entropy": 1.0672967433929443,
+ "epoch": 1.2323232323232323,
+ "grad_norm": 2.0819947719573975,
+ "learning_rate": 3.76969696969697e-06,
+ "loss": 1.0929367542266846,
+ "mean_token_accuracy": 0.715254008769989,
+ "num_tokens": 9994240.0,
+ "step": 610
+ },
+ {
+ "entropy": 1.4465869665145874,
+ "epoch": 1.2343434343434343,
+ "grad_norm": 2.1786868572235107,
+ "learning_rate": 3.767676767676768e-06,
+ "loss": 1.436307668685913,
+ "mean_token_accuracy": 0.6642037034034729,
+ "num_tokens": 10010624.0,
+ "step": 611
+ },
+ {
+ "entropy": 1.6424144506454468,
+ "epoch": 1.2363636363636363,
+ "grad_norm": 2.2582032680511475,
+ "learning_rate": 3.765656565656566e-06,
+ "loss": 1.6621832847595215,
+ "mean_token_accuracy": 0.6218246221542358,
+ "num_tokens": 10027008.0,
+ "step": 612
+ },
+ {
+ "entropy": 1.546712875366211,
+ "epoch": 1.2383838383838384,
+ "grad_norm": 2.1685879230499268,
+ "learning_rate": 3.7636363636363637e-06,
+ "loss": 1.5617464780807495,
+ "mean_token_accuracy": 0.6325110197067261,
+ "num_tokens": 10043392.0,
+ "step": 613
+ },
+ {
+ "entropy": 1.5758557319641113,
+ "epoch": 1.2404040404040404,
+ "grad_norm": 2.0911128520965576,
+ "learning_rate": 3.7616161616161616e-06,
+ "loss": 1.5830271244049072,
+ "mean_token_accuracy": 0.6349536180496216,
+ "num_tokens": 10059776.0,
+ "step": 614
+ },
+ {
+ "entropy": 1.5223995447158813,
+ "epoch": 1.2424242424242424,
+ "grad_norm": 2.0333359241485596,
+ "learning_rate": 3.7595959595959595e-06,
+ "loss": 1.5132982730865479,
+ "mean_token_accuracy": 0.6518075466156006,
+ "num_tokens": 10076160.0,
+ "step": 615
+ },
+ {
+ "entropy": 1.0950185060501099,
+ "epoch": 1.2444444444444445,
+ "grad_norm": 2.0048129558563232,
+ "learning_rate": 3.757575757575758e-06,
+ "loss": 1.0901896953582764,
+ "mean_token_accuracy": 0.723375678062439,
+ "num_tokens": 10092544.0,
+ "step": 616
+ },
+ {
+ "entropy": 1.340641736984253,
+ "epoch": 1.2464646464646465,
+ "grad_norm": 2.079256534576416,
+ "learning_rate": 3.7555555555555557e-06,
+ "loss": 1.3583415746688843,
+ "mean_token_accuracy": 0.671775758266449,
+ "num_tokens": 10108928.0,
+ "step": 617
+ },
+ {
+ "entropy": 1.1836222410202026,
+ "epoch": 1.2484848484848485,
+ "grad_norm": 1.9266347885131836,
+ "learning_rate": 3.7535353535353536e-06,
+ "loss": 1.171565294265747,
+ "mean_token_accuracy": 0.7046287059783936,
+ "num_tokens": 10125312.0,
+ "step": 618
+ },
+ {
+ "entropy": 1.6007641553878784,
+ "epoch": 1.2505050505050506,
+ "grad_norm": 2.25252103805542,
+ "learning_rate": 3.7515151515151515e-06,
+ "loss": 1.5955660343170166,
+ "mean_token_accuracy": 0.6306790709495544,
+ "num_tokens": 10141696.0,
+ "step": 619
+ },
+ {
+ "entropy": 1.8159959316253662,
+ "epoch": 1.2525252525252526,
+ "grad_norm": 2.255959987640381,
+ "learning_rate": 3.74949494949495e-06,
+ "loss": 1.8223116397857666,
+ "mean_token_accuracy": 0.5867122411727905,
+ "num_tokens": 10158080.0,
+ "step": 620
+ },
+ {
+ "entropy": 1.6469950675964355,
+ "epoch": 1.2545454545454544,
+ "grad_norm": 1.9183777570724487,
+ "learning_rate": 3.747474747474748e-06,
+ "loss": 1.6623587608337402,
+ "mean_token_accuracy": 0.6283586025238037,
+ "num_tokens": 10174464.0,
+ "step": 621
+ },
+ {
+ "entropy": 1.3507136106491089,
+ "epoch": 1.2565656565656567,
+ "grad_norm": 1.9393805265426636,
+ "learning_rate": 3.745454545454546e-06,
+ "loss": 1.3371413946151733,
+ "mean_token_accuracy": 0.6808744668960571,
+ "num_tokens": 10190848.0,
+ "step": 622
+ },
+ {
+ "entropy": 1.5176761150360107,
+ "epoch": 1.2585858585858585,
+ "grad_norm": 2.0721757411956787,
+ "learning_rate": 3.743434343434344e-06,
+ "loss": 1.5010406970977783,
+ "mean_token_accuracy": 0.6603566408157349,
+ "num_tokens": 10207232.0,
+ "step": 623
+ },
+ {
+ "entropy": 1.1409586668014526,
+ "epoch": 1.2606060606060607,
+ "grad_norm": 1.90805983543396,
+ "learning_rate": 3.7414141414141418e-06,
+ "loss": 1.1339551210403442,
+ "mean_token_accuracy": 0.7151318788528442,
+ "num_tokens": 10223616.0,
+ "step": 624
+ },
+ {
+ "entropy": 1.2258127927780151,
+ "epoch": 1.2626262626262625,
+ "grad_norm": 1.9276432991027832,
+ "learning_rate": 3.73939393939394e-06,
+ "loss": 1.2232249975204468,
+ "mean_token_accuracy": 0.7053614854812622,
+ "num_tokens": 10240000.0,
+ "step": 625
+ },
+ {
+ "entropy": 1.236358642578125,
+ "epoch": 1.2646464646464646,
+ "grad_norm": 2.0792832374572754,
+ "learning_rate": 3.737373737373738e-06,
+ "loss": 1.2570738792419434,
+ "mean_token_accuracy": 0.6869809627532959,
+ "num_tokens": 10256384.0,
+ "step": 626
+ },
+ {
+ "entropy": 1.0995185375213623,
+ "epoch": 1.2666666666666666,
+ "grad_norm": 2.035024881362915,
+ "learning_rate": 3.735353535353536e-06,
+ "loss": 1.10158109664917,
+ "mean_token_accuracy": 0.7248412370681763,
+ "num_tokens": 10272768.0,
+ "step": 627
+ },
+ {
+ "entropy": 1.8980706930160522,
+ "epoch": 1.2686868686868686,
+ "grad_norm": 2.040414333343506,
+ "learning_rate": 3.7333333333333337e-06,
+ "loss": 1.9260807037353516,
+ "mean_token_accuracy": 0.5953834652900696,
+ "num_tokens": 10289152.0,
+ "step": 628
+ },
+ {
+ "entropy": 1.4265168905258179,
+ "epoch": 1.2707070707070707,
+ "grad_norm": 2.418179750442505,
+ "learning_rate": 3.7313131313131316e-06,
+ "loss": 1.4415385723114014,
+ "mean_token_accuracy": 0.6535784006118774,
+ "num_tokens": 10305536.0,
+ "step": 629
+ },
+ {
+ "entropy": 1.217471957206726,
+ "epoch": 1.2727272727272727,
+ "grad_norm": 2.072441577911377,
+ "learning_rate": 3.72929292929293e-06,
+ "loss": 1.2361960411071777,
+ "mean_token_accuracy": 0.6947972774505615,
+ "num_tokens": 10321920.0,
+ "step": 630
+ },
+ {
+ "entropy": 1.0714137554168701,
+ "epoch": 1.2747474747474747,
+ "grad_norm": 3.0428647994995117,
+ "learning_rate": 3.727272727272728e-06,
+ "loss": 1.112711787223816,
+ "mean_token_accuracy": 0.7243527173995972,
+ "num_tokens": 10338304.0,
+ "step": 631
+ },
+ {
+ "entropy": 1.5873851776123047,
+ "epoch": 1.2767676767676768,
+ "grad_norm": 1.945098638534546,
+ "learning_rate": 3.7252525252525257e-06,
+ "loss": 1.595362663269043,
+ "mean_token_accuracy": 0.639594554901123,
+ "num_tokens": 10354688.0,
+ "step": 632
+ },
+ {
+ "entropy": 1.9688408374786377,
+ "epoch": 1.2787878787878788,
+ "grad_norm": 2.0450353622436523,
+ "learning_rate": 3.7232323232323236e-06,
+ "loss": 1.9989259243011475,
+ "mean_token_accuracy": 0.5685759782791138,
+ "num_tokens": 10371072.0,
+ "step": 633
+ },
+ {
+ "entropy": 1.2927459478378296,
+ "epoch": 1.2808080808080808,
+ "grad_norm": 1.959108591079712,
+ "learning_rate": 3.7212121212121215e-06,
+ "loss": 1.3025646209716797,
+ "mean_token_accuracy": 0.6853932738304138,
+ "num_tokens": 10387456.0,
+ "step": 634
+ },
+ {
+ "entropy": 1.6645994186401367,
+ "epoch": 1.2828282828282829,
+ "grad_norm": 2.041923999786377,
+ "learning_rate": 3.7191919191919194e-06,
+ "loss": 1.6959854364395142,
+ "mean_token_accuracy": 0.6319003701210022,
+ "num_tokens": 10403840.0,
+ "step": 635
+ },
+ {
+ "entropy": 1.401423692703247,
+ "epoch": 1.284848484848485,
+ "grad_norm": 2.040456533432007,
+ "learning_rate": 3.7171717171717177e-06,
+ "loss": 1.3859784603118896,
+ "mean_token_accuracy": 0.6740962266921997,
+ "num_tokens": 10420224.0,
+ "step": 636
+ },
+ {
+ "entropy": 1.0597256422042847,
+ "epoch": 1.286868686868687,
+ "grad_norm": 2.053690195083618,
+ "learning_rate": 3.7151515151515156e-06,
+ "loss": 1.052672266960144,
+ "mean_token_accuracy": 0.7290546894073486,
+ "num_tokens": 10436608.0,
+ "step": 637
+ },
+ {
+ "entropy": 1.280874252319336,
+ "epoch": 1.2888888888888888,
+ "grad_norm": 2.0566959381103516,
+ "learning_rate": 3.7131313131313135e-06,
+ "loss": 1.2883412837982178,
+ "mean_token_accuracy": 0.6822178959846497,
+ "num_tokens": 10452992.0,
+ "step": 638
+ },
+ {
+ "entropy": 1.299896240234375,
+ "epoch": 1.290909090909091,
+ "grad_norm": 1.8896070718765259,
+ "learning_rate": 3.7111111111111113e-06,
+ "loss": 1.3038103580474854,
+ "mean_token_accuracy": 0.6712872385978699,
+ "num_tokens": 10469376.0,
+ "step": 639
+ },
+ {
+ "entropy": 1.1713769435882568,
+ "epoch": 1.2929292929292928,
+ "grad_norm": 2.1309149265289307,
+ "learning_rate": 3.7090909090909092e-06,
+ "loss": 1.1899120807647705,
+ "mean_token_accuracy": 0.700537383556366,
+ "num_tokens": 10485760.0,
+ "step": 640
+ },
+ {
+ "entropy": 1.3056974411010742,
+ "epoch": 1.294949494949495,
+ "grad_norm": 1.9990391731262207,
+ "learning_rate": 3.7070707070707075e-06,
+ "loss": 1.3287105560302734,
+ "mean_token_accuracy": 0.6773326992988586,
+ "num_tokens": 10502144.0,
+ "step": 641
+ },
+ {
+ "entropy": 1.7352585792541504,
+ "epoch": 1.2969696969696969,
+ "grad_norm": 2.100043535232544,
+ "learning_rate": 3.7050505050505054e-06,
+ "loss": 1.7806944847106934,
+ "mean_token_accuracy": 0.60332190990448,
+ "num_tokens": 10518528.0,
+ "step": 642
+ },
+ {
+ "entropy": 1.3748377561569214,
+ "epoch": 1.298989898989899,
+ "grad_norm": 2.1280689239501953,
+ "learning_rate": 3.7030303030303033e-06,
+ "loss": 1.379891037940979,
+ "mean_token_accuracy": 0.6617611050605774,
+ "num_tokens": 10534912.0,
+ "step": 643
+ },
+ {
+ "entropy": 0.9991571307182312,
+ "epoch": 1.301010101010101,
+ "grad_norm": 1.7343134880065918,
+ "learning_rate": 3.701010101010101e-06,
+ "loss": 1.0082780122756958,
+ "mean_token_accuracy": 0.740229606628418,
+ "num_tokens": 10551296.0,
+ "step": 644
+ },
+ {
+ "entropy": 1.6313605308532715,
+ "epoch": 1.303030303030303,
+ "grad_norm": 2.216167449951172,
+ "learning_rate": 3.698989898989899e-06,
+ "loss": 1.6183781623840332,
+ "mean_token_accuracy": 0.6164509057998657,
+ "num_tokens": 10567680.0,
+ "step": 645
+ },
+ {
+ "entropy": 1.4174962043762207,
+ "epoch": 1.305050505050505,
+ "grad_norm": 2.0921854972839355,
+ "learning_rate": 3.6969696969696974e-06,
+ "loss": 1.4087945222854614,
+ "mean_token_accuracy": 0.6705544590950012,
+ "num_tokens": 10584064.0,
+ "step": 646
+ },
+ {
+ "entropy": 1.8302873373031616,
+ "epoch": 1.307070707070707,
+ "grad_norm": 2.0742204189300537,
+ "learning_rate": 3.6949494949494953e-06,
+ "loss": 1.8456642627716064,
+ "mean_token_accuracy": 0.5914142727851868,
+ "num_tokens": 10600448.0,
+ "step": 647
+ },
+ {
+ "entropy": 1.29628324508667,
+ "epoch": 1.309090909090909,
+ "grad_norm": 2.1446709632873535,
+ "learning_rate": 3.692929292929293e-06,
+ "loss": 1.3066375255584717,
+ "mean_token_accuracy": 0.6798363327980042,
+ "num_tokens": 10616832.0,
+ "step": 648
+ },
+ {
+ "entropy": 1.396378993988037,
+ "epoch": 1.3111111111111111,
+ "grad_norm": 1.9578899145126343,
+ "learning_rate": 3.690909090909091e-06,
+ "loss": 1.3943686485290527,
+ "mean_token_accuracy": 0.6772105693817139,
+ "num_tokens": 10633216.0,
+ "step": 649
+ },
+ {
+ "entropy": 1.6457722187042236,
+ "epoch": 1.3131313131313131,
+ "grad_norm": 1.9999926090240479,
+ "learning_rate": 3.688888888888889e-06,
+ "loss": 1.6266872882843018,
+ "mean_token_accuracy": 0.630923330783844,
+ "num_tokens": 10649600.0,
+ "step": 650
+ },
+ {
+ "epoch": 1.3131313131313131,
+ "eval_entropy": 1.456459879875183,
+ "eval_loss": 1.539737582206726,
+ "eval_mean_token_accuracy": 0.643713459853203,
+ "eval_num_tokens": 10649600.0,
+ "eval_runtime": 43.7637,
+ "eval_samples_per_second": 22.621,
+ "eval_steps_per_second": 2.833,
+ "step": 650
+ },
+ {
+ "entropy": 1.1152859926223755,
+ "epoch": 1.3151515151515152,
+ "grad_norm": 2.9232723712921143,
+ "learning_rate": 3.686868686868687e-06,
+ "loss": 1.099653959274292,
+ "mean_token_accuracy": 0.7227039337158203,
+ "num_tokens": 10665984.0,
+ "step": 651
+ },
+ {
+ "entropy": 1.6141386032104492,
+ "epoch": 1.3171717171717172,
+ "grad_norm": 2.1557929515838623,
+ "learning_rate": 3.684848484848485e-06,
+ "loss": 1.6156055927276611,
+ "mean_token_accuracy": 0.6288471221923828,
+ "num_tokens": 10682368.0,
+ "step": 652
+ },
+ {
+ "entropy": 1.282158374786377,
+ "epoch": 1.3191919191919192,
+ "grad_norm": 2.061983346939087,
+ "learning_rate": 3.682828282828283e-06,
+ "loss": 1.2867733240127563,
+ "mean_token_accuracy": 0.6869198679924011,
+ "num_tokens": 10698752.0,
+ "step": 653
+ },
+ {
+ "entropy": 0.9986366033554077,
+ "epoch": 1.3212121212121213,
+ "grad_norm": 2.0446274280548096,
+ "learning_rate": 3.680808080808081e-06,
+ "loss": 1.019629716873169,
+ "mean_token_accuracy": 0.7427943348884583,
+ "num_tokens": 10715136.0,
+ "step": 654
+ },
+ {
+ "entropy": 1.4256011247634888,
+ "epoch": 1.3232323232323233,
+ "grad_norm": 2.1024749279022217,
+ "learning_rate": 3.678787878787879e-06,
+ "loss": 1.4269766807556152,
+ "mean_token_accuracy": 0.6674401760101318,
+ "num_tokens": 10731520.0,
+ "step": 655
+ },
+ {
+ "entropy": 1.4913274049758911,
+ "epoch": 1.3252525252525253,
+ "grad_norm": 2.068432569503784,
+ "learning_rate": 3.6767676767676767e-06,
+ "loss": 1.4930779933929443,
+ "mean_token_accuracy": 0.6474719047546387,
+ "num_tokens": 10747904.0,
+ "step": 656
+ },
+ {
+ "entropy": 1.2957285642623901,
+ "epoch": 1.3272727272727272,
+ "grad_norm": 2.2480008602142334,
+ "learning_rate": 3.674747474747475e-06,
+ "loss": 1.3001320362091064,
+ "mean_token_accuracy": 0.679286777973175,
+ "num_tokens": 10764288.0,
+ "step": 657
+ },
+ {
+ "entropy": 1.328946828842163,
+ "epoch": 1.3292929292929294,
+ "grad_norm": 2.311176300048828,
+ "learning_rate": 3.672727272727273e-06,
+ "loss": 1.3286024332046509,
+ "mean_token_accuracy": 0.662432849407196,
+ "num_tokens": 10780672.0,
+ "step": 658
+ },
+ {
+ "entropy": 1.458139419555664,
+ "epoch": 1.3313131313131312,
+ "grad_norm": 2.1967899799346924,
+ "learning_rate": 3.670707070707071e-06,
+ "loss": 1.4384279251098633,
+ "mean_token_accuracy": 0.6435637474060059,
+ "num_tokens": 10797056.0,
+ "step": 659
+ },
+ {
+ "entropy": 1.43666672706604,
+ "epoch": 1.3333333333333333,
+ "grad_norm": 2.163083076477051,
+ "learning_rate": 3.6686868686868687e-06,
+ "loss": 1.4396371841430664,
+ "mean_token_accuracy": 0.6531509757041931,
+ "num_tokens": 10813440.0,
+ "step": 660
+ },
+ {
+ "entropy": 1.6210501194000244,
+ "epoch": 1.3353535353535353,
+ "grad_norm": 2.3601770401000977,
+ "learning_rate": 3.6666666666666666e-06,
+ "loss": 1.6370227336883545,
+ "mean_token_accuracy": 0.6149242520332336,
+ "num_tokens": 10829824.0,
+ "step": 661
+ },
+ {
+ "entropy": 0.9602832794189453,
+ "epoch": 1.3373737373737373,
+ "grad_norm": 1.8452314138412476,
+ "learning_rate": 3.664646464646465e-06,
+ "loss": 0.9445997476577759,
+ "mean_token_accuracy": 0.759709358215332,
+ "num_tokens": 10846208.0,
+ "step": 662
+ },
+ {
+ "entropy": 1.2215200662612915,
+ "epoch": 1.3393939393939394,
+ "grad_norm": 2.019989252090454,
+ "learning_rate": 3.662626262626263e-06,
+ "loss": 1.2451637983322144,
+ "mean_token_accuracy": 0.7051172256469727,
+ "num_tokens": 10862592.0,
+ "step": 663
+ },
+ {
+ "entropy": 1.3253310918807983,
+ "epoch": 1.3414141414141414,
+ "grad_norm": 1.9700425863265991,
+ "learning_rate": 3.660606060606061e-06,
+ "loss": 1.3407385349273682,
+ "mean_token_accuracy": 0.6988885998725891,
+ "num_tokens": 10878976.0,
+ "step": 664
+ },
+ {
+ "entropy": 1.1397737264633179,
+ "epoch": 1.3434343434343434,
+ "grad_norm": 2.1194040775299072,
+ "learning_rate": 3.658585858585859e-06,
+ "loss": 1.1545679569244385,
+ "mean_token_accuracy": 0.7071323990821838,
+ "num_tokens": 10895360.0,
+ "step": 665
+ },
+ {
+ "entropy": 1.1714493036270142,
+ "epoch": 1.3454545454545455,
+ "grad_norm": 2.078051805496216,
+ "learning_rate": 3.6565656565656573e-06,
+ "loss": 1.1751537322998047,
+ "mean_token_accuracy": 0.6963238716125488,
+ "num_tokens": 10911744.0,
+ "step": 666
+ },
+ {
+ "entropy": 1.2954860925674438,
+ "epoch": 1.3474747474747475,
+ "grad_norm": 1.9348593950271606,
+ "learning_rate": 3.654545454545455e-06,
+ "loss": 1.2961801290512085,
+ "mean_token_accuracy": 0.6831338405609131,
+ "num_tokens": 10928128.0,
+ "step": 667
+ },
+ {
+ "entropy": 1.401628851890564,
+ "epoch": 1.3494949494949495,
+ "grad_norm": 2.1757614612579346,
+ "learning_rate": 3.652525252525253e-06,
+ "loss": 1.4025013446807861,
+ "mean_token_accuracy": 0.6590132117271423,
+ "num_tokens": 10944512.0,
+ "step": 668
+ },
+ {
+ "entropy": 1.4164557456970215,
+ "epoch": 1.3515151515151516,
+ "grad_norm": 2.0192055702209473,
+ "learning_rate": 3.650505050505051e-06,
+ "loss": 1.43379545211792,
+ "mean_token_accuracy": 0.6554714441299438,
+ "num_tokens": 10960896.0,
+ "step": 669
+ },
+ {
+ "entropy": 1.4888815879821777,
+ "epoch": 1.3535353535353536,
+ "grad_norm": 2.2166643142700195,
+ "learning_rate": 3.648484848484849e-06,
+ "loss": 1.494372844696045,
+ "mean_token_accuracy": 0.6402052044868469,
+ "num_tokens": 10977280.0,
+ "step": 670
+ },
+ {
+ "entropy": 1.215625286102295,
+ "epoch": 1.3555555555555556,
+ "grad_norm": 1.8059836626052856,
+ "learning_rate": 3.6464646464646467e-06,
+ "loss": 1.2074902057647705,
+ "mean_token_accuracy": 0.7123228907585144,
+ "num_tokens": 10993664.0,
+ "step": 671
+ },
+ {
+ "entropy": 0.9577685594558716,
+ "epoch": 1.3575757575757577,
+ "grad_norm": 1.8803631067276,
+ "learning_rate": 3.644444444444445e-06,
+ "loss": 0.956764817237854,
+ "mean_token_accuracy": 0.759709358215332,
+ "num_tokens": 11010048.0,
+ "step": 672
+ },
+ {
+ "entropy": 1.7215807437896729,
+ "epoch": 1.3595959595959597,
+ "grad_norm": 2.1421561241149902,
+ "learning_rate": 3.642424242424243e-06,
+ "loss": 1.7220706939697266,
+ "mean_token_accuracy": 0.6187102794647217,
+ "num_tokens": 11026432.0,
+ "step": 673
+ },
+ {
+ "entropy": 1.057277798652649,
+ "epoch": 1.3616161616161615,
+ "grad_norm": 1.9014254808425903,
+ "learning_rate": 3.640404040404041e-06,
+ "loss": 1.0582892894744873,
+ "mean_token_accuracy": 0.7327185869216919,
+ "num_tokens": 11042816.0,
+ "step": 674
+ },
+ {
+ "entropy": 1.220097541809082,
+ "epoch": 1.3636363636363638,
+ "grad_norm": 2.1406402587890625,
+ "learning_rate": 3.6383838383838387e-06,
+ "loss": 1.2152808904647827,
+ "mean_token_accuracy": 0.6976673007011414,
+ "num_tokens": 11059200.0,
+ "step": 675
+ },
+ {
+ "entropy": 1.4171489477157593,
+ "epoch": 1.3656565656565656,
+ "grad_norm": 2.1809890270233154,
+ "learning_rate": 3.6363636363636366e-06,
+ "loss": 1.4846035242080688,
+ "mean_token_accuracy": 0.6681118607521057,
+ "num_tokens": 11075584.0,
+ "step": 676
+ },
+ {
+ "entropy": 1.2915124893188477,
+ "epoch": 1.3676767676767676,
+ "grad_norm": 2.0533218383789062,
+ "learning_rate": 3.634343434343435e-06,
+ "loss": 1.3094089031219482,
+ "mean_token_accuracy": 0.6841719746589661,
+ "num_tokens": 11091968.0,
+ "step": 677
+ },
+ {
+ "entropy": 1.4717603921890259,
+ "epoch": 1.3696969696969696,
+ "grad_norm": 2.125450849533081,
+ "learning_rate": 3.6323232323232328e-06,
+ "loss": 1.479896903038025,
+ "mean_token_accuracy": 0.6601734161376953,
+ "num_tokens": 11108352.0,
+ "step": 678
+ },
+ {
+ "entropy": 1.0841981172561646,
+ "epoch": 1.3717171717171717,
+ "grad_norm": 1.936905026435852,
+ "learning_rate": 3.6303030303030307e-06,
+ "loss": 1.0895963907241821,
+ "mean_token_accuracy": 0.7307645082473755,
+ "num_tokens": 11124736.0,
+ "step": 679
+ },
+ {
+ "entropy": 0.85239177942276,
+ "epoch": 1.3737373737373737,
+ "grad_norm": 1.7117112874984741,
+ "learning_rate": 3.6282828282828286e-06,
+ "loss": 0.8716775178909302,
+ "mean_token_accuracy": 0.7721666097640991,
+ "num_tokens": 11141120.0,
+ "step": 680
+ },
+ {
+ "entropy": 1.4737738370895386,
+ "epoch": 1.3757575757575757,
+ "grad_norm": 2.0140671730041504,
+ "learning_rate": 3.6262626262626264e-06,
+ "loss": 1.5029723644256592,
+ "mean_token_accuracy": 0.653822660446167,
+ "num_tokens": 11157504.0,
+ "step": 681
+ },
+ {
+ "entropy": 1.260698914527893,
+ "epoch": 1.3777777777777778,
+ "grad_norm": 1.9868851900100708,
+ "learning_rate": 3.6242424242424248e-06,
+ "loss": 1.256290316581726,
+ "mean_token_accuracy": 0.6891182065010071,
+ "num_tokens": 11173888.0,
+ "step": 682
+ },
+ {
+ "entropy": 1.7418453693389893,
+ "epoch": 1.3797979797979798,
+ "grad_norm": 2.0905439853668213,
+ "learning_rate": 3.6222222222222226e-06,
+ "loss": 1.7568984031677246,
+ "mean_token_accuracy": 0.6291524171829224,
+ "num_tokens": 11190272.0,
+ "step": 683
+ },
+ {
+ "entropy": 1.5552523136138916,
+ "epoch": 1.3818181818181818,
+ "grad_norm": 2.2071683406829834,
+ "learning_rate": 3.6202020202020205e-06,
+ "loss": 1.555542230606079,
+ "mean_token_accuracy": 0.6403883695602417,
+ "num_tokens": 11206656.0,
+ "step": 684
+ },
+ {
+ "entropy": 1.4404902458190918,
+ "epoch": 1.3838383838383839,
+ "grad_norm": 2.276245594024658,
+ "learning_rate": 3.6181818181818184e-06,
+ "loss": 1.4155148267745972,
+ "mean_token_accuracy": 0.6630434989929199,
+ "num_tokens": 11223040.0,
+ "step": 685
+ },
+ {
+ "entropy": 1.1282018423080444,
+ "epoch": 1.385858585858586,
+ "grad_norm": 1.9824057817459106,
+ "learning_rate": 3.6161616161616163e-06,
+ "loss": 1.099952220916748,
+ "mean_token_accuracy": 0.7234978079795837,
+ "num_tokens": 11239424.0,
+ "step": 686
+ },
+ {
+ "entropy": 1.5559885501861572,
+ "epoch": 1.387878787878788,
+ "grad_norm": 2.243121862411499,
+ "learning_rate": 3.614141414141414e-06,
+ "loss": 1.520850658416748,
+ "mean_token_accuracy": 0.6340987086296082,
+ "num_tokens": 11255808.0,
+ "step": 687
+ },
+ {
+ "entropy": 1.322407841682434,
+ "epoch": 1.38989898989899,
+ "grad_norm": 2.049233913421631,
+ "learning_rate": 3.6121212121212125e-06,
+ "loss": 1.322263479232788,
+ "mean_token_accuracy": 0.6778212189674377,
+ "num_tokens": 11272192.0,
+ "step": 688
+ },
+ {
+ "entropy": 1.3112859725952148,
+ "epoch": 1.391919191919192,
+ "grad_norm": 2.0666465759277344,
+ "learning_rate": 3.6101010101010104e-06,
+ "loss": 1.3103243112564087,
+ "mean_token_accuracy": 0.6828285455703735,
+ "num_tokens": 11288576.0,
+ "step": 689
+ },
+ {
+ "entropy": 1.3321104049682617,
+ "epoch": 1.393939393939394,
+ "grad_norm": 2.5464279651641846,
+ "learning_rate": 3.6080808080808083e-06,
+ "loss": 1.3247697353363037,
+ "mean_token_accuracy": 0.6785539984703064,
+ "num_tokens": 11304960.0,
+ "step": 690
+ },
+ {
+ "entropy": 1.0779680013656616,
+ "epoch": 1.3959595959595958,
+ "grad_norm": 2.1386117935180664,
+ "learning_rate": 3.606060606060606e-06,
+ "loss": 1.0994013547897339,
+ "mean_token_accuracy": 0.7129946351051331,
+ "num_tokens": 11321344.0,
+ "step": 691
+ },
+ {
+ "entropy": 1.0701864957809448,
+ "epoch": 1.397979797979798,
+ "grad_norm": 1.9387527704238892,
+ "learning_rate": 3.604040404040404e-06,
+ "loss": 1.0633184909820557,
+ "mean_token_accuracy": 0.7378480434417725,
+ "num_tokens": 11337728.0,
+ "step": 692
+ },
+ {
+ "entropy": 1.2258752584457397,
+ "epoch": 1.4,
+ "grad_norm": 2.0486111640930176,
+ "learning_rate": 3.6020202020202024e-06,
+ "loss": 1.2098207473754883,
+ "mean_token_accuracy": 0.6999877691268921,
+ "num_tokens": 11354112.0,
+ "step": 693
+ },
+ {
+ "entropy": 1.4652774333953857,
+ "epoch": 1.402020202020202,
+ "grad_norm": 2.043079376220703,
+ "learning_rate": 3.6000000000000003e-06,
+ "loss": 1.4780974388122559,
+ "mean_token_accuracy": 0.6681729555130005,
+ "num_tokens": 11370496.0,
+ "step": 694
+ },
+ {
+ "entropy": 1.3215711116790771,
+ "epoch": 1.404040404040404,
+ "grad_norm": 2.1327438354492188,
+ "learning_rate": 3.597979797979798e-06,
+ "loss": 1.3116662502288818,
+ "mean_token_accuracy": 0.6757450103759766,
+ "num_tokens": 11386880.0,
+ "step": 695
+ },
+ {
+ "entropy": 1.624266266822815,
+ "epoch": 1.406060606060606,
+ "grad_norm": 2.114398717880249,
+ "learning_rate": 3.595959595959596e-06,
+ "loss": 1.628610372543335,
+ "mean_token_accuracy": 0.6487542986869812,
+ "num_tokens": 11403264.0,
+ "step": 696
+ },
+ {
+ "entropy": 1.6059489250183105,
+ "epoch": 1.408080808080808,
+ "grad_norm": 1.952991008758545,
+ "learning_rate": 3.593939393939394e-06,
+ "loss": 1.6145906448364258,
+ "mean_token_accuracy": 0.6257938742637634,
+ "num_tokens": 11419648.0,
+ "step": 697
+ },
+ {
+ "entropy": 1.0662391185760498,
+ "epoch": 1.41010101010101,
+ "grad_norm": 2.0804851055145264,
+ "learning_rate": 3.5919191919191922e-06,
+ "loss": 1.0772000551223755,
+ "mean_token_accuracy": 0.7187347412109375,
+ "num_tokens": 11436032.0,
+ "step": 698
+ },
+ {
+ "entropy": 1.221283197402954,
+ "epoch": 1.412121212121212,
+ "grad_norm": 1.9502744674682617,
+ "learning_rate": 3.58989898989899e-06,
+ "loss": 1.2352337837219238,
+ "mean_token_accuracy": 0.6918050646781921,
+ "num_tokens": 11452416.0,
+ "step": 699
+ },
+ {
+ "entropy": 1.4205158948898315,
+ "epoch": 1.4141414141414141,
+ "grad_norm": 2.2222518920898438,
+ "learning_rate": 3.587878787878788e-06,
+ "loss": 1.451093077659607,
+ "mean_token_accuracy": 0.6648754477500916,
+ "num_tokens": 11468800.0,
+ "step": 700
+ },
+ {
+ "epoch": 1.4141414141414141,
+ "eval_entropy": 1.4370074204860195,
+ "eval_loss": 1.5383458137512207,
+ "eval_mean_token_accuracy": 0.6441497793120723,
+ "eval_num_tokens": 11468800.0,
+ "eval_runtime": 43.744,
+ "eval_samples_per_second": 22.632,
+ "eval_steps_per_second": 2.835,
+ "step": 700
+ },
+ {
+ "entropy": 1.5432499647140503,
+ "epoch": 1.4161616161616162,
+ "grad_norm": 2.084463596343994,
+ "learning_rate": 3.585858585858586e-06,
+ "loss": 1.5958356857299805,
+ "mean_token_accuracy": 0.6377015113830566,
+ "num_tokens": 11485184.0,
+ "step": 701
+ },
+ {
+ "entropy": 1.297119379043579,
+ "epoch": 1.4181818181818182,
+ "grad_norm": 2.1996524333953857,
+ "learning_rate": 3.5838383838383838e-06,
+ "loss": 1.3346054553985596,
+ "mean_token_accuracy": 0.6801416873931885,
+ "num_tokens": 11501568.0,
+ "step": 702
+ },
+ {
+ "entropy": 1.3905766010284424,
+ "epoch": 1.4202020202020202,
+ "grad_norm": 2.1683545112609863,
+ "learning_rate": 3.5818181818181817e-06,
+ "loss": 1.40826416015625,
+ "mean_token_accuracy": 0.6653639674186707,
+ "num_tokens": 11517952.0,
+ "step": 703
+ },
+ {
+ "entropy": 0.9598695635795593,
+ "epoch": 1.4222222222222223,
+ "grad_norm": 1.9240537881851196,
+ "learning_rate": 3.57979797979798e-06,
+ "loss": 0.9487459659576416,
+ "mean_token_accuracy": 0.7556179761886597,
+ "num_tokens": 11534336.0,
+ "step": 704
+ },
+ {
+ "entropy": 1.2008687257766724,
+ "epoch": 1.4242424242424243,
+ "grad_norm": 1.9687083959579468,
+ "learning_rate": 3.577777777777778e-06,
+ "loss": 1.2234617471694946,
+ "mean_token_accuracy": 0.6982779502868652,
+ "num_tokens": 11550720.0,
+ "step": 705
+ },
+ {
+ "entropy": 1.1168856620788574,
+ "epoch": 1.4262626262626263,
+ "grad_norm": 2.1796388626098633,
+ "learning_rate": 3.575757575757576e-06,
+ "loss": 1.1320589780807495,
+ "mean_token_accuracy": 0.7057889699935913,
+ "num_tokens": 11567104.0,
+ "step": 706
+ },
+ {
+ "entropy": 1.3408682346343994,
+ "epoch": 1.4282828282828284,
+ "grad_norm": 1.9048689603805542,
+ "learning_rate": 3.573737373737374e-06,
+ "loss": 1.3476686477661133,
+ "mean_token_accuracy": 0.6839887499809265,
+ "num_tokens": 11583488.0,
+ "step": 707
+ },
+ {
+ "entropy": 1.4392427206039429,
+ "epoch": 1.4303030303030302,
+ "grad_norm": 2.183016777038574,
+ "learning_rate": 3.5717171717171724e-06,
+ "loss": 1.4443378448486328,
+ "mean_token_accuracy": 0.6546775698661804,
+ "num_tokens": 11599872.0,
+ "step": 708
+ },
+ {
+ "entropy": 1.1428641080856323,
+ "epoch": 1.4323232323232324,
+ "grad_norm": 2.0477187633514404,
+ "learning_rate": 3.5696969696969703e-06,
+ "loss": 1.1438124179840088,
+ "mean_token_accuracy": 0.7126282453536987,
+ "num_tokens": 11616256.0,
+ "step": 709
+ },
+ {
+ "entropy": 1.6266967058181763,
+ "epoch": 1.4343434343434343,
+ "grad_norm": 2.322098731994629,
+ "learning_rate": 3.567676767676768e-06,
+ "loss": 1.6161916255950928,
+ "mean_token_accuracy": 0.6202979683876038,
+ "num_tokens": 11632640.0,
+ "step": 710
+ },
+ {
+ "entropy": 1.349686622619629,
+ "epoch": 1.4363636363636363,
+ "grad_norm": 2.021871328353882,
+ "learning_rate": 3.565656565656566e-06,
+ "loss": 1.3713490962982178,
+ "mean_token_accuracy": 0.6596238613128662,
+ "num_tokens": 11649024.0,
+ "step": 711
+ },
+ {
+ "entropy": 1.4476866722106934,
+ "epoch": 1.4383838383838383,
+ "grad_norm": 2.194718837738037,
+ "learning_rate": 3.563636363636364e-06,
+ "loss": 1.4593651294708252,
+ "mean_token_accuracy": 0.6476551294326782,
+ "num_tokens": 11665408.0,
+ "step": 712
+ },
+ {
+ "entropy": 1.625157356262207,
+ "epoch": 1.4404040404040404,
+ "grad_norm": 2.0966169834136963,
+ "learning_rate": 3.5616161616161622e-06,
+ "loss": 1.6374845504760742,
+ "mean_token_accuracy": 0.6293356418609619,
+ "num_tokens": 11681792.0,
+ "step": 713
+ },
+ {
+ "entropy": 1.985729455947876,
+ "epoch": 1.4424242424242424,
+ "grad_norm": 2.129556655883789,
+ "learning_rate": 3.55959595959596e-06,
+ "loss": 2.0454561710357666,
+ "mean_token_accuracy": 0.5728505253791809,
+ "num_tokens": 11698176.0,
+ "step": 714
+ },
+ {
+ "entropy": 1.546321153640747,
+ "epoch": 1.4444444444444444,
+ "grad_norm": 2.1565628051757812,
+ "learning_rate": 3.557575757575758e-06,
+ "loss": 1.5467555522918701,
+ "mean_token_accuracy": 0.6351978778839111,
+ "num_tokens": 11714560.0,
+ "step": 715
+ },
+ {
+ "entropy": 1.3690776824951172,
+ "epoch": 1.4464646464646465,
+ "grad_norm": 2.206984519958496,
+ "learning_rate": 3.555555555555556e-06,
+ "loss": 1.3877894878387451,
+ "mean_token_accuracy": 0.6911333799362183,
+ "num_tokens": 11730944.0,
+ "step": 716
+ },
+ {
+ "entropy": 1.3643333911895752,
+ "epoch": 1.4484848484848485,
+ "grad_norm": 2.0049662590026855,
+ "learning_rate": 3.553535353535354e-06,
+ "loss": 1.364513635635376,
+ "mean_token_accuracy": 0.6781876087188721,
+ "num_tokens": 11747328.0,
+ "step": 717
+ },
+ {
+ "entropy": 1.337018370628357,
+ "epoch": 1.4505050505050505,
+ "grad_norm": 2.0919127464294434,
+ "learning_rate": 3.551515151515152e-06,
+ "loss": 1.3578065633773804,
+ "mean_token_accuracy": 0.672874927520752,
+ "num_tokens": 11763712.0,
+ "step": 718
+ },
+ {
+ "entropy": 1.3421508073806763,
+ "epoch": 1.4525252525252526,
+ "grad_norm": 2.0651159286499023,
+ "learning_rate": 3.54949494949495e-06,
+ "loss": 1.33928382396698,
+ "mean_token_accuracy": 0.6866145730018616,
+ "num_tokens": 11780096.0,
+ "step": 719
+ },
+ {
+ "entropy": 1.2841005325317383,
+ "epoch": 1.4545454545454546,
+ "grad_norm": 2.124370813369751,
+ "learning_rate": 3.547474747474748e-06,
+ "loss": 1.258314609527588,
+ "mean_token_accuracy": 0.6806302070617676,
+ "num_tokens": 11796480.0,
+ "step": 720
+ },
+ {
+ "entropy": 1.56289803981781,
+ "epoch": 1.4565656565656566,
+ "grad_norm": 1.9271100759506226,
+ "learning_rate": 3.5454545454545458e-06,
+ "loss": 1.5867946147918701,
+ "mean_token_accuracy": 0.6430752277374268,
+ "num_tokens": 11812864.0,
+ "step": 721
+ },
+ {
+ "entropy": 1.3374838829040527,
+ "epoch": 1.4585858585858587,
+ "grad_norm": 2.1113364696502686,
+ "learning_rate": 3.5434343434343437e-06,
+ "loss": 1.327169418334961,
+ "mean_token_accuracy": 0.6806912422180176,
+ "num_tokens": 11829248.0,
+ "step": 722
+ },
+ {
+ "entropy": 1.7233415842056274,
+ "epoch": 1.4606060606060607,
+ "grad_norm": 2.1862399578094482,
+ "learning_rate": 3.5414141414141416e-06,
+ "loss": 1.7290821075439453,
+ "mean_token_accuracy": 0.6056423783302307,
+ "num_tokens": 11845632.0,
+ "step": 723
+ },
+ {
+ "entropy": 1.28484046459198,
+ "epoch": 1.4626262626262627,
+ "grad_norm": 2.0059030055999756,
+ "learning_rate": 3.53939393939394e-06,
+ "loss": 1.2958555221557617,
+ "mean_token_accuracy": 0.6774548292160034,
+ "num_tokens": 11862016.0,
+ "step": 724
+ },
+ {
+ "entropy": 1.2592295408248901,
+ "epoch": 1.4646464646464645,
+ "grad_norm": 1.9422065019607544,
+ "learning_rate": 3.5373737373737378e-06,
+ "loss": 1.2501411437988281,
+ "mean_token_accuracy": 0.700537383556366,
+ "num_tokens": 11878400.0,
+ "step": 725
+ },
+ {
+ "entropy": 1.3708571195602417,
+ "epoch": 1.4666666666666668,
+ "grad_norm": 2.164499044418335,
+ "learning_rate": 3.5353535353535356e-06,
+ "loss": 1.3901662826538086,
+ "mean_token_accuracy": 0.6618832349777222,
+ "num_tokens": 11894784.0,
+ "step": 726
+ },
+ {
+ "entropy": 1.239045262336731,
+ "epoch": 1.4686868686868686,
+ "grad_norm": 1.9581060409545898,
+ "learning_rate": 3.5333333333333335e-06,
+ "loss": 1.2436164617538452,
+ "mean_token_accuracy": 0.6939423680305481,
+ "num_tokens": 11911168.0,
+ "step": 727
+ },
+ {
+ "entropy": 1.5965995788574219,
+ "epoch": 1.4707070707070706,
+ "grad_norm": 2.343669891357422,
+ "learning_rate": 3.5313131313131314e-06,
+ "loss": 1.5918347835540771,
+ "mean_token_accuracy": 0.6274425983428955,
+ "num_tokens": 11927552.0,
+ "step": 728
+ },
+ {
+ "entropy": 1.1196811199188232,
+ "epoch": 1.4727272727272727,
+ "grad_norm": 2.181840181350708,
+ "learning_rate": 3.5292929292929297e-06,
+ "loss": 1.1142621040344238,
+ "mean_token_accuracy": 0.7192842960357666,
+ "num_tokens": 11943936.0,
+ "step": 729
+ },
+ {
+ "entropy": 1.129361629486084,
+ "epoch": 1.4747474747474747,
+ "grad_norm": 2.065730571746826,
+ "learning_rate": 3.5272727272727276e-06,
+ "loss": 1.143613576889038,
+ "mean_token_accuracy": 0.7145212292671204,
+ "num_tokens": 11960320.0,
+ "step": 730
+ },
+ {
+ "entropy": 1.4940218925476074,
+ "epoch": 1.4767676767676767,
+ "grad_norm": 2.083453893661499,
+ "learning_rate": 3.5252525252525255e-06,
+ "loss": 1.4957321882247925,
+ "mean_token_accuracy": 0.6481436491012573,
+ "num_tokens": 11976704.0,
+ "step": 731
+ },
+ {
+ "entropy": 1.3427786827087402,
+ "epoch": 1.4787878787878788,
+ "grad_norm": 2.0897183418273926,
+ "learning_rate": 3.5232323232323234e-06,
+ "loss": 1.354066014289856,
+ "mean_token_accuracy": 0.6660356521606445,
+ "num_tokens": 11993088.0,
+ "step": 732
+ },
+ {
+ "entropy": 1.3300065994262695,
+ "epoch": 1.4808080808080808,
+ "grad_norm": 2.0537912845611572,
+ "learning_rate": 3.5212121212121213e-06,
+ "loss": 1.3332751989364624,
+ "mean_token_accuracy": 0.6817293763160706,
+ "num_tokens": 12009472.0,
+ "step": 733
+ },
+ {
+ "entropy": 1.0657827854156494,
+ "epoch": 1.4828282828282828,
+ "grad_norm": 1.9099130630493164,
+ "learning_rate": 3.5191919191919196e-06,
+ "loss": 1.067232608795166,
+ "mean_token_accuracy": 0.734245240688324,
+ "num_tokens": 12025856.0,
+ "step": 734
+ },
+ {
+ "entropy": 1.5265862941741943,
+ "epoch": 1.4848484848484849,
+ "grad_norm": 2.100236177444458,
+ "learning_rate": 3.5171717171717175e-06,
+ "loss": 1.5169599056243896,
+ "mean_token_accuracy": 0.6382511258125305,
+ "num_tokens": 12042240.0,
+ "step": 735
+ },
+ {
+ "entropy": 1.4428536891937256,
+ "epoch": 1.486868686868687,
+ "grad_norm": 1.9902187585830688,
+ "learning_rate": 3.5151515151515154e-06,
+ "loss": 1.4398434162139893,
+ "mean_token_accuracy": 0.675500750541687,
+ "num_tokens": 12058624.0,
+ "step": 736
+ },
+ {
+ "entropy": 1.528032660484314,
+ "epoch": 1.488888888888889,
+ "grad_norm": 2.043238878250122,
+ "learning_rate": 3.5131313131313133e-06,
+ "loss": 1.5280101299285889,
+ "mean_token_accuracy": 0.6431363224983215,
+ "num_tokens": 12075008.0,
+ "step": 737
+ },
+ {
+ "entropy": 1.4250329732894897,
+ "epoch": 1.490909090909091,
+ "grad_norm": 1.791640281677246,
+ "learning_rate": 3.511111111111111e-06,
+ "loss": 1.426466464996338,
+ "mean_token_accuracy": 0.6765388250350952,
+ "num_tokens": 12091392.0,
+ "step": 738
+ },
+ {
+ "entropy": 1.1264820098876953,
+ "epoch": 1.492929292929293,
+ "grad_norm": 2.0549607276916504,
+ "learning_rate": 3.509090909090909e-06,
+ "loss": 1.1271553039550781,
+ "mean_token_accuracy": 0.6996213793754578,
+ "num_tokens": 12107776.0,
+ "step": 739
+ },
+ {
+ "entropy": 1.538375735282898,
+ "epoch": 1.494949494949495,
+ "grad_norm": 2.22322940826416,
+ "learning_rate": 3.5070707070707073e-06,
+ "loss": 1.5588887929916382,
+ "mean_token_accuracy": 0.6384953856468201,
+ "num_tokens": 12124160.0,
+ "step": 740
+ },
+ {
+ "entropy": 1.2698255777359009,
+ "epoch": 1.496969696969697,
+ "grad_norm": 2.17710542678833,
+ "learning_rate": 3.5050505050505052e-06,
+ "loss": 1.29567289352417,
+ "mean_token_accuracy": 0.6762945652008057,
+ "num_tokens": 12140544.0,
+ "step": 741
+ },
+ {
+ "entropy": 1.523453950881958,
+ "epoch": 1.4989898989898989,
+ "grad_norm": 2.0350148677825928,
+ "learning_rate": 3.503030303030303e-06,
+ "loss": 1.5341038703918457,
+ "mean_token_accuracy": 0.6367855668067932,
+ "num_tokens": 12156928.0,
+ "step": 742
+ },
+ {
+ "entropy": 1.4250539541244507,
+ "epoch": 1.5010101010101011,
+ "grad_norm": 2.1306707859039307,
+ "learning_rate": 3.501010101010101e-06,
+ "loss": 1.4353611469268799,
+ "mean_token_accuracy": 0.6626160144805908,
+ "num_tokens": 12173312.0,
+ "step": 743
+ },
+ {
+ "entropy": 1.1440296173095703,
+ "epoch": 1.503030303030303,
+ "grad_norm": 2.053969383239746,
+ "learning_rate": 3.498989898989899e-06,
+ "loss": 1.1151821613311768,
+ "mean_token_accuracy": 0.7080483436584473,
+ "num_tokens": 12189696.0,
+ "step": 744
+ },
+ {
+ "entropy": 1.5290073156356812,
+ "epoch": 1.5050505050505052,
+ "grad_norm": 2.0550003051757812,
+ "learning_rate": 3.496969696969697e-06,
+ "loss": 1.5260624885559082,
+ "mean_token_accuracy": 0.6493038535118103,
+ "num_tokens": 12206080.0,
+ "step": 745
+ },
+ {
+ "entropy": 1.1096605062484741,
+ "epoch": 1.507070707070707,
+ "grad_norm": 1.8131766319274902,
+ "learning_rate": 3.494949494949495e-06,
+ "loss": 1.1221846342086792,
+ "mean_token_accuracy": 0.7263678312301636,
+ "num_tokens": 12222464.0,
+ "step": 746
+ },
+ {
+ "entropy": 1.282676339149475,
+ "epoch": 1.509090909090909,
+ "grad_norm": 2.3051960468292236,
+ "learning_rate": 3.492929292929293e-06,
+ "loss": 1.2863187789916992,
+ "mean_token_accuracy": 0.6765388250350952,
+ "num_tokens": 12238848.0,
+ "step": 747
+ },
+ {
+ "entropy": 1.2971303462982178,
+ "epoch": 1.511111111111111,
+ "grad_norm": 2.0721139907836914,
+ "learning_rate": 3.4909090909090913e-06,
+ "loss": 1.3010127544403076,
+ "mean_token_accuracy": 0.6753175258636475,
+ "num_tokens": 12255232.0,
+ "step": 748
+ },
+ {
+ "entropy": 1.3711751699447632,
+ "epoch": 1.513131313131313,
+ "grad_norm": 1.9657210111618042,
+ "learning_rate": 3.4888888888888896e-06,
+ "loss": 1.3703930377960205,
+ "mean_token_accuracy": 0.6686614751815796,
+ "num_tokens": 12271616.0,
+ "step": 749
+ },
+ {
+ "entropy": 1.512154459953308,
+ "epoch": 1.5151515151515151,
+ "grad_norm": 2.214172124862671,
+ "learning_rate": 3.4868686868686875e-06,
+ "loss": 1.5243256092071533,
+ "mean_token_accuracy": 0.6485100388526917,
+ "num_tokens": 12288000.0,
+ "step": 750
+ },
+ {
+ "epoch": 1.5151515151515151,
+ "eval_entropy": 1.4458443743567313,
+ "eval_loss": 1.535510540008545,
+ "eval_mean_token_accuracy": 0.6444821915318889,
+ "eval_num_tokens": 12288000.0,
+ "eval_runtime": 43.8043,
+ "eval_samples_per_second": 22.601,
+ "eval_steps_per_second": 2.831,
+ "step": 750
+ },
+ {
+ "entropy": 1.2052820920944214,
+ "epoch": 1.5171717171717172,
+ "grad_norm": 2.104870319366455,
+ "learning_rate": 3.4848484848484854e-06,
+ "loss": 1.2207554578781128,
+ "mean_token_accuracy": 0.695713222026825,
+ "num_tokens": 12304384.0,
+ "step": 751
+ },
+ {
+ "entropy": 1.732282280921936,
+ "epoch": 1.5191919191919192,
+ "grad_norm": 2.101760149002075,
+ "learning_rate": 3.4828282828282833e-06,
+ "loss": 1.7461241483688354,
+ "mean_token_accuracy": 0.6505251526832581,
+ "num_tokens": 12320768.0,
+ "step": 752
+ },
+ {
+ "entropy": 1.5805878639221191,
+ "epoch": 1.5212121212121212,
+ "grad_norm": 2.139244318008423,
+ "learning_rate": 3.480808080808081e-06,
+ "loss": 1.5813164710998535,
+ "mean_token_accuracy": 0.6281143426895142,
+ "num_tokens": 12337152.0,
+ "step": 753
+ },
+ {
+ "entropy": 1.1882269382476807,
+ "epoch": 1.5232323232323233,
+ "grad_norm": 2.187206745147705,
+ "learning_rate": 3.4787878787878795e-06,
+ "loss": 1.1732902526855469,
+ "mean_token_accuracy": 0.7063385248184204,
+ "num_tokens": 12353536.0,
+ "step": 754
+ },
+ {
+ "entropy": 1.4743424654006958,
+ "epoch": 1.5252525252525253,
+ "grad_norm": 2.12687349319458,
+ "learning_rate": 3.4767676767676774e-06,
+ "loss": 1.4681663513183594,
+ "mean_token_accuracy": 0.6558378338813782,
+ "num_tokens": 12369920.0,
+ "step": 755
+ },
+ {
+ "entropy": 1.5533041954040527,
+ "epoch": 1.5272727272727273,
+ "grad_norm": 2.1306610107421875,
+ "learning_rate": 3.4747474747474752e-06,
+ "loss": 1.5599431991577148,
+ "mean_token_accuracy": 0.6399609446525574,
+ "num_tokens": 12386304.0,
+ "step": 756
+ },
+ {
+ "entropy": 0.8933209180831909,
+ "epoch": 1.5292929292929291,
+ "grad_norm": 2.425602912902832,
+ "learning_rate": 3.472727272727273e-06,
+ "loss": 0.8840203285217285,
+ "mean_token_accuracy": 0.7638006806373596,
+ "num_tokens": 12402688.0,
+ "step": 757
+ },
+ {
+ "entropy": 1.6764189004898071,
+ "epoch": 1.5313131313131314,
+ "grad_norm": 2.0684356689453125,
+ "learning_rate": 3.470707070707071e-06,
+ "loss": 1.6948609352111816,
+ "mean_token_accuracy": 0.6087567210197449,
+ "num_tokens": 12419072.0,
+ "step": 758
+ },
+ {
+ "entropy": 1.1875263452529907,
+ "epoch": 1.5333333333333332,
+ "grad_norm": 2.0625953674316406,
+ "learning_rate": 3.468686868686869e-06,
+ "loss": 1.1840041875839233,
+ "mean_token_accuracy": 0.7067660093307495,
+ "num_tokens": 12435456.0,
+ "step": 759
+ },
+ {
+ "entropy": 1.1589192152023315,
+ "epoch": 1.5353535353535355,
+ "grad_norm": 1.9212846755981445,
+ "learning_rate": 3.4666666666666672e-06,
+ "loss": 1.1537678241729736,
+ "mean_token_accuracy": 0.7107962965965271,
+ "num_tokens": 12451840.0,
+ "step": 760
+ },
+ {
+ "entropy": 0.9687032699584961,
+ "epoch": 1.5373737373737373,
+ "grad_norm": 1.9402192831039429,
+ "learning_rate": 3.464646464646465e-06,
+ "loss": 0.9954442977905273,
+ "mean_token_accuracy": 0.7412676811218262,
+ "num_tokens": 12468224.0,
+ "step": 761
+ },
+ {
+ "entropy": 1.4129019975662231,
+ "epoch": 1.5393939393939395,
+ "grad_norm": 2.0903310775756836,
+ "learning_rate": 3.462626262626263e-06,
+ "loss": 1.4131944179534912,
+ "mean_token_accuracy": 0.6676844358444214,
+ "num_tokens": 12484608.0,
+ "step": 762
+ },
+ {
+ "entropy": 1.5170351266860962,
+ "epoch": 1.5414141414141413,
+ "grad_norm": 2.12807035446167,
+ "learning_rate": 3.460606060606061e-06,
+ "loss": 1.5342246294021606,
+ "mean_token_accuracy": 0.6621885895729065,
+ "num_tokens": 12500992.0,
+ "step": 763
+ },
+ {
+ "entropy": 1.1815718412399292,
+ "epoch": 1.5434343434343434,
+ "grad_norm": 2.2491719722747803,
+ "learning_rate": 3.4585858585858588e-06,
+ "loss": 1.1669261455535889,
+ "mean_token_accuracy": 0.694614052772522,
+ "num_tokens": 12517376.0,
+ "step": 764
+ },
+ {
+ "entropy": 1.547598958015442,
+ "epoch": 1.5454545454545454,
+ "grad_norm": 2.136096954345703,
+ "learning_rate": 3.456565656565657e-06,
+ "loss": 1.5515470504760742,
+ "mean_token_accuracy": 0.648876428604126,
+ "num_tokens": 12533760.0,
+ "step": 765
+ },
+ {
+ "entropy": 0.7156143188476562,
+ "epoch": 1.5474747474747474,
+ "grad_norm": 1.8139498233795166,
+ "learning_rate": 3.454545454545455e-06,
+ "loss": 0.7052675485610962,
+ "mean_token_accuracy": 0.8103932738304138,
+ "num_tokens": 12550144.0,
+ "step": 766
+ },
+ {
+ "entropy": 1.400156855583191,
+ "epoch": 1.5494949494949495,
+ "grad_norm": 2.14924693107605,
+ "learning_rate": 3.452525252525253e-06,
+ "loss": 1.4047740697860718,
+ "mean_token_accuracy": 0.6719589829444885,
+ "num_tokens": 12566528.0,
+ "step": 767
+ },
+ {
+ "entropy": 1.8254657983779907,
+ "epoch": 1.5515151515151515,
+ "grad_norm": 2.1882269382476807,
+ "learning_rate": 3.4505050505050507e-06,
+ "loss": 1.8615440130233765,
+ "mean_token_accuracy": 0.5776746273040771,
+ "num_tokens": 12582912.0,
+ "step": 768
+ },
+ {
+ "entropy": 1.5947710275650024,
+ "epoch": 1.5535353535353535,
+ "grad_norm": 2.3221426010131836,
+ "learning_rate": 3.4484848484848486e-06,
+ "loss": 1.6178712844848633,
+ "mean_token_accuracy": 0.6210918426513672,
+ "num_tokens": 12599296.0,
+ "step": 769
+ },
+ {
+ "entropy": 1.2678714990615845,
+ "epoch": 1.5555555555555556,
+ "grad_norm": 1.970064640045166,
+ "learning_rate": 3.446464646464647e-06,
+ "loss": 1.2646088600158691,
+ "mean_token_accuracy": 0.6990107297897339,
+ "num_tokens": 12615680.0,
+ "step": 770
+ },
+ {
+ "entropy": 1.3723604679107666,
+ "epoch": 1.5575757575757576,
+ "grad_norm": 2.171534776687622,
+ "learning_rate": 3.444444444444445e-06,
+ "loss": 1.3890197277069092,
+ "mean_token_accuracy": 0.6709819436073303,
+ "num_tokens": 12632064.0,
+ "step": 771
+ },
+ {
+ "entropy": 1.0273096561431885,
+ "epoch": 1.5595959595959596,
+ "grad_norm": 2.072798490524292,
+ "learning_rate": 3.4424242424242427e-06,
+ "loss": 1.029193639755249,
+ "mean_token_accuracy": 0.7348558902740479,
+ "num_tokens": 12648448.0,
+ "step": 772
+ },
+ {
+ "entropy": 1.4004037380218506,
+ "epoch": 1.5616161616161617,
+ "grad_norm": 2.111480474472046,
+ "learning_rate": 3.4404040404040406e-06,
+ "loss": 1.4213340282440186,
+ "mean_token_accuracy": 0.660906195640564,
+ "num_tokens": 12664832.0,
+ "step": 773
+ },
+ {
+ "entropy": 1.533424735069275,
+ "epoch": 1.5636363636363635,
+ "grad_norm": 2.0949435234069824,
+ "learning_rate": 3.4383838383838385e-06,
+ "loss": 1.5259253978729248,
+ "mean_token_accuracy": 0.6544333100318909,
+ "num_tokens": 12681216.0,
+ "step": 774
+ },
+ {
+ "entropy": 1.5620275735855103,
+ "epoch": 1.5656565656565657,
+ "grad_norm": 2.339731454849243,
+ "learning_rate": 3.4363636363636364e-06,
+ "loss": 1.5596071481704712,
+ "mean_token_accuracy": 0.6290302872657776,
+ "num_tokens": 12697600.0,
+ "step": 775
+ },
+ {
+ "entropy": 1.3272082805633545,
+ "epoch": 1.5676767676767676,
+ "grad_norm": 2.0871188640594482,
+ "learning_rate": 3.4343434343434347e-06,
+ "loss": 1.3413164615631104,
+ "mean_token_accuracy": 0.6809965968132019,
+ "num_tokens": 12713984.0,
+ "step": 776
+ },
+ {
+ "entropy": 1.202813982963562,
+ "epoch": 1.5696969696969698,
+ "grad_norm": 2.0538578033447266,
+ "learning_rate": 3.4323232323232326e-06,
+ "loss": 1.1947424411773682,
+ "mean_token_accuracy": 0.6969956159591675,
+ "num_tokens": 12730368.0,
+ "step": 777
+ },
+ {
+ "entropy": 1.3332773447036743,
+ "epoch": 1.5717171717171716,
+ "grad_norm": 1.9941202402114868,
+ "learning_rate": 3.4303030303030305e-06,
+ "loss": 1.3345956802368164,
+ "mean_token_accuracy": 0.6761724352836609,
+ "num_tokens": 12746752.0,
+ "step": 778
+ },
+ {
+ "entropy": 1.1724364757537842,
+ "epoch": 1.5737373737373739,
+ "grad_norm": 2.213365316390991,
+ "learning_rate": 3.4282828282828284e-06,
+ "loss": 1.1479787826538086,
+ "mean_token_accuracy": 0.7132999300956726,
+ "num_tokens": 12763136.0,
+ "step": 779
+ },
+ {
+ "entropy": 1.4718358516693115,
+ "epoch": 1.5757575757575757,
+ "grad_norm": 1.9743510484695435,
+ "learning_rate": 3.4262626262626262e-06,
+ "loss": 1.4844720363616943,
+ "mean_token_accuracy": 0.656448483467102,
+ "num_tokens": 12779520.0,
+ "step": 780
+ },
+ {
+ "entropy": 1.362882375717163,
+ "epoch": 1.5777777777777777,
+ "grad_norm": 2.370483875274658,
+ "learning_rate": 3.4242424242424246e-06,
+ "loss": 1.350801706314087,
+ "mean_token_accuracy": 0.6707376837730408,
+ "num_tokens": 12795904.0,
+ "step": 781
+ },
+ {
+ "entropy": 1.8302308320999146,
+ "epoch": 1.5797979797979798,
+ "grad_norm": 2.19045090675354,
+ "learning_rate": 3.4222222222222224e-06,
+ "loss": 1.8371320962905884,
+ "mean_token_accuracy": 0.5929408669471741,
+ "num_tokens": 12812288.0,
+ "step": 782
+ },
+ {
+ "entropy": 1.3435529470443726,
+ "epoch": 1.5818181818181818,
+ "grad_norm": 2.1254351139068604,
+ "learning_rate": 3.4202020202020203e-06,
+ "loss": 1.3621928691864014,
+ "mean_token_accuracy": 0.6867367029190063,
+ "num_tokens": 12828672.0,
+ "step": 783
+ },
+ {
+ "entropy": 1.8402602672576904,
+ "epoch": 1.5838383838383838,
+ "grad_norm": 2.123666524887085,
+ "learning_rate": 3.4181818181818182e-06,
+ "loss": 1.8564848899841309,
+ "mean_token_accuracy": 0.5871397256851196,
+ "num_tokens": 12845056.0,
+ "step": 784
+ },
+ {
+ "entropy": 1.441917896270752,
+ "epoch": 1.5858585858585859,
+ "grad_norm": 2.3286895751953125,
+ "learning_rate": 3.416161616161616e-06,
+ "loss": 1.4724184274673462,
+ "mean_token_accuracy": 0.6438080072402954,
+ "num_tokens": 12861440.0,
+ "step": 785
+ },
+ {
+ "entropy": 1.3071452379226685,
+ "epoch": 1.587878787878788,
+ "grad_norm": 2.149143934249878,
+ "learning_rate": 3.414141414141414e-06,
+ "loss": 1.331390142440796,
+ "mean_token_accuracy": 0.679347813129425,
+ "num_tokens": 12877824.0,
+ "step": 786
+ },
+ {
+ "entropy": 1.3547145128250122,
+ "epoch": 1.58989898989899,
+ "grad_norm": 2.2531180381774902,
+ "learning_rate": 3.4121212121212123e-06,
+ "loss": 1.361851453781128,
+ "mean_token_accuracy": 0.6767830848693848,
+ "num_tokens": 12894208.0,
+ "step": 787
+ },
+ {
+ "entropy": 0.9811291098594666,
+ "epoch": 1.591919191919192,
+ "grad_norm": 2.1053760051727295,
+ "learning_rate": 3.41010101010101e-06,
+ "loss": 0.9885507822036743,
+ "mean_token_accuracy": 0.7480459213256836,
+ "num_tokens": 12910592.0,
+ "step": 788
+ },
+ {
+ "entropy": 1.4940402507781982,
+ "epoch": 1.593939393939394,
+ "grad_norm": 2.240074634552002,
+ "learning_rate": 3.408080808080808e-06,
+ "loss": 1.5014076232910156,
+ "mean_token_accuracy": 0.6441743969917297,
+ "num_tokens": 12926976.0,
+ "step": 789
+ },
+ {
+ "entropy": 1.8134219646453857,
+ "epoch": 1.595959595959596,
+ "grad_norm": 2.040239095687866,
+ "learning_rate": 3.406060606060606e-06,
+ "loss": 1.8437519073486328,
+ "mean_token_accuracy": 0.5904372334480286,
+ "num_tokens": 12943360.0,
+ "step": 790
+ },
+ {
+ "entropy": 1.7233681678771973,
+ "epoch": 1.5979797979797978,
+ "grad_norm": 2.381786823272705,
+ "learning_rate": 3.4040404040404047e-06,
+ "loss": 1.7328217029571533,
+ "mean_token_accuracy": 0.6004518866539001,
+ "num_tokens": 12959744.0,
+ "step": 791
+ },
+ {
+ "entropy": 1.3966271877288818,
+ "epoch": 1.6,
+ "grad_norm": 2.17006778717041,
+ "learning_rate": 3.4020202020202026e-06,
+ "loss": 1.4065337181091309,
+ "mean_token_accuracy": 0.6629824042320251,
+ "num_tokens": 12976128.0,
+ "step": 792
+ },
+ {
+ "entropy": 1.5002557039260864,
+ "epoch": 1.602020202020202,
+ "grad_norm": 2.558037281036377,
+ "learning_rate": 3.4000000000000005e-06,
+ "loss": 1.4841307401657104,
+ "mean_token_accuracy": 0.647838294506073,
+ "num_tokens": 12992512.0,
+ "step": 793
+ },
+ {
+ "entropy": 1.7432687282562256,
+ "epoch": 1.6040404040404042,
+ "grad_norm": 2.134734630584717,
+ "learning_rate": 3.3979797979797984e-06,
+ "loss": 1.7539207935333252,
+ "mean_token_accuracy": 0.6028333902359009,
+ "num_tokens": 13008896.0,
+ "step": 794
+ },
+ {
+ "entropy": 1.4490517377853394,
+ "epoch": 1.606060606060606,
+ "grad_norm": 2.1590232849121094,
+ "learning_rate": 3.3959595959595963e-06,
+ "loss": 1.468411922454834,
+ "mean_token_accuracy": 0.656509518623352,
+ "num_tokens": 13025280.0,
+ "step": 795
+ },
+ {
+ "entropy": 1.3823497295379639,
+ "epoch": 1.6080808080808082,
+ "grad_norm": 2.1508285999298096,
+ "learning_rate": 3.3939393939393946e-06,
+ "loss": 1.4011330604553223,
+ "mean_token_accuracy": 0.6530898809432983,
+ "num_tokens": 13041664.0,
+ "step": 796
+ },
+ {
+ "entropy": 1.6982436180114746,
+ "epoch": 1.61010101010101,
+ "grad_norm": 2.077131986618042,
+ "learning_rate": 3.3919191919191925e-06,
+ "loss": 1.727889060974121,
+ "mean_token_accuracy": 0.6232290863990784,
+ "num_tokens": 13058048.0,
+ "step": 797
+ },
+ {
+ "entropy": 1.7268915176391602,
+ "epoch": 1.612121212121212,
+ "grad_norm": 2.0703542232513428,
+ "learning_rate": 3.3898989898989903e-06,
+ "loss": 1.7333730459213257,
+ "mean_token_accuracy": 0.6110160946846008,
+ "num_tokens": 13074432.0,
+ "step": 798
+ },
+ {
+ "entropy": 1.6253595352172852,
+ "epoch": 1.614141414141414,
+ "grad_norm": 2.241154432296753,
+ "learning_rate": 3.3878787878787882e-06,
+ "loss": 1.6275582313537598,
+ "mean_token_accuracy": 0.6198094487190247,
+ "num_tokens": 13090816.0,
+ "step": 799
+ },
+ {
+ "entropy": 1.8784692287445068,
+ "epoch": 1.6161616161616161,
+ "grad_norm": 2.505871057510376,
+ "learning_rate": 3.385858585858586e-06,
+ "loss": 1.8784008026123047,
+ "mean_token_accuracy": 0.5944064259529114,
+ "num_tokens": 13107200.0,
+ "step": 800
+ },
+ {
+ "epoch": 1.6161616161616161,
+ "eval_entropy": 1.4455043437980837,
+ "eval_loss": 1.5330992937088013,
+ "eval_mean_token_accuracy": 0.6448845310557273,
+ "eval_num_tokens": 13107200.0,
+ "eval_runtime": 43.766,
+ "eval_samples_per_second": 22.62,
+ "eval_steps_per_second": 2.833,
+ "step": 800
+ },
+ {
+ "entropy": 1.921440601348877,
+ "epoch": 1.6181818181818182,
+ "grad_norm": 2.1760783195495605,
+ "learning_rate": 3.3838383838383844e-06,
+ "loss": 1.9410110712051392,
+ "mean_token_accuracy": 0.5776746273040771,
+ "num_tokens": 13123584.0,
+ "step": 801
+ },
+ {
+ "entropy": 1.4233160018920898,
+ "epoch": 1.6202020202020202,
+ "grad_norm": 2.2687666416168213,
+ "learning_rate": 3.3818181818181823e-06,
+ "loss": 1.4465446472167969,
+ "mean_token_accuracy": 0.6488153338432312,
+ "num_tokens": 13139968.0,
+ "step": 802
+ },
+ {
+ "entropy": 1.697847843170166,
+ "epoch": 1.6222222222222222,
+ "grad_norm": 2.0649187564849854,
+ "learning_rate": 3.37979797979798e-06,
+ "loss": 1.7235385179519653,
+ "mean_token_accuracy": 0.6196262836456299,
+ "num_tokens": 13156352.0,
+ "step": 803
+ },
+ {
+ "entropy": 1.2027701139450073,
+ "epoch": 1.6242424242424243,
+ "grad_norm": 2.0115890502929688,
+ "learning_rate": 3.377777777777778e-06,
+ "loss": 1.1890676021575928,
+ "mean_token_accuracy": 0.6928431987762451,
+ "num_tokens": 13172736.0,
+ "step": 804
+ },
+ {
+ "entropy": 1.4847105741500854,
+ "epoch": 1.6262626262626263,
+ "grad_norm": 2.145098924636841,
+ "learning_rate": 3.375757575757576e-06,
+ "loss": 1.4817116260528564,
+ "mean_token_accuracy": 0.6415486335754395,
+ "num_tokens": 13189120.0,
+ "step": 805
+ },
+ {
+ "entropy": 1.3548893928527832,
+ "epoch": 1.6282828282828283,
+ "grad_norm": 2.3037521839141846,
+ "learning_rate": 3.3737373737373743e-06,
+ "loss": 1.3433952331542969,
+ "mean_token_accuracy": 0.6681729555130005,
+ "num_tokens": 13205504.0,
+ "step": 806
+ },
+ {
+ "entropy": 1.375793218612671,
+ "epoch": 1.6303030303030304,
+ "grad_norm": 2.0814852714538574,
+ "learning_rate": 3.371717171717172e-06,
+ "loss": 1.3506405353546143,
+ "mean_token_accuracy": 0.6634098887443542,
+ "num_tokens": 13221888.0,
+ "step": 807
+ },
+ {
+ "entropy": 1.1417323350906372,
+ "epoch": 1.6323232323232322,
+ "grad_norm": 2.0166866779327393,
+ "learning_rate": 3.36969696969697e-06,
+ "loss": 1.1731860637664795,
+ "mean_token_accuracy": 0.7123839855194092,
+ "num_tokens": 13238272.0,
+ "step": 808
+ },
+ {
+ "entropy": 1.3159022331237793,
+ "epoch": 1.6343434343434344,
+ "grad_norm": 1.9626281261444092,
+ "learning_rate": 3.367676767676768e-06,
+ "loss": 1.3286499977111816,
+ "mean_token_accuracy": 0.675561785697937,
+ "num_tokens": 13254656.0,
+ "step": 809
+ },
+ {
+ "entropy": 1.325775384902954,
+ "epoch": 1.6363636363636362,
+ "grad_norm": 2.1239242553710938,
+ "learning_rate": 3.365656565656566e-06,
+ "loss": 1.345514178276062,
+ "mean_token_accuracy": 0.6780043840408325,
+ "num_tokens": 13271040.0,
+ "step": 810
+ },
+ {
+ "entropy": 1.8307870626449585,
+ "epoch": 1.6383838383838385,
+ "grad_norm": 2.208296298980713,
+ "learning_rate": 3.3636363636363637e-06,
+ "loss": 1.8402234315872192,
+ "mean_token_accuracy": 0.5958719849586487,
+ "num_tokens": 13287424.0,
+ "step": 811
+ },
+ {
+ "entropy": 1.2596150636672974,
+ "epoch": 1.6404040404040403,
+ "grad_norm": 2.0936427116394043,
+ "learning_rate": 3.361616161616162e-06,
+ "loss": 1.2552804946899414,
+ "mean_token_accuracy": 0.6874083876609802,
+ "num_tokens": 13303808.0,
+ "step": 812
+ },
+ {
+ "entropy": 1.2575111389160156,
+ "epoch": 1.6424242424242426,
+ "grad_norm": 2.0232772827148438,
+ "learning_rate": 3.35959595959596e-06,
+ "loss": 1.2805989980697632,
+ "mean_token_accuracy": 0.6916829347610474,
+ "num_tokens": 13320192.0,
+ "step": 813
+ },
+ {
+ "entropy": 1.0270155668258667,
+ "epoch": 1.6444444444444444,
+ "grad_norm": 1.9090715646743774,
+ "learning_rate": 3.357575757575758e-06,
+ "loss": 1.0277824401855469,
+ "mean_token_accuracy": 0.7379091382026672,
+ "num_tokens": 13336576.0,
+ "step": 814
+ },
+ {
+ "entropy": 1.2992243766784668,
+ "epoch": 1.6464646464646466,
+ "grad_norm": 2.1718814373016357,
+ "learning_rate": 3.3555555555555557e-06,
+ "loss": 1.3278638124465942,
+ "mean_token_accuracy": 0.6764777898788452,
+ "num_tokens": 13352960.0,
+ "step": 815
+ },
+ {
+ "entropy": 1.5365362167358398,
+ "epoch": 1.6484848484848484,
+ "grad_norm": 2.0298495292663574,
+ "learning_rate": 3.3535353535353536e-06,
+ "loss": 1.5257508754730225,
+ "mean_token_accuracy": 0.6511358022689819,
+ "num_tokens": 13369344.0,
+ "step": 816
+ },
+ {
+ "entropy": 1.4384456872940063,
+ "epoch": 1.6505050505050505,
+ "grad_norm": 2.375277042388916,
+ "learning_rate": 3.351515151515152e-06,
+ "loss": 1.4287049770355225,
+ "mean_token_accuracy": 0.645639955997467,
+ "num_tokens": 13385728.0,
+ "step": 817
+ },
+ {
+ "entropy": 1.7104265689849854,
+ "epoch": 1.6525252525252525,
+ "grad_norm": 2.052535057067871,
+ "learning_rate": 3.34949494949495e-06,
+ "loss": 1.6971676349639893,
+ "mean_token_accuracy": 0.6159623861312866,
+ "num_tokens": 13402112.0,
+ "step": 818
+ },
+ {
+ "entropy": 1.5908539295196533,
+ "epoch": 1.6545454545454545,
+ "grad_norm": 2.047088861465454,
+ "learning_rate": 3.3474747474747477e-06,
+ "loss": 1.5640246868133545,
+ "mean_token_accuracy": 0.640754759311676,
+ "num_tokens": 13418496.0,
+ "step": 819
+ },
+ {
+ "entropy": 1.3660207986831665,
+ "epoch": 1.6565656565656566,
+ "grad_norm": 2.2711079120635986,
+ "learning_rate": 3.3454545454545456e-06,
+ "loss": 1.3601036071777344,
+ "mean_token_accuracy": 0.6577919125556946,
+ "num_tokens": 13434880.0,
+ "step": 820
+ },
+ {
+ "entropy": 1.5897634029388428,
+ "epoch": 1.6585858585858586,
+ "grad_norm": 2.0869712829589844,
+ "learning_rate": 3.3434343434343435e-06,
+ "loss": 1.5801420211791992,
+ "mean_token_accuracy": 0.6312897205352783,
+ "num_tokens": 13451264.0,
+ "step": 821
+ },
+ {
+ "entropy": 1.428406834602356,
+ "epoch": 1.6606060606060606,
+ "grad_norm": 2.049370765686035,
+ "learning_rate": 3.3414141414141413e-06,
+ "loss": 1.4145596027374268,
+ "mean_token_accuracy": 0.6649975776672363,
+ "num_tokens": 13467648.0,
+ "step": 822
+ },
+ {
+ "entropy": 1.5026675462722778,
+ "epoch": 1.6626262626262627,
+ "grad_norm": 2.157031297683716,
+ "learning_rate": 3.3393939393939397e-06,
+ "loss": 1.4902830123901367,
+ "mean_token_accuracy": 0.6524181962013245,
+ "num_tokens": 13484032.0,
+ "step": 823
+ },
+ {
+ "entropy": 1.4907052516937256,
+ "epoch": 1.6646464646464647,
+ "grad_norm": 2.1476478576660156,
+ "learning_rate": 3.3373737373737375e-06,
+ "loss": 1.5102436542510986,
+ "mean_token_accuracy": 0.6485100388526917,
+ "num_tokens": 13500416.0,
+ "step": 824
+ },
+ {
+ "entropy": 1.4277782440185547,
+ "epoch": 1.6666666666666665,
+ "grad_norm": 2.2880661487579346,
+ "learning_rate": 3.3353535353535354e-06,
+ "loss": 1.4358019828796387,
+ "mean_token_accuracy": 0.6522960662841797,
+ "num_tokens": 13516800.0,
+ "step": 825
+ },
+ {
+ "entropy": 1.5560128688812256,
+ "epoch": 1.6686868686868688,
+ "grad_norm": 2.048403024673462,
+ "learning_rate": 3.3333333333333333e-06,
+ "loss": 1.5472501516342163,
+ "mean_token_accuracy": 0.6488153338432312,
+ "num_tokens": 13533184.0,
+ "step": 826
+ },
+ {
+ "entropy": 1.665900707244873,
+ "epoch": 1.6707070707070706,
+ "grad_norm": 2.0828492641448975,
+ "learning_rate": 3.331313131313131e-06,
+ "loss": 1.6905372142791748,
+ "mean_token_accuracy": 0.629885196685791,
+ "num_tokens": 13549568.0,
+ "step": 827
+ },
+ {
+ "entropy": 1.3484426736831665,
+ "epoch": 1.6727272727272728,
+ "grad_norm": 2.1873531341552734,
+ "learning_rate": 3.3292929292929295e-06,
+ "loss": 1.3510258197784424,
+ "mean_token_accuracy": 0.6721421480178833,
+ "num_tokens": 13565952.0,
+ "step": 828
+ },
+ {
+ "entropy": 1.7577860355377197,
+ "epoch": 1.6747474747474747,
+ "grad_norm": 2.0467135906219482,
+ "learning_rate": 3.3272727272727274e-06,
+ "loss": 1.7791969776153564,
+ "mean_token_accuracy": 0.6028944849967957,
+ "num_tokens": 13582336.0,
+ "step": 829
+ },
+ {
+ "entropy": 1.4549124240875244,
+ "epoch": 1.676767676767677,
+ "grad_norm": 2.2004988193511963,
+ "learning_rate": 3.3252525252525253e-06,
+ "loss": 1.4512457847595215,
+ "mean_token_accuracy": 0.6521128416061401,
+ "num_tokens": 13598720.0,
+ "step": 830
+ },
+ {
+ "entropy": 1.2226758003234863,
+ "epoch": 1.6787878787878787,
+ "grad_norm": 2.0866682529449463,
+ "learning_rate": 3.323232323232323e-06,
+ "loss": 1.226635456085205,
+ "mean_token_accuracy": 0.6969345211982727,
+ "num_tokens": 13615104.0,
+ "step": 831
+ },
+ {
+ "entropy": 0.9826563000679016,
+ "epoch": 1.680808080808081,
+ "grad_norm": 1.9865771532058716,
+ "learning_rate": 3.321212121212121e-06,
+ "loss": 1.009648323059082,
+ "mean_token_accuracy": 0.7507327795028687,
+ "num_tokens": 13631488.0,
+ "step": 832
+ },
+ {
+ "entropy": 1.620038390159607,
+ "epoch": 1.6828282828282828,
+ "grad_norm": 2.1745760440826416,
+ "learning_rate": 3.3191919191919194e-06,
+ "loss": 1.615234136581421,
+ "mean_token_accuracy": 0.6305569410324097,
+ "num_tokens": 13647872.0,
+ "step": 833
+ },
+ {
+ "entropy": 1.4803876876831055,
+ "epoch": 1.6848484848484848,
+ "grad_norm": 2.328029155731201,
+ "learning_rate": 3.3171717171717177e-06,
+ "loss": 1.509049892425537,
+ "mean_token_accuracy": 0.6353810429573059,
+ "num_tokens": 13664256.0,
+ "step": 834
+ },
+ {
+ "entropy": 1.2290453910827637,
+ "epoch": 1.6868686868686869,
+ "grad_norm": 2.1777281761169434,
+ "learning_rate": 3.3151515151515156e-06,
+ "loss": 1.222726583480835,
+ "mean_token_accuracy": 0.6842330098152161,
+ "num_tokens": 13680640.0,
+ "step": 835
+ },
+ {
+ "entropy": 1.1965430974960327,
+ "epoch": 1.6888888888888889,
+ "grad_norm": 2.211397886276245,
+ "learning_rate": 3.3131313131313135e-06,
+ "loss": 1.2027852535247803,
+ "mean_token_accuracy": 0.7028578519821167,
+ "num_tokens": 13697024.0,
+ "step": 836
+ },
+ {
+ "entropy": 1.5326080322265625,
+ "epoch": 1.690909090909091,
+ "grad_norm": 2.8396589756011963,
+ "learning_rate": 3.3111111111111118e-06,
+ "loss": 1.5819613933563232,
+ "mean_token_accuracy": 0.625,
+ "num_tokens": 13713408.0,
+ "step": 837
+ },
+ {
+ "entropy": 1.2514865398406982,
+ "epoch": 1.692929292929293,
+ "grad_norm": 1.923771858215332,
+ "learning_rate": 3.3090909090909097e-06,
+ "loss": 1.2518126964569092,
+ "mean_token_accuracy": 0.6955300569534302,
+ "num_tokens": 13729792.0,
+ "step": 838
+ },
+ {
+ "entropy": 1.3717304468154907,
+ "epoch": 1.694949494949495,
+ "grad_norm": 2.0393056869506836,
+ "learning_rate": 3.3070707070707076e-06,
+ "loss": 1.400814175605774,
+ "mean_token_accuracy": 0.6618832349777222,
+ "num_tokens": 13746176.0,
+ "step": 839
+ },
+ {
+ "entropy": 1.2871394157409668,
+ "epoch": 1.696969696969697,
+ "grad_norm": 2.114823579788208,
+ "learning_rate": 3.3050505050505054e-06,
+ "loss": 1.302154541015625,
+ "mean_token_accuracy": 0.6790425181388855,
+ "num_tokens": 13762560.0,
+ "step": 840
+ },
+ {
+ "entropy": 1.518404483795166,
+ "epoch": 1.698989898989899,
+ "grad_norm": 2.186727523803711,
+ "learning_rate": 3.3030303030303033e-06,
+ "loss": 1.5067068338394165,
+ "mean_token_accuracy": 0.638006865978241,
+ "num_tokens": 13778944.0,
+ "step": 841
+ },
+ {
+ "entropy": 1.5698254108428955,
+ "epoch": 1.7010101010101009,
+ "grad_norm": 2.159721851348877,
+ "learning_rate": 3.3010101010101016e-06,
+ "loss": 1.5568101406097412,
+ "mean_token_accuracy": 0.6398998498916626,
+ "num_tokens": 13795328.0,
+ "step": 842
+ },
+ {
+ "entropy": 1.5159670114517212,
+ "epoch": 1.7030303030303031,
+ "grad_norm": 2.0655386447906494,
+ "learning_rate": 3.2989898989898995e-06,
+ "loss": 1.5152846574783325,
+ "mean_token_accuracy": 0.6441133618354797,
+ "num_tokens": 13811712.0,
+ "step": 843
+ },
+ {
+ "entropy": 1.231048822402954,
+ "epoch": 1.705050505050505,
+ "grad_norm": 1.99764084815979,
+ "learning_rate": 3.2969696969696974e-06,
+ "loss": 1.2419885396957397,
+ "mean_token_accuracy": 0.6936370134353638,
+ "num_tokens": 13828096.0,
+ "step": 844
+ },
+ {
+ "entropy": 1.1431403160095215,
+ "epoch": 1.7070707070707072,
+ "grad_norm": 2.0046544075012207,
+ "learning_rate": 3.2949494949494953e-06,
+ "loss": 1.147315502166748,
+ "mean_token_accuracy": 0.7155593633651733,
+ "num_tokens": 13844480.0,
+ "step": 845
+ },
+ {
+ "entropy": 1.903164267539978,
+ "epoch": 1.709090909090909,
+ "grad_norm": 2.068286418914795,
+ "learning_rate": 3.292929292929293e-06,
+ "loss": 1.9086098670959473,
+ "mean_token_accuracy": 0.5785295367240906,
+ "num_tokens": 13860864.0,
+ "step": 846
+ },
+ {
+ "entropy": 1.028931736946106,
+ "epoch": 1.7111111111111112,
+ "grad_norm": 2.008197546005249,
+ "learning_rate": 3.290909090909091e-06,
+ "loss": 1.0251559019088745,
+ "mean_token_accuracy": 0.7308866381645203,
+ "num_tokens": 13877248.0,
+ "step": 847
+ },
+ {
+ "entropy": 1.8438282012939453,
+ "epoch": 1.713131313131313,
+ "grad_norm": 2.180811882019043,
+ "learning_rate": 3.2888888888888894e-06,
+ "loss": 1.8572561740875244,
+ "mean_token_accuracy": 0.5788959264755249,
+ "num_tokens": 13893632.0,
+ "step": 848
+ },
+ {
+ "entropy": 1.437190294265747,
+ "epoch": 1.7151515151515153,
+ "grad_norm": 2.1013762950897217,
+ "learning_rate": 3.2868686868686873e-06,
+ "loss": 1.4337413311004639,
+ "mean_token_accuracy": 0.6489985585212708,
+ "num_tokens": 13910016.0,
+ "step": 849
+ },
+ {
+ "entropy": 1.482985496520996,
+ "epoch": 1.7171717171717171,
+ "grad_norm": 2.290910243988037,
+ "learning_rate": 3.284848484848485e-06,
+ "loss": 1.4959741830825806,
+ "mean_token_accuracy": 0.6447850465774536,
+ "num_tokens": 13926400.0,
+ "step": 850
+ },
+ {
+ "epoch": 1.7171717171717171,
+ "eval_entropy": 1.447427170411233,
+ "eval_loss": 1.5309957265853882,
+ "eval_mean_token_accuracy": 0.6452888370521607,
+ "eval_num_tokens": 13926400.0,
+ "eval_runtime": 43.7619,
+ "eval_samples_per_second": 22.622,
+ "eval_steps_per_second": 2.834,
+ "step": 850
+ },
+ {
+ "entropy": 1.4470106363296509,
+ "epoch": 1.7191919191919192,
+ "grad_norm": 2.2015624046325684,
+ "learning_rate": 3.282828282828283e-06,
+ "loss": 1.462576150894165,
+ "mean_token_accuracy": 0.643991231918335,
+ "num_tokens": 13942784.0,
+ "step": 851
+ },
+ {
+ "entropy": 1.4063832759857178,
+ "epoch": 1.7212121212121212,
+ "grad_norm": 2.1743109226226807,
+ "learning_rate": 3.280808080808081e-06,
+ "loss": 1.402491569519043,
+ "mean_token_accuracy": 0.6533952355384827,
+ "num_tokens": 13959168.0,
+ "step": 852
+ },
+ {
+ "entropy": 1.3735251426696777,
+ "epoch": 1.7232323232323232,
+ "grad_norm": 2.1350960731506348,
+ "learning_rate": 3.2787878787878793e-06,
+ "loss": 1.383405327796936,
+ "mean_token_accuracy": 0.6767830848693848,
+ "num_tokens": 13975552.0,
+ "step": 853
+ },
+ {
+ "entropy": 1.359702229499817,
+ "epoch": 1.7252525252525253,
+ "grad_norm": 2.1547393798828125,
+ "learning_rate": 3.276767676767677e-06,
+ "loss": 1.3454554080963135,
+ "mean_token_accuracy": 0.6712262034416199,
+ "num_tokens": 13991936.0,
+ "step": 854
+ },
+ {
+ "entropy": 1.0641778707504272,
+ "epoch": 1.7272727272727273,
+ "grad_norm": 2.0301826000213623,
+ "learning_rate": 3.274747474747475e-06,
+ "loss": 1.0626115798950195,
+ "mean_token_accuracy": 0.7188568711280823,
+ "num_tokens": 14008320.0,
+ "step": 855
+ },
+ {
+ "entropy": 1.598792552947998,
+ "epoch": 1.7292929292929293,
+ "grad_norm": 2.3324217796325684,
+ "learning_rate": 3.272727272727273e-06,
+ "loss": 1.6028941869735718,
+ "mean_token_accuracy": 0.6462506055831909,
+ "num_tokens": 14024704.0,
+ "step": 856
+ },
+ {
+ "entropy": 1.6304137706756592,
+ "epoch": 1.7313131313131314,
+ "grad_norm": 1.9968358278274536,
+ "learning_rate": 3.270707070707071e-06,
+ "loss": 1.6449415683746338,
+ "mean_token_accuracy": 0.6322056651115417,
+ "num_tokens": 14041088.0,
+ "step": 857
+ },
+ {
+ "entropy": 1.3219174146652222,
+ "epoch": 1.7333333333333334,
+ "grad_norm": 2.059927225112915,
+ "learning_rate": 3.2686868686868687e-06,
+ "loss": 1.3487168550491333,
+ "mean_token_accuracy": 0.6769052147865295,
+ "num_tokens": 14057472.0,
+ "step": 858
+ },
+ {
+ "entropy": 1.4645651578903198,
+ "epoch": 1.7353535353535352,
+ "grad_norm": 2.068962812423706,
+ "learning_rate": 3.266666666666667e-06,
+ "loss": 1.4480202198028564,
+ "mean_token_accuracy": 0.6521128416061401,
+ "num_tokens": 14073856.0,
+ "step": 859
+ },
+ {
+ "entropy": 1.5005300045013428,
+ "epoch": 1.7373737373737375,
+ "grad_norm": 2.228736162185669,
+ "learning_rate": 3.264646464646465e-06,
+ "loss": 1.5110201835632324,
+ "mean_token_accuracy": 0.6482657790184021,
+ "num_tokens": 14090240.0,
+ "step": 860
+ },
+ {
+ "entropy": 1.1641852855682373,
+ "epoch": 1.7393939393939393,
+ "grad_norm": 2.0611488819122314,
+ "learning_rate": 3.262626262626263e-06,
+ "loss": 1.1905086040496826,
+ "mean_token_accuracy": 0.7104909420013428,
+ "num_tokens": 14106624.0,
+ "step": 861
+ },
+ {
+ "entropy": 1.4315950870513916,
+ "epoch": 1.7414141414141415,
+ "grad_norm": 1.9914237260818481,
+ "learning_rate": 3.2606060606060607e-06,
+ "loss": 1.467012882232666,
+ "mean_token_accuracy": 0.657608687877655,
+ "num_tokens": 14123008.0,
+ "step": 862
+ },
+ {
+ "entropy": 1.5196973085403442,
+ "epoch": 1.7434343434343433,
+ "grad_norm": 2.3258447647094727,
+ "learning_rate": 3.2585858585858586e-06,
+ "loss": 1.5382301807403564,
+ "mean_token_accuracy": 0.6370298266410828,
+ "num_tokens": 14139392.0,
+ "step": 863
+ },
+ {
+ "entropy": 1.5098228454589844,
+ "epoch": 1.7454545454545456,
+ "grad_norm": 2.146327495574951,
+ "learning_rate": 3.256565656565657e-06,
+ "loss": 1.540649652481079,
+ "mean_token_accuracy": 0.6439301371574402,
+ "num_tokens": 14155776.0,
+ "step": 864
+ },
+ {
+ "entropy": 1.128011703491211,
+ "epoch": 1.7474747474747474,
+ "grad_norm": 1.967443823814392,
+ "learning_rate": 3.2545454545454548e-06,
+ "loss": 1.1707985401153564,
+ "mean_token_accuracy": 0.7298485636711121,
+ "num_tokens": 14172160.0,
+ "step": 865
+ },
+ {
+ "entropy": 1.9117931127548218,
+ "epoch": 1.7494949494949497,
+ "grad_norm": 2.097781181335449,
+ "learning_rate": 3.2525252525252527e-06,
+ "loss": 1.9291374683380127,
+ "mean_token_accuracy": 0.580483615398407,
+ "num_tokens": 14188544.0,
+ "step": 866
+ },
+ {
+ "entropy": 1.9053187370300293,
+ "epoch": 1.7515151515151515,
+ "grad_norm": 1.9865039587020874,
+ "learning_rate": 3.2505050505050505e-06,
+ "loss": 1.932790756225586,
+ "mean_token_accuracy": 0.5770639777183533,
+ "num_tokens": 14204928.0,
+ "step": 867
+ },
+ {
+ "entropy": 1.2647870779037476,
+ "epoch": 1.7535353535353535,
+ "grad_norm": 2.099891185760498,
+ "learning_rate": 3.2484848484848484e-06,
+ "loss": 1.281562328338623,
+ "mean_token_accuracy": 0.6938812732696533,
+ "num_tokens": 14221312.0,
+ "step": 868
+ },
+ {
+ "entropy": 1.5446428060531616,
+ "epoch": 1.7555555555555555,
+ "grad_norm": 1.9044189453125,
+ "learning_rate": 3.2464646464646467e-06,
+ "loss": 1.5440425872802734,
+ "mean_token_accuracy": 0.6547996997833252,
+ "num_tokens": 14237696.0,
+ "step": 869
+ },
+ {
+ "entropy": 1.179898977279663,
+ "epoch": 1.7575757575757576,
+ "grad_norm": 2.0257933139801025,
+ "learning_rate": 3.2444444444444446e-06,
+ "loss": 1.1607933044433594,
+ "mean_token_accuracy": 0.7053614854812622,
+ "num_tokens": 14254080.0,
+ "step": 870
+ },
+ {
+ "entropy": 1.0462855100631714,
+ "epoch": 1.7595959595959596,
+ "grad_norm": 1.8995661735534668,
+ "learning_rate": 3.2424242424242425e-06,
+ "loss": 1.0395057201385498,
+ "mean_token_accuracy": 0.7405959963798523,
+ "num_tokens": 14270464.0,
+ "step": 871
+ },
+ {
+ "entropy": 0.8866081833839417,
+ "epoch": 1.7616161616161616,
+ "grad_norm": 1.7807546854019165,
+ "learning_rate": 3.2404040404040404e-06,
+ "loss": 0.8795047998428345,
+ "mean_token_accuracy": 0.7721666097640991,
+ "num_tokens": 14286848.0,
+ "step": 872
+ },
+ {
+ "entropy": 1.9576365947723389,
+ "epoch": 1.7636363636363637,
+ "grad_norm": 2.1867618560791016,
+ "learning_rate": 3.2383838383838383e-06,
+ "loss": 1.9605103731155396,
+ "mean_token_accuracy": 0.5809110999107361,
+ "num_tokens": 14303232.0,
+ "step": 873
+ },
+ {
+ "entropy": 1.010751485824585,
+ "epoch": 1.7656565656565657,
+ "grad_norm": 1.9643300771713257,
+ "learning_rate": 3.236363636363636e-06,
+ "loss": 0.9990894794464111,
+ "mean_token_accuracy": 0.7386419177055359,
+ "num_tokens": 14319616.0,
+ "step": 874
+ },
+ {
+ "entropy": 1.3587232828140259,
+ "epoch": 1.7676767676767677,
+ "grad_norm": 2.2632040977478027,
+ "learning_rate": 3.2343434343434345e-06,
+ "loss": 1.3631021976470947,
+ "mean_token_accuracy": 0.665791392326355,
+ "num_tokens": 14336000.0,
+ "step": 875
+ },
+ {
+ "entropy": 1.6577962636947632,
+ "epoch": 1.7696969696969695,
+ "grad_norm": 2.3661980628967285,
+ "learning_rate": 3.232323232323233e-06,
+ "loss": 1.6802719831466675,
+ "mean_token_accuracy": 0.6024059653282166,
+ "num_tokens": 14352384.0,
+ "step": 876
+ },
+ {
+ "entropy": 1.4332902431488037,
+ "epoch": 1.7717171717171718,
+ "grad_norm": 2.2296342849731445,
+ "learning_rate": 3.2303030303030307e-06,
+ "loss": 1.4129014015197754,
+ "mean_token_accuracy": 0.6511358022689819,
+ "num_tokens": 14368768.0,
+ "step": 877
+ },
+ {
+ "entropy": 1.2260215282440186,
+ "epoch": 1.7737373737373736,
+ "grad_norm": 2.0659983158111572,
+ "learning_rate": 3.228282828282829e-06,
+ "loss": 1.2417192459106445,
+ "mean_token_accuracy": 0.693514883518219,
+ "num_tokens": 14385152.0,
+ "step": 878
+ },
+ {
+ "entropy": 1.5402687788009644,
+ "epoch": 1.7757575757575759,
+ "grad_norm": 2.2024054527282715,
+ "learning_rate": 3.226262626262627e-06,
+ "loss": 1.5535321235656738,
+ "mean_token_accuracy": 0.6331827044487,
+ "num_tokens": 14401536.0,
+ "step": 879
+ },
+ {
+ "entropy": 1.2205268144607544,
+ "epoch": 1.7777777777777777,
+ "grad_norm": 2.0056662559509277,
+ "learning_rate": 3.2242424242424248e-06,
+ "loss": 1.2244641780853271,
+ "mean_token_accuracy": 0.7037127614021301,
+ "num_tokens": 14417920.0,
+ "step": 880
+ },
+ {
+ "entropy": 1.542244791984558,
+ "epoch": 1.77979797979798,
+ "grad_norm": 2.0935254096984863,
+ "learning_rate": 3.2222222222222227e-06,
+ "loss": 1.5638453960418701,
+ "mean_token_accuracy": 0.648937463760376,
+ "num_tokens": 14434304.0,
+ "step": 881
+ },
+ {
+ "entropy": 1.4249905347824097,
+ "epoch": 1.7818181818181817,
+ "grad_norm": 1.9853826761245728,
+ "learning_rate": 3.2202020202020206e-06,
+ "loss": 1.4453399181365967,
+ "mean_token_accuracy": 0.6869809627532959,
+ "num_tokens": 14450688.0,
+ "step": 882
+ },
+ {
+ "entropy": 1.4777165651321411,
+ "epoch": 1.783838383838384,
+ "grad_norm": 2.064542055130005,
+ "learning_rate": 3.2181818181818184e-06,
+ "loss": 1.4756921529769897,
+ "mean_token_accuracy": 0.6640205383300781,
+ "num_tokens": 14467072.0,
+ "step": 883
+ },
+ {
+ "entropy": 1.3442282676696777,
+ "epoch": 1.7858585858585858,
+ "grad_norm": 2.043712854385376,
+ "learning_rate": 3.2161616161616168e-06,
+ "loss": 1.3412827253341675,
+ "mean_token_accuracy": 0.6821568012237549,
+ "num_tokens": 14483456.0,
+ "step": 884
+ },
+ {
+ "entropy": 1.5201579332351685,
+ "epoch": 1.7878787878787878,
+ "grad_norm": 2.060661792755127,
+ "learning_rate": 3.2141414141414146e-06,
+ "loss": 1.5367834568023682,
+ "mean_token_accuracy": 0.6442354917526245,
+ "num_tokens": 14499840.0,
+ "step": 885
+ },
+ {
+ "entropy": 1.1536649465560913,
+ "epoch": 1.7898989898989899,
+ "grad_norm": 2.1190037727355957,
+ "learning_rate": 3.2121212121212125e-06,
+ "loss": 1.1588165760040283,
+ "mean_token_accuracy": 0.7039570212364197,
+ "num_tokens": 14516224.0,
+ "step": 886
+ },
+ {
+ "entropy": 1.109966516494751,
+ "epoch": 1.791919191919192,
+ "grad_norm": 2.0320141315460205,
+ "learning_rate": 3.2101010101010104e-06,
+ "loss": 1.1113003492355347,
+ "mean_token_accuracy": 0.7123228907585144,
+ "num_tokens": 14532608.0,
+ "step": 887
+ },
+ {
+ "entropy": 1.5709099769592285,
+ "epoch": 1.793939393939394,
+ "grad_norm": 2.0467336177825928,
+ "learning_rate": 3.2080808080808083e-06,
+ "loss": 1.5823577642440796,
+ "mean_token_accuracy": 0.6404494643211365,
+ "num_tokens": 14548992.0,
+ "step": 888
+ },
+ {
+ "entropy": 1.4305545091629028,
+ "epoch": 1.795959595959596,
+ "grad_norm": 2.0764269828796387,
+ "learning_rate": 3.2060606060606066e-06,
+ "loss": 1.438415765762329,
+ "mean_token_accuracy": 0.6574255228042603,
+ "num_tokens": 14565376.0,
+ "step": 889
+ },
+ {
+ "entropy": 1.3523615598678589,
+ "epoch": 1.797979797979798,
+ "grad_norm": 2.167036294937134,
+ "learning_rate": 3.2040404040404045e-06,
+ "loss": 1.3782477378845215,
+ "mean_token_accuracy": 0.6645090579986572,
+ "num_tokens": 14581760.0,
+ "step": 890
+ },
+ {
+ "entropy": 1.491297721862793,
+ "epoch": 1.8,
+ "grad_norm": 2.3090412616729736,
+ "learning_rate": 3.2020202020202024e-06,
+ "loss": 1.4965641498565674,
+ "mean_token_accuracy": 0.636907696723938,
+ "num_tokens": 14598144.0,
+ "step": 891
+ },
+ {
+ "entropy": 1.2584501504898071,
+ "epoch": 1.802020202020202,
+ "grad_norm": 2.1699554920196533,
+ "learning_rate": 3.2000000000000003e-06,
+ "loss": 1.2622835636138916,
+ "mean_token_accuracy": 0.6901563405990601,
+ "num_tokens": 14614528.0,
+ "step": 892
+ },
+ {
+ "entropy": 2.149350643157959,
+ "epoch": 1.8040404040404039,
+ "grad_norm": 2.2669718265533447,
+ "learning_rate": 3.197979797979798e-06,
+ "loss": 2.167891263961792,
+ "mean_token_accuracy": 0.5497679710388184,
+ "num_tokens": 14630912.0,
+ "step": 893
+ },
+ {
+ "entropy": 1.1999561786651611,
+ "epoch": 1.8060606060606061,
+ "grad_norm": 2.163228988647461,
+ "learning_rate": 3.195959595959596e-06,
+ "loss": 1.2024329900741577,
+ "mean_token_accuracy": 0.7035906314849854,
+ "num_tokens": 14647296.0,
+ "step": 894
+ },
+ {
+ "entropy": 1.770652413368225,
+ "epoch": 1.808080808080808,
+ "grad_norm": 2.1631994247436523,
+ "learning_rate": 3.1939393939393944e-06,
+ "loss": 1.7568567991256714,
+ "mean_token_accuracy": 0.6030776500701904,
+ "num_tokens": 14663680.0,
+ "step": 895
+ },
+ {
+ "entropy": 1.2487529516220093,
+ "epoch": 1.8101010101010102,
+ "grad_norm": 1.987622857093811,
+ "learning_rate": 3.1919191919191923e-06,
+ "loss": 1.255242109298706,
+ "mean_token_accuracy": 0.6929042339324951,
+ "num_tokens": 14680064.0,
+ "step": 896
+ },
+ {
+ "entropy": 1.138524055480957,
+ "epoch": 1.812121212121212,
+ "grad_norm": 2.0367984771728516,
+ "learning_rate": 3.18989898989899e-06,
+ "loss": 1.1514946222305298,
+ "mean_token_accuracy": 0.7093917727470398,
+ "num_tokens": 14696448.0,
+ "step": 897
+ },
+ {
+ "entropy": 1.3337515592575073,
+ "epoch": 1.8141414141414143,
+ "grad_norm": 1.9483140707015991,
+ "learning_rate": 3.187878787878788e-06,
+ "loss": 1.3251439332962036,
+ "mean_token_accuracy": 0.6861260533332825,
+ "num_tokens": 14712832.0,
+ "step": 898
+ },
+ {
+ "entropy": 1.047120451927185,
+ "epoch": 1.816161616161616,
+ "grad_norm": 2.0481390953063965,
+ "learning_rate": 3.185858585858586e-06,
+ "loss": 1.071772575378418,
+ "mean_token_accuracy": 0.7275891304016113,
+ "num_tokens": 14729216.0,
+ "step": 899
+ },
+ {
+ "entropy": 1.3311363458633423,
+ "epoch": 1.8181818181818183,
+ "grad_norm": 2.3203413486480713,
+ "learning_rate": 3.1838383838383842e-06,
+ "loss": 1.334134817123413,
+ "mean_token_accuracy": 0.6643869280815125,
+ "num_tokens": 14745600.0,
+ "step": 900
+ },
+ {
+ "epoch": 1.8181818181818183,
+ "eval_entropy": 1.4476436400605786,
+ "eval_loss": 1.5290467739105225,
+ "eval_mean_token_accuracy": 0.6456143543604882,
+ "eval_num_tokens": 14745600.0,
+ "eval_runtime": 43.8113,
+ "eval_samples_per_second": 22.597,
+ "eval_steps_per_second": 2.83,
+ "step": 900
+ },
+ {
+ "entropy": 1.6842114925384521,
+ "epoch": 1.8202020202020202,
+ "grad_norm": 2.1709823608398438,
+ "learning_rate": 3.181818181818182e-06,
+ "loss": 1.6816682815551758,
+ "mean_token_accuracy": 0.6033830046653748,
+ "num_tokens": 14761984.0,
+ "step": 901
+ },
+ {
+ "entropy": 1.2661798000335693,
+ "epoch": 1.8222222222222222,
+ "grad_norm": 2.113783836364746,
+ "learning_rate": 3.17979797979798e-06,
+ "loss": 1.249183177947998,
+ "mean_token_accuracy": 0.6865534782409668,
+ "num_tokens": 14778368.0,
+ "step": 902
+ },
+ {
+ "entropy": 1.2666339874267578,
+ "epoch": 1.8242424242424242,
+ "grad_norm": 2.0210978984832764,
+ "learning_rate": 3.177777777777778e-06,
+ "loss": 1.2548515796661377,
+ "mean_token_accuracy": 0.686431348323822,
+ "num_tokens": 14794752.0,
+ "step": 903
+ },
+ {
+ "entropy": 1.1660066843032837,
+ "epoch": 1.8262626262626263,
+ "grad_norm": 2.0998034477233887,
+ "learning_rate": 3.1757575757575758e-06,
+ "loss": 1.1654324531555176,
+ "mean_token_accuracy": 0.695713222026825,
+ "num_tokens": 14811136.0,
+ "step": 904
+ },
+ {
+ "entropy": 1.2902735471725464,
+ "epoch": 1.8282828282828283,
+ "grad_norm": 2.1510303020477295,
+ "learning_rate": 3.173737373737374e-06,
+ "loss": 1.2879812717437744,
+ "mean_token_accuracy": 0.6693942546844482,
+ "num_tokens": 14827520.0,
+ "step": 905
+ },
+ {
+ "entropy": 1.1441799402236938,
+ "epoch": 1.8303030303030303,
+ "grad_norm": 1.989449143409729,
+ "learning_rate": 3.171717171717172e-06,
+ "loss": 1.1552281379699707,
+ "mean_token_accuracy": 0.7127503752708435,
+ "num_tokens": 14843904.0,
+ "step": 906
+ },
+ {
+ "entropy": 1.08464515209198,
+ "epoch": 1.8323232323232324,
+ "grad_norm": 2.094696044921875,
+ "learning_rate": 3.16969696969697e-06,
+ "loss": 1.0758531093597412,
+ "mean_token_accuracy": 0.7164142727851868,
+ "num_tokens": 14860288.0,
+ "step": 907
+ },
+ {
+ "entropy": 1.3847618103027344,
+ "epoch": 1.8343434343434344,
+ "grad_norm": 2.207976818084717,
+ "learning_rate": 3.1676767676767678e-06,
+ "loss": 1.3722931146621704,
+ "mean_token_accuracy": 0.6741573214530945,
+ "num_tokens": 14876672.0,
+ "step": 908
+ },
+ {
+ "entropy": 1.3558534383773804,
+ "epoch": 1.8363636363636364,
+ "grad_norm": 2.166674852371216,
+ "learning_rate": 3.1656565656565656e-06,
+ "loss": 1.3612618446350098,
+ "mean_token_accuracy": 0.6669516563415527,
+ "num_tokens": 14893056.0,
+ "step": 909
+ },
+ {
+ "entropy": 1.4160822629928589,
+ "epoch": 1.8383838383838382,
+ "grad_norm": 2.2242727279663086,
+ "learning_rate": 3.1636363636363635e-06,
+ "loss": 1.408278226852417,
+ "mean_token_accuracy": 0.6634098887443542,
+ "num_tokens": 14909440.0,
+ "step": 910
+ },
+ {
+ "entropy": 1.1337573528289795,
+ "epoch": 1.8404040404040405,
+ "grad_norm": 2.1882550716400146,
+ "learning_rate": 3.161616161616162e-06,
+ "loss": 1.1503205299377441,
+ "mean_token_accuracy": 0.7059721350669861,
+ "num_tokens": 14925824.0,
+ "step": 911
+ },
+ {
+ "entropy": 1.4373403787612915,
+ "epoch": 1.8424242424242423,
+ "grad_norm": 2.1595654487609863,
+ "learning_rate": 3.1595959595959597e-06,
+ "loss": 1.4344165325164795,
+ "mean_token_accuracy": 0.6599902510643005,
+ "num_tokens": 14942208.0,
+ "step": 912
+ },
+ {
+ "entropy": 1.0798827409744263,
+ "epoch": 1.8444444444444446,
+ "grad_norm": 2.101088047027588,
+ "learning_rate": 3.1575757575757576e-06,
+ "loss": 1.0873513221740723,
+ "mean_token_accuracy": 0.7169027924537659,
+ "num_tokens": 14958592.0,
+ "step": 913
+ },
+ {
+ "entropy": 1.7024599313735962,
+ "epoch": 1.8464646464646464,
+ "grad_norm": 2.291907548904419,
+ "learning_rate": 3.1555555555555555e-06,
+ "loss": 1.7014999389648438,
+ "mean_token_accuracy": 0.6105275750160217,
+ "num_tokens": 14974976.0,
+ "step": 914
+ },
+ {
+ "entropy": 1.7765953540802002,
+ "epoch": 1.8484848484848486,
+ "grad_norm": 2.1465439796447754,
+ "learning_rate": 3.1535353535353534e-06,
+ "loss": 1.788269281387329,
+ "mean_token_accuracy": 0.6066194176673889,
+ "num_tokens": 14991360.0,
+ "step": 915
+ },
+ {
+ "entropy": 1.5223884582519531,
+ "epoch": 1.8505050505050504,
+ "grad_norm": 2.313291549682617,
+ "learning_rate": 3.1515151515151517e-06,
+ "loss": 1.55397367477417,
+ "mean_token_accuracy": 0.6377626061439514,
+ "num_tokens": 15007744.0,
+ "step": 916
+ },
+ {
+ "entropy": 1.5820955038070679,
+ "epoch": 1.8525252525252527,
+ "grad_norm": 2.094886541366577,
+ "learning_rate": 3.1494949494949496e-06,
+ "loss": 1.58372962474823,
+ "mean_token_accuracy": 0.6339154839515686,
+ "num_tokens": 15024128.0,
+ "step": 917
+ },
+ {
+ "entropy": 1.7399076223373413,
+ "epoch": 1.8545454545454545,
+ "grad_norm": 2.0311105251312256,
+ "learning_rate": 3.1474747474747475e-06,
+ "loss": 1.7362079620361328,
+ "mean_token_accuracy": 0.604421079158783,
+ "num_tokens": 15040512.0,
+ "step": 918
+ },
+ {
+ "entropy": 1.4311537742614746,
+ "epoch": 1.8565656565656565,
+ "grad_norm": 2.13740873336792,
+ "learning_rate": 3.145454545454546e-06,
+ "loss": 1.4574000835418701,
+ "mean_token_accuracy": 0.648876428604126,
+ "num_tokens": 15056896.0,
+ "step": 919
+ },
+ {
+ "entropy": 1.6997709274291992,
+ "epoch": 1.8585858585858586,
+ "grad_norm": 2.1604959964752197,
+ "learning_rate": 3.143434343434344e-06,
+ "loss": 1.694093942642212,
+ "mean_token_accuracy": 0.6144357323646545,
+ "num_tokens": 15073280.0,
+ "step": 920
+ },
+ {
+ "entropy": 1.5796921253204346,
+ "epoch": 1.8606060606060606,
+ "grad_norm": 2.3852932453155518,
+ "learning_rate": 3.141414141414142e-06,
+ "loss": 1.5799579620361328,
+ "mean_token_accuracy": 0.6322667598724365,
+ "num_tokens": 15089664.0,
+ "step": 921
+ },
+ {
+ "entropy": 1.6185977458953857,
+ "epoch": 1.8626262626262626,
+ "grad_norm": 2.0631208419799805,
+ "learning_rate": 3.13939393939394e-06,
+ "loss": 1.6541715860366821,
+ "mean_token_accuracy": 0.632083535194397,
+ "num_tokens": 15106048.0,
+ "step": 922
+ },
+ {
+ "entropy": 1.5872372388839722,
+ "epoch": 1.8646464646464647,
+ "grad_norm": 2.154554843902588,
+ "learning_rate": 3.1373737373737378e-06,
+ "loss": 1.5946767330169678,
+ "mean_token_accuracy": 0.6294577717781067,
+ "num_tokens": 15122432.0,
+ "step": 923
+ },
+ {
+ "entropy": 1.6059986352920532,
+ "epoch": 1.8666666666666667,
+ "grad_norm": 1.9893742799758911,
+ "learning_rate": 3.1353535353535357e-06,
+ "loss": 1.6153039932250977,
+ "mean_token_accuracy": 0.6373962163925171,
+ "num_tokens": 15138816.0,
+ "step": 924
+ },
+ {
+ "entropy": 1.3143881559371948,
+ "epoch": 1.8686868686868687,
+ "grad_norm": 1.8989832401275635,
+ "learning_rate": 3.133333333333334e-06,
+ "loss": 1.3332319259643555,
+ "mean_token_accuracy": 0.6842330098152161,
+ "num_tokens": 15155200.0,
+ "step": 925
+ },
+ {
+ "entropy": 1.0474469661712646,
+ "epoch": 1.8707070707070708,
+ "grad_norm": 1.9129916429519653,
+ "learning_rate": 3.131313131313132e-06,
+ "loss": 1.0238264799118042,
+ "mean_token_accuracy": 0.7459697127342224,
+ "num_tokens": 15171584.0,
+ "step": 926
+ },
+ {
+ "entropy": 1.581508994102478,
+ "epoch": 1.8727272727272726,
+ "grad_norm": 2.0047285556793213,
+ "learning_rate": 3.1292929292929297e-06,
+ "loss": 1.6087100505828857,
+ "mean_token_accuracy": 0.6282364726066589,
+ "num_tokens": 15187968.0,
+ "step": 927
+ },
+ {
+ "entropy": 1.0985661745071411,
+ "epoch": 1.8747474747474748,
+ "grad_norm": 2.1207540035247803,
+ "learning_rate": 3.1272727272727276e-06,
+ "loss": 1.1158297061920166,
+ "mean_token_accuracy": 0.7111626863479614,
+ "num_tokens": 15204352.0,
+ "step": 928
+ },
+ {
+ "entropy": 1.4591490030288696,
+ "epoch": 1.8767676767676766,
+ "grad_norm": 2.2879691123962402,
+ "learning_rate": 3.1252525252525255e-06,
+ "loss": 1.4775316715240479,
+ "mean_token_accuracy": 0.6477161645889282,
+ "num_tokens": 15220736.0,
+ "step": 929
+ },
+ {
+ "entropy": 1.5469937324523926,
+ "epoch": 1.878787878787879,
+ "grad_norm": 2.0972726345062256,
+ "learning_rate": 3.1232323232323234e-06,
+ "loss": 1.5949021577835083,
+ "mean_token_accuracy": 0.6547996997833252,
+ "num_tokens": 15237120.0,
+ "step": 930
+ },
+ {
+ "entropy": 1.363472819328308,
+ "epoch": 1.8808080808080807,
+ "grad_norm": 2.149606943130493,
+ "learning_rate": 3.1212121212121217e-06,
+ "loss": 1.358612298965454,
+ "mean_token_accuracy": 0.6780654788017273,
+ "num_tokens": 15253504.0,
+ "step": 931
+ },
+ {
+ "entropy": 1.189648985862732,
+ "epoch": 1.882828282828283,
+ "grad_norm": 2.2575035095214844,
+ "learning_rate": 3.1191919191919196e-06,
+ "loss": 1.209303379058838,
+ "mean_token_accuracy": 0.7002320289611816,
+ "num_tokens": 15269888.0,
+ "step": 932
+ },
+ {
+ "entropy": 1.3054684400558472,
+ "epoch": 1.8848484848484848,
+ "grad_norm": 2.243915319442749,
+ "learning_rate": 3.1171717171717175e-06,
+ "loss": 1.333367109298706,
+ "mean_token_accuracy": 0.6703712940216064,
+ "num_tokens": 15286272.0,
+ "step": 933
+ },
+ {
+ "entropy": 1.6298167705535889,
+ "epoch": 1.886868686868687,
+ "grad_norm": 1.9654396772384644,
+ "learning_rate": 3.1151515151515154e-06,
+ "loss": 1.6496453285217285,
+ "mean_token_accuracy": 0.6248778700828552,
+ "num_tokens": 15302656.0,
+ "step": 934
+ },
+ {
+ "entropy": 1.1228219270706177,
+ "epoch": 1.8888888888888888,
+ "grad_norm": 2.2492377758026123,
+ "learning_rate": 3.1131313131313133e-06,
+ "loss": 1.1560063362121582,
+ "mean_token_accuracy": 0.699499249458313,
+ "num_tokens": 15319040.0,
+ "step": 935
+ },
+ {
+ "entropy": 1.7481759786605835,
+ "epoch": 1.8909090909090909,
+ "grad_norm": 2.1635665893554688,
+ "learning_rate": 3.1111111111111116e-06,
+ "loss": 1.737868070602417,
+ "mean_token_accuracy": 0.6030776500701904,
+ "num_tokens": 15335424.0,
+ "step": 936
+ },
+ {
+ "entropy": 1.3964993953704834,
+ "epoch": 1.892929292929293,
+ "grad_norm": 2.262946367263794,
+ "learning_rate": 3.1090909090909095e-06,
+ "loss": 1.3927817344665527,
+ "mean_token_accuracy": 0.6546775698661804,
+ "num_tokens": 15351808.0,
+ "step": 937
+ },
+ {
+ "entropy": 1.2503907680511475,
+ "epoch": 1.894949494949495,
+ "grad_norm": 1.9837188720703125,
+ "learning_rate": 3.1070707070707074e-06,
+ "loss": 1.2529189586639404,
+ "mean_token_accuracy": 0.6965681314468384,
+ "num_tokens": 15368192.0,
+ "step": 938
+ },
+ {
+ "entropy": 0.9774144291877747,
+ "epoch": 1.896969696969697,
+ "grad_norm": 1.9201257228851318,
+ "learning_rate": 3.1050505050505052e-06,
+ "loss": 0.9864459037780762,
+ "mean_token_accuracy": 0.7534807324409485,
+ "num_tokens": 15384576.0,
+ "step": 939
+ },
+ {
+ "entropy": 1.4065781831741333,
+ "epoch": 1.898989898989899,
+ "grad_norm": 2.357865571975708,
+ "learning_rate": 3.103030303030303e-06,
+ "loss": 1.428115725517273,
+ "mean_token_accuracy": 0.6493649482727051,
+ "num_tokens": 15400960.0,
+ "step": 940
+ },
+ {
+ "entropy": 1.1632903814315796,
+ "epoch": 1.901010101010101,
+ "grad_norm": 1.8824918270111084,
+ "learning_rate": 3.1010101010101014e-06,
+ "loss": 1.1478252410888672,
+ "mean_token_accuracy": 0.7158036231994629,
+ "num_tokens": 15417344.0,
+ "step": 941
+ },
+ {
+ "entropy": 1.7470908164978027,
+ "epoch": 1.903030303030303,
+ "grad_norm": 2.2958669662475586,
+ "learning_rate": 3.0989898989898993e-06,
+ "loss": 1.7697877883911133,
+ "mean_token_accuracy": 0.5892769694328308,
+ "num_tokens": 15433728.0,
+ "step": 942
+ },
+ {
+ "entropy": 1.2897151708602905,
+ "epoch": 1.905050505050505,
+ "grad_norm": 2.155731439590454,
+ "learning_rate": 3.0969696969696972e-06,
+ "loss": 1.2971919775009155,
+ "mean_token_accuracy": 0.6731802821159363,
+ "num_tokens": 15450112.0,
+ "step": 943
+ },
+ {
+ "entropy": 1.2687026262283325,
+ "epoch": 1.907070707070707,
+ "grad_norm": 2.0896284580230713,
+ "learning_rate": 3.094949494949495e-06,
+ "loss": 1.2423906326293945,
+ "mean_token_accuracy": 0.6988885998725891,
+ "num_tokens": 15466496.0,
+ "step": 944
+ },
+ {
+ "entropy": 1.5710495710372925,
+ "epoch": 1.9090909090909092,
+ "grad_norm": 2.200758457183838,
+ "learning_rate": 3.092929292929293e-06,
+ "loss": 1.5329574346542358,
+ "mean_token_accuracy": 0.6560820937156677,
+ "num_tokens": 15482880.0,
+ "step": 945
+ },
+ {
+ "entropy": 1.1555253267288208,
+ "epoch": 1.911111111111111,
+ "grad_norm": 2.278230905532837,
+ "learning_rate": 3.090909090909091e-06,
+ "loss": 1.1469731330871582,
+ "mean_token_accuracy": 0.7220932841300964,
+ "num_tokens": 15499264.0,
+ "step": 946
+ },
+ {
+ "entropy": 1.2870018482208252,
+ "epoch": 1.9131313131313132,
+ "grad_norm": 1.8973854780197144,
+ "learning_rate": 3.088888888888889e-06,
+ "loss": 1.2787907123565674,
+ "mean_token_accuracy": 0.6885075569152832,
+ "num_tokens": 15515648.0,
+ "step": 947
+ },
+ {
+ "entropy": 1.324872612953186,
+ "epoch": 1.915151515151515,
+ "grad_norm": 2.1889214515686035,
+ "learning_rate": 3.086868686868687e-06,
+ "loss": 1.3286981582641602,
+ "mean_token_accuracy": 0.6743404865264893,
+ "num_tokens": 15532032.0,
+ "step": 948
+ },
+ {
+ "entropy": 1.5601656436920166,
+ "epoch": 1.9171717171717173,
+ "grad_norm": 2.304993152618408,
+ "learning_rate": 3.084848484848485e-06,
+ "loss": 1.5766493082046509,
+ "mean_token_accuracy": 0.6172447204589844,
+ "num_tokens": 15548416.0,
+ "step": 949
+ },
+ {
+ "entropy": 1.4633477926254272,
+ "epoch": 1.9191919191919191,
+ "grad_norm": 1.9083963632583618,
+ "learning_rate": 3.082828282828283e-06,
+ "loss": 1.4554777145385742,
+ "mean_token_accuracy": 0.6667073965072632,
+ "num_tokens": 15564800.0,
+ "step": 950
+ },
+ {
+ "epoch": 1.9191919191919191,
+ "eval_entropy": 1.4443931824737979,
+ "eval_loss": 1.5272752046585083,
+ "eval_mean_token_accuracy": 0.6459201723337173,
+ "eval_num_tokens": 15564800.0,
+ "eval_runtime": 43.7243,
+ "eval_samples_per_second": 22.642,
+ "eval_steps_per_second": 2.836,
+ "step": 950
+ },
+ {
+ "entropy": 1.3350679874420166,
+ "epoch": 1.9212121212121214,
+ "grad_norm": 2.139172315597534,
+ "learning_rate": 3.0808080808080807e-06,
+ "loss": 1.3285927772521973,
+ "mean_token_accuracy": 0.6833781003952026,
+ "num_tokens": 15581184.0,
+ "step": 951
+ },
+ {
+ "entropy": 1.2373814582824707,
+ "epoch": 1.9232323232323232,
+ "grad_norm": 2.1068115234375,
+ "learning_rate": 3.078787878787879e-06,
+ "loss": 1.2527921199798584,
+ "mean_token_accuracy": 0.6969345211982727,
+ "num_tokens": 15597568.0,
+ "step": 952
+ },
+ {
+ "entropy": 1.2182631492614746,
+ "epoch": 1.9252525252525252,
+ "grad_norm": 2.1326634883880615,
+ "learning_rate": 3.076767676767677e-06,
+ "loss": 1.2174904346466064,
+ "mean_token_accuracy": 0.6905227303504944,
+ "num_tokens": 15613952.0,
+ "step": 953
+ },
+ {
+ "entropy": 0.9768888354301453,
+ "epoch": 1.9272727272727272,
+ "grad_norm": 1.9833085536956787,
+ "learning_rate": 3.074747474747475e-06,
+ "loss": 0.9956569671630859,
+ "mean_token_accuracy": 0.7401685118675232,
+ "num_tokens": 15630336.0,
+ "step": 954
+ },
+ {
+ "entropy": 0.7818430066108704,
+ "epoch": 1.9292929292929293,
+ "grad_norm": 1.9219205379486084,
+ "learning_rate": 3.0727272727272727e-06,
+ "loss": 0.7922182083129883,
+ "mean_token_accuracy": 0.7827919125556946,
+ "num_tokens": 15646720.0,
+ "step": 955
+ },
+ {
+ "entropy": 1.6172020435333252,
+ "epoch": 1.9313131313131313,
+ "grad_norm": 1.9390695095062256,
+ "learning_rate": 3.0707070707070706e-06,
+ "loss": 1.6629729270935059,
+ "mean_token_accuracy": 0.6192598938941956,
+ "num_tokens": 15663104.0,
+ "step": 956
+ },
+ {
+ "entropy": 1.291207194328308,
+ "epoch": 1.9333333333333333,
+ "grad_norm": 1.9242857694625854,
+ "learning_rate": 3.068686868686869e-06,
+ "loss": 1.2721552848815918,
+ "mean_token_accuracy": 0.6867367029190063,
+ "num_tokens": 15679488.0,
+ "step": 957
+ },
+ {
+ "entropy": 1.3730298280715942,
+ "epoch": 1.9353535353535354,
+ "grad_norm": 1.9483041763305664,
+ "learning_rate": 3.066666666666667e-06,
+ "loss": 1.3835055828094482,
+ "mean_token_accuracy": 0.6745237112045288,
+ "num_tokens": 15695872.0,
+ "step": 958
+ },
+ {
+ "entropy": 1.1266423463821411,
+ "epoch": 1.9373737373737374,
+ "grad_norm": 2.1663992404937744,
+ "learning_rate": 3.0646464646464647e-06,
+ "loss": 1.1091173887252808,
+ "mean_token_accuracy": 0.7151318788528442,
+ "num_tokens": 15712256.0,
+ "step": 959
+ },
+ {
+ "entropy": 1.298812747001648,
+ "epoch": 1.9393939393939394,
+ "grad_norm": 2.0564448833465576,
+ "learning_rate": 3.0626262626262626e-06,
+ "loss": 1.3167932033538818,
+ "mean_token_accuracy": 0.6812408566474915,
+ "num_tokens": 15728640.0,
+ "step": 960
+ },
+ {
+ "entropy": 1.7007938623428345,
+ "epoch": 1.9414141414141413,
+ "grad_norm": 2.1043670177459717,
+ "learning_rate": 3.0606060606060605e-06,
+ "loss": 1.6956250667572021,
+ "mean_token_accuracy": 0.6194430589675903,
+ "num_tokens": 15745024.0,
+ "step": 961
+ },
+ {
+ "entropy": 1.408955693244934,
+ "epoch": 1.9434343434343435,
+ "grad_norm": 2.0432121753692627,
+ "learning_rate": 3.058585858585859e-06,
+ "loss": 1.3905062675476074,
+ "mean_token_accuracy": 0.6662799119949341,
+ "num_tokens": 15761408.0,
+ "step": 962
+ },
+ {
+ "entropy": 1.938867211341858,
+ "epoch": 1.9454545454545453,
+ "grad_norm": 2.3359744548797607,
+ "learning_rate": 3.056565656565657e-06,
+ "loss": 1.9670305252075195,
+ "mean_token_accuracy": 0.5845139026641846,
+ "num_tokens": 15777792.0,
+ "step": 963
+ },
+ {
+ "entropy": 1.5049448013305664,
+ "epoch": 1.9474747474747476,
+ "grad_norm": 2.1907997131347656,
+ "learning_rate": 3.054545454545455e-06,
+ "loss": 1.534651756286621,
+ "mean_token_accuracy": 0.652723491191864,
+ "num_tokens": 15794176.0,
+ "step": 964
+ },
+ {
+ "entropy": 1.1916700601577759,
+ "epoch": 1.9494949494949494,
+ "grad_norm": 2.098696231842041,
+ "learning_rate": 3.052525252525253e-06,
+ "loss": 1.191361904144287,
+ "mean_token_accuracy": 0.715192973613739,
+ "num_tokens": 15810560.0,
+ "step": 965
+ },
+ {
+ "entropy": 1.6707743406295776,
+ "epoch": 1.9515151515151516,
+ "grad_norm": 2.106846570968628,
+ "learning_rate": 3.0505050505050508e-06,
+ "loss": 1.6789159774780273,
+ "mean_token_accuracy": 0.6133365631103516,
+ "num_tokens": 15826944.0,
+ "step": 966
+ },
+ {
+ "entropy": 1.4679991006851196,
+ "epoch": 1.9535353535353535,
+ "grad_norm": 2.1568050384521484,
+ "learning_rate": 3.048484848484849e-06,
+ "loss": 1.4427556991577148,
+ "mean_token_accuracy": 0.6526013612747192,
+ "num_tokens": 15843328.0,
+ "step": 967
+ },
+ {
+ "entropy": 0.9881705641746521,
+ "epoch": 1.9555555555555557,
+ "grad_norm": 2.029680013656616,
+ "learning_rate": 3.046464646464647e-06,
+ "loss": 0.9961811304092407,
+ "mean_token_accuracy": 0.7408402562141418,
+ "num_tokens": 15859712.0,
+ "step": 968
+ },
+ {
+ "entropy": 1.0721628665924072,
+ "epoch": 1.9575757575757575,
+ "grad_norm": 1.9763140678405762,
+ "learning_rate": 3.044444444444445e-06,
+ "loss": 1.067029356956482,
+ "mean_token_accuracy": 0.733146071434021,
+ "num_tokens": 15876096.0,
+ "step": 969
+ },
+ {
+ "entropy": 1.7053834199905396,
+ "epoch": 1.9595959595959596,
+ "grad_norm": 2.3659849166870117,
+ "learning_rate": 3.0424242424242427e-06,
+ "loss": 1.7629203796386719,
+ "mean_token_accuracy": 0.6105275750160217,
+ "num_tokens": 15892480.0,
+ "step": 970
+ },
+ {
+ "entropy": 1.8083471059799194,
+ "epoch": 1.9616161616161616,
+ "grad_norm": 2.081869602203369,
+ "learning_rate": 3.0404040404040406e-06,
+ "loss": 1.8430367708206177,
+ "mean_token_accuracy": 0.6009404063224792,
+ "num_tokens": 15908864.0,
+ "step": 971
+ },
+ {
+ "entropy": 1.2576755285263062,
+ "epoch": 1.9636363636363636,
+ "grad_norm": 2.128230571746826,
+ "learning_rate": 3.038383838383839e-06,
+ "loss": 1.2493822574615479,
+ "mean_token_accuracy": 0.6913776397705078,
+ "num_tokens": 15925248.0,
+ "step": 972
+ },
+ {
+ "entropy": 0.8995128273963928,
+ "epoch": 1.9656565656565657,
+ "grad_norm": 2.0216031074523926,
+ "learning_rate": 3.036363636363637e-06,
+ "loss": 0.9069615602493286,
+ "mean_token_accuracy": 0.7620908617973328,
+ "num_tokens": 15941632.0,
+ "step": 973
+ },
+ {
+ "entropy": 1.615832805633545,
+ "epoch": 1.9676767676767677,
+ "grad_norm": 2.26552152633667,
+ "learning_rate": 3.0343434343434347e-06,
+ "loss": 1.6284873485565186,
+ "mean_token_accuracy": 0.6193209290504456,
+ "num_tokens": 15958016.0,
+ "step": 974
+ },
+ {
+ "entropy": 1.404674768447876,
+ "epoch": 1.9696969696969697,
+ "grad_norm": 2.1790690422058105,
+ "learning_rate": 3.0323232323232326e-06,
+ "loss": 1.4130914211273193,
+ "mean_token_accuracy": 0.6594406366348267,
+ "num_tokens": 15974400.0,
+ "step": 975
+ },
+ {
+ "entropy": 1.3440049886703491,
+ "epoch": 1.9717171717171718,
+ "grad_norm": 2.121338367462158,
+ "learning_rate": 3.0303030303030305e-06,
+ "loss": 1.3652238845825195,
+ "mean_token_accuracy": 0.6794699430465698,
+ "num_tokens": 15990784.0,
+ "step": 976
+ },
+ {
+ "entropy": 1.0894689559936523,
+ "epoch": 1.9737373737373738,
+ "grad_norm": 2.0155327320098877,
+ "learning_rate": 3.028282828282829e-06,
+ "loss": 1.0642526149749756,
+ "mean_token_accuracy": 0.7223986387252808,
+ "num_tokens": 16007168.0,
+ "step": 977
+ },
+ {
+ "entropy": 1.469588041305542,
+ "epoch": 1.9757575757575756,
+ "grad_norm": 2.2047550678253174,
+ "learning_rate": 3.0262626262626267e-06,
+ "loss": 1.498295545578003,
+ "mean_token_accuracy": 0.6441133618354797,
+ "num_tokens": 16023552.0,
+ "step": 978
+ },
+ {
+ "entropy": 1.6617093086242676,
+ "epoch": 1.9777777777777779,
+ "grad_norm": 1.9770722389221191,
+ "learning_rate": 3.0242424242424246e-06,
+ "loss": 1.6753556728363037,
+ "mean_token_accuracy": 0.6284807324409485,
+ "num_tokens": 16039936.0,
+ "step": 979
+ },
+ {
+ "entropy": 1.8029206991195679,
+ "epoch": 1.9797979797979797,
+ "grad_norm": 2.2365968227386475,
+ "learning_rate": 3.0222222222222225e-06,
+ "loss": 1.7957369089126587,
+ "mean_token_accuracy": 0.601062536239624,
+ "num_tokens": 16056320.0,
+ "step": 980
+ },
+ {
+ "entropy": 1.6331384181976318,
+ "epoch": 1.981818181818182,
+ "grad_norm": 2.0565884113311768,
+ "learning_rate": 3.0202020202020203e-06,
+ "loss": 1.673048973083496,
+ "mean_token_accuracy": 0.6203590631484985,
+ "num_tokens": 16072704.0,
+ "step": 981
+ },
+ {
+ "entropy": 1.1730273962020874,
+ "epoch": 1.9838383838383837,
+ "grad_norm": 2.0347228050231934,
+ "learning_rate": 3.0181818181818182e-06,
+ "loss": 1.1709872484207153,
+ "mean_token_accuracy": 0.7086589932441711,
+ "num_tokens": 16089088.0,
+ "step": 982
+ },
+ {
+ "entropy": 1.1109111309051514,
+ "epoch": 1.985858585858586,
+ "grad_norm": 1.9992051124572754,
+ "learning_rate": 3.0161616161616165e-06,
+ "loss": 1.097399353981018,
+ "mean_token_accuracy": 0.7245969772338867,
+ "num_tokens": 16105472.0,
+ "step": 983
+ },
+ {
+ "entropy": 1.2580686807632446,
+ "epoch": 1.9878787878787878,
+ "grad_norm": 2.1748640537261963,
+ "learning_rate": 3.0141414141414144e-06,
+ "loss": 1.2503113746643066,
+ "mean_token_accuracy": 0.6883854269981384,
+ "num_tokens": 16121856.0,
+ "step": 984
+ },
+ {
+ "entropy": 1.4884002208709717,
+ "epoch": 1.98989898989899,
+ "grad_norm": 2.3283042907714844,
+ "learning_rate": 3.0121212121212123e-06,
+ "loss": 1.4856922626495361,
+ "mean_token_accuracy": 0.6461895704269409,
+ "num_tokens": 16138240.0,
+ "step": 985
+ },
+ {
+ "entropy": 1.6224603652954102,
+ "epoch": 1.9919191919191919,
+ "grad_norm": 2.0596823692321777,
+ "learning_rate": 3.0101010101010102e-06,
+ "loss": 1.6202584505081177,
+ "mean_token_accuracy": 0.6388617753982544,
+ "num_tokens": 16154624.0,
+ "step": 986
+ },
+ {
+ "entropy": 1.559553623199463,
+ "epoch": 1.993939393939394,
+ "grad_norm": 2.100667953491211,
+ "learning_rate": 3.008080808080808e-06,
+ "loss": 1.5781179666519165,
+ "mean_token_accuracy": 0.6409379839897156,
+ "num_tokens": 16171008.0,
+ "step": 987
+ },
+ {
+ "entropy": 1.299710750579834,
+ "epoch": 1.995959595959596,
+ "grad_norm": 2.2846767902374268,
+ "learning_rate": 3.0060606060606064e-06,
+ "loss": 1.2758262157440186,
+ "mean_token_accuracy": 0.6882632970809937,
+ "num_tokens": 16187392.0,
+ "step": 988
+ },
+ {
+ "entropy": 1.3966935873031616,
+ "epoch": 1.997979797979798,
+ "grad_norm": 2.1244044303894043,
+ "learning_rate": 3.0040404040404043e-06,
+ "loss": 1.420769214630127,
+ "mean_token_accuracy": 0.6670737862586975,
+ "num_tokens": 16203776.0,
+ "step": 989
+ },
+ {
+ "entropy": 1.5851658582687378,
+ "epoch": 2.0,
+ "grad_norm": 2.277179002761841,
+ "learning_rate": 3.002020202020202e-06,
+ "loss": 1.5963382720947266,
+ "mean_token_accuracy": 0.6276868581771851,
+ "num_tokens": 16220160.0,
+ "step": 990
+ },
+ {
+ "entropy": 1.9146819114685059,
+ "epoch": 2.002020202020202,
+ "grad_norm": 2.0008981227874756,
+ "learning_rate": 3e-06,
+ "loss": 1.8820562362670898,
+ "mean_token_accuracy": 0.5895212292671204,
+ "num_tokens": 16236544.0,
+ "step": 991
+ },
+ {
+ "entropy": 1.3451478481292725,
+ "epoch": 2.004040404040404,
+ "grad_norm": 2.236388921737671,
+ "learning_rate": 2.997979797979798e-06,
+ "loss": 1.2709381580352783,
+ "mean_token_accuracy": 0.6848436594009399,
+ "num_tokens": 16252928.0,
+ "step": 992
+ },
+ {
+ "entropy": 1.4454528093338013,
+ "epoch": 2.006060606060606,
+ "grad_norm": 2.220324754714966,
+ "learning_rate": 2.9959595959595963e-06,
+ "loss": 1.387961506843567,
+ "mean_token_accuracy": 0.6642647981643677,
+ "num_tokens": 16269312.0,
+ "step": 993
+ },
+ {
+ "entropy": 1.2083219289779663,
+ "epoch": 2.008080808080808,
+ "grad_norm": 2.092430830001831,
+ "learning_rate": 2.993939393939394e-06,
+ "loss": 1.1450953483581543,
+ "mean_token_accuracy": 0.7159257531166077,
+ "num_tokens": 16285696.0,
+ "step": 994
+ },
+ {
+ "entropy": 1.3030426502227783,
+ "epoch": 2.01010101010101,
+ "grad_norm": 2.0722665786743164,
+ "learning_rate": 2.991919191919192e-06,
+ "loss": 1.2463462352752686,
+ "mean_token_accuracy": 0.6780654788017273,
+ "num_tokens": 16302080.0,
+ "step": 995
+ },
+ {
+ "entropy": 1.3418742418289185,
+ "epoch": 2.012121212121212,
+ "grad_norm": 1.9004424810409546,
+ "learning_rate": 2.98989898989899e-06,
+ "loss": 1.3043954372406006,
+ "mean_token_accuracy": 0.6850268840789795,
+ "num_tokens": 16318464.0,
+ "step": 996
+ },
+ {
+ "entropy": 1.6736825704574585,
+ "epoch": 2.014141414141414,
+ "grad_norm": 2.132983446121216,
+ "learning_rate": 2.987878787878788e-06,
+ "loss": 1.631580114364624,
+ "mean_token_accuracy": 0.619076669216156,
+ "num_tokens": 16334848.0,
+ "step": 997
+ },
+ {
+ "entropy": 1.435375690460205,
+ "epoch": 2.0161616161616163,
+ "grad_norm": 2.2508206367492676,
+ "learning_rate": 2.9858585858585857e-06,
+ "loss": 1.3867809772491455,
+ "mean_token_accuracy": 0.662432849407196,
+ "num_tokens": 16351232.0,
+ "step": 998
+ },
+ {
+ "entropy": 1.050812005996704,
+ "epoch": 2.018181818181818,
+ "grad_norm": 1.878668189048767,
+ "learning_rate": 2.983838383838384e-06,
+ "loss": 1.0428903102874756,
+ "mean_token_accuracy": 0.7329018115997314,
+ "num_tokens": 16367616.0,
+ "step": 999
+ },
+ {
+ "entropy": 1.325859546661377,
+ "epoch": 2.0202020202020203,
+ "grad_norm": 1.8450349569320679,
+ "learning_rate": 2.981818181818182e-06,
+ "loss": 1.302099347114563,
+ "mean_token_accuracy": 0.6926599740982056,
+ "num_tokens": 16384000.0,
+ "step": 1000
+ },
+ {
+ "epoch": 2.0202020202020203,
+ "eval_entropy": 1.3899660999736478,
+ "eval_loss": 1.5310550928115845,
+ "eval_mean_token_accuracy": 0.6458596015168775,
+ "eval_num_tokens": 16384000.0,
+ "eval_runtime": 43.8845,
+ "eval_samples_per_second": 22.559,
+ "eval_steps_per_second": 2.826,
+ "step": 1000
+ },
+ {
+ "entropy": 1.5226235389709473,
+ "epoch": 2.022222222222222,
+ "grad_norm": 2.0676803588867188,
+ "learning_rate": 2.97979797979798e-06,
+ "loss": 1.4971344470977783,
+ "mean_token_accuracy": 0.6517464518547058,
+ "num_tokens": 16400384.0,
+ "step": 1001
+ },
+ {
+ "entropy": 0.9506940841674805,
+ "epoch": 2.0242424242424244,
+ "grad_norm": 1.972718596458435,
+ "learning_rate": 2.9777777777777777e-06,
+ "loss": 0.9229776859283447,
+ "mean_token_accuracy": 0.7554958462715149,
+ "num_tokens": 16416768.0,
+ "step": 1002
+ },
+ {
+ "entropy": 1.8095428943634033,
+ "epoch": 2.026262626262626,
+ "grad_norm": 2.119502305984497,
+ "learning_rate": 2.9757575757575756e-06,
+ "loss": 1.8333206176757812,
+ "mean_token_accuracy": 0.5888495445251465,
+ "num_tokens": 16433152.0,
+ "step": 1003
+ },
+ {
+ "entropy": 1.110617756843567,
+ "epoch": 2.0282828282828285,
+ "grad_norm": 2.1480863094329834,
+ "learning_rate": 2.9737373737373743e-06,
+ "loss": 1.0999674797058105,
+ "mean_token_accuracy": 0.7213605046272278,
+ "num_tokens": 16449536.0,
+ "step": 1004
+ },
+ {
+ "entropy": 1.1889066696166992,
+ "epoch": 2.0303030303030303,
+ "grad_norm": 2.1086478233337402,
+ "learning_rate": 2.971717171717172e-06,
+ "loss": 1.2093305587768555,
+ "mean_token_accuracy": 0.7071323990821838,
+ "num_tokens": 16465920.0,
+ "step": 1005
+ },
+ {
+ "entropy": 1.2134279012680054,
+ "epoch": 2.0323232323232325,
+ "grad_norm": 2.020108461380005,
+ "learning_rate": 2.96969696969697e-06,
+ "loss": 1.2015900611877441,
+ "mean_token_accuracy": 0.7037127614021301,
+ "num_tokens": 16482304.0,
+ "step": 1006
+ },
+ {
+ "entropy": 1.1034245491027832,
+ "epoch": 2.0343434343434343,
+ "grad_norm": 2.1722943782806396,
+ "learning_rate": 2.967676767676768e-06,
+ "loss": 1.1243445873260498,
+ "mean_token_accuracy": 0.7051172256469727,
+ "num_tokens": 16498688.0,
+ "step": 1007
+ },
+ {
+ "entropy": 1.6888902187347412,
+ "epoch": 2.036363636363636,
+ "grad_norm": 2.1574206352233887,
+ "learning_rate": 2.9656565656565663e-06,
+ "loss": 1.7164931297302246,
+ "mean_token_accuracy": 0.6174889802932739,
+ "num_tokens": 16515072.0,
+ "step": 1008
+ },
+ {
+ "entropy": 0.8055605292320251,
+ "epoch": 2.0383838383838384,
+ "grad_norm": 1.8896156549453735,
+ "learning_rate": 2.963636363636364e-06,
+ "loss": 0.7961650490760803,
+ "mean_token_accuracy": 0.7895700931549072,
+ "num_tokens": 16531456.0,
+ "step": 1009
+ },
+ {
+ "entropy": 1.7698009014129639,
+ "epoch": 2.04040404040404,
+ "grad_norm": 2.1700897216796875,
+ "learning_rate": 2.961616161616162e-06,
+ "loss": 1.7871267795562744,
+ "mean_token_accuracy": 0.6139472126960754,
+ "num_tokens": 16547840.0,
+ "step": 1010
+ },
+ {
+ "entropy": 1.414292335510254,
+ "epoch": 2.0424242424242425,
+ "grad_norm": 2.0185277462005615,
+ "learning_rate": 2.95959595959596e-06,
+ "loss": 1.4295077323913574,
+ "mean_token_accuracy": 0.6670126914978027,
+ "num_tokens": 16564224.0,
+ "step": 1011
+ },
+ {
+ "entropy": 1.4013340473175049,
+ "epoch": 2.0444444444444443,
+ "grad_norm": 2.1133840084075928,
+ "learning_rate": 2.957575757575758e-06,
+ "loss": 1.4482200145721436,
+ "mean_token_accuracy": 0.6780654788017273,
+ "num_tokens": 16580608.0,
+ "step": 1012
+ },
+ {
+ "entropy": 1.342740774154663,
+ "epoch": 2.0464646464646465,
+ "grad_norm": 2.0520355701446533,
+ "learning_rate": 2.955555555555556e-06,
+ "loss": 1.3550419807434082,
+ "mean_token_accuracy": 0.6850879192352295,
+ "num_tokens": 16596992.0,
+ "step": 1013
+ },
+ {
+ "entropy": 1.2301024198532104,
+ "epoch": 2.0484848484848484,
+ "grad_norm": 2.1948137283325195,
+ "learning_rate": 2.953535353535354e-06,
+ "loss": 1.243154525756836,
+ "mean_token_accuracy": 0.6838055849075317,
+ "num_tokens": 16613376.0,
+ "step": 1014
+ },
+ {
+ "entropy": 1.3916606903076172,
+ "epoch": 2.0505050505050506,
+ "grad_norm": 2.241595506668091,
+ "learning_rate": 2.951515151515152e-06,
+ "loss": 1.4011318683624268,
+ "mean_token_accuracy": 0.664631187915802,
+ "num_tokens": 16629760.0,
+ "step": 1015
+ },
+ {
+ "entropy": 1.4715481996536255,
+ "epoch": 2.0525252525252524,
+ "grad_norm": 2.224256753921509,
+ "learning_rate": 2.94949494949495e-06,
+ "loss": 1.5011882781982422,
+ "mean_token_accuracy": 0.6493038535118103,
+ "num_tokens": 16646144.0,
+ "step": 1016
+ },
+ {
+ "entropy": 1.1261032819747925,
+ "epoch": 2.0545454545454547,
+ "grad_norm": 2.2784500122070312,
+ "learning_rate": 2.9474747474747477e-06,
+ "loss": 1.1250882148742676,
+ "mean_token_accuracy": 0.7178798317909241,
+ "num_tokens": 16662528.0,
+ "step": 1017
+ },
+ {
+ "entropy": 1.782008171081543,
+ "epoch": 2.0565656565656565,
+ "grad_norm": 2.1711416244506836,
+ "learning_rate": 2.9454545454545456e-06,
+ "loss": 1.8037612438201904,
+ "mean_token_accuracy": 0.6113824844360352,
+ "num_tokens": 16678912.0,
+ "step": 1018
+ },
+ {
+ "entropy": 1.521897315979004,
+ "epoch": 2.0585858585858587,
+ "grad_norm": 2.1903202533721924,
+ "learning_rate": 2.943434343434344e-06,
+ "loss": 1.5377610921859741,
+ "mean_token_accuracy": 0.642953097820282,
+ "num_tokens": 16695296.0,
+ "step": 1019
+ },
+ {
+ "entropy": 1.3112438917160034,
+ "epoch": 2.0606060606060606,
+ "grad_norm": 2.0299105644226074,
+ "learning_rate": 2.941414141414142e-06,
+ "loss": 1.3218681812286377,
+ "mean_token_accuracy": 0.6842941045761108,
+ "num_tokens": 16711680.0,
+ "step": 1020
+ },
+ {
+ "entropy": 1.3634895086288452,
+ "epoch": 2.062626262626263,
+ "grad_norm": 2.2818808555603027,
+ "learning_rate": 2.9393939393939397e-06,
+ "loss": 1.3611259460449219,
+ "mean_token_accuracy": 0.6758060455322266,
+ "num_tokens": 16728064.0,
+ "step": 1021
+ },
+ {
+ "entropy": 1.361467957496643,
+ "epoch": 2.0646464646464646,
+ "grad_norm": 2.290698289871216,
+ "learning_rate": 2.9373737373737376e-06,
+ "loss": 1.3082963228225708,
+ "mean_token_accuracy": 0.6809965968132019,
+ "num_tokens": 16744448.0,
+ "step": 1022
+ },
+ {
+ "entropy": 1.633399486541748,
+ "epoch": 2.066666666666667,
+ "grad_norm": 2.1775712966918945,
+ "learning_rate": 2.9353535353535355e-06,
+ "loss": 1.6423053741455078,
+ "mean_token_accuracy": 0.639655590057373,
+ "num_tokens": 16760832.0,
+ "step": 1023
+ },
+ {
+ "entropy": 1.4717872142791748,
+ "epoch": 2.0686868686868687,
+ "grad_norm": 2.2070701122283936,
+ "learning_rate": 2.9333333333333338e-06,
+ "loss": 1.508943796157837,
+ "mean_token_accuracy": 0.6469223499298096,
+ "num_tokens": 16777216.0,
+ "step": 1024
+ },
+ {
+ "entropy": 1.1494653224945068,
+ "epoch": 2.0707070707070705,
+ "grad_norm": 2.0239098072052,
+ "learning_rate": 2.9313131313131317e-06,
+ "loss": 1.1367695331573486,
+ "mean_token_accuracy": 0.7148265838623047,
+ "num_tokens": 16793600.0,
+ "step": 1025
+ },
+ {
+ "entropy": 1.0235252380371094,
+ "epoch": 2.0727272727272728,
+ "grad_norm": 2.071176052093506,
+ "learning_rate": 2.9292929292929295e-06,
+ "loss": 1.0369395017623901,
+ "mean_token_accuracy": 0.7396799921989441,
+ "num_tokens": 16809984.0,
+ "step": 1026
+ },
+ {
+ "entropy": 1.2325578927993774,
+ "epoch": 2.0747474747474746,
+ "grad_norm": 2.212671995162964,
+ "learning_rate": 2.9272727272727274e-06,
+ "loss": 1.2429314851760864,
+ "mean_token_accuracy": 0.688568651676178,
+ "num_tokens": 16826368.0,
+ "step": 1027
+ },
+ {
+ "entropy": 1.2879300117492676,
+ "epoch": 2.076767676767677,
+ "grad_norm": 2.0366690158843994,
+ "learning_rate": 2.9252525252525253e-06,
+ "loss": 1.2931241989135742,
+ "mean_token_accuracy": 0.6813018918037415,
+ "num_tokens": 16842752.0,
+ "step": 1028
+ },
+ {
+ "entropy": 0.9473312497138977,
+ "epoch": 2.0787878787878786,
+ "grad_norm": 2.0870468616485596,
+ "learning_rate": 2.9232323232323236e-06,
+ "loss": 0.9292516708374023,
+ "mean_token_accuracy": 0.7525036931037903,
+ "num_tokens": 16859136.0,
+ "step": 1029
+ },
+ {
+ "entropy": 1.3882765769958496,
+ "epoch": 2.080808080808081,
+ "grad_norm": 2.1285617351531982,
+ "learning_rate": 2.9212121212121215e-06,
+ "loss": 1.3798363208770752,
+ "mean_token_accuracy": 0.6768441796302795,
+ "num_tokens": 16875520.0,
+ "step": 1030
+ },
+ {
+ "entropy": 1.3588547706604004,
+ "epoch": 2.0828282828282827,
+ "grad_norm": 2.2275242805480957,
+ "learning_rate": 2.9191919191919194e-06,
+ "loss": 1.3538521528244019,
+ "mean_token_accuracy": 0.6615168452262878,
+ "num_tokens": 16891904.0,
+ "step": 1031
+ },
+ {
+ "entropy": 1.0415440797805786,
+ "epoch": 2.084848484848485,
+ "grad_norm": 2.2782297134399414,
+ "learning_rate": 2.9171717171717173e-06,
+ "loss": 1.03328537940979,
+ "mean_token_accuracy": 0.7577552795410156,
+ "num_tokens": 16908288.0,
+ "step": 1032
+ },
+ {
+ "entropy": 1.241741418838501,
+ "epoch": 2.0868686868686868,
+ "grad_norm": 2.141939163208008,
+ "learning_rate": 2.915151515151515e-06,
+ "loss": 1.2248635292053223,
+ "mean_token_accuracy": 0.6950415372848511,
+ "num_tokens": 16924672.0,
+ "step": 1033
+ },
+ {
+ "entropy": 1.1192725896835327,
+ "epoch": 2.088888888888889,
+ "grad_norm": 1.9980862140655518,
+ "learning_rate": 2.913131313131313e-06,
+ "loss": 1.1176412105560303,
+ "mean_token_accuracy": 0.7272838354110718,
+ "num_tokens": 16941056.0,
+ "step": 1034
+ },
+ {
+ "entropy": 1.1871322393417358,
+ "epoch": 2.090909090909091,
+ "grad_norm": 2.1212220191955566,
+ "learning_rate": 2.9111111111111114e-06,
+ "loss": 1.1670379638671875,
+ "mean_token_accuracy": 0.7048119306564331,
+ "num_tokens": 16957440.0,
+ "step": 1035
+ },
+ {
+ "entropy": 1.5194982290267944,
+ "epoch": 2.092929292929293,
+ "grad_norm": 2.2593677043914795,
+ "learning_rate": 2.9090909090909093e-06,
+ "loss": 1.50520920753479,
+ "mean_token_accuracy": 0.6475940346717834,
+ "num_tokens": 16973824.0,
+ "step": 1036
+ },
+ {
+ "entropy": 1.212785243988037,
+ "epoch": 2.094949494949495,
+ "grad_norm": 2.221132278442383,
+ "learning_rate": 2.907070707070707e-06,
+ "loss": 1.206315040588379,
+ "mean_token_accuracy": 0.7046898007392883,
+ "num_tokens": 16990208.0,
+ "step": 1037
+ },
+ {
+ "entropy": 1.2913068532943726,
+ "epoch": 2.096969696969697,
+ "grad_norm": 2.0525012016296387,
+ "learning_rate": 2.905050505050505e-06,
+ "loss": 1.2987749576568604,
+ "mean_token_accuracy": 0.6894235610961914,
+ "num_tokens": 17006592.0,
+ "step": 1038
+ },
+ {
+ "entropy": 1.139485239982605,
+ "epoch": 2.098989898989899,
+ "grad_norm": 2.093764543533325,
+ "learning_rate": 2.903030303030303e-06,
+ "loss": 1.1460036039352417,
+ "mean_token_accuracy": 0.7205055952072144,
+ "num_tokens": 17022976.0,
+ "step": 1039
+ },
+ {
+ "entropy": 1.3781968355178833,
+ "epoch": 2.101010101010101,
+ "grad_norm": 2.095311164855957,
+ "learning_rate": 2.9010101010101012e-06,
+ "loss": 1.3901969194412231,
+ "mean_token_accuracy": 0.679286777973175,
+ "num_tokens": 17039360.0,
+ "step": 1040
+ },
+ {
+ "entropy": 1.1370116472244263,
+ "epoch": 2.103030303030303,
+ "grad_norm": 2.139094352722168,
+ "learning_rate": 2.898989898989899e-06,
+ "loss": 1.1491400003433228,
+ "mean_token_accuracy": 0.72007817029953,
+ "num_tokens": 17055744.0,
+ "step": 1041
+ },
+ {
+ "entropy": 1.0845732688903809,
+ "epoch": 2.105050505050505,
+ "grad_norm": 2.180278778076172,
+ "learning_rate": 2.896969696969697e-06,
+ "loss": 1.0764763355255127,
+ "mean_token_accuracy": 0.7365657091140747,
+ "num_tokens": 17072128.0,
+ "step": 1042
+ },
+ {
+ "entropy": 1.4869064092636108,
+ "epoch": 2.107070707070707,
+ "grad_norm": 2.053727388381958,
+ "learning_rate": 2.894949494949495e-06,
+ "loss": 1.4862562417984009,
+ "mean_token_accuracy": 0.6643258333206177,
+ "num_tokens": 17088512.0,
+ "step": 1043
+ },
+ {
+ "entropy": 1.048888921737671,
+ "epoch": 2.109090909090909,
+ "grad_norm": 2.1302237510681152,
+ "learning_rate": 2.892929292929293e-06,
+ "loss": 1.0331683158874512,
+ "mean_token_accuracy": 0.7388250827789307,
+ "num_tokens": 17104896.0,
+ "step": 1044
+ },
+ {
+ "entropy": 1.2839856147766113,
+ "epoch": 2.111111111111111,
+ "grad_norm": 2.8510677814483643,
+ "learning_rate": 2.8909090909090907e-06,
+ "loss": 1.3016979694366455,
+ "mean_token_accuracy": 0.6904616355895996,
+ "num_tokens": 17121280.0,
+ "step": 1045
+ },
+ {
+ "entropy": 1.050045371055603,
+ "epoch": 2.113131313131313,
+ "grad_norm": 2.2457358837127686,
+ "learning_rate": 2.888888888888889e-06,
+ "loss": 1.0580981969833374,
+ "mean_token_accuracy": 0.7221543788909912,
+ "num_tokens": 17137664.0,
+ "step": 1046
+ },
+ {
+ "entropy": 1.3087437152862549,
+ "epoch": 2.1151515151515152,
+ "grad_norm": 2.0929996967315674,
+ "learning_rate": 2.8868686868686873e-06,
+ "loss": 1.2914986610412598,
+ "mean_token_accuracy": 0.6939423680305481,
+ "num_tokens": 17154048.0,
+ "step": 1047
+ },
+ {
+ "entropy": 1.7247086763381958,
+ "epoch": 2.117171717171717,
+ "grad_norm": 2.2230091094970703,
+ "learning_rate": 2.884848484848485e-06,
+ "loss": 1.7394956350326538,
+ "mean_token_accuracy": 0.6131533980369568,
+ "num_tokens": 17170432.0,
+ "step": 1048
+ },
+ {
+ "entropy": 1.5816357135772705,
+ "epoch": 2.1191919191919193,
+ "grad_norm": 2.196422576904297,
+ "learning_rate": 2.8828282828282835e-06,
+ "loss": 1.6226823329925537,
+ "mean_token_accuracy": 0.6375183463096619,
+ "num_tokens": 17186816.0,
+ "step": 1049
+ },
+ {
+ "entropy": 1.2316261529922485,
+ "epoch": 2.121212121212121,
+ "grad_norm": 2.173909902572632,
+ "learning_rate": 2.8808080808080814e-06,
+ "loss": 1.2228440046310425,
+ "mean_token_accuracy": 0.6940034031867981,
+ "num_tokens": 17203200.0,
+ "step": 1050
+ },
+ {
+ "epoch": 2.121212121212121,
+ "eval_entropy": 1.3722701356295617,
+ "eval_loss": 1.5385833978652954,
+ "eval_mean_token_accuracy": 0.6450051809510877,
+ "eval_num_tokens": 17203200.0,
+ "eval_runtime": 43.9186,
+ "eval_samples_per_second": 22.542,
+ "eval_steps_per_second": 2.823,
+ "step": 1050
+ },
+ {
+ "entropy": 1.3023537397384644,
+ "epoch": 2.1232323232323234,
+ "grad_norm": 2.1490018367767334,
+ "learning_rate": 2.8787878787878793e-06,
+ "loss": 1.284008264541626,
+ "mean_token_accuracy": 0.7057278752326965,
+ "num_tokens": 17219584.0,
+ "step": 1051
+ },
+ {
+ "entropy": 1.3198270797729492,
+ "epoch": 2.125252525252525,
+ "grad_norm": 2.582298994064331,
+ "learning_rate": 2.876767676767677e-06,
+ "loss": 1.3185001611709595,
+ "mean_token_accuracy": 0.6772105693817139,
+ "num_tokens": 17235968.0,
+ "step": 1052
+ },
+ {
+ "entropy": 0.8889825940132141,
+ "epoch": 2.1272727272727274,
+ "grad_norm": 2.0617194175720215,
+ "learning_rate": 2.874747474747475e-06,
+ "loss": 0.889773964881897,
+ "mean_token_accuracy": 0.767892062664032,
+ "num_tokens": 17252352.0,
+ "step": 1053
+ },
+ {
+ "entropy": 1.5485399961471558,
+ "epoch": 2.1292929292929292,
+ "grad_norm": 2.2752370834350586,
+ "learning_rate": 2.872727272727273e-06,
+ "loss": 1.5542051792144775,
+ "mean_token_accuracy": 0.6397166848182678,
+ "num_tokens": 17268736.0,
+ "step": 1054
+ },
+ {
+ "entropy": 1.3949565887451172,
+ "epoch": 2.1313131313131315,
+ "grad_norm": 2.0112996101379395,
+ "learning_rate": 2.8707070707070713e-06,
+ "loss": 1.4111721515655518,
+ "mean_token_accuracy": 0.6659746170043945,
+ "num_tokens": 17285120.0,
+ "step": 1055
+ },
+ {
+ "entropy": 1.0529720783233643,
+ "epoch": 2.1333333333333333,
+ "grad_norm": 2.2219948768615723,
+ "learning_rate": 2.868686868686869e-06,
+ "loss": 1.0484942197799683,
+ "mean_token_accuracy": 0.7224596738815308,
+ "num_tokens": 17301504.0,
+ "step": 1056
+ },
+ {
+ "entropy": 1.5362433195114136,
+ "epoch": 2.1353535353535356,
+ "grad_norm": 2.4097044467926025,
+ "learning_rate": 2.866666666666667e-06,
+ "loss": 1.5809426307678223,
+ "mean_token_accuracy": 0.6266487836837769,
+ "num_tokens": 17317888.0,
+ "step": 1057
+ },
+ {
+ "entropy": 1.6682685613632202,
+ "epoch": 2.1373737373737374,
+ "grad_norm": 2.121086359024048,
+ "learning_rate": 2.864646464646465e-06,
+ "loss": 1.6636815071105957,
+ "mean_token_accuracy": 0.6317782402038574,
+ "num_tokens": 17334272.0,
+ "step": 1058
+ },
+ {
+ "entropy": 1.5480221509933472,
+ "epoch": 2.1393939393939396,
+ "grad_norm": 2.2226176261901855,
+ "learning_rate": 2.862626262626263e-06,
+ "loss": 1.535983681678772,
+ "mean_token_accuracy": 0.6623717546463013,
+ "num_tokens": 17350656.0,
+ "step": 1059
+ },
+ {
+ "entropy": 1.3323101997375488,
+ "epoch": 2.1414141414141414,
+ "grad_norm": 2.2644572257995605,
+ "learning_rate": 2.860606060606061e-06,
+ "loss": 1.3316677808761597,
+ "mean_token_accuracy": 0.6741573214530945,
+ "num_tokens": 17367040.0,
+ "step": 1060
+ },
+ {
+ "entropy": 1.5107028484344482,
+ "epoch": 2.1434343434343432,
+ "grad_norm": 2.1291613578796387,
+ "learning_rate": 2.858585858585859e-06,
+ "loss": 1.4996232986450195,
+ "mean_token_accuracy": 0.6537005305290222,
+ "num_tokens": 17383424.0,
+ "step": 1061
+ },
+ {
+ "entropy": 1.5927387475967407,
+ "epoch": 2.1454545454545455,
+ "grad_norm": 2.1977617740631104,
+ "learning_rate": 2.856565656565657e-06,
+ "loss": 1.582688331604004,
+ "mean_token_accuracy": 0.6306790709495544,
+ "num_tokens": 17399808.0,
+ "step": 1062
+ },
+ {
+ "entropy": 1.0717219114303589,
+ "epoch": 2.1474747474747473,
+ "grad_norm": 2.2494494915008545,
+ "learning_rate": 2.8545454545454548e-06,
+ "loss": 1.0780384540557861,
+ "mean_token_accuracy": 0.7214826345443726,
+ "num_tokens": 17416192.0,
+ "step": 1063
+ },
+ {
+ "entropy": 1.484608769416809,
+ "epoch": 2.1494949494949496,
+ "grad_norm": 2.20664644241333,
+ "learning_rate": 2.8525252525252527e-06,
+ "loss": 1.5074207782745361,
+ "mean_token_accuracy": 0.6502808928489685,
+ "num_tokens": 17432576.0,
+ "step": 1064
+ },
+ {
+ "entropy": 1.6607396602630615,
+ "epoch": 2.1515151515151514,
+ "grad_norm": 2.123538017272949,
+ "learning_rate": 2.850505050505051e-06,
+ "loss": 1.659307837486267,
+ "mean_token_accuracy": 0.619076669216156,
+ "num_tokens": 17448960.0,
+ "step": 1065
+ },
+ {
+ "entropy": 1.1195216178894043,
+ "epoch": 2.1535353535353536,
+ "grad_norm": 2.037261486053467,
+ "learning_rate": 2.848484848484849e-06,
+ "loss": 1.127397894859314,
+ "mean_token_accuracy": 0.7209330797195435,
+ "num_tokens": 17465344.0,
+ "step": 1066
+ },
+ {
+ "entropy": 1.0512653589248657,
+ "epoch": 2.1555555555555554,
+ "grad_norm": 2.209764003753662,
+ "learning_rate": 2.8464646464646468e-06,
+ "loss": 1.073946475982666,
+ "mean_token_accuracy": 0.717391312122345,
+ "num_tokens": 17481728.0,
+ "step": 1067
+ },
+ {
+ "entropy": 1.131946325302124,
+ "epoch": 2.1575757575757577,
+ "grad_norm": 1.9188926219940186,
+ "learning_rate": 2.8444444444444446e-06,
+ "loss": 1.11629056930542,
+ "mean_token_accuracy": 0.7220322489738464,
+ "num_tokens": 17498112.0,
+ "step": 1068
+ },
+ {
+ "entropy": 1.4745967388153076,
+ "epoch": 2.1595959595959595,
+ "grad_norm": 2.285099983215332,
+ "learning_rate": 2.8424242424242425e-06,
+ "loss": 1.4795483350753784,
+ "mean_token_accuracy": 0.6466169953346252,
+ "num_tokens": 17514496.0,
+ "step": 1069
+ },
+ {
+ "entropy": 1.2614645957946777,
+ "epoch": 2.1616161616161618,
+ "grad_norm": 2.4019627571105957,
+ "learning_rate": 2.8404040404040404e-06,
+ "loss": 1.2659732103347778,
+ "mean_token_accuracy": 0.6797142028808594,
+ "num_tokens": 17530880.0,
+ "step": 1070
+ },
+ {
+ "entropy": 1.4247770309448242,
+ "epoch": 2.1636363636363636,
+ "grad_norm": 2.350632905960083,
+ "learning_rate": 2.8383838383838387e-06,
+ "loss": 1.415325403213501,
+ "mean_token_accuracy": 0.6655471324920654,
+ "num_tokens": 17547264.0,
+ "step": 1071
+ },
+ {
+ "entropy": 1.5746078491210938,
+ "epoch": 2.165656565656566,
+ "grad_norm": 2.18448543548584,
+ "learning_rate": 2.8363636363636366e-06,
+ "loss": 1.6022642850875854,
+ "mean_token_accuracy": 0.6324499249458313,
+ "num_tokens": 17563648.0,
+ "step": 1072
+ },
+ {
+ "entropy": 1.5869743824005127,
+ "epoch": 2.1676767676767676,
+ "grad_norm": 2.3553452491760254,
+ "learning_rate": 2.8343434343434345e-06,
+ "loss": 1.6163790225982666,
+ "mean_token_accuracy": 0.6199926733970642,
+ "num_tokens": 17580032.0,
+ "step": 1073
+ },
+ {
+ "entropy": 1.2412667274475098,
+ "epoch": 2.16969696969697,
+ "grad_norm": 2.209017038345337,
+ "learning_rate": 2.8323232323232324e-06,
+ "loss": 1.2491700649261475,
+ "mean_token_accuracy": 0.6911333799362183,
+ "num_tokens": 17596416.0,
+ "step": 1074
+ },
+ {
+ "entropy": 0.9127920866012573,
+ "epoch": 2.1717171717171717,
+ "grad_norm": 1.9761381149291992,
+ "learning_rate": 2.8303030303030303e-06,
+ "loss": 0.8961890339851379,
+ "mean_token_accuracy": 0.7523815631866455,
+ "num_tokens": 17612800.0,
+ "step": 1075
+ },
+ {
+ "entropy": 1.2798824310302734,
+ "epoch": 2.1737373737373735,
+ "grad_norm": 2.397031784057617,
+ "learning_rate": 2.8282828282828286e-06,
+ "loss": 1.286920189857483,
+ "mean_token_accuracy": 0.6805080771446228,
+ "num_tokens": 17629184.0,
+ "step": 1076
+ },
+ {
+ "entropy": 1.7863305807113647,
+ "epoch": 2.175757575757576,
+ "grad_norm": 2.2798852920532227,
+ "learning_rate": 2.8262626262626265e-06,
+ "loss": 1.7654080390930176,
+ "mean_token_accuracy": 0.5806057453155518,
+ "num_tokens": 17645568.0,
+ "step": 1077
+ },
+ {
+ "entropy": 1.327653408050537,
+ "epoch": 2.1777777777777776,
+ "grad_norm": 2.3473095893859863,
+ "learning_rate": 2.8242424242424244e-06,
+ "loss": 1.3175032138824463,
+ "mean_token_accuracy": 0.6750732660293579,
+ "num_tokens": 17661952.0,
+ "step": 1078
+ },
+ {
+ "entropy": 1.4917371273040771,
+ "epoch": 2.17979797979798,
+ "grad_norm": 2.181729793548584,
+ "learning_rate": 2.8222222222222223e-06,
+ "loss": 1.4738472700119019,
+ "mean_token_accuracy": 0.6619443297386169,
+ "num_tokens": 17678336.0,
+ "step": 1079
+ },
+ {
+ "entropy": 1.4644533395767212,
+ "epoch": 2.1818181818181817,
+ "grad_norm": 2.2558112144470215,
+ "learning_rate": 2.82020202020202e-06,
+ "loss": 1.4563480615615845,
+ "mean_token_accuracy": 0.6484489440917969,
+ "num_tokens": 17694720.0,
+ "step": 1080
+ },
+ {
+ "entropy": 0.7626141905784607,
+ "epoch": 2.183838383838384,
+ "grad_norm": 1.8687561750411987,
+ "learning_rate": 2.818181818181818e-06,
+ "loss": 0.7718763947486877,
+ "mean_token_accuracy": 0.7982413172721863,
+ "num_tokens": 17711104.0,
+ "step": 1081
+ },
+ {
+ "entropy": 1.0613173246383667,
+ "epoch": 2.1858585858585857,
+ "grad_norm": 2.072643995285034,
+ "learning_rate": 2.8161616161616163e-06,
+ "loss": 1.0554629564285278,
+ "mean_token_accuracy": 0.734306275844574,
+ "num_tokens": 17727488.0,
+ "step": 1082
+ },
+ {
+ "entropy": 1.0348066091537476,
+ "epoch": 2.187878787878788,
+ "grad_norm": 1.8455089330673218,
+ "learning_rate": 2.8141414141414142e-06,
+ "loss": 1.0231983661651611,
+ "mean_token_accuracy": 0.7382755279541016,
+ "num_tokens": 17743872.0,
+ "step": 1083
+ },
+ {
+ "entropy": 1.271000862121582,
+ "epoch": 2.18989898989899,
+ "grad_norm": 2.3292477130889893,
+ "learning_rate": 2.812121212121212e-06,
+ "loss": 1.2822595834732056,
+ "mean_token_accuracy": 0.6816682815551758,
+ "num_tokens": 17760256.0,
+ "step": 1084
+ },
+ {
+ "entropy": 1.62147855758667,
+ "epoch": 2.191919191919192,
+ "grad_norm": 2.3269946575164795,
+ "learning_rate": 2.81010101010101e-06,
+ "loss": 1.6017215251922607,
+ "mean_token_accuracy": 0.635808527469635,
+ "num_tokens": 17776640.0,
+ "step": 1085
+ },
+ {
+ "entropy": 1.4834544658660889,
+ "epoch": 2.193939393939394,
+ "grad_norm": 2.223590135574341,
+ "learning_rate": 2.808080808080808e-06,
+ "loss": 1.496992826461792,
+ "mean_token_accuracy": 0.6494870781898499,
+ "num_tokens": 17793024.0,
+ "step": 1086
+ },
+ {
+ "entropy": 1.4784536361694336,
+ "epoch": 2.195959595959596,
+ "grad_norm": 2.169416666030884,
+ "learning_rate": 2.806060606060606e-06,
+ "loss": 1.4766731262207031,
+ "mean_token_accuracy": 0.6543722748756409,
+ "num_tokens": 17809408.0,
+ "step": 1087
+ },
+ {
+ "entropy": 1.3650349378585815,
+ "epoch": 2.197979797979798,
+ "grad_norm": 2.1369845867156982,
+ "learning_rate": 2.804040404040404e-06,
+ "loss": 1.3475263118743896,
+ "mean_token_accuracy": 0.6712262034416199,
+ "num_tokens": 17825792.0,
+ "step": 1088
+ },
+ {
+ "entropy": 1.052316665649414,
+ "epoch": 2.2,
+ "grad_norm": 2.3176774978637695,
+ "learning_rate": 2.802020202020202e-06,
+ "loss": 1.056478500366211,
+ "mean_token_accuracy": 0.7374206185340881,
+ "num_tokens": 17842176.0,
+ "step": 1089
+ },
+ {
+ "entropy": 1.1928741931915283,
+ "epoch": 2.202020202020202,
+ "grad_norm": 2.2518808841705322,
+ "learning_rate": 2.8000000000000003e-06,
+ "loss": 1.2238969802856445,
+ "mean_token_accuracy": 0.7048119306564331,
+ "num_tokens": 17858560.0,
+ "step": 1090
+ },
+ {
+ "entropy": 1.135977864265442,
+ "epoch": 2.2040404040404042,
+ "grad_norm": 2.2688262462615967,
+ "learning_rate": 2.7979797979797986e-06,
+ "loss": 1.1330978870391846,
+ "mean_token_accuracy": 0.7053004503250122,
+ "num_tokens": 17874944.0,
+ "step": 1091
+ },
+ {
+ "entropy": 1.3542495965957642,
+ "epoch": 2.206060606060606,
+ "grad_norm": 2.1078014373779297,
+ "learning_rate": 2.7959595959595965e-06,
+ "loss": 1.3983874320983887,
+ "mean_token_accuracy": 0.6840498447418213,
+ "num_tokens": 17891328.0,
+ "step": 1092
+ },
+ {
+ "entropy": 1.1861188411712646,
+ "epoch": 2.2080808080808083,
+ "grad_norm": 2.348931312561035,
+ "learning_rate": 2.7939393939393944e-06,
+ "loss": 1.1982170343399048,
+ "mean_token_accuracy": 0.7014533281326294,
+ "num_tokens": 17907712.0,
+ "step": 1093
+ },
+ {
+ "entropy": 1.2858752012252808,
+ "epoch": 2.21010101010101,
+ "grad_norm": 2.1746175289154053,
+ "learning_rate": 2.7919191919191923e-06,
+ "loss": 1.302664041519165,
+ "mean_token_accuracy": 0.6818515062332153,
+ "num_tokens": 17924096.0,
+ "step": 1094
+ },
+ {
+ "entropy": 1.4817839860916138,
+ "epoch": 2.212121212121212,
+ "grad_norm": 1.9842815399169922,
+ "learning_rate": 2.78989898989899e-06,
+ "loss": 1.5031331777572632,
+ "mean_token_accuracy": 0.6546165347099304,
+ "num_tokens": 17940480.0,
+ "step": 1095
+ },
+ {
+ "entropy": 1.2977368831634521,
+ "epoch": 2.214141414141414,
+ "grad_norm": 2.0394232273101807,
+ "learning_rate": 2.7878787878787885e-06,
+ "loss": 1.3185319900512695,
+ "mean_token_accuracy": 0.6885075569152832,
+ "num_tokens": 17956864.0,
+ "step": 1096
+ },
+ {
+ "entropy": 1.010884404182434,
+ "epoch": 2.216161616161616,
+ "grad_norm": 2.1113393306732178,
+ "learning_rate": 2.7858585858585864e-06,
+ "loss": 1.0239849090576172,
+ "mean_token_accuracy": 0.7426111102104187,
+ "num_tokens": 17973248.0,
+ "step": 1097
+ },
+ {
+ "entropy": 1.6405178308486938,
+ "epoch": 2.2181818181818183,
+ "grad_norm": 2.220865249633789,
+ "learning_rate": 2.7838383838383842e-06,
+ "loss": 1.660733938217163,
+ "mean_token_accuracy": 0.6250610947608948,
+ "num_tokens": 17989632.0,
+ "step": 1098
+ },
+ {
+ "entropy": 1.2822861671447754,
+ "epoch": 2.22020202020202,
+ "grad_norm": 2.1409878730773926,
+ "learning_rate": 2.781818181818182e-06,
+ "loss": 1.2675114870071411,
+ "mean_token_accuracy": 0.7099413871765137,
+ "num_tokens": 18006016.0,
+ "step": 1099
+ },
+ {
+ "entropy": 1.5835071802139282,
+ "epoch": 2.2222222222222223,
+ "grad_norm": 2.32753586769104,
+ "learning_rate": 2.77979797979798e-06,
+ "loss": 1.585759162902832,
+ "mean_token_accuracy": 0.6321446299552917,
+ "num_tokens": 18022400.0,
+ "step": 1100
+ },
+ {
+ "epoch": 2.2222222222222223,
+ "eval_entropy": 1.3726552551792515,
+ "eval_loss": 1.5380089282989502,
+ "eval_mean_token_accuracy": 0.645188374384757,
+ "eval_num_tokens": 18022400.0,
+ "eval_runtime": 43.7591,
+ "eval_samples_per_second": 22.624,
+ "eval_steps_per_second": 2.834,
+ "step": 1100
+ },
+ {
+ "entropy": 1.0727043151855469,
+ "epoch": 2.224242424242424,
+ "grad_norm": 2.2024266719818115,
+ "learning_rate": 2.7777777777777783e-06,
+ "loss": 1.066201090812683,
+ "mean_token_accuracy": 0.7272838354110718,
+ "num_tokens": 18038784.0,
+ "step": 1101
+ },
+ {
+ "entropy": 1.1188048124313354,
+ "epoch": 2.2262626262626264,
+ "grad_norm": 2.032862663269043,
+ "learning_rate": 2.7757575757575762e-06,
+ "loss": 1.1292006969451904,
+ "mean_token_accuracy": 0.7198949456214905,
+ "num_tokens": 18055168.0,
+ "step": 1102
+ },
+ {
+ "entropy": 1.3658474683761597,
+ "epoch": 2.228282828282828,
+ "grad_norm": 2.081120491027832,
+ "learning_rate": 2.773737373737374e-06,
+ "loss": 1.3503609895706177,
+ "mean_token_accuracy": 0.6908280253410339,
+ "num_tokens": 18071552.0,
+ "step": 1103
+ },
+ {
+ "entropy": 1.5833252668380737,
+ "epoch": 2.2303030303030305,
+ "grad_norm": 2.174055576324463,
+ "learning_rate": 2.771717171717172e-06,
+ "loss": 1.5991246700286865,
+ "mean_token_accuracy": 0.6322056651115417,
+ "num_tokens": 18087936.0,
+ "step": 1104
+ },
+ {
+ "entropy": 1.2178122997283936,
+ "epoch": 2.2323232323232323,
+ "grad_norm": 2.328620195388794,
+ "learning_rate": 2.76969696969697e-06,
+ "loss": 1.2162861824035645,
+ "mean_token_accuracy": 0.6916218996047974,
+ "num_tokens": 18104320.0,
+ "step": 1105
+ },
+ {
+ "entropy": 1.2894606590270996,
+ "epoch": 2.2343434343434345,
+ "grad_norm": 2.236239433288574,
+ "learning_rate": 2.7676767676767678e-06,
+ "loss": 1.2695279121398926,
+ "mean_token_accuracy": 0.689728856086731,
+ "num_tokens": 18120704.0,
+ "step": 1106
+ },
+ {
+ "entropy": 1.0141552686691284,
+ "epoch": 2.2363636363636363,
+ "grad_norm": 2.000847339630127,
+ "learning_rate": 2.765656565656566e-06,
+ "loss": 1.0147396326065063,
+ "mean_token_accuracy": 0.7469467520713806,
+ "num_tokens": 18137088.0,
+ "step": 1107
+ },
+ {
+ "entropy": 1.150480031967163,
+ "epoch": 2.2383838383838386,
+ "grad_norm": 2.15156888961792,
+ "learning_rate": 2.763636363636364e-06,
+ "loss": 1.1593115329742432,
+ "mean_token_accuracy": 0.7129335403442383,
+ "num_tokens": 18153472.0,
+ "step": 1108
+ },
+ {
+ "entropy": 1.4387518167495728,
+ "epoch": 2.2404040404040404,
+ "grad_norm": 2.3262319564819336,
+ "learning_rate": 2.761616161616162e-06,
+ "loss": 1.4393596649169922,
+ "mean_token_accuracy": 0.6572422981262207,
+ "num_tokens": 18169856.0,
+ "step": 1109
+ },
+ {
+ "entropy": 1.205459475517273,
+ "epoch": 2.242424242424242,
+ "grad_norm": 2.223987579345703,
+ "learning_rate": 2.7595959595959597e-06,
+ "loss": 1.1871377229690552,
+ "mean_token_accuracy": 0.7018197178840637,
+ "num_tokens": 18186240.0,
+ "step": 1110
+ },
+ {
+ "entropy": 1.428817868232727,
+ "epoch": 2.2444444444444445,
+ "grad_norm": 2.091627836227417,
+ "learning_rate": 2.7575757575757576e-06,
+ "loss": 1.4262187480926514,
+ "mean_token_accuracy": 0.6708598136901855,
+ "num_tokens": 18202624.0,
+ "step": 1111
+ },
+ {
+ "entropy": 1.420682668685913,
+ "epoch": 2.2464646464646463,
+ "grad_norm": 2.30232310295105,
+ "learning_rate": 2.755555555555556e-06,
+ "loss": 1.439821720123291,
+ "mean_token_accuracy": 0.6508305072784424,
+ "num_tokens": 18219008.0,
+ "step": 1112
+ },
+ {
+ "entropy": 1.2365189790725708,
+ "epoch": 2.2484848484848485,
+ "grad_norm": 2.1747143268585205,
+ "learning_rate": 2.753535353535354e-06,
+ "loss": 1.2286595106124878,
+ "mean_token_accuracy": 0.7011480331420898,
+ "num_tokens": 18235392.0,
+ "step": 1113
+ },
+ {
+ "entropy": 1.0271798372268677,
+ "epoch": 2.2505050505050503,
+ "grad_norm": 2.1767842769622803,
+ "learning_rate": 2.7515151515151517e-06,
+ "loss": 1.018047571182251,
+ "mean_token_accuracy": 0.7297264337539673,
+ "num_tokens": 18251776.0,
+ "step": 1114
+ },
+ {
+ "entropy": 1.229053258895874,
+ "epoch": 2.2525252525252526,
+ "grad_norm": 2.390345811843872,
+ "learning_rate": 2.7494949494949496e-06,
+ "loss": 1.250126838684082,
+ "mean_token_accuracy": 0.6869809627532959,
+ "num_tokens": 18268160.0,
+ "step": 1115
+ },
+ {
+ "entropy": 1.4062843322753906,
+ "epoch": 2.2545454545454544,
+ "grad_norm": 2.220144271850586,
+ "learning_rate": 2.7474747474747475e-06,
+ "loss": 1.4262535572052002,
+ "mean_token_accuracy": 0.6592574715614319,
+ "num_tokens": 18284544.0,
+ "step": 1116
+ },
+ {
+ "entropy": 1.1991937160491943,
+ "epoch": 2.2565656565656567,
+ "grad_norm": 1.9910556077957153,
+ "learning_rate": 2.7454545454545454e-06,
+ "loss": 1.1869618892669678,
+ "mean_token_accuracy": 0.7049340605735779,
+ "num_tokens": 18300928.0,
+ "step": 1117
+ },
+ {
+ "entropy": 1.3147934675216675,
+ "epoch": 2.2585858585858585,
+ "grad_norm": 2.033942222595215,
+ "learning_rate": 2.7434343434343437e-06,
+ "loss": 1.3236720561981201,
+ "mean_token_accuracy": 0.6854543089866638,
+ "num_tokens": 18317312.0,
+ "step": 1118
+ },
+ {
+ "entropy": 1.2828413248062134,
+ "epoch": 2.2606060606060607,
+ "grad_norm": 2.2729172706604004,
+ "learning_rate": 2.7414141414141416e-06,
+ "loss": 1.2601265907287598,
+ "mean_token_accuracy": 0.6866145730018616,
+ "num_tokens": 18333696.0,
+ "step": 1119
+ },
+ {
+ "entropy": 1.9708930253982544,
+ "epoch": 2.2626262626262625,
+ "grad_norm": 2.4053614139556885,
+ "learning_rate": 2.7393939393939395e-06,
+ "loss": 1.9953765869140625,
+ "mean_token_accuracy": 0.5719345211982727,
+ "num_tokens": 18350080.0,
+ "step": 1120
+ },
+ {
+ "entropy": 1.071018099784851,
+ "epoch": 2.264646464646465,
+ "grad_norm": 2.0399091243743896,
+ "learning_rate": 2.7373737373737374e-06,
+ "loss": 1.0672862529754639,
+ "mean_token_accuracy": 0.72007817029953,
+ "num_tokens": 18366464.0,
+ "step": 1121
+ },
+ {
+ "entropy": 1.8741599321365356,
+ "epoch": 2.2666666666666666,
+ "grad_norm": 2.0918753147125244,
+ "learning_rate": 2.7353535353535353e-06,
+ "loss": 1.8604744672775269,
+ "mean_token_accuracy": 0.598436713218689,
+ "num_tokens": 18382848.0,
+ "step": 1122
+ },
+ {
+ "entropy": 1.0931968688964844,
+ "epoch": 2.268686868686869,
+ "grad_norm": 2.161102294921875,
+ "learning_rate": 2.7333333333333336e-06,
+ "loss": 1.1152517795562744,
+ "mean_token_accuracy": 0.7209330797195435,
+ "num_tokens": 18399232.0,
+ "step": 1123
+ },
+ {
+ "entropy": 1.3318616151809692,
+ "epoch": 2.2707070707070707,
+ "grad_norm": 2.200199842453003,
+ "learning_rate": 2.7313131313131315e-06,
+ "loss": 1.3456424474716187,
+ "mean_token_accuracy": 0.6789203882217407,
+ "num_tokens": 18415616.0,
+ "step": 1124
+ },
+ {
+ "entropy": 1.3777986764907837,
+ "epoch": 2.2727272727272725,
+ "grad_norm": 2.328887701034546,
+ "learning_rate": 2.7292929292929293e-06,
+ "loss": 1.3955169916152954,
+ "mean_token_accuracy": 0.6679897308349609,
+ "num_tokens": 18432000.0,
+ "step": 1125
+ },
+ {
+ "entropy": 1.0748711824417114,
+ "epoch": 2.2747474747474747,
+ "grad_norm": 2.151282787322998,
+ "learning_rate": 2.7272727272727272e-06,
+ "loss": 1.0380895137786865,
+ "mean_token_accuracy": 0.736932098865509,
+ "num_tokens": 18448384.0,
+ "step": 1126
+ },
+ {
+ "entropy": 1.4420548677444458,
+ "epoch": 2.276767676767677,
+ "grad_norm": 2.133190870285034,
+ "learning_rate": 2.725252525252525e-06,
+ "loss": 1.4465012550354004,
+ "mean_token_accuracy": 0.6784929037094116,
+ "num_tokens": 18464768.0,
+ "step": 1127
+ },
+ {
+ "entropy": 0.9478949904441833,
+ "epoch": 2.278787878787879,
+ "grad_norm": 2.0902628898620605,
+ "learning_rate": 2.7232323232323234e-06,
+ "loss": 0.9504396915435791,
+ "mean_token_accuracy": 0.7458475828170776,
+ "num_tokens": 18481152.0,
+ "step": 1128
+ },
+ {
+ "entropy": 1.367720127105713,
+ "epoch": 2.2808080808080806,
+ "grad_norm": 2.0413167476654053,
+ "learning_rate": 2.7212121212121213e-06,
+ "loss": 1.3884644508361816,
+ "mean_token_accuracy": 0.6762945652008057,
+ "num_tokens": 18497536.0,
+ "step": 1129
+ },
+ {
+ "entropy": 1.6931284666061401,
+ "epoch": 2.282828282828283,
+ "grad_norm": 2.188426971435547,
+ "learning_rate": 2.719191919191919e-06,
+ "loss": 1.7233154773712158,
+ "mean_token_accuracy": 0.6323277950286865,
+ "num_tokens": 18513920.0,
+ "step": 1130
+ },
+ {
+ "entropy": 1.563072681427002,
+ "epoch": 2.2848484848484847,
+ "grad_norm": 2.241102695465088,
+ "learning_rate": 2.717171717171717e-06,
+ "loss": 1.5677220821380615,
+ "mean_token_accuracy": 0.640693724155426,
+ "num_tokens": 18530304.0,
+ "step": 1131
+ },
+ {
+ "entropy": 1.3332083225250244,
+ "epoch": 2.286868686868687,
+ "grad_norm": 2.209522008895874,
+ "learning_rate": 2.715151515151516e-06,
+ "loss": 1.337693452835083,
+ "mean_token_accuracy": 0.6715925931930542,
+ "num_tokens": 18546688.0,
+ "step": 1132
+ },
+ {
+ "entropy": 1.5389035940170288,
+ "epoch": 2.2888888888888888,
+ "grad_norm": 2.212047815322876,
+ "learning_rate": 2.7131313131313137e-06,
+ "loss": 1.52158784866333,
+ "mean_token_accuracy": 0.6447850465774536,
+ "num_tokens": 18563072.0,
+ "step": 1133
+ },
+ {
+ "entropy": 1.2687056064605713,
+ "epoch": 2.290909090909091,
+ "grad_norm": 2.1673405170440674,
+ "learning_rate": 2.7111111111111116e-06,
+ "loss": 1.2689636945724487,
+ "mean_token_accuracy": 0.6911333799362183,
+ "num_tokens": 18579456.0,
+ "step": 1134
+ },
+ {
+ "entropy": 1.1464369297027588,
+ "epoch": 2.292929292929293,
+ "grad_norm": 1.9283273220062256,
+ "learning_rate": 2.7090909090909095e-06,
+ "loss": 1.1613389253616333,
+ "mean_token_accuracy": 0.7145212292671204,
+ "num_tokens": 18595840.0,
+ "step": 1135
+ },
+ {
+ "entropy": 1.327767252922058,
+ "epoch": 2.294949494949495,
+ "grad_norm": 2.2193424701690674,
+ "learning_rate": 2.7070707070707074e-06,
+ "loss": 1.3209152221679688,
+ "mean_token_accuracy": 0.6740962266921997,
+ "num_tokens": 18612224.0,
+ "step": 1136
+ },
+ {
+ "entropy": 1.5820066928863525,
+ "epoch": 2.296969696969697,
+ "grad_norm": 2.2020492553710938,
+ "learning_rate": 2.7050505050505057e-06,
+ "loss": 1.603272795677185,
+ "mean_token_accuracy": 0.6400830745697021,
+ "num_tokens": 18628608.0,
+ "step": 1137
+ },
+ {
+ "entropy": 1.279178500175476,
+ "epoch": 2.298989898989899,
+ "grad_norm": 2.1509714126586914,
+ "learning_rate": 2.7030303030303036e-06,
+ "loss": 1.2742427587509155,
+ "mean_token_accuracy": 0.689667820930481,
+ "num_tokens": 18644992.0,
+ "step": 1138
+ },
+ {
+ "entropy": 1.2199708223342896,
+ "epoch": 2.301010101010101,
+ "grad_norm": 2.0223684310913086,
+ "learning_rate": 2.7010101010101015e-06,
+ "loss": 1.2254796028137207,
+ "mean_token_accuracy": 0.6971787810325623,
+ "num_tokens": 18661376.0,
+ "step": 1139
+ },
+ {
+ "entropy": 1.1640081405639648,
+ "epoch": 2.303030303030303,
+ "grad_norm": 2.1907923221588135,
+ "learning_rate": 2.6989898989898994e-06,
+ "loss": 1.170613408088684,
+ "mean_token_accuracy": 0.6979726552963257,
+ "num_tokens": 18677760.0,
+ "step": 1140
+ },
+ {
+ "entropy": 1.3213385343551636,
+ "epoch": 2.305050505050505,
+ "grad_norm": 2.170276641845703,
+ "learning_rate": 2.6969696969696972e-06,
+ "loss": 1.3338474035263062,
+ "mean_token_accuracy": 0.672874927520752,
+ "num_tokens": 18694144.0,
+ "step": 1141
+ },
+ {
+ "entropy": 1.6036078929901123,
+ "epoch": 2.3070707070707073,
+ "grad_norm": 2.3499386310577393,
+ "learning_rate": 2.694949494949495e-06,
+ "loss": 1.5859179496765137,
+ "mean_token_accuracy": 0.6349536180496216,
+ "num_tokens": 18710528.0,
+ "step": 1142
+ },
+ {
+ "entropy": 1.2171586751937866,
+ "epoch": 2.309090909090909,
+ "grad_norm": 2.0380377769470215,
+ "learning_rate": 2.6929292929292934e-06,
+ "loss": 1.2191288471221924,
+ "mean_token_accuracy": 0.7063385248184204,
+ "num_tokens": 18726912.0,
+ "step": 1143
+ },
+ {
+ "entropy": 1.165902018547058,
+ "epoch": 2.311111111111111,
+ "grad_norm": 2.077310085296631,
+ "learning_rate": 2.6909090909090913e-06,
+ "loss": 1.1728768348693848,
+ "mean_token_accuracy": 0.7075598239898682,
+ "num_tokens": 18743296.0,
+ "step": 1144
+ },
+ {
+ "entropy": 1.5172539949417114,
+ "epoch": 2.313131313131313,
+ "grad_norm": 2.0132439136505127,
+ "learning_rate": 2.6888888888888892e-06,
+ "loss": 1.5073471069335938,
+ "mean_token_accuracy": 0.6546775698661804,
+ "num_tokens": 18759680.0,
+ "step": 1145
+ },
+ {
+ "entropy": 1.3588616847991943,
+ "epoch": 2.315151515151515,
+ "grad_norm": 2.1851465702056885,
+ "learning_rate": 2.686868686868687e-06,
+ "loss": 1.4111744165420532,
+ "mean_token_accuracy": 0.6750732660293579,
+ "num_tokens": 18776064.0,
+ "step": 1146
+ },
+ {
+ "entropy": 1.1948320865631104,
+ "epoch": 2.317171717171717,
+ "grad_norm": 2.1817259788513184,
+ "learning_rate": 2.684848484848485e-06,
+ "loss": 1.188377857208252,
+ "mean_token_accuracy": 0.7122007608413696,
+ "num_tokens": 18792448.0,
+ "step": 1147
+ },
+ {
+ "entropy": 1.7049702405929565,
+ "epoch": 2.319191919191919,
+ "grad_norm": 2.309352159500122,
+ "learning_rate": 2.6828282828282833e-06,
+ "loss": 1.6966536045074463,
+ "mean_token_accuracy": 0.6052759885787964,
+ "num_tokens": 18808832.0,
+ "step": 1148
+ },
+ {
+ "entropy": 1.0853440761566162,
+ "epoch": 2.3212121212121213,
+ "grad_norm": 2.1501059532165527,
+ "learning_rate": 2.680808080808081e-06,
+ "loss": 1.0747997760772705,
+ "mean_token_accuracy": 0.7195896506309509,
+ "num_tokens": 18825216.0,
+ "step": 1149
+ },
+ {
+ "entropy": 1.209170937538147,
+ "epoch": 2.323232323232323,
+ "grad_norm": 2.105555772781372,
+ "learning_rate": 2.678787878787879e-06,
+ "loss": 1.2055076360702515,
+ "mean_token_accuracy": 0.7062774896621704,
+ "num_tokens": 18841600.0,
+ "step": 1150
+ },
+ {
+ "epoch": 2.323232323232323,
+ "eval_entropy": 1.3799022854335847,
+ "eval_loss": 1.5365934371948242,
+ "eval_mean_token_accuracy": 0.6452573234996488,
+ "eval_num_tokens": 18841600.0,
+ "eval_runtime": 43.7362,
+ "eval_samples_per_second": 22.636,
+ "eval_steps_per_second": 2.835,
+ "step": 1150
+ },
+ {
+ "entropy": 0.9533048868179321,
+ "epoch": 2.3252525252525253,
+ "grad_norm": 1.9681413173675537,
+ "learning_rate": 2.676767676767677e-06,
+ "loss": 0.9284776449203491,
+ "mean_token_accuracy": 0.7609916925430298,
+ "num_tokens": 18857984.0,
+ "step": 1151
+ },
+ {
+ "entropy": 1.575872778892517,
+ "epoch": 2.327272727272727,
+ "grad_norm": 2.15085768699646,
+ "learning_rate": 2.674747474747475e-06,
+ "loss": 1.5716705322265625,
+ "mean_token_accuracy": 0.6415486335754395,
+ "num_tokens": 18874368.0,
+ "step": 1152
+ },
+ {
+ "entropy": 1.1680493354797363,
+ "epoch": 2.3292929292929294,
+ "grad_norm": 2.189589023590088,
+ "learning_rate": 2.6727272727272727e-06,
+ "loss": 1.1651779413223267,
+ "mean_token_accuracy": 0.7156203985214233,
+ "num_tokens": 18890752.0,
+ "step": 1153
+ },
+ {
+ "entropy": 1.1636664867401123,
+ "epoch": 2.3313131313131312,
+ "grad_norm": 1.9992915391921997,
+ "learning_rate": 2.670707070707071e-06,
+ "loss": 1.1650042533874512,
+ "mean_token_accuracy": 0.7116512060165405,
+ "num_tokens": 18907136.0,
+ "step": 1154
+ },
+ {
+ "entropy": 1.205094337463379,
+ "epoch": 2.3333333333333335,
+ "grad_norm": 2.070323944091797,
+ "learning_rate": 2.668686868686869e-06,
+ "loss": 1.2071596384048462,
+ "mean_token_accuracy": 0.7018197178840637,
+ "num_tokens": 18923520.0,
+ "step": 1155
+ },
+ {
+ "entropy": 1.2256643772125244,
+ "epoch": 2.3353535353535353,
+ "grad_norm": 2.144339084625244,
+ "learning_rate": 2.666666666666667e-06,
+ "loss": 1.2168457508087158,
+ "mean_token_accuracy": 0.7010869383811951,
+ "num_tokens": 18939904.0,
+ "step": 1156
+ },
+ {
+ "entropy": 1.631039023399353,
+ "epoch": 2.3373737373737375,
+ "grad_norm": 2.2886807918548584,
+ "learning_rate": 2.6646464646464647e-06,
+ "loss": 1.6534450054168701,
+ "mean_token_accuracy": 0.6248778700828552,
+ "num_tokens": 18956288.0,
+ "step": 1157
+ },
+ {
+ "entropy": 1.0360562801361084,
+ "epoch": 2.3393939393939394,
+ "grad_norm": 2.137491226196289,
+ "learning_rate": 2.6626262626262626e-06,
+ "loss": 1.0167723894119263,
+ "mean_token_accuracy": 0.7330239415168762,
+ "num_tokens": 18972672.0,
+ "step": 1158
+ },
+ {
+ "entropy": 1.2292720079421997,
+ "epoch": 2.341414141414141,
+ "grad_norm": 3.1518330574035645,
+ "learning_rate": 2.660606060606061e-06,
+ "loss": 1.2180495262145996,
+ "mean_token_accuracy": 0.6957743167877197,
+ "num_tokens": 18989056.0,
+ "step": 1159
+ },
+ {
+ "entropy": 1.2210739850997925,
+ "epoch": 2.3434343434343434,
+ "grad_norm": 2.1314170360565186,
+ "learning_rate": 2.658585858585859e-06,
+ "loss": 1.2399296760559082,
+ "mean_token_accuracy": 0.7047508358955383,
+ "num_tokens": 19005440.0,
+ "step": 1160
+ },
+ {
+ "entropy": 1.2762846946716309,
+ "epoch": 2.3454545454545457,
+ "grad_norm": 2.1767046451568604,
+ "learning_rate": 2.6565656565656567e-06,
+ "loss": 1.2640776634216309,
+ "mean_token_accuracy": 0.6954079270362854,
+ "num_tokens": 19021824.0,
+ "step": 1161
+ },
+ {
+ "entropy": 1.271180272102356,
+ "epoch": 2.3474747474747475,
+ "grad_norm": 2.121891975402832,
+ "learning_rate": 2.6545454545454546e-06,
+ "loss": 1.2666800022125244,
+ "mean_token_accuracy": 0.6880801320075989,
+ "num_tokens": 19038208.0,
+ "step": 1162
+ },
+ {
+ "entropy": 1.5604509115219116,
+ "epoch": 2.3494949494949493,
+ "grad_norm": 2.320765495300293,
+ "learning_rate": 2.6525252525252525e-06,
+ "loss": 1.5687410831451416,
+ "mean_token_accuracy": 0.627198338508606,
+ "num_tokens": 19054592.0,
+ "step": 1163
+ },
+ {
+ "entropy": 1.1606905460357666,
+ "epoch": 2.3515151515151516,
+ "grad_norm": 2.1167874336242676,
+ "learning_rate": 2.6505050505050508e-06,
+ "loss": 1.1879122257232666,
+ "mean_token_accuracy": 0.7130556702613831,
+ "num_tokens": 19070976.0,
+ "step": 1164
+ },
+ {
+ "entropy": 1.5421797037124634,
+ "epoch": 2.3535353535353534,
+ "grad_norm": 2.144657850265503,
+ "learning_rate": 2.6484848484848487e-06,
+ "loss": 1.5562596321105957,
+ "mean_token_accuracy": 0.6350146532058716,
+ "num_tokens": 19087360.0,
+ "step": 1165
+ },
+ {
+ "entropy": 1.2803337574005127,
+ "epoch": 2.3555555555555556,
+ "grad_norm": 2.0440914630889893,
+ "learning_rate": 2.6464646464646466e-06,
+ "loss": 1.3008983135223389,
+ "mean_token_accuracy": 0.6868588328361511,
+ "num_tokens": 19103744.0,
+ "step": 1166
+ },
+ {
+ "entropy": 1.1863176822662354,
+ "epoch": 2.3575757575757574,
+ "grad_norm": 2.231771469116211,
+ "learning_rate": 2.6444444444444444e-06,
+ "loss": 1.1863257884979248,
+ "mean_token_accuracy": 0.7069491744041443,
+ "num_tokens": 19120128.0,
+ "step": 1167
+ },
+ {
+ "entropy": 1.1186378002166748,
+ "epoch": 2.3595959595959597,
+ "grad_norm": 2.0704708099365234,
+ "learning_rate": 2.6424242424242423e-06,
+ "loss": 1.1196620464324951,
+ "mean_token_accuracy": 0.726062536239624,
+ "num_tokens": 19136512.0,
+ "step": 1168
+ },
+ {
+ "entropy": 1.2110928297042847,
+ "epoch": 2.3616161616161615,
+ "grad_norm": 2.251551866531372,
+ "learning_rate": 2.6404040404040402e-06,
+ "loss": 1.2020606994628906,
+ "mean_token_accuracy": 0.7058500051498413,
+ "num_tokens": 19152896.0,
+ "step": 1169
+ },
+ {
+ "entropy": 1.302309513092041,
+ "epoch": 2.3636363636363638,
+ "grad_norm": 2.171961545944214,
+ "learning_rate": 2.6383838383838385e-06,
+ "loss": 1.3029515743255615,
+ "mean_token_accuracy": 0.673363447189331,
+ "num_tokens": 19169280.0,
+ "step": 1170
+ },
+ {
+ "entropy": 1.6036607027053833,
+ "epoch": 2.3656565656565656,
+ "grad_norm": 2.336979627609253,
+ "learning_rate": 2.6363636363636364e-06,
+ "loss": 1.6076864004135132,
+ "mean_token_accuracy": 0.630984365940094,
+ "num_tokens": 19185664.0,
+ "step": 1171
+ },
+ {
+ "entropy": 1.431620478630066,
+ "epoch": 2.367676767676768,
+ "grad_norm": 2.115891218185425,
+ "learning_rate": 2.6343434343434343e-06,
+ "loss": 1.4436556100845337,
+ "mean_token_accuracy": 0.6623717546463013,
+ "num_tokens": 19202048.0,
+ "step": 1172
+ },
+ {
+ "entropy": 1.4537419080734253,
+ "epoch": 2.3696969696969696,
+ "grad_norm": 2.497891664505005,
+ "learning_rate": 2.632323232323232e-06,
+ "loss": 1.48027503490448,
+ "mean_token_accuracy": 0.6352589130401611,
+ "num_tokens": 19218432.0,
+ "step": 1173
+ },
+ {
+ "entropy": 1.292363166809082,
+ "epoch": 2.371717171717172,
+ "grad_norm": 2.0235633850097656,
+ "learning_rate": 2.63030303030303e-06,
+ "loss": 1.3111207485198975,
+ "mean_token_accuracy": 0.6873473525047302,
+ "num_tokens": 19234816.0,
+ "step": 1174
+ },
+ {
+ "entropy": 1.2336944341659546,
+ "epoch": 2.3737373737373737,
+ "grad_norm": 2.342643976211548,
+ "learning_rate": 2.628282828282829e-06,
+ "loss": 1.2427266836166382,
+ "mean_token_accuracy": 0.6844772696495056,
+ "num_tokens": 19251200.0,
+ "step": 1175
+ },
+ {
+ "entropy": 1.166039228439331,
+ "epoch": 2.375757575757576,
+ "grad_norm": 2.157937526702881,
+ "learning_rate": 2.6262626262626267e-06,
+ "loss": 1.167286992073059,
+ "mean_token_accuracy": 0.7115290760993958,
+ "num_tokens": 19267584.0,
+ "step": 1176
+ },
+ {
+ "entropy": 1.3824102878570557,
+ "epoch": 2.3777777777777778,
+ "grad_norm": 2.2874608039855957,
+ "learning_rate": 2.6242424242424246e-06,
+ "loss": 1.404619574546814,
+ "mean_token_accuracy": 0.6707376837730408,
+ "num_tokens": 19283968.0,
+ "step": 1177
+ },
+ {
+ "entropy": 0.8266427516937256,
+ "epoch": 2.3797979797979796,
+ "grad_norm": 2.098217010498047,
+ "learning_rate": 2.6222222222222225e-06,
+ "loss": 0.821276068687439,
+ "mean_token_accuracy": 0.7726551294326782,
+ "num_tokens": 19300352.0,
+ "step": 1178
+ },
+ {
+ "entropy": 1.4335079193115234,
+ "epoch": 2.381818181818182,
+ "grad_norm": 2.3051438331604004,
+ "learning_rate": 2.620202020202021e-06,
+ "loss": 1.437370777130127,
+ "mean_token_accuracy": 0.6520518064498901,
+ "num_tokens": 19316736.0,
+ "step": 1179
+ },
+ {
+ "entropy": 1.0816318988800049,
+ "epoch": 2.3838383838383836,
+ "grad_norm": 2.0723016262054443,
+ "learning_rate": 2.6181818181818187e-06,
+ "loss": 1.0697057247161865,
+ "mean_token_accuracy": 0.7304592132568359,
+ "num_tokens": 19333120.0,
+ "step": 1180
+ },
+ {
+ "entropy": 1.2515255212783813,
+ "epoch": 2.385858585858586,
+ "grad_norm": 2.32125186920166,
+ "learning_rate": 2.6161616161616166e-06,
+ "loss": 1.256905198097229,
+ "mean_token_accuracy": 0.697300910949707,
+ "num_tokens": 19349504.0,
+ "step": 1181
+ },
+ {
+ "entropy": 1.0271868705749512,
+ "epoch": 2.3878787878787877,
+ "grad_norm": 2.0145103931427,
+ "learning_rate": 2.6141414141414145e-06,
+ "loss": 1.0158754587173462,
+ "mean_token_accuracy": 0.7518930435180664,
+ "num_tokens": 19365888.0,
+ "step": 1182
+ },
+ {
+ "entropy": 1.2139984369277954,
+ "epoch": 2.38989898989899,
+ "grad_norm": 2.1393911838531494,
+ "learning_rate": 2.6121212121212123e-06,
+ "loss": 1.2150651216506958,
+ "mean_token_accuracy": 0.7135441899299622,
+ "num_tokens": 19382272.0,
+ "step": 1183
+ },
+ {
+ "entropy": 1.076101303100586,
+ "epoch": 2.391919191919192,
+ "grad_norm": 2.1664907932281494,
+ "learning_rate": 2.6101010101010107e-06,
+ "loss": 1.0744431018829346,
+ "mean_token_accuracy": 0.7267953157424927,
+ "num_tokens": 19398656.0,
+ "step": 1184
+ },
+ {
+ "entropy": 1.5176570415496826,
+ "epoch": 2.393939393939394,
+ "grad_norm": 2.0756311416625977,
+ "learning_rate": 2.6080808080808085e-06,
+ "loss": 1.5209624767303467,
+ "mean_token_accuracy": 0.647777259349823,
+ "num_tokens": 19415040.0,
+ "step": 1185
+ },
+ {
+ "entropy": 1.0946495532989502,
+ "epoch": 2.395959595959596,
+ "grad_norm": 2.0545496940612793,
+ "learning_rate": 2.6060606060606064e-06,
+ "loss": 1.0858100652694702,
+ "mean_token_accuracy": 0.7226428985595703,
+ "num_tokens": 19431424.0,
+ "step": 1186
+ },
+ {
+ "entropy": 0.7713431715965271,
+ "epoch": 2.397979797979798,
+ "grad_norm": 2.0606112480163574,
+ "learning_rate": 2.6040404040404043e-06,
+ "loss": 0.7782045006752014,
+ "mean_token_accuracy": 0.7804103493690491,
+ "num_tokens": 19447808.0,
+ "step": 1187
+ },
+ {
+ "entropy": 1.2803305387496948,
+ "epoch": 2.4,
+ "grad_norm": 2.2125155925750732,
+ "learning_rate": 2.602020202020202e-06,
+ "loss": 1.2971004247665405,
+ "mean_token_accuracy": 0.6787371635437012,
+ "num_tokens": 19464192.0,
+ "step": 1188
+ },
+ {
+ "entropy": 0.9156424403190613,
+ "epoch": 2.402020202020202,
+ "grad_norm": 2.032531499862671,
+ "learning_rate": 2.6e-06,
+ "loss": 0.9119267463684082,
+ "mean_token_accuracy": 0.7627015113830566,
+ "num_tokens": 19480576.0,
+ "step": 1189
+ },
+ {
+ "entropy": 1.2698813676834106,
+ "epoch": 2.404040404040404,
+ "grad_norm": 2.2204320430755615,
+ "learning_rate": 2.5979797979797984e-06,
+ "loss": 1.2934629917144775,
+ "mean_token_accuracy": 0.682584285736084,
+ "num_tokens": 19496960.0,
+ "step": 1190
+ },
+ {
+ "entropy": 1.5677871704101562,
+ "epoch": 2.4060606060606062,
+ "grad_norm": 2.229994773864746,
+ "learning_rate": 2.5959595959595963e-06,
+ "loss": 1.591191053390503,
+ "mean_token_accuracy": 0.6297020316123962,
+ "num_tokens": 19513344.0,
+ "step": 1191
+ },
+ {
+ "entropy": 1.3057011365890503,
+ "epoch": 2.408080808080808,
+ "grad_norm": 2.4158618450164795,
+ "learning_rate": 2.593939393939394e-06,
+ "loss": 1.2989580631256104,
+ "mean_token_accuracy": 0.689728856086731,
+ "num_tokens": 19529728.0,
+ "step": 1192
+ },
+ {
+ "entropy": 1.3805862665176392,
+ "epoch": 2.41010101010101,
+ "grad_norm": 2.2611825466156006,
+ "learning_rate": 2.591919191919192e-06,
+ "loss": 1.385383129119873,
+ "mean_token_accuracy": 0.6634709239006042,
+ "num_tokens": 19546112.0,
+ "step": 1193
+ },
+ {
+ "entropy": 1.8123539686203003,
+ "epoch": 2.412121212121212,
+ "grad_norm": 2.3038923740386963,
+ "learning_rate": 2.58989898989899e-06,
+ "loss": 1.80764901638031,
+ "mean_token_accuracy": 0.5925744771957397,
+ "num_tokens": 19562496.0,
+ "step": 1194
+ },
+ {
+ "entropy": 1.1476300954818726,
+ "epoch": 2.4141414141414144,
+ "grad_norm": 2.2417445182800293,
+ "learning_rate": 2.5878787878787883e-06,
+ "loss": 1.1802024841308594,
+ "mean_token_accuracy": 0.7130556702613831,
+ "num_tokens": 19578880.0,
+ "step": 1195
+ },
+ {
+ "entropy": 1.6678205728530884,
+ "epoch": 2.416161616161616,
+ "grad_norm": 2.399136781692505,
+ "learning_rate": 2.585858585858586e-06,
+ "loss": 1.6780188083648682,
+ "mean_token_accuracy": 0.6161455512046814,
+ "num_tokens": 19595264.0,
+ "step": 1196
+ },
+ {
+ "entropy": 1.053451418876648,
+ "epoch": 2.418181818181818,
+ "grad_norm": 1.9980698823928833,
+ "learning_rate": 2.583838383838384e-06,
+ "loss": 1.0241587162017822,
+ "mean_token_accuracy": 0.7372984886169434,
+ "num_tokens": 19611648.0,
+ "step": 1197
+ },
+ {
+ "entropy": 1.5389430522918701,
+ "epoch": 2.4202020202020202,
+ "grad_norm": 2.171046733856201,
+ "learning_rate": 2.581818181818182e-06,
+ "loss": 1.5499310493469238,
+ "mean_token_accuracy": 0.6422203183174133,
+ "num_tokens": 19628032.0,
+ "step": 1198
+ },
+ {
+ "entropy": 1.4151558876037598,
+ "epoch": 2.422222222222222,
+ "grad_norm": 2.185184955596924,
+ "learning_rate": 2.57979797979798e-06,
+ "loss": 1.4296989440917969,
+ "mean_token_accuracy": 0.663593053817749,
+ "num_tokens": 19644416.0,
+ "step": 1199
+ },
+ {
+ "entropy": 1.1455156803131104,
+ "epoch": 2.4242424242424243,
+ "grad_norm": 2.224477767944336,
+ "learning_rate": 2.577777777777778e-06,
+ "loss": 1.1682045459747314,
+ "mean_token_accuracy": 0.7084147334098816,
+ "num_tokens": 19660800.0,
+ "step": 1200
+ },
+ {
+ "epoch": 2.4242424242424243,
+ "eval_entropy": 1.3761802339746105,
+ "eval_loss": 1.53595769405365,
+ "eval_mean_token_accuracy": 0.6453439944213436,
+ "eval_num_tokens": 19660800.0,
+ "eval_runtime": 43.8176,
+ "eval_samples_per_second": 22.594,
+ "eval_steps_per_second": 2.83,
+ "step": 1200
+ },
+ {
+ "entropy": 1.6039199829101562,
+ "epoch": 2.426262626262626,
+ "grad_norm": 2.122072696685791,
+ "learning_rate": 2.575757575757576e-06,
+ "loss": 1.625109314918518,
+ "mean_token_accuracy": 0.6306179761886597,
+ "num_tokens": 19677184.0,
+ "step": 1201
+ },
+ {
+ "entropy": 1.2364906072616577,
+ "epoch": 2.4282828282828284,
+ "grad_norm": 2.175671339035034,
+ "learning_rate": 2.573737373737374e-06,
+ "loss": 1.2398111820220947,
+ "mean_token_accuracy": 0.6886907815933228,
+ "num_tokens": 19693568.0,
+ "step": 1202
+ },
+ {
+ "entropy": 1.2937804460525513,
+ "epoch": 2.43030303030303,
+ "grad_norm": 2.1206040382385254,
+ "learning_rate": 2.571717171717172e-06,
+ "loss": 1.3073123693466187,
+ "mean_token_accuracy": 0.6762945652008057,
+ "num_tokens": 19709952.0,
+ "step": 1203
+ },
+ {
+ "entropy": 1.0239282846450806,
+ "epoch": 2.4323232323232324,
+ "grad_norm": 1.863510012626648,
+ "learning_rate": 2.5696969696969697e-06,
+ "loss": 1.015241026878357,
+ "mean_token_accuracy": 0.7452369332313538,
+ "num_tokens": 19726336.0,
+ "step": 1204
+ },
+ {
+ "entropy": 1.2327015399932861,
+ "epoch": 2.4343434343434343,
+ "grad_norm": 1.968019962310791,
+ "learning_rate": 2.5676767676767676e-06,
+ "loss": 1.2275054454803467,
+ "mean_token_accuracy": 0.6974841356277466,
+ "num_tokens": 19742720.0,
+ "step": 1205
+ },
+ {
+ "entropy": 1.130920171737671,
+ "epoch": 2.4363636363636365,
+ "grad_norm": 2.096205949783325,
+ "learning_rate": 2.565656565656566e-06,
+ "loss": 1.1229937076568604,
+ "mean_token_accuracy": 0.7102466821670532,
+ "num_tokens": 19759104.0,
+ "step": 1206
+ },
+ {
+ "entropy": 0.9204118251800537,
+ "epoch": 2.4383838383838383,
+ "grad_norm": 1.7956496477127075,
+ "learning_rate": 2.5636363636363638e-06,
+ "loss": 0.9276518821716309,
+ "mean_token_accuracy": 0.7644113302230835,
+ "num_tokens": 19775488.0,
+ "step": 1207
+ },
+ {
+ "entropy": 0.9552480578422546,
+ "epoch": 2.4404040404040406,
+ "grad_norm": 2.0838167667388916,
+ "learning_rate": 2.5616161616161617e-06,
+ "loss": 0.9676632881164551,
+ "mean_token_accuracy": 0.7421225905418396,
+ "num_tokens": 19791872.0,
+ "step": 1208
+ },
+ {
+ "entropy": 1.0109010934829712,
+ "epoch": 2.4424242424242424,
+ "grad_norm": 2.1739392280578613,
+ "learning_rate": 2.5595959595959595e-06,
+ "loss": 1.0201005935668945,
+ "mean_token_accuracy": 0.7366267442703247,
+ "num_tokens": 19808256.0,
+ "step": 1209
+ },
+ {
+ "entropy": 1.4037907123565674,
+ "epoch": 2.4444444444444446,
+ "grad_norm": 2.151865005493164,
+ "learning_rate": 2.5575757575757574e-06,
+ "loss": 1.4110395908355713,
+ "mean_token_accuracy": 0.6682339906692505,
+ "num_tokens": 19824640.0,
+ "step": 1210
+ },
+ {
+ "entropy": 1.2449947595596313,
+ "epoch": 2.4464646464646465,
+ "grad_norm": 2.065934896469116,
+ "learning_rate": 2.5555555555555557e-06,
+ "loss": 1.2521324157714844,
+ "mean_token_accuracy": 0.6952247023582458,
+ "num_tokens": 19841024.0,
+ "step": 1211
+ },
+ {
+ "entropy": 1.2075072526931763,
+ "epoch": 2.4484848484848483,
+ "grad_norm": 2.197885274887085,
+ "learning_rate": 2.5535353535353536e-06,
+ "loss": 1.2041654586791992,
+ "mean_token_accuracy": 0.69840008020401,
+ "num_tokens": 19857408.0,
+ "step": 1212
+ },
+ {
+ "entropy": 1.0454095602035522,
+ "epoch": 2.4505050505050505,
+ "grad_norm": 2.0778188705444336,
+ "learning_rate": 2.5515151515151515e-06,
+ "loss": 1.0392906665802002,
+ "mean_token_accuracy": 0.74187833070755,
+ "num_tokens": 19873792.0,
+ "step": 1213
+ },
+ {
+ "entropy": 1.3138911724090576,
+ "epoch": 2.4525252525252528,
+ "grad_norm": 2.1871910095214844,
+ "learning_rate": 2.5494949494949494e-06,
+ "loss": 1.317048192024231,
+ "mean_token_accuracy": 0.6720811128616333,
+ "num_tokens": 19890176.0,
+ "step": 1214
+ },
+ {
+ "entropy": 1.263680338859558,
+ "epoch": 2.4545454545454546,
+ "grad_norm": 2.129873752593994,
+ "learning_rate": 2.5474747474747473e-06,
+ "loss": 1.277045726776123,
+ "mean_token_accuracy": 0.7011480331420898,
+ "num_tokens": 19906560.0,
+ "step": 1215
+ },
+ {
+ "entropy": 0.9653311967849731,
+ "epoch": 2.4565656565656564,
+ "grad_norm": 2.1161508560180664,
+ "learning_rate": 2.5454545454545456e-06,
+ "loss": 0.9699352979660034,
+ "mean_token_accuracy": 0.7437713742256165,
+ "num_tokens": 19922944.0,
+ "step": 1216
+ },
+ {
+ "entropy": 1.350392460823059,
+ "epoch": 2.4585858585858587,
+ "grad_norm": 2.0958914756774902,
+ "learning_rate": 2.5434343434343435e-06,
+ "loss": 1.3442802429199219,
+ "mean_token_accuracy": 0.6768441796302795,
+ "num_tokens": 19939328.0,
+ "step": 1217
+ },
+ {
+ "entropy": 1.2484683990478516,
+ "epoch": 2.4606060606060605,
+ "grad_norm": 2.191668748855591,
+ "learning_rate": 2.541414141414142e-06,
+ "loss": 1.2663497924804688,
+ "mean_token_accuracy": 0.6958964467048645,
+ "num_tokens": 19955712.0,
+ "step": 1218
+ },
+ {
+ "entropy": 0.9751343131065369,
+ "epoch": 2.4626262626262627,
+ "grad_norm": 2.0976815223693848,
+ "learning_rate": 2.5393939393939397e-06,
+ "loss": 0.9734562635421753,
+ "mean_token_accuracy": 0.7372373938560486,
+ "num_tokens": 19972096.0,
+ "step": 1219
+ },
+ {
+ "entropy": 1.1655267477035522,
+ "epoch": 2.4646464646464645,
+ "grad_norm": 2.3504767417907715,
+ "learning_rate": 2.537373737373738e-06,
+ "loss": 1.1775636672973633,
+ "mean_token_accuracy": 0.7023082375526428,
+ "num_tokens": 19988480.0,
+ "step": 1220
+ },
+ {
+ "entropy": 1.3543325662612915,
+ "epoch": 2.466666666666667,
+ "grad_norm": 2.038313627243042,
+ "learning_rate": 2.535353535353536e-06,
+ "loss": 1.3447773456573486,
+ "mean_token_accuracy": 0.6779433488845825,
+ "num_tokens": 20004864.0,
+ "step": 1221
+ },
+ {
+ "entropy": 1.1814380884170532,
+ "epoch": 2.4686868686868686,
+ "grad_norm": 2.186947822570801,
+ "learning_rate": 2.5333333333333338e-06,
+ "loss": 1.186347246170044,
+ "mean_token_accuracy": 0.7064606547355652,
+ "num_tokens": 20021248.0,
+ "step": 1222
+ },
+ {
+ "entropy": 0.9775183796882629,
+ "epoch": 2.470707070707071,
+ "grad_norm": 2.144923210144043,
+ "learning_rate": 2.5313131313131317e-06,
+ "loss": 1.0010652542114258,
+ "mean_token_accuracy": 0.7385808229446411,
+ "num_tokens": 20037632.0,
+ "step": 1223
+ },
+ {
+ "entropy": 1.6076546907424927,
+ "epoch": 2.4727272727272727,
+ "grad_norm": 2.1664817333221436,
+ "learning_rate": 2.5292929292929296e-06,
+ "loss": 1.618837594985962,
+ "mean_token_accuracy": 0.6297630667686462,
+ "num_tokens": 20054016.0,
+ "step": 1224
+ },
+ {
+ "entropy": 1.0457805395126343,
+ "epoch": 2.474747474747475,
+ "grad_norm": 2.154000997543335,
+ "learning_rate": 2.5272727272727274e-06,
+ "loss": 1.0406007766723633,
+ "mean_token_accuracy": 0.7322911620140076,
+ "num_tokens": 20070400.0,
+ "step": 1225
+ },
+ {
+ "entropy": 1.065863847732544,
+ "epoch": 2.4767676767676767,
+ "grad_norm": 2.062626361846924,
+ "learning_rate": 2.5252525252525258e-06,
+ "loss": 1.0432696342468262,
+ "mean_token_accuracy": 0.7371152639389038,
+ "num_tokens": 20086784.0,
+ "step": 1226
+ },
+ {
+ "entropy": 1.5166304111480713,
+ "epoch": 2.4787878787878785,
+ "grad_norm": 2.089587926864624,
+ "learning_rate": 2.5232323232323236e-06,
+ "loss": 1.5205962657928467,
+ "mean_token_accuracy": 0.653822660446167,
+ "num_tokens": 20103168.0,
+ "step": 1227
+ },
+ {
+ "entropy": 1.5969332456588745,
+ "epoch": 2.480808080808081,
+ "grad_norm": 2.297184705734253,
+ "learning_rate": 2.5212121212121215e-06,
+ "loss": 1.611903190612793,
+ "mean_token_accuracy": 0.6246336102485657,
+ "num_tokens": 20119552.0,
+ "step": 1228
+ },
+ {
+ "entropy": 1.8385323286056519,
+ "epoch": 2.482828282828283,
+ "grad_norm": 2.0568299293518066,
+ "learning_rate": 2.5191919191919194e-06,
+ "loss": 1.848689079284668,
+ "mean_token_accuracy": 0.6079018115997314,
+ "num_tokens": 20135936.0,
+ "step": 1229
+ },
+ {
+ "entropy": 1.1586394309997559,
+ "epoch": 2.484848484848485,
+ "grad_norm": 1.91037917137146,
+ "learning_rate": 2.5171717171717173e-06,
+ "loss": 1.1482139825820923,
+ "mean_token_accuracy": 0.7165364027023315,
+ "num_tokens": 20152320.0,
+ "step": 1230
+ },
+ {
+ "entropy": 1.6905426979064941,
+ "epoch": 2.4868686868686867,
+ "grad_norm": 2.2317121028900146,
+ "learning_rate": 2.5151515151515156e-06,
+ "loss": 1.7028450965881348,
+ "mean_token_accuracy": 0.6125427484512329,
+ "num_tokens": 20168704.0,
+ "step": 1231
+ },
+ {
+ "entropy": 1.1329548358917236,
+ "epoch": 2.488888888888889,
+ "grad_norm": 2.112380027770996,
+ "learning_rate": 2.5131313131313135e-06,
+ "loss": 1.120601773262024,
+ "mean_token_accuracy": 0.7181240916252136,
+ "num_tokens": 20185088.0,
+ "step": 1232
+ },
+ {
+ "entropy": 1.2013109922409058,
+ "epoch": 2.4909090909090907,
+ "grad_norm": 2.087054491043091,
+ "learning_rate": 2.5111111111111114e-06,
+ "loss": 1.2169899940490723,
+ "mean_token_accuracy": 0.7089032530784607,
+ "num_tokens": 20201472.0,
+ "step": 1233
+ },
+ {
+ "entropy": 1.187458872795105,
+ "epoch": 2.492929292929293,
+ "grad_norm": 2.431793689727783,
+ "learning_rate": 2.5090909090909093e-06,
+ "loss": 1.2183618545532227,
+ "mean_token_accuracy": 0.6925378441810608,
+ "num_tokens": 20217856.0,
+ "step": 1234
+ },
+ {
+ "entropy": 1.2123541831970215,
+ "epoch": 2.494949494949495,
+ "grad_norm": 2.1715121269226074,
+ "learning_rate": 2.507070707070707e-06,
+ "loss": 1.2254973649978638,
+ "mean_token_accuracy": 0.703285276889801,
+ "num_tokens": 20234240.0,
+ "step": 1235
+ },
+ {
+ "entropy": 1.0582001209259033,
+ "epoch": 2.496969696969697,
+ "grad_norm": 2.222578763961792,
+ "learning_rate": 2.5050505050505055e-06,
+ "loss": 1.061441421508789,
+ "mean_token_accuracy": 0.7247191071510315,
+ "num_tokens": 20250624.0,
+ "step": 1236
+ },
+ {
+ "entropy": 1.5102840662002563,
+ "epoch": 2.498989898989899,
+ "grad_norm": 2.27860951423645,
+ "learning_rate": 2.5030303030303034e-06,
+ "loss": 1.519880771636963,
+ "mean_token_accuracy": 0.6581583023071289,
+ "num_tokens": 20267008.0,
+ "step": 1237
+ },
+ {
+ "entropy": 1.222317099571228,
+ "epoch": 2.501010101010101,
+ "grad_norm": 2.182908773422241,
+ "learning_rate": 2.5010101010101013e-06,
+ "loss": 1.2382556200027466,
+ "mean_token_accuracy": 0.6908891201019287,
+ "num_tokens": 20283392.0,
+ "step": 1238
+ },
+ {
+ "entropy": 1.252193570137024,
+ "epoch": 2.503030303030303,
+ "grad_norm": 2.2065556049346924,
+ "learning_rate": 2.498989898989899e-06,
+ "loss": 1.2510521411895752,
+ "mean_token_accuracy": 0.6914386749267578,
+ "num_tokens": 20299776.0,
+ "step": 1239
+ },
+ {
+ "entropy": 1.2909793853759766,
+ "epoch": 2.505050505050505,
+ "grad_norm": 2.374338150024414,
+ "learning_rate": 2.496969696969697e-06,
+ "loss": 1.3144972324371338,
+ "mean_token_accuracy": 0.6847215294837952,
+ "num_tokens": 20316160.0,
+ "step": 1240
+ },
+ {
+ "entropy": 1.5218966007232666,
+ "epoch": 2.507070707070707,
+ "grad_norm": 2.3914759159088135,
+ "learning_rate": 2.494949494949495e-06,
+ "loss": 1.5194602012634277,
+ "mean_token_accuracy": 0.6398998498916626,
+ "num_tokens": 20332544.0,
+ "step": 1241
+ },
+ {
+ "entropy": 1.2130569219589233,
+ "epoch": 2.509090909090909,
+ "grad_norm": 2.2857706546783447,
+ "learning_rate": 2.4929292929292932e-06,
+ "loss": 1.2035828828811646,
+ "mean_token_accuracy": 0.6977283954620361,
+ "num_tokens": 20348928.0,
+ "step": 1242
+ },
+ {
+ "entropy": 1.623070478439331,
+ "epoch": 2.511111111111111,
+ "grad_norm": 2.3757028579711914,
+ "learning_rate": 2.490909090909091e-06,
+ "loss": 1.654674768447876,
+ "mean_token_accuracy": 0.6242061257362366,
+ "num_tokens": 20365312.0,
+ "step": 1243
+ },
+ {
+ "entropy": 1.6762864589691162,
+ "epoch": 2.5131313131313133,
+ "grad_norm": 2.239466667175293,
+ "learning_rate": 2.488888888888889e-06,
+ "loss": 1.6712524890899658,
+ "mean_token_accuracy": 0.6280532479286194,
+ "num_tokens": 20381696.0,
+ "step": 1244
+ },
+ {
+ "entropy": 1.6754344701766968,
+ "epoch": 2.515151515151515,
+ "grad_norm": 2.2599003314971924,
+ "learning_rate": 2.486868686868687e-06,
+ "loss": 1.6697289943695068,
+ "mean_token_accuracy": 0.6158402562141418,
+ "num_tokens": 20398080.0,
+ "step": 1245
+ },
+ {
+ "entropy": 1.6282587051391602,
+ "epoch": 2.517171717171717,
+ "grad_norm": 2.0555050373077393,
+ "learning_rate": 2.4848484848484848e-06,
+ "loss": 1.6177103519439697,
+ "mean_token_accuracy": 0.63312166929245,
+ "num_tokens": 20414464.0,
+ "step": 1246
+ },
+ {
+ "entropy": 1.3457770347595215,
+ "epoch": 2.519191919191919,
+ "grad_norm": 2.3396496772766113,
+ "learning_rate": 2.482828282828283e-06,
+ "loss": 1.3288640975952148,
+ "mean_token_accuracy": 0.6720811128616333,
+ "num_tokens": 20430848.0,
+ "step": 1247
+ },
+ {
+ "entropy": 1.5269931554794312,
+ "epoch": 2.5212121212121215,
+ "grad_norm": 2.303053617477417,
+ "learning_rate": 2.480808080808081e-06,
+ "loss": 1.4953217506408691,
+ "mean_token_accuracy": 0.639167070388794,
+ "num_tokens": 20447232.0,
+ "step": 1248
+ },
+ {
+ "entropy": 1.3399546146392822,
+ "epoch": 2.5232323232323233,
+ "grad_norm": 2.3238017559051514,
+ "learning_rate": 2.478787878787879e-06,
+ "loss": 1.337552547454834,
+ "mean_token_accuracy": 0.6790425181388855,
+ "num_tokens": 20463616.0,
+ "step": 1249
+ },
+ {
+ "entropy": 1.0954687595367432,
+ "epoch": 2.525252525252525,
+ "grad_norm": 2.2266361713409424,
+ "learning_rate": 2.476767676767677e-06,
+ "loss": 1.074552297592163,
+ "mean_token_accuracy": 0.7192232608795166,
+ "num_tokens": 20480000.0,
+ "step": 1250
+ },
+ {
+ "epoch": 2.525252525252525,
+ "eval_entropy": 1.375552624464035,
+ "eval_loss": 1.5356626510620117,
+ "eval_mean_token_accuracy": 0.6455749580937047,
+ "eval_num_tokens": 20480000.0,
+ "eval_runtime": 43.8024,
+ "eval_samples_per_second": 22.602,
+ "eval_steps_per_second": 2.831,
+ "step": 1250
+ },
+ {
+ "entropy": 1.329379916191101,
+ "epoch": 2.5272727272727273,
+ "grad_norm": 2.288421392440796,
+ "learning_rate": 2.474747474747475e-06,
+ "loss": 1.3166120052337646,
+ "mean_token_accuracy": 0.6743404865264893,
+ "num_tokens": 20496384.0,
+ "step": 1251
+ },
+ {
+ "entropy": 1.3155947923660278,
+ "epoch": 2.529292929292929,
+ "grad_norm": 2.2791004180908203,
+ "learning_rate": 2.472727272727273e-06,
+ "loss": 1.3281152248382568,
+ "mean_token_accuracy": 0.6623717546463013,
+ "num_tokens": 20512768.0,
+ "step": 1252
+ },
+ {
+ "entropy": 1.2875827550888062,
+ "epoch": 2.5313131313131314,
+ "grad_norm": 2.193981409072876,
+ "learning_rate": 2.470707070707071e-06,
+ "loss": 1.280174732208252,
+ "mean_token_accuracy": 0.6996824741363525,
+ "num_tokens": 20529152.0,
+ "step": 1253
+ },
+ {
+ "entropy": 1.1551588773727417,
+ "epoch": 2.533333333333333,
+ "grad_norm": 2.297534942626953,
+ "learning_rate": 2.468686868686869e-06,
+ "loss": 1.1548980474472046,
+ "mean_token_accuracy": 0.7089643478393555,
+ "num_tokens": 20545536.0,
+ "step": 1254
+ },
+ {
+ "entropy": 1.1836578845977783,
+ "epoch": 2.5353535353535355,
+ "grad_norm": 2.1724929809570312,
+ "learning_rate": 2.466666666666667e-06,
+ "loss": 1.1787464618682861,
+ "mean_token_accuracy": 0.705422580242157,
+ "num_tokens": 20561920.0,
+ "step": 1255
+ },
+ {
+ "entropy": 1.0938197374343872,
+ "epoch": 2.5373737373737373,
+ "grad_norm": 2.247507095336914,
+ "learning_rate": 2.464646464646465e-06,
+ "loss": 1.0977723598480225,
+ "mean_token_accuracy": 0.7205055952072144,
+ "num_tokens": 20578304.0,
+ "step": 1256
+ },
+ {
+ "entropy": 1.790277361869812,
+ "epoch": 2.5393939393939395,
+ "grad_norm": 2.314159631729126,
+ "learning_rate": 2.462626262626263e-06,
+ "loss": 1.792272925376892,
+ "mean_token_accuracy": 0.6231069564819336,
+ "num_tokens": 20594688.0,
+ "step": 1257
+ },
+ {
+ "entropy": 1.385352611541748,
+ "epoch": 2.5414141414141413,
+ "grad_norm": 2.2065250873565674,
+ "learning_rate": 2.4606060606060607e-06,
+ "loss": 1.4161381721496582,
+ "mean_token_accuracy": 0.673363447189331,
+ "num_tokens": 20611072.0,
+ "step": 1258
+ },
+ {
+ "entropy": 1.5186140537261963,
+ "epoch": 2.5434343434343436,
+ "grad_norm": 2.277163028717041,
+ "learning_rate": 2.4585858585858586e-06,
+ "loss": 1.5157082080841064,
+ "mean_token_accuracy": 0.6420371532440186,
+ "num_tokens": 20627456.0,
+ "step": 1259
+ },
+ {
+ "entropy": 1.6179429292678833,
+ "epoch": 2.5454545454545454,
+ "grad_norm": 2.1880221366882324,
+ "learning_rate": 2.456565656565657e-06,
+ "loss": 1.6365107297897339,
+ "mean_token_accuracy": 0.6164509057998657,
+ "num_tokens": 20643840.0,
+ "step": 1260
+ },
+ {
+ "entropy": 1.5839602947235107,
+ "epoch": 2.5474747474747472,
+ "grad_norm": 2.3658924102783203,
+ "learning_rate": 2.454545454545455e-06,
+ "loss": 1.593346118927002,
+ "mean_token_accuracy": 0.6339765787124634,
+ "num_tokens": 20660224.0,
+ "step": 1261
+ },
+ {
+ "entropy": 1.3825792074203491,
+ "epoch": 2.5494949494949495,
+ "grad_norm": 2.3293564319610596,
+ "learning_rate": 2.4525252525252527e-06,
+ "loss": 1.44181489944458,
+ "mean_token_accuracy": 0.6660356521606445,
+ "num_tokens": 20676608.0,
+ "step": 1262
+ },
+ {
+ "entropy": 1.0796198844909668,
+ "epoch": 2.5515151515151517,
+ "grad_norm": 2.1055870056152344,
+ "learning_rate": 2.4505050505050506e-06,
+ "loss": 1.0861892700195312,
+ "mean_token_accuracy": 0.7263067960739136,
+ "num_tokens": 20692992.0,
+ "step": 1263
+ },
+ {
+ "entropy": 1.4059629440307617,
+ "epoch": 2.5535353535353535,
+ "grad_norm": 2.137063980102539,
+ "learning_rate": 2.4484848484848485e-06,
+ "loss": 1.4127235412597656,
+ "mean_token_accuracy": 0.6654250025749207,
+ "num_tokens": 20709376.0,
+ "step": 1264
+ },
+ {
+ "entropy": 1.2902467250823975,
+ "epoch": 2.5555555555555554,
+ "grad_norm": 2.175401210784912,
+ "learning_rate": 2.4464646464646468e-06,
+ "loss": 1.2854642868041992,
+ "mean_token_accuracy": 0.6966292262077332,
+ "num_tokens": 20725760.0,
+ "step": 1265
+ },
+ {
+ "entropy": 1.7958943843841553,
+ "epoch": 2.5575757575757576,
+ "grad_norm": 2.3500523567199707,
+ "learning_rate": 2.4444444444444447e-06,
+ "loss": 1.811906337738037,
+ "mean_token_accuracy": 0.5961162447929382,
+ "num_tokens": 20742144.0,
+ "step": 1266
+ },
+ {
+ "entropy": 1.7406131029129028,
+ "epoch": 2.55959595959596,
+ "grad_norm": 2.3911163806915283,
+ "learning_rate": 2.4424242424242426e-06,
+ "loss": 1.7608356475830078,
+ "mean_token_accuracy": 0.6002076268196106,
+ "num_tokens": 20758528.0,
+ "step": 1267
+ },
+ {
+ "entropy": 1.440137267112732,
+ "epoch": 2.5616161616161617,
+ "grad_norm": 2.057145833969116,
+ "learning_rate": 2.4404040404040404e-06,
+ "loss": 1.4450485706329346,
+ "mean_token_accuracy": 0.6570591330528259,
+ "num_tokens": 20774912.0,
+ "step": 1268
+ },
+ {
+ "entropy": 1.510685682296753,
+ "epoch": 2.5636363636363635,
+ "grad_norm": 2.379920244216919,
+ "learning_rate": 2.4383838383838383e-06,
+ "loss": 1.5113954544067383,
+ "mean_token_accuracy": 0.6450293064117432,
+ "num_tokens": 20791296.0,
+ "step": 1269
+ },
+ {
+ "entropy": 1.3595963716506958,
+ "epoch": 2.5656565656565657,
+ "grad_norm": 2.226229190826416,
+ "learning_rate": 2.4363636363636366e-06,
+ "loss": 1.3598928451538086,
+ "mean_token_accuracy": 0.6723864078521729,
+ "num_tokens": 20807680.0,
+ "step": 1270
+ },
+ {
+ "entropy": 1.3053373098373413,
+ "epoch": 2.5676767676767676,
+ "grad_norm": 2.181692361831665,
+ "learning_rate": 2.4343434343434345e-06,
+ "loss": 1.283501148223877,
+ "mean_token_accuracy": 0.672874927520752,
+ "num_tokens": 20824064.0,
+ "step": 1271
+ },
+ {
+ "entropy": 0.9631388187408447,
+ "epoch": 2.56969696969697,
+ "grad_norm": 2.083442449569702,
+ "learning_rate": 2.432323232323233e-06,
+ "loss": 0.9463216066360474,
+ "mean_token_accuracy": 0.7476184368133545,
+ "num_tokens": 20840448.0,
+ "step": 1272
+ },
+ {
+ "entropy": 1.0792899131774902,
+ "epoch": 2.5717171717171716,
+ "grad_norm": 2.204257011413574,
+ "learning_rate": 2.4303030303030307e-06,
+ "loss": 1.0787214040756226,
+ "mean_token_accuracy": 0.7241694927215576,
+ "num_tokens": 20856832.0,
+ "step": 1273
+ },
+ {
+ "entropy": 1.0843226909637451,
+ "epoch": 2.573737373737374,
+ "grad_norm": 2.0145976543426514,
+ "learning_rate": 2.4282828282828286e-06,
+ "loss": 1.0722554922103882,
+ "mean_token_accuracy": 0.7263678312301636,
+ "num_tokens": 20873216.0,
+ "step": 1274
+ },
+ {
+ "entropy": 1.5040221214294434,
+ "epoch": 2.5757575757575757,
+ "grad_norm": 2.3076000213623047,
+ "learning_rate": 2.4262626262626265e-06,
+ "loss": 1.5152506828308105,
+ "mean_token_accuracy": 0.6420371532440186,
+ "num_tokens": 20889600.0,
+ "step": 1275
+ },
+ {
+ "entropy": 1.5882561206817627,
+ "epoch": 2.5777777777777775,
+ "grad_norm": 2.093400716781616,
+ "learning_rate": 2.4242424242424244e-06,
+ "loss": 1.5827500820159912,
+ "mean_token_accuracy": 0.6293356418609619,
+ "num_tokens": 20905984.0,
+ "step": 1276
+ },
+ {
+ "entropy": 1.1761770248413086,
+ "epoch": 2.5797979797979798,
+ "grad_norm": 1.9723676443099976,
+ "learning_rate": 2.4222222222222223e-06,
+ "loss": 1.1729084253311157,
+ "mean_token_accuracy": 0.7160478830337524,
+ "num_tokens": 20922368.0,
+ "step": 1277
+ },
+ {
+ "entropy": 1.5020796060562134,
+ "epoch": 2.581818181818182,
+ "grad_norm": 2.198739767074585,
+ "learning_rate": 2.4202020202020206e-06,
+ "loss": 1.5012025833129883,
+ "mean_token_accuracy": 0.6408768892288208,
+ "num_tokens": 20938752.0,
+ "step": 1278
+ },
+ {
+ "entropy": 1.0234061479568481,
+ "epoch": 2.583838383838384,
+ "grad_norm": 2.0784571170806885,
+ "learning_rate": 2.4181818181818185e-06,
+ "loss": 1.0343588590621948,
+ "mean_token_accuracy": 0.7372984886169434,
+ "num_tokens": 20955136.0,
+ "step": 1279
+ },
+ {
+ "entropy": 1.6372766494750977,
+ "epoch": 2.5858585858585856,
+ "grad_norm": 2.339104652404785,
+ "learning_rate": 2.4161616161616164e-06,
+ "loss": 1.6200227737426758,
+ "mean_token_accuracy": 0.6249389052391052,
+ "num_tokens": 20971520.0,
+ "step": 1280
+ },
+ {
+ "entropy": 1.3991668224334717,
+ "epoch": 2.587878787878788,
+ "grad_norm": 2.33329439163208,
+ "learning_rate": 2.4141414141414143e-06,
+ "loss": 1.434525489807129,
+ "mean_token_accuracy": 0.6656692624092102,
+ "num_tokens": 20987904.0,
+ "step": 1281
+ },
+ {
+ "entropy": 0.9680843949317932,
+ "epoch": 2.58989898989899,
+ "grad_norm": 2.0454914569854736,
+ "learning_rate": 2.412121212121212e-06,
+ "loss": 0.9875590801239014,
+ "mean_token_accuracy": 0.7512823939323425,
+ "num_tokens": 21004288.0,
+ "step": 1282
+ },
+ {
+ "entropy": 1.1564005613327026,
+ "epoch": 2.591919191919192,
+ "grad_norm": 2.0994997024536133,
+ "learning_rate": 2.4101010101010105e-06,
+ "loss": 1.1473734378814697,
+ "mean_token_accuracy": 0.709208607673645,
+ "num_tokens": 21020672.0,
+ "step": 1283
+ },
+ {
+ "entropy": 1.435438871383667,
+ "epoch": 2.5939393939393938,
+ "grad_norm": 2.369999408721924,
+ "learning_rate": 2.4080808080808083e-06,
+ "loss": 1.4502406120300293,
+ "mean_token_accuracy": 0.6505251526832581,
+ "num_tokens": 21037056.0,
+ "step": 1284
+ },
+ {
+ "entropy": 1.358853816986084,
+ "epoch": 2.595959595959596,
+ "grad_norm": 4.034506320953369,
+ "learning_rate": 2.4060606060606062e-06,
+ "loss": 1.3811532258987427,
+ "mean_token_accuracy": 0.6720200181007385,
+ "num_tokens": 21053440.0,
+ "step": 1285
+ },
+ {
+ "entropy": 1.2741261720657349,
+ "epoch": 2.597979797979798,
+ "grad_norm": 2.231471061706543,
+ "learning_rate": 2.404040404040404e-06,
+ "loss": 1.2811863422393799,
+ "mean_token_accuracy": 0.6806302070617676,
+ "num_tokens": 21069824.0,
+ "step": 1286
+ },
+ {
+ "entropy": 1.3789101839065552,
+ "epoch": 2.6,
+ "grad_norm": 2.287457227706909,
+ "learning_rate": 2.402020202020202e-06,
+ "loss": 1.389050006866455,
+ "mean_token_accuracy": 0.6687836050987244,
+ "num_tokens": 21086208.0,
+ "step": 1287
+ },
+ {
+ "entropy": 1.0021989345550537,
+ "epoch": 2.602020202020202,
+ "grad_norm": 1.9996925592422485,
+ "learning_rate": 2.4000000000000003e-06,
+ "loss": 1.028086543083191,
+ "mean_token_accuracy": 0.7479237914085388,
+ "num_tokens": 21102592.0,
+ "step": 1288
+ },
+ {
+ "entropy": 1.4363199472427368,
+ "epoch": 2.604040404040404,
+ "grad_norm": 2.203239917755127,
+ "learning_rate": 2.397979797979798e-06,
+ "loss": 1.4645861387252808,
+ "mean_token_accuracy": 0.6590132117271423,
+ "num_tokens": 21118976.0,
+ "step": 1289
+ },
+ {
+ "entropy": 1.307484745979309,
+ "epoch": 2.606060606060606,
+ "grad_norm": 1.9925199747085571,
+ "learning_rate": 2.395959595959596e-06,
+ "loss": 1.3217129707336426,
+ "mean_token_accuracy": 0.6787371635437012,
+ "num_tokens": 21135360.0,
+ "step": 1290
+ },
+ {
+ "entropy": 1.2430635690689087,
+ "epoch": 2.608080808080808,
+ "grad_norm": 2.1385529041290283,
+ "learning_rate": 2.393939393939394e-06,
+ "loss": 1.238807201385498,
+ "mean_token_accuracy": 0.6949804425239563,
+ "num_tokens": 21151744.0,
+ "step": 1291
+ },
+ {
+ "entropy": 1.0351316928863525,
+ "epoch": 2.61010101010101,
+ "grad_norm": 2.2897744178771973,
+ "learning_rate": 2.3919191919191923e-06,
+ "loss": 1.032408595085144,
+ "mean_token_accuracy": 0.7267953157424927,
+ "num_tokens": 21168128.0,
+ "step": 1292
+ },
+ {
+ "entropy": 1.2167868614196777,
+ "epoch": 2.6121212121212123,
+ "grad_norm": 2.021771192550659,
+ "learning_rate": 2.38989898989899e-06,
+ "loss": 1.223731279373169,
+ "mean_token_accuracy": 0.7067049145698547,
+ "num_tokens": 21184512.0,
+ "step": 1293
+ },
+ {
+ "entropy": 1.0386477708816528,
+ "epoch": 2.614141414141414,
+ "grad_norm": 2.048285722732544,
+ "learning_rate": 2.387878787878788e-06,
+ "loss": 1.0437978506088257,
+ "mean_token_accuracy": 0.7332682013511658,
+ "num_tokens": 21200896.0,
+ "step": 1294
+ },
+ {
+ "entropy": 1.5832154750823975,
+ "epoch": 2.616161616161616,
+ "grad_norm": 2.214010715484619,
+ "learning_rate": 2.385858585858586e-06,
+ "loss": 1.5869958400726318,
+ "mean_token_accuracy": 0.6240839958190918,
+ "num_tokens": 21217280.0,
+ "step": 1295
+ },
+ {
+ "entropy": 1.4032398462295532,
+ "epoch": 2.618181818181818,
+ "grad_norm": 2.069399833679199,
+ "learning_rate": 2.3838383838383843e-06,
+ "loss": 1.40470290184021,
+ "mean_token_accuracy": 0.6671348214149475,
+ "num_tokens": 21233664.0,
+ "step": 1296
+ },
+ {
+ "entropy": 1.1057566404342651,
+ "epoch": 2.6202020202020204,
+ "grad_norm": 2.090847969055176,
+ "learning_rate": 2.381818181818182e-06,
+ "loss": 1.099064826965332,
+ "mean_token_accuracy": 0.7186736464500427,
+ "num_tokens": 21250048.0,
+ "step": 1297
+ },
+ {
+ "entropy": 1.1217924356460571,
+ "epoch": 2.6222222222222222,
+ "grad_norm": 2.2802088260650635,
+ "learning_rate": 2.37979797979798e-06,
+ "loss": 1.1425195932388306,
+ "mean_token_accuracy": 0.717452347278595,
+ "num_tokens": 21266432.0,
+ "step": 1298
+ },
+ {
+ "entropy": 1.3103245496749878,
+ "epoch": 2.624242424242424,
+ "grad_norm": 2.2961838245391846,
+ "learning_rate": 2.377777777777778e-06,
+ "loss": 1.325084924697876,
+ "mean_token_accuracy": 0.6704323291778564,
+ "num_tokens": 21282816.0,
+ "step": 1299
+ },
+ {
+ "entropy": 1.397556185722351,
+ "epoch": 2.6262626262626263,
+ "grad_norm": 2.14385986328125,
+ "learning_rate": 2.375757575757576e-06,
+ "loss": 1.3891851902008057,
+ "mean_token_accuracy": 0.6722642779350281,
+ "num_tokens": 21299200.0,
+ "step": 1300
+ },
+ {
+ "epoch": 2.6262626262626263,
+ "eval_entropy": 1.377457697064646,
+ "eval_loss": 1.5343767404556274,
+ "eval_mean_token_accuracy": 0.6456591679203895,
+ "eval_num_tokens": 21299200.0,
+ "eval_runtime": 43.8661,
+ "eval_samples_per_second": 22.569,
+ "eval_steps_per_second": 2.827,
+ "step": 1300
+ },
+ {
+ "entropy": 1.4542138576507568,
+ "epoch": 2.6282828282828286,
+ "grad_norm": 2.2326884269714355,
+ "learning_rate": 2.373737373737374e-06,
+ "loss": 1.459869384765625,
+ "mean_token_accuracy": 0.6545554399490356,
+ "num_tokens": 21315584.0,
+ "step": 1301
+ },
+ {
+ "entropy": 1.144904613494873,
+ "epoch": 2.6303030303030304,
+ "grad_norm": 2.0662593841552734,
+ "learning_rate": 2.371717171717172e-06,
+ "loss": 1.151012897491455,
+ "mean_token_accuracy": 0.7232535481452942,
+ "num_tokens": 21331968.0,
+ "step": 1302
+ },
+ {
+ "entropy": 1.2142722606658936,
+ "epoch": 2.632323232323232,
+ "grad_norm": 2.206763982772827,
+ "learning_rate": 2.36969696969697e-06,
+ "loss": 1.2040069103240967,
+ "mean_token_accuracy": 0.7002320289611816,
+ "num_tokens": 21348352.0,
+ "step": 1303
+ },
+ {
+ "entropy": 1.7695715427398682,
+ "epoch": 2.6343434343434344,
+ "grad_norm": 2.1346793174743652,
+ "learning_rate": 2.367676767676768e-06,
+ "loss": 1.7356187105178833,
+ "mean_token_accuracy": 0.6051538586616516,
+ "num_tokens": 21364736.0,
+ "step": 1304
+ },
+ {
+ "entropy": 1.351786494255066,
+ "epoch": 2.6363636363636362,
+ "grad_norm": 2.291290283203125,
+ "learning_rate": 2.3656565656565657e-06,
+ "loss": 1.3471777439117432,
+ "mean_token_accuracy": 0.6797142028808594,
+ "num_tokens": 21381120.0,
+ "step": 1305
+ },
+ {
+ "entropy": 1.202124834060669,
+ "epoch": 2.6383838383838385,
+ "grad_norm": 2.1541965007781982,
+ "learning_rate": 2.363636363636364e-06,
+ "loss": 1.2085479497909546,
+ "mean_token_accuracy": 0.7031631469726562,
+ "num_tokens": 21397504.0,
+ "step": 1306
+ },
+ {
+ "entropy": 0.9269154667854309,
+ "epoch": 2.6404040404040403,
+ "grad_norm": 2.1681666374206543,
+ "learning_rate": 2.361616161616162e-06,
+ "loss": 0.9319549798965454,
+ "mean_token_accuracy": 0.7553126811981201,
+ "num_tokens": 21413888.0,
+ "step": 1307
+ },
+ {
+ "entropy": 1.0244951248168945,
+ "epoch": 2.6424242424242426,
+ "grad_norm": 1.9392226934432983,
+ "learning_rate": 2.3595959595959598e-06,
+ "loss": 0.9993175268173218,
+ "mean_token_accuracy": 0.7588544487953186,
+ "num_tokens": 21430272.0,
+ "step": 1308
+ },
+ {
+ "entropy": 1.256042718887329,
+ "epoch": 2.6444444444444444,
+ "grad_norm": 2.346327304840088,
+ "learning_rate": 2.3575757575757577e-06,
+ "loss": 1.2512779235839844,
+ "mean_token_accuracy": 0.6858206987380981,
+ "num_tokens": 21446656.0,
+ "step": 1309
+ },
+ {
+ "entropy": 1.0318644046783447,
+ "epoch": 2.6464646464646466,
+ "grad_norm": 1.988356113433838,
+ "learning_rate": 2.3555555555555555e-06,
+ "loss": 1.0428593158721924,
+ "mean_token_accuracy": 0.7383365631103516,
+ "num_tokens": 21463040.0,
+ "step": 1310
+ },
+ {
+ "entropy": 1.3145314455032349,
+ "epoch": 2.6484848484848484,
+ "grad_norm": 2.1777143478393555,
+ "learning_rate": 2.3535353535353534e-06,
+ "loss": 1.3207441568374634,
+ "mean_token_accuracy": 0.6881411671638489,
+ "num_tokens": 21479424.0,
+ "step": 1311
+ },
+ {
+ "entropy": 1.2858450412750244,
+ "epoch": 2.6505050505050507,
+ "grad_norm": 2.2654671669006348,
+ "learning_rate": 2.3515151515151517e-06,
+ "loss": 1.2940181493759155,
+ "mean_token_accuracy": 0.6831338405609131,
+ "num_tokens": 21495808.0,
+ "step": 1312
+ },
+ {
+ "entropy": 1.4315612316131592,
+ "epoch": 2.6525252525252525,
+ "grad_norm": 2.140456438064575,
+ "learning_rate": 2.3494949494949496e-06,
+ "loss": 1.4298607110977173,
+ "mean_token_accuracy": 0.6585246920585632,
+ "num_tokens": 21512192.0,
+ "step": 1313
+ },
+ {
+ "entropy": 1.2628042697906494,
+ "epoch": 2.6545454545454543,
+ "grad_norm": 2.387079954147339,
+ "learning_rate": 2.347474747474748e-06,
+ "loss": 1.2526830434799194,
+ "mean_token_accuracy": 0.6882632970809937,
+ "num_tokens": 21528576.0,
+ "step": 1314
+ },
+ {
+ "entropy": 1.3501923084259033,
+ "epoch": 2.6565656565656566,
+ "grad_norm": 2.17873215675354,
+ "learning_rate": 2.345454545454546e-06,
+ "loss": 1.3459683656692505,
+ "mean_token_accuracy": 0.6783707737922668,
+ "num_tokens": 21544960.0,
+ "step": 1315
+ },
+ {
+ "entropy": 1.4369540214538574,
+ "epoch": 2.658585858585859,
+ "grad_norm": 2.335825204849243,
+ "learning_rate": 2.3434343434343437e-06,
+ "loss": 1.4296600818634033,
+ "mean_token_accuracy": 0.6534562706947327,
+ "num_tokens": 21561344.0,
+ "step": 1316
+ },
+ {
+ "entropy": 1.1612765789031982,
+ "epoch": 2.6606060606060606,
+ "grad_norm": 2.194164276123047,
+ "learning_rate": 2.3414141414141416e-06,
+ "loss": 1.1527385711669922,
+ "mean_token_accuracy": 0.7140327095985413,
+ "num_tokens": 21577728.0,
+ "step": 1317
+ },
+ {
+ "entropy": 1.0844523906707764,
+ "epoch": 2.6626262626262625,
+ "grad_norm": 2.1018459796905518,
+ "learning_rate": 2.3393939393939395e-06,
+ "loss": 1.0732126235961914,
+ "mean_token_accuracy": 0.7245969772338867,
+ "num_tokens": 21594112.0,
+ "step": 1318
+ },
+ {
+ "entropy": 1.1760655641555786,
+ "epoch": 2.6646464646464647,
+ "grad_norm": 2.1674582958221436,
+ "learning_rate": 2.337373737373738e-06,
+ "loss": 1.1928012371063232,
+ "mean_token_accuracy": 0.693453848361969,
+ "num_tokens": 21610496.0,
+ "step": 1319
+ },
+ {
+ "entropy": 1.3627854585647583,
+ "epoch": 2.6666666666666665,
+ "grad_norm": 2.2469325065612793,
+ "learning_rate": 2.3353535353535357e-06,
+ "loss": 1.375096321105957,
+ "mean_token_accuracy": 0.6670737862586975,
+ "num_tokens": 21626880.0,
+ "step": 1320
+ },
+ {
+ "entropy": 1.0562385320663452,
+ "epoch": 2.6686868686868688,
+ "grad_norm": 2.3479180335998535,
+ "learning_rate": 2.3333333333333336e-06,
+ "loss": 1.0513578653335571,
+ "mean_token_accuracy": 0.7401685118675232,
+ "num_tokens": 21643264.0,
+ "step": 1321
+ },
+ {
+ "entropy": 1.1622484922409058,
+ "epoch": 2.6707070707070706,
+ "grad_norm": 2.218021869659424,
+ "learning_rate": 2.3313131313131315e-06,
+ "loss": 1.172579288482666,
+ "mean_token_accuracy": 0.7042623162269592,
+ "num_tokens": 21659648.0,
+ "step": 1322
+ },
+ {
+ "entropy": 1.2139835357666016,
+ "epoch": 2.672727272727273,
+ "grad_norm": 2.2875492572784424,
+ "learning_rate": 2.3292929292929294e-06,
+ "loss": 1.2015337944030762,
+ "mean_token_accuracy": 0.7027357220649719,
+ "num_tokens": 21676032.0,
+ "step": 1323
+ },
+ {
+ "entropy": 1.7094130516052246,
+ "epoch": 2.6747474747474747,
+ "grad_norm": 2.2344024181365967,
+ "learning_rate": 2.3272727272727277e-06,
+ "loss": 1.7565287351608276,
+ "mean_token_accuracy": 0.6025891304016113,
+ "num_tokens": 21692416.0,
+ "step": 1324
+ },
+ {
+ "entropy": 1.1033966541290283,
+ "epoch": 2.676767676767677,
+ "grad_norm": 2.2697556018829346,
+ "learning_rate": 2.3252525252525256e-06,
+ "loss": 1.1173789501190186,
+ "mean_token_accuracy": 0.7180629968643188,
+ "num_tokens": 21708800.0,
+ "step": 1325
+ },
+ {
+ "entropy": 0.9456706643104553,
+ "epoch": 2.6787878787878787,
+ "grad_norm": 2.1194941997528076,
+ "learning_rate": 2.3232323232323234e-06,
+ "loss": 0.9613161087036133,
+ "mean_token_accuracy": 0.7573277950286865,
+ "num_tokens": 21725184.0,
+ "step": 1326
+ },
+ {
+ "entropy": 1.098917841911316,
+ "epoch": 2.680808080808081,
+ "grad_norm": 2.2719058990478516,
+ "learning_rate": 2.3212121212121213e-06,
+ "loss": 1.116957664489746,
+ "mean_token_accuracy": 0.709269642829895,
+ "num_tokens": 21741568.0,
+ "step": 1327
+ },
+ {
+ "entropy": 1.5637823343276978,
+ "epoch": 2.682828282828283,
+ "grad_norm": 2.257596731185913,
+ "learning_rate": 2.3191919191919192e-06,
+ "loss": 1.5726745128631592,
+ "mean_token_accuracy": 0.6359306573867798,
+ "num_tokens": 21757952.0,
+ "step": 1328
+ },
+ {
+ "entropy": 0.9321528673171997,
+ "epoch": 2.6848484848484846,
+ "grad_norm": 2.0641798973083496,
+ "learning_rate": 2.317171717171717e-06,
+ "loss": 0.9297301769256592,
+ "mean_token_accuracy": 0.7602589130401611,
+ "num_tokens": 21774336.0,
+ "step": 1329
+ },
+ {
+ "entropy": 1.2314496040344238,
+ "epoch": 2.686868686868687,
+ "grad_norm": 2.143329620361328,
+ "learning_rate": 2.3151515151515154e-06,
+ "loss": 1.2292556762695312,
+ "mean_token_accuracy": 0.6940034031867981,
+ "num_tokens": 21790720.0,
+ "step": 1330
+ },
+ {
+ "entropy": 1.195270299911499,
+ "epoch": 2.688888888888889,
+ "grad_norm": 2.291335105895996,
+ "learning_rate": 2.3131313131313133e-06,
+ "loss": 1.1904417276382446,
+ "mean_token_accuracy": 0.7023082375526428,
+ "num_tokens": 21807104.0,
+ "step": 1331
+ },
+ {
+ "entropy": 0.8783636093139648,
+ "epoch": 2.690909090909091,
+ "grad_norm": 2.069147825241089,
+ "learning_rate": 2.311111111111111e-06,
+ "loss": 0.8936513066291809,
+ "mean_token_accuracy": 0.7703346610069275,
+ "num_tokens": 21823488.0,
+ "step": 1332
+ },
+ {
+ "entropy": 1.3100625276565552,
+ "epoch": 2.6929292929292927,
+ "grad_norm": 2.396043300628662,
+ "learning_rate": 2.309090909090909e-06,
+ "loss": 1.3054237365722656,
+ "mean_token_accuracy": 0.6758671402931213,
+ "num_tokens": 21839872.0,
+ "step": 1333
+ },
+ {
+ "entropy": 0.9413285255432129,
+ "epoch": 2.694949494949495,
+ "grad_norm": 2.0738699436187744,
+ "learning_rate": 2.307070707070707e-06,
+ "loss": 0.9171029329299927,
+ "mean_token_accuracy": 0.7571446299552917,
+ "num_tokens": 21856256.0,
+ "step": 1334
+ },
+ {
+ "entropy": 1.268250584602356,
+ "epoch": 2.6969696969696972,
+ "grad_norm": 2.0988986492156982,
+ "learning_rate": 2.3050505050505053e-06,
+ "loss": 1.257792353630066,
+ "mean_token_accuracy": 0.6866145730018616,
+ "num_tokens": 21872640.0,
+ "step": 1335
+ },
+ {
+ "entropy": 1.3402502536773682,
+ "epoch": 2.698989898989899,
+ "grad_norm": 2.08109712600708,
+ "learning_rate": 2.303030303030303e-06,
+ "loss": 1.3316476345062256,
+ "mean_token_accuracy": 0.6869198679924011,
+ "num_tokens": 21889024.0,
+ "step": 1336
+ },
+ {
+ "entropy": 1.5612709522247314,
+ "epoch": 2.701010101010101,
+ "grad_norm": 2.2410786151885986,
+ "learning_rate": 2.3010101010101015e-06,
+ "loss": 1.5604108572006226,
+ "mean_token_accuracy": 0.6339765787124634,
+ "num_tokens": 21905408.0,
+ "step": 1337
+ },
+ {
+ "entropy": 1.0962555408477783,
+ "epoch": 2.703030303030303,
+ "grad_norm": 2.289069890975952,
+ "learning_rate": 2.2989898989898994e-06,
+ "loss": 1.1016948223114014,
+ "mean_token_accuracy": 0.714093804359436,
+ "num_tokens": 21921792.0,
+ "step": 1338
+ },
+ {
+ "entropy": 1.600198745727539,
+ "epoch": 2.705050505050505,
+ "grad_norm": 2.152244806289673,
+ "learning_rate": 2.2969696969696973e-06,
+ "loss": 1.608877420425415,
+ "mean_token_accuracy": 0.6394724249839783,
+ "num_tokens": 21938176.0,
+ "step": 1339
+ },
+ {
+ "entropy": 1.4823815822601318,
+ "epoch": 2.707070707070707,
+ "grad_norm": 2.142822742462158,
+ "learning_rate": 2.294949494949495e-06,
+ "loss": 1.4775569438934326,
+ "mean_token_accuracy": 0.6656082272529602,
+ "num_tokens": 21954560.0,
+ "step": 1340
+ },
+ {
+ "entropy": 1.1100481748580933,
+ "epoch": 2.709090909090909,
+ "grad_norm": 2.271402597427368,
+ "learning_rate": 2.292929292929293e-06,
+ "loss": 1.1097328662872314,
+ "mean_token_accuracy": 0.7238031029701233,
+ "num_tokens": 21970944.0,
+ "step": 1341
+ },
+ {
+ "entropy": 1.3482670783996582,
+ "epoch": 2.7111111111111112,
+ "grad_norm": 2.136296510696411,
+ "learning_rate": 2.2909090909090913e-06,
+ "loss": 1.3715920448303223,
+ "mean_token_accuracy": 0.670615553855896,
+ "num_tokens": 21987328.0,
+ "step": 1342
+ },
+ {
+ "entropy": 1.3454313278198242,
+ "epoch": 2.713131313131313,
+ "grad_norm": 1.8919764757156372,
+ "learning_rate": 2.2888888888888892e-06,
+ "loss": 1.3545129299163818,
+ "mean_token_accuracy": 0.6950415372848511,
+ "num_tokens": 22003712.0,
+ "step": 1343
+ },
+ {
+ "entropy": 1.314072847366333,
+ "epoch": 2.7151515151515153,
+ "grad_norm": 2.0495684146881104,
+ "learning_rate": 2.286868686868687e-06,
+ "loss": 1.3117344379425049,
+ "mean_token_accuracy": 0.6800195574760437,
+ "num_tokens": 22020096.0,
+ "step": 1344
+ },
+ {
+ "entropy": 0.7977169156074524,
+ "epoch": 2.717171717171717,
+ "grad_norm": 2.050006866455078,
+ "learning_rate": 2.284848484848485e-06,
+ "loss": 0.7982609868049622,
+ "mean_token_accuracy": 0.7774181962013245,
+ "num_tokens": 22036480.0,
+ "step": 1345
+ },
+ {
+ "entropy": 1.6121824979782104,
+ "epoch": 2.7191919191919194,
+ "grad_norm": 2.1058602333068848,
+ "learning_rate": 2.282828282828283e-06,
+ "loss": 1.6340572834014893,
+ "mean_token_accuracy": 0.6278700828552246,
+ "num_tokens": 22052864.0,
+ "step": 1346
+ },
+ {
+ "entropy": 1.2698943614959717,
+ "epoch": 2.721212121212121,
+ "grad_norm": 1.9856007099151611,
+ "learning_rate": 2.2808080808080808e-06,
+ "loss": 1.2857555150985718,
+ "mean_token_accuracy": 0.7088422179222107,
+ "num_tokens": 22069248.0,
+ "step": 1347
+ },
+ {
+ "entropy": 1.6614608764648438,
+ "epoch": 2.723232323232323,
+ "grad_norm": 2.1818666458129883,
+ "learning_rate": 2.278787878787879e-06,
+ "loss": 1.6615056991577148,
+ "mean_token_accuracy": 0.6217635273933411,
+ "num_tokens": 22085632.0,
+ "step": 1348
+ },
+ {
+ "entropy": 1.542864203453064,
+ "epoch": 2.7252525252525253,
+ "grad_norm": 2.1022093296051025,
+ "learning_rate": 2.276767676767677e-06,
+ "loss": 1.5696978569030762,
+ "mean_token_accuracy": 0.6462506055831909,
+ "num_tokens": 22102016.0,
+ "step": 1349
+ },
+ {
+ "entropy": 1.0613912343978882,
+ "epoch": 2.7272727272727275,
+ "grad_norm": 2.213463068008423,
+ "learning_rate": 2.274747474747475e-06,
+ "loss": 1.0789921283721924,
+ "mean_token_accuracy": 0.7288104295730591,
+ "num_tokens": 22118400.0,
+ "step": 1350
+ },
+ {
+ "epoch": 2.7272727272727275,
+ "eval_entropy": 1.362602738603469,
+ "eval_loss": 1.5345665216445923,
+ "eval_mean_token_accuracy": 0.6458812678052533,
+ "eval_num_tokens": 22118400.0,
+ "eval_runtime": 43.734,
+ "eval_samples_per_second": 22.637,
+ "eval_steps_per_second": 2.835,
+ "step": 1350
+ },
+ {
+ "entropy": 1.175439715385437,
+ "epoch": 2.7292929292929293,
+ "grad_norm": 2.3636603355407715,
+ "learning_rate": 2.2727272727272728e-06,
+ "loss": 1.2032190561294556,
+ "mean_token_accuracy": 0.689667820930481,
+ "num_tokens": 22134784.0,
+ "step": 1351
+ },
+ {
+ "entropy": 1.056649923324585,
+ "epoch": 2.731313131313131,
+ "grad_norm": 2.111971139907837,
+ "learning_rate": 2.2707070707070706e-06,
+ "loss": 1.0646127462387085,
+ "mean_token_accuracy": 0.7132999300956726,
+ "num_tokens": 22151168.0,
+ "step": 1352
+ },
+ {
+ "entropy": 1.401559591293335,
+ "epoch": 2.7333333333333334,
+ "grad_norm": 2.269573211669922,
+ "learning_rate": 2.268686868686869e-06,
+ "loss": 1.405207633972168,
+ "mean_token_accuracy": 0.6612115502357483,
+ "num_tokens": 22167552.0,
+ "step": 1353
+ },
+ {
+ "entropy": 0.9708455204963684,
+ "epoch": 2.735353535353535,
+ "grad_norm": 2.3020989894866943,
+ "learning_rate": 2.266666666666667e-06,
+ "loss": 0.9864540100097656,
+ "mean_token_accuracy": 0.7361993193626404,
+ "num_tokens": 22183936.0,
+ "step": 1354
+ },
+ {
+ "entropy": 1.2840858697891235,
+ "epoch": 2.7373737373737375,
+ "grad_norm": 2.4489457607269287,
+ "learning_rate": 2.2646464646464647e-06,
+ "loss": 1.2992041110992432,
+ "mean_token_accuracy": 0.6783097386360168,
+ "num_tokens": 22200320.0,
+ "step": 1355
+ },
+ {
+ "entropy": 1.6704803705215454,
+ "epoch": 2.7393939393939393,
+ "grad_norm": 2.213287591934204,
+ "learning_rate": 2.262626262626263e-06,
+ "loss": 1.7022172212600708,
+ "mean_token_accuracy": 0.6433194875717163,
+ "num_tokens": 22216704.0,
+ "step": 1356
+ },
+ {
+ "entropy": 1.2570815086364746,
+ "epoch": 2.7414141414141415,
+ "grad_norm": 2.250062942504883,
+ "learning_rate": 2.260606060606061e-06,
+ "loss": 1.2799533605575562,
+ "mean_token_accuracy": 0.6823400259017944,
+ "num_tokens": 22233088.0,
+ "step": 1357
+ },
+ {
+ "entropy": 1.0966694355010986,
+ "epoch": 2.7434343434343433,
+ "grad_norm": 2.24507212638855,
+ "learning_rate": 2.258585858585859e-06,
+ "loss": 1.0954806804656982,
+ "mean_token_accuracy": 0.7264289259910583,
+ "num_tokens": 22249472.0,
+ "step": 1358
+ },
+ {
+ "entropy": 1.6675825119018555,
+ "epoch": 2.7454545454545456,
+ "grad_norm": 2.1572930812835693,
+ "learning_rate": 2.2565656565656567e-06,
+ "loss": 1.6607719659805298,
+ "mean_token_accuracy": 0.6113214492797852,
+ "num_tokens": 22265856.0,
+ "step": 1359
+ },
+ {
+ "entropy": 1.247696042060852,
+ "epoch": 2.7474747474747474,
+ "grad_norm": 2.105797290802002,
+ "learning_rate": 2.254545454545455e-06,
+ "loss": 1.2314106225967407,
+ "mean_token_accuracy": 0.6982169151306152,
+ "num_tokens": 22282240.0,
+ "step": 1360
+ },
+ {
+ "entropy": 1.6081417798995972,
+ "epoch": 2.7494949494949497,
+ "grad_norm": 2.1624395847320557,
+ "learning_rate": 2.252525252525253e-06,
+ "loss": 1.6011556386947632,
+ "mean_token_accuracy": 0.6282364726066589,
+ "num_tokens": 22298624.0,
+ "step": 1361
+ },
+ {
+ "entropy": 1.1124155521392822,
+ "epoch": 2.7515151515151515,
+ "grad_norm": 2.141758918762207,
+ "learning_rate": 2.250505050505051e-06,
+ "loss": 1.1052089929580688,
+ "mean_token_accuracy": 0.7202003002166748,
+ "num_tokens": 22315008.0,
+ "step": 1362
+ },
+ {
+ "entropy": 1.093716025352478,
+ "epoch": 2.7535353535353533,
+ "grad_norm": 2.2610108852386475,
+ "learning_rate": 2.2484848484848487e-06,
+ "loss": 1.095299482345581,
+ "mean_token_accuracy": 0.714154839515686,
+ "num_tokens": 22331392.0,
+ "step": 1363
+ },
+ {
+ "entropy": 1.353977918624878,
+ "epoch": 2.7555555555555555,
+ "grad_norm": 2.171783447265625,
+ "learning_rate": 2.2464646464646466e-06,
+ "loss": 1.3497436046600342,
+ "mean_token_accuracy": 0.6914386749267578,
+ "num_tokens": 22347776.0,
+ "step": 1364
+ },
+ {
+ "entropy": 1.4595965147018433,
+ "epoch": 2.757575757575758,
+ "grad_norm": 2.2337677478790283,
+ "learning_rate": 2.2444444444444445e-06,
+ "loss": 1.4730737209320068,
+ "mean_token_accuracy": 0.6578529477119446,
+ "num_tokens": 22364160.0,
+ "step": 1365
+ },
+ {
+ "entropy": 1.5668566226959229,
+ "epoch": 2.7595959595959596,
+ "grad_norm": 2.2195982933044434,
+ "learning_rate": 2.2424242424242428e-06,
+ "loss": 1.5661592483520508,
+ "mean_token_accuracy": 0.6397166848182678,
+ "num_tokens": 22380544.0,
+ "step": 1366
+ },
+ {
+ "entropy": 1.4419786930084229,
+ "epoch": 2.7616161616161614,
+ "grad_norm": 2.202221393585205,
+ "learning_rate": 2.2404040404040407e-06,
+ "loss": 1.4466135501861572,
+ "mean_token_accuracy": 0.6698827743530273,
+ "num_tokens": 22396928.0,
+ "step": 1367
+ },
+ {
+ "entropy": 1.43394136428833,
+ "epoch": 2.7636363636363637,
+ "grad_norm": 2.1969001293182373,
+ "learning_rate": 2.2383838383838385e-06,
+ "loss": 1.433483362197876,
+ "mean_token_accuracy": 0.6655471324920654,
+ "num_tokens": 22413312.0,
+ "step": 1368
+ },
+ {
+ "entropy": 1.048527717590332,
+ "epoch": 2.765656565656566,
+ "grad_norm": 2.142869472503662,
+ "learning_rate": 2.2363636363636364e-06,
+ "loss": 1.0373283624649048,
+ "mean_token_accuracy": 0.7396189570426941,
+ "num_tokens": 22429696.0,
+ "step": 1369
+ },
+ {
+ "entropy": 1.4282819032669067,
+ "epoch": 2.7676767676767677,
+ "grad_norm": 2.1550097465515137,
+ "learning_rate": 2.2343434343434343e-06,
+ "loss": 1.419884443283081,
+ "mean_token_accuracy": 0.6712262034416199,
+ "num_tokens": 22446080.0,
+ "step": 1370
+ },
+ {
+ "entropy": 1.5361018180847168,
+ "epoch": 2.7696969696969695,
+ "grad_norm": 2.1175451278686523,
+ "learning_rate": 2.2323232323232326e-06,
+ "loss": 1.5433859825134277,
+ "mean_token_accuracy": 0.636907696723938,
+ "num_tokens": 22462464.0,
+ "step": 1371
+ },
+ {
+ "entropy": 1.326242208480835,
+ "epoch": 2.771717171717172,
+ "grad_norm": 2.084632635116577,
+ "learning_rate": 2.2303030303030305e-06,
+ "loss": 1.3364207744598389,
+ "mean_token_accuracy": 0.6809965968132019,
+ "num_tokens": 22478848.0,
+ "step": 1372
+ },
+ {
+ "entropy": 1.255878210067749,
+ "epoch": 2.7737373737373736,
+ "grad_norm": 2.142777919769287,
+ "learning_rate": 2.2282828282828284e-06,
+ "loss": 1.264795184135437,
+ "mean_token_accuracy": 0.6875916123390198,
+ "num_tokens": 22495232.0,
+ "step": 1373
+ },
+ {
+ "entropy": 1.2916159629821777,
+ "epoch": 2.775757575757576,
+ "grad_norm": 2.115644693374634,
+ "learning_rate": 2.2262626262626263e-06,
+ "loss": 1.2945363521575928,
+ "mean_token_accuracy": 0.6895456910133362,
+ "num_tokens": 22511616.0,
+ "step": 1374
+ },
+ {
+ "entropy": 1.4490385055541992,
+ "epoch": 2.7777777777777777,
+ "grad_norm": 2.2078094482421875,
+ "learning_rate": 2.224242424242424e-06,
+ "loss": 1.4401804208755493,
+ "mean_token_accuracy": 0.6637151837348938,
+ "num_tokens": 22528000.0,
+ "step": 1375
+ },
+ {
+ "entropy": 1.2710940837860107,
+ "epoch": 2.77979797979798,
+ "grad_norm": 2.1877286434173584,
+ "learning_rate": 2.222222222222222e-06,
+ "loss": 1.2672982215881348,
+ "mean_token_accuracy": 0.6958353519439697,
+ "num_tokens": 22544384.0,
+ "step": 1376
+ },
+ {
+ "entropy": 1.3981144428253174,
+ "epoch": 2.7818181818181817,
+ "grad_norm": 2.2816240787506104,
+ "learning_rate": 2.2202020202020204e-06,
+ "loss": 1.4073718786239624,
+ "mean_token_accuracy": 0.6583414673805237,
+ "num_tokens": 22560768.0,
+ "step": 1377
+ },
+ {
+ "entropy": 1.541896104812622,
+ "epoch": 2.783838383838384,
+ "grad_norm": 2.3664891719818115,
+ "learning_rate": 2.2181818181818187e-06,
+ "loss": 1.5782896280288696,
+ "mean_token_accuracy": 0.6264045238494873,
+ "num_tokens": 22577152.0,
+ "step": 1378
+ },
+ {
+ "entropy": 1.1470069885253906,
+ "epoch": 2.785858585858586,
+ "grad_norm": 2.334867000579834,
+ "learning_rate": 2.2161616161616166e-06,
+ "loss": 1.1734097003936768,
+ "mean_token_accuracy": 0.6981558203697205,
+ "num_tokens": 22593536.0,
+ "step": 1379
+ },
+ {
+ "entropy": 1.3938566446304321,
+ "epoch": 2.787878787878788,
+ "grad_norm": 2.493093967437744,
+ "learning_rate": 2.2141414141414145e-06,
+ "loss": 1.4161372184753418,
+ "mean_token_accuracy": 0.6610283255577087,
+ "num_tokens": 22609920.0,
+ "step": 1380
+ },
+ {
+ "entropy": 1.2948062419891357,
+ "epoch": 2.78989898989899,
+ "grad_norm": 2.227708578109741,
+ "learning_rate": 2.2121212121212124e-06,
+ "loss": 1.3157434463500977,
+ "mean_token_accuracy": 0.6835002303123474,
+ "num_tokens": 22626304.0,
+ "step": 1381
+ },
+ {
+ "entropy": 0.9506068229675293,
+ "epoch": 2.7919191919191917,
+ "grad_norm": 2.1482889652252197,
+ "learning_rate": 2.2101010101010102e-06,
+ "loss": 0.950823187828064,
+ "mean_token_accuracy": 0.7500610947608948,
+ "num_tokens": 22642688.0,
+ "step": 1382
+ },
+ {
+ "entropy": 1.4429880380630493,
+ "epoch": 2.793939393939394,
+ "grad_norm": 2.8189656734466553,
+ "learning_rate": 2.208080808080808e-06,
+ "loss": 1.4801222085952759,
+ "mean_token_accuracy": 0.6469223499298096,
+ "num_tokens": 22659072.0,
+ "step": 1383
+ },
+ {
+ "entropy": 1.3517380952835083,
+ "epoch": 2.795959595959596,
+ "grad_norm": 2.047236442565918,
+ "learning_rate": 2.2060606060606064e-06,
+ "loss": 1.3715345859527588,
+ "mean_token_accuracy": 0.6921714544296265,
+ "num_tokens": 22675456.0,
+ "step": 1384
+ },
+ {
+ "entropy": 1.3647958040237427,
+ "epoch": 2.797979797979798,
+ "grad_norm": 2.296548366546631,
+ "learning_rate": 2.2040404040404043e-06,
+ "loss": 1.3669397830963135,
+ "mean_token_accuracy": 0.6769663095474243,
+ "num_tokens": 22691840.0,
+ "step": 1385
+ },
+ {
+ "entropy": 1.2830337285995483,
+ "epoch": 2.8,
+ "grad_norm": 2.222038745880127,
+ "learning_rate": 2.2020202020202022e-06,
+ "loss": 1.2938398122787476,
+ "mean_token_accuracy": 0.6786150336265564,
+ "num_tokens": 22708224.0,
+ "step": 1386
+ },
+ {
+ "entropy": 1.1581721305847168,
+ "epoch": 2.802020202020202,
+ "grad_norm": 1.9979426860809326,
+ "learning_rate": 2.2e-06,
+ "loss": 1.1363141536712646,
+ "mean_token_accuracy": 0.7106130719184875,
+ "num_tokens": 22724608.0,
+ "step": 1387
+ },
+ {
+ "entropy": 1.4855602979660034,
+ "epoch": 2.804040404040404,
+ "grad_norm": 2.2763257026672363,
+ "learning_rate": 2.197979797979798e-06,
+ "loss": 1.5020172595977783,
+ "mean_token_accuracy": 0.6478993892669678,
+ "num_tokens": 22740992.0,
+ "step": 1388
+ },
+ {
+ "entropy": 1.1942899227142334,
+ "epoch": 2.806060606060606,
+ "grad_norm": 2.1685962677001953,
+ "learning_rate": 2.1959595959595963e-06,
+ "loss": 1.188063621520996,
+ "mean_token_accuracy": 0.705483615398407,
+ "num_tokens": 22757376.0,
+ "step": 1389
+ },
+ {
+ "entropy": 1.1369596719741821,
+ "epoch": 2.808080808080808,
+ "grad_norm": 2.1820614337921143,
+ "learning_rate": 2.193939393939394e-06,
+ "loss": 1.1348206996917725,
+ "mean_token_accuracy": 0.7253297567367554,
+ "num_tokens": 22773760.0,
+ "step": 1390
+ },
+ {
+ "entropy": 1.2539693117141724,
+ "epoch": 2.81010101010101,
+ "grad_norm": 2.0384390354156494,
+ "learning_rate": 2.191919191919192e-06,
+ "loss": 1.250205636024475,
+ "mean_token_accuracy": 0.6993771195411682,
+ "num_tokens": 22790144.0,
+ "step": 1391
+ },
+ {
+ "entropy": 1.7445363998413086,
+ "epoch": 2.812121212121212,
+ "grad_norm": 2.55062198638916,
+ "learning_rate": 2.18989898989899e-06,
+ "loss": 1.7788689136505127,
+ "mean_token_accuracy": 0.607107937335968,
+ "num_tokens": 22806528.0,
+ "step": 1392
+ },
+ {
+ "entropy": 1.2282429933547974,
+ "epoch": 2.8141414141414143,
+ "grad_norm": 2.1791000366210938,
+ "learning_rate": 2.187878787878788e-06,
+ "loss": 1.2460708618164062,
+ "mean_token_accuracy": 0.697239875793457,
+ "num_tokens": 22822912.0,
+ "step": 1393
+ },
+ {
+ "entropy": 1.2007265090942383,
+ "epoch": 2.816161616161616,
+ "grad_norm": 2.4067599773406982,
+ "learning_rate": 2.1858585858585858e-06,
+ "loss": 1.2013460397720337,
+ "mean_token_accuracy": 0.6963238716125488,
+ "num_tokens": 22839296.0,
+ "step": 1394
+ },
+ {
+ "entropy": 1.7315549850463867,
+ "epoch": 2.8181818181818183,
+ "grad_norm": 2.15682053565979,
+ "learning_rate": 2.183838383838384e-06,
+ "loss": 1.6833488941192627,
+ "mean_token_accuracy": 0.6170004606246948,
+ "num_tokens": 22855680.0,
+ "step": 1395
+ },
+ {
+ "entropy": 1.325459599494934,
+ "epoch": 2.82020202020202,
+ "grad_norm": 2.0464699268341064,
+ "learning_rate": 2.181818181818182e-06,
+ "loss": 1.3265891075134277,
+ "mean_token_accuracy": 0.6802638173103333,
+ "num_tokens": 22872064.0,
+ "step": 1396
+ },
+ {
+ "entropy": 1.0391966104507446,
+ "epoch": 2.822222222222222,
+ "grad_norm": 2.3595499992370605,
+ "learning_rate": 2.17979797979798e-06,
+ "loss": 1.0515706539154053,
+ "mean_token_accuracy": 0.7299706935882568,
+ "num_tokens": 22888448.0,
+ "step": 1397
+ },
+ {
+ "entropy": 1.5484130382537842,
+ "epoch": 2.824242424242424,
+ "grad_norm": 2.24381422996521,
+ "learning_rate": 2.1777777777777777e-06,
+ "loss": 1.5656720399856567,
+ "mean_token_accuracy": 0.6390449404716492,
+ "num_tokens": 22904832.0,
+ "step": 1398
+ },
+ {
+ "entropy": 1.2196807861328125,
+ "epoch": 2.8262626262626265,
+ "grad_norm": 2.3043432235717773,
+ "learning_rate": 2.175757575757576e-06,
+ "loss": 1.2172417640686035,
+ "mean_token_accuracy": 0.6985833048820496,
+ "num_tokens": 22921216.0,
+ "step": 1399
+ },
+ {
+ "entropy": 1.171323537826538,
+ "epoch": 2.8282828282828283,
+ "grad_norm": 2.34149169921875,
+ "learning_rate": 2.173737373737374e-06,
+ "loss": 1.1752846240997314,
+ "mean_token_accuracy": 0.6979726552963257,
+ "num_tokens": 22937600.0,
+ "step": 1400
+ },
+ {
+ "epoch": 2.8282828282828283,
+ "eval_entropy": 1.3776377598124165,
+ "eval_loss": 1.533128261566162,
+ "eval_mean_token_accuracy": 0.645915245336871,
+ "eval_num_tokens": 22937600.0,
+ "eval_runtime": 43.7952,
+ "eval_samples_per_second": 22.605,
+ "eval_steps_per_second": 2.831,
+ "step": 1400
+ },
+ {
+ "entropy": 1.2173237800598145,
+ "epoch": 2.83030303030303,
+ "grad_norm": 2.245490789413452,
+ "learning_rate": 2.171717171717172e-06,
+ "loss": 1.1928956508636475,
+ "mean_token_accuracy": 0.6988885998725891,
+ "num_tokens": 22953984.0,
+ "step": 1401
+ },
+ {
+ "entropy": 1.4213385581970215,
+ "epoch": 2.8323232323232324,
+ "grad_norm": 2.192051649093628,
+ "learning_rate": 2.16969696969697e-06,
+ "loss": 1.4340262413024902,
+ "mean_token_accuracy": 0.6622496247291565,
+ "num_tokens": 22970368.0,
+ "step": 1402
+ },
+ {
+ "entropy": 1.922193169593811,
+ "epoch": 2.8343434343434346,
+ "grad_norm": 2.0441250801086426,
+ "learning_rate": 2.167676767676768e-06,
+ "loss": 1.9164612293243408,
+ "mean_token_accuracy": 0.5942232608795166,
+ "num_tokens": 22986752.0,
+ "step": 1403
+ },
+ {
+ "entropy": 1.3124938011169434,
+ "epoch": 2.8363636363636364,
+ "grad_norm": 2.245614528656006,
+ "learning_rate": 2.165656565656566e-06,
+ "loss": 1.3055529594421387,
+ "mean_token_accuracy": 0.680385947227478,
+ "num_tokens": 23003136.0,
+ "step": 1404
+ },
+ {
+ "entropy": 0.9893879294395447,
+ "epoch": 2.8383838383838382,
+ "grad_norm": 2.1041340827941895,
+ "learning_rate": 2.163636363636364e-06,
+ "loss": 0.9646933078765869,
+ "mean_token_accuracy": 0.7441377639770508,
+ "num_tokens": 23019520.0,
+ "step": 1405
+ },
+ {
+ "entropy": 1.4011882543563843,
+ "epoch": 2.8404040404040405,
+ "grad_norm": 2.5371105670928955,
+ "learning_rate": 2.1616161616161617e-06,
+ "loss": 1.381878137588501,
+ "mean_token_accuracy": 0.6522960662841797,
+ "num_tokens": 23035904.0,
+ "step": 1406
+ },
+ {
+ "entropy": 1.4563852548599243,
+ "epoch": 2.8424242424242423,
+ "grad_norm": 2.1308720111846924,
+ "learning_rate": 2.15959595959596e-06,
+ "loss": 1.4763004779815674,
+ "mean_token_accuracy": 0.6792256832122803,
+ "num_tokens": 23052288.0,
+ "step": 1407
+ },
+ {
+ "entropy": 0.8676514625549316,
+ "epoch": 2.8444444444444446,
+ "grad_norm": 2.016085147857666,
+ "learning_rate": 2.157575757575758e-06,
+ "loss": 0.8672611713409424,
+ "mean_token_accuracy": 0.7660601139068604,
+ "num_tokens": 23068672.0,
+ "step": 1408
+ },
+ {
+ "entropy": 1.2011700868606567,
+ "epoch": 2.8464646464646464,
+ "grad_norm": 2.1641454696655273,
+ "learning_rate": 2.1555555555555558e-06,
+ "loss": 1.2113897800445557,
+ "mean_token_accuracy": 0.7180019617080688,
+ "num_tokens": 23085056.0,
+ "step": 1409
+ },
+ {
+ "entropy": 1.7813633680343628,
+ "epoch": 2.8484848484848486,
+ "grad_norm": 2.1768839359283447,
+ "learning_rate": 2.1535353535353537e-06,
+ "loss": 1.7897224426269531,
+ "mean_token_accuracy": 0.6129701733589172,
+ "num_tokens": 23101440.0,
+ "step": 1410
+ },
+ {
+ "entropy": 1.0219851732254028,
+ "epoch": 2.8505050505050504,
+ "grad_norm": 2.299229145050049,
+ "learning_rate": 2.1515151515151515e-06,
+ "loss": 1.0073617696762085,
+ "mean_token_accuracy": 0.7240473628044128,
+ "num_tokens": 23117824.0,
+ "step": 1411
+ },
+ {
+ "entropy": 1.3342534303665161,
+ "epoch": 2.8525252525252527,
+ "grad_norm": 2.165161371231079,
+ "learning_rate": 2.1494949494949494e-06,
+ "loss": 1.3219788074493408,
+ "mean_token_accuracy": 0.6764777898788452,
+ "num_tokens": 23134208.0,
+ "step": 1412
+ },
+ {
+ "entropy": 1.1618846654891968,
+ "epoch": 2.8545454545454545,
+ "grad_norm": 2.1276957988739014,
+ "learning_rate": 2.1474747474747477e-06,
+ "loss": 1.1667187213897705,
+ "mean_token_accuracy": 0.7214215993881226,
+ "num_tokens": 23150592.0,
+ "step": 1413
+ },
+ {
+ "entropy": 1.3673025369644165,
+ "epoch": 2.8565656565656568,
+ "grad_norm": 2.249702215194702,
+ "learning_rate": 2.1454545454545456e-06,
+ "loss": 1.380744218826294,
+ "mean_token_accuracy": 0.6656082272529602,
+ "num_tokens": 23166976.0,
+ "step": 1414
+ },
+ {
+ "entropy": 1.4118674993515015,
+ "epoch": 2.8585858585858586,
+ "grad_norm": 2.1488327980041504,
+ "learning_rate": 2.1434343434343435e-06,
+ "loss": 1.4007214307785034,
+ "mean_token_accuracy": 0.6663410067558289,
+ "num_tokens": 23183360.0,
+ "step": 1415
+ },
+ {
+ "entropy": 1.4398424625396729,
+ "epoch": 2.8606060606060604,
+ "grad_norm": 2.2609732151031494,
+ "learning_rate": 2.1414141414141414e-06,
+ "loss": 1.4657073020935059,
+ "mean_token_accuracy": 0.6483268141746521,
+ "num_tokens": 23199744.0,
+ "step": 1416
+ },
+ {
+ "entropy": 1.2922199964523315,
+ "epoch": 2.8626262626262626,
+ "grad_norm": 2.1911425590515137,
+ "learning_rate": 2.1393939393939393e-06,
+ "loss": 1.3118690252304077,
+ "mean_token_accuracy": 0.6951025724411011,
+ "num_tokens": 23216128.0,
+ "step": 1417
+ },
+ {
+ "entropy": 1.6888933181762695,
+ "epoch": 2.864646464646465,
+ "grad_norm": 2.2805919647216797,
+ "learning_rate": 2.1373737373737376e-06,
+ "loss": 1.6726739406585693,
+ "mean_token_accuracy": 0.6136419177055359,
+ "num_tokens": 23232512.0,
+ "step": 1418
+ },
+ {
+ "entropy": 1.289890170097351,
+ "epoch": 2.8666666666666667,
+ "grad_norm": 2.092263698577881,
+ "learning_rate": 2.1353535353535355e-06,
+ "loss": 1.2876317501068115,
+ "mean_token_accuracy": 0.6835613250732422,
+ "num_tokens": 23248896.0,
+ "step": 1419
+ },
+ {
+ "entropy": 1.361393690109253,
+ "epoch": 2.8686868686868685,
+ "grad_norm": 2.1863481998443604,
+ "learning_rate": 2.133333333333334e-06,
+ "loss": 1.3719209432601929,
+ "mean_token_accuracy": 0.663532018661499,
+ "num_tokens": 23265280.0,
+ "step": 1420
+ },
+ {
+ "entropy": 1.1187894344329834,
+ "epoch": 2.8707070707070708,
+ "grad_norm": 2.256434679031372,
+ "learning_rate": 2.1313131313131317e-06,
+ "loss": 1.1177198886871338,
+ "mean_token_accuracy": 0.7082926034927368,
+ "num_tokens": 23281664.0,
+ "step": 1421
+ },
+ {
+ "entropy": 0.8779795169830322,
+ "epoch": 2.8727272727272726,
+ "grad_norm": 2.095226526260376,
+ "learning_rate": 2.1292929292929296e-06,
+ "loss": 0.8675153851509094,
+ "mean_token_accuracy": 0.7647777199745178,
+ "num_tokens": 23298048.0,
+ "step": 1422
+ },
+ {
+ "entropy": 1.458146095275879,
+ "epoch": 2.874747474747475,
+ "grad_norm": 2.225179433822632,
+ "learning_rate": 2.1272727272727275e-06,
+ "loss": 1.4488263130187988,
+ "mean_token_accuracy": 0.6640205383300781,
+ "num_tokens": 23314432.0,
+ "step": 1423
+ },
+ {
+ "entropy": 1.187269926071167,
+ "epoch": 2.8767676767676766,
+ "grad_norm": 2.023710250854492,
+ "learning_rate": 2.1252525252525254e-06,
+ "loss": 1.182060718536377,
+ "mean_token_accuracy": 0.7017586827278137,
+ "num_tokens": 23330816.0,
+ "step": 1424
+ },
+ {
+ "entropy": 1.1446877717971802,
+ "epoch": 2.878787878787879,
+ "grad_norm": 2.301314115524292,
+ "learning_rate": 2.1232323232323237e-06,
+ "loss": 1.1507803201675415,
+ "mean_token_accuracy": 0.7048119306564331,
+ "num_tokens": 23347200.0,
+ "step": 1425
+ },
+ {
+ "entropy": 1.3453346490859985,
+ "epoch": 2.8808080808080807,
+ "grad_norm": 2.0338571071624756,
+ "learning_rate": 2.1212121212121216e-06,
+ "loss": 1.345799207687378,
+ "mean_token_accuracy": 0.6966292262077332,
+ "num_tokens": 23363584.0,
+ "step": 1426
+ },
+ {
+ "entropy": 1.2638238668441772,
+ "epoch": 2.882828282828283,
+ "grad_norm": 2.1575632095336914,
+ "learning_rate": 2.1191919191919194e-06,
+ "loss": 1.2882521152496338,
+ "mean_token_accuracy": 0.6901563405990601,
+ "num_tokens": 23379968.0,
+ "step": 1427
+ },
+ {
+ "entropy": 1.4490493535995483,
+ "epoch": 2.8848484848484848,
+ "grad_norm": 2.1735994815826416,
+ "learning_rate": 2.1171717171717173e-06,
+ "loss": 1.455702781677246,
+ "mean_token_accuracy": 0.6538837552070618,
+ "num_tokens": 23396352.0,
+ "step": 1428
+ },
+ {
+ "entropy": 1.3873618841171265,
+ "epoch": 2.886868686868687,
+ "grad_norm": 2.3870506286621094,
+ "learning_rate": 2.1151515151515152e-06,
+ "loss": 1.3657323122024536,
+ "mean_token_accuracy": 0.6692110300064087,
+ "num_tokens": 23412736.0,
+ "step": 1429
+ },
+ {
+ "entropy": 1.298308253288269,
+ "epoch": 2.888888888888889,
+ "grad_norm": 2.079481363296509,
+ "learning_rate": 2.113131313131313e-06,
+ "loss": 1.30775785446167,
+ "mean_token_accuracy": 0.6834391951560974,
+ "num_tokens": 23429120.0,
+ "step": 1430
+ },
+ {
+ "entropy": 1.1544564962387085,
+ "epoch": 2.8909090909090907,
+ "grad_norm": 2.2400357723236084,
+ "learning_rate": 2.1111111111111114e-06,
+ "loss": 1.17466139793396,
+ "mean_token_accuracy": 0.7093307375907898,
+ "num_tokens": 23445504.0,
+ "step": 1431
+ },
+ {
+ "entropy": 1.4711883068084717,
+ "epoch": 2.892929292929293,
+ "grad_norm": 2.1779074668884277,
+ "learning_rate": 2.1090909090909093e-06,
+ "loss": 1.4626137018203735,
+ "mean_token_accuracy": 0.6602955460548401,
+ "num_tokens": 23461888.0,
+ "step": 1432
+ },
+ {
+ "entropy": 1.4876407384872437,
+ "epoch": 2.894949494949495,
+ "grad_norm": 2.278554677963257,
+ "learning_rate": 2.107070707070707e-06,
+ "loss": 1.5031921863555908,
+ "mean_token_accuracy": 0.6570591330528259,
+ "num_tokens": 23478272.0,
+ "step": 1433
+ },
+ {
+ "entropy": 1.328355073928833,
+ "epoch": 2.896969696969697,
+ "grad_norm": 2.034842014312744,
+ "learning_rate": 2.105050505050505e-06,
+ "loss": 1.3163859844207764,
+ "mean_token_accuracy": 0.6878358721733093,
+ "num_tokens": 23494656.0,
+ "step": 1434
+ },
+ {
+ "entropy": 1.487100601196289,
+ "epoch": 2.898989898989899,
+ "grad_norm": 2.1980724334716797,
+ "learning_rate": 2.103030303030303e-06,
+ "loss": 1.4886071681976318,
+ "mean_token_accuracy": 0.6521739363670349,
+ "num_tokens": 23511040.0,
+ "step": 1435
+ },
+ {
+ "entropy": 1.4826854467391968,
+ "epoch": 2.901010101010101,
+ "grad_norm": 2.5214874744415283,
+ "learning_rate": 2.1010101010101013e-06,
+ "loss": 1.4610626697540283,
+ "mean_token_accuracy": 0.6503419876098633,
+ "num_tokens": 23527424.0,
+ "step": 1436
+ },
+ {
+ "entropy": 1.00849187374115,
+ "epoch": 2.9030303030303033,
+ "grad_norm": 2.1452033519744873,
+ "learning_rate": 2.098989898989899e-06,
+ "loss": 0.9949107766151428,
+ "mean_token_accuracy": 0.7457864880561829,
+ "num_tokens": 23543808.0,
+ "step": 1437
+ },
+ {
+ "entropy": 1.0621757507324219,
+ "epoch": 2.905050505050505,
+ "grad_norm": 2.1871836185455322,
+ "learning_rate": 2.096969696969697e-06,
+ "loss": 1.0466980934143066,
+ "mean_token_accuracy": 0.7275280952453613,
+ "num_tokens": 23560192.0,
+ "step": 1438
+ },
+ {
+ "entropy": 1.3413732051849365,
+ "epoch": 2.907070707070707,
+ "grad_norm": 2.2954704761505127,
+ "learning_rate": 2.094949494949495e-06,
+ "loss": 1.3602681159973145,
+ "mean_token_accuracy": 0.6686003804206848,
+ "num_tokens": 23576576.0,
+ "step": 1439
+ },
+ {
+ "entropy": 0.8138323426246643,
+ "epoch": 2.909090909090909,
+ "grad_norm": 1.9476791620254517,
+ "learning_rate": 2.092929292929293e-06,
+ "loss": 0.8208089470863342,
+ "mean_token_accuracy": 0.7796775698661804,
+ "num_tokens": 23592960.0,
+ "step": 1440
+ },
+ {
+ "entropy": 0.9035854339599609,
+ "epoch": 2.911111111111111,
+ "grad_norm": 1.9753291606903076,
+ "learning_rate": 2.090909090909091e-06,
+ "loss": 0.8889603614807129,
+ "mean_token_accuracy": 0.7610527873039246,
+ "num_tokens": 23609344.0,
+ "step": 1441
+ },
+ {
+ "entropy": 0.9301351308822632,
+ "epoch": 2.9131313131313132,
+ "grad_norm": 2.244597911834717,
+ "learning_rate": 2.088888888888889e-06,
+ "loss": 0.9369313716888428,
+ "mean_token_accuracy": 0.7603810429573059,
+ "num_tokens": 23625728.0,
+ "step": 1442
+ },
+ {
+ "entropy": 0.8740416765213013,
+ "epoch": 2.915151515151515,
+ "grad_norm": 2.0207667350769043,
+ "learning_rate": 2.0868686868686873e-06,
+ "loss": 0.8751406073570251,
+ "mean_token_accuracy": 0.7634342908859253,
+ "num_tokens": 23642112.0,
+ "step": 1443
+ },
+ {
+ "entropy": 1.044225811958313,
+ "epoch": 2.9171717171717173,
+ "grad_norm": 2.8100409507751465,
+ "learning_rate": 2.0848484848484852e-06,
+ "loss": 1.0568115711212158,
+ "mean_token_accuracy": 0.7291157841682434,
+ "num_tokens": 23658496.0,
+ "step": 1444
+ },
+ {
+ "entropy": 1.1093257665634155,
+ "epoch": 2.919191919191919,
+ "grad_norm": 2.2957236766815186,
+ "learning_rate": 2.082828282828283e-06,
+ "loss": 1.1235377788543701,
+ "mean_token_accuracy": 0.7108573317527771,
+ "num_tokens": 23674880.0,
+ "step": 1445
+ },
+ {
+ "entropy": 1.2211518287658691,
+ "epoch": 2.9212121212121214,
+ "grad_norm": 2.022287130355835,
+ "learning_rate": 2.080808080808081e-06,
+ "loss": 1.241473913192749,
+ "mean_token_accuracy": 0.7020029425621033,
+ "num_tokens": 23691264.0,
+ "step": 1446
+ },
+ {
+ "entropy": 1.3636398315429688,
+ "epoch": 2.923232323232323,
+ "grad_norm": 2.1484291553497314,
+ "learning_rate": 2.078787878787879e-06,
+ "loss": 1.3876409530639648,
+ "mean_token_accuracy": 0.6714093685150146,
+ "num_tokens": 23707648.0,
+ "step": 1447
+ },
+ {
+ "entropy": 1.1615006923675537,
+ "epoch": 2.9252525252525254,
+ "grad_norm": 2.775181293487549,
+ "learning_rate": 2.0767676767676768e-06,
+ "loss": 1.1792278289794922,
+ "mean_token_accuracy": 0.7070102691650391,
+ "num_tokens": 23724032.0,
+ "step": 1448
+ },
+ {
+ "entropy": 1.6308530569076538,
+ "epoch": 2.9272727272727272,
+ "grad_norm": 2.3813729286193848,
+ "learning_rate": 2.074747474747475e-06,
+ "loss": 1.6642422676086426,
+ "mean_token_accuracy": 0.6171225905418396,
+ "num_tokens": 23740416.0,
+ "step": 1449
+ },
+ {
+ "entropy": 1.2764222621917725,
+ "epoch": 2.929292929292929,
+ "grad_norm": 2.234309196472168,
+ "learning_rate": 2.072727272727273e-06,
+ "loss": 1.2756493091583252,
+ "mean_token_accuracy": 0.6824010610580444,
+ "num_tokens": 23756800.0,
+ "step": 1450
+ },
+ {
+ "epoch": 2.929292929292929,
+ "eval_entropy": 1.366065975639128,
+ "eval_loss": 1.5328131914138794,
+ "eval_mean_token_accuracy": 0.6461491642459747,
+ "eval_num_tokens": 23756800.0,
+ "eval_runtime": 43.8548,
+ "eval_samples_per_second": 22.574,
+ "eval_steps_per_second": 2.828,
+ "step": 1450
+ },
+ {
+ "entropy": 1.178619623184204,
+ "epoch": 2.9313131313131313,
+ "grad_norm": 2.0946547985076904,
+ "learning_rate": 2.070707070707071e-06,
+ "loss": 1.1751608848571777,
+ "mean_token_accuracy": 0.704384446144104,
+ "num_tokens": 23773184.0,
+ "step": 1451
+ },
+ {
+ "entropy": 1.319455862045288,
+ "epoch": 2.9333333333333336,
+ "grad_norm": 2.1925816535949707,
+ "learning_rate": 2.0686868686868688e-06,
+ "loss": 1.3229246139526367,
+ "mean_token_accuracy": 0.6867977380752563,
+ "num_tokens": 23789568.0,
+ "step": 1452
+ },
+ {
+ "entropy": 1.787271499633789,
+ "epoch": 2.9353535353535354,
+ "grad_norm": 2.374494791030884,
+ "learning_rate": 2.0666666666666666e-06,
+ "loss": 1.7788429260253906,
+ "mean_token_accuracy": 0.589154839515686,
+ "num_tokens": 23805952.0,
+ "step": 1453
+ },
+ {
+ "entropy": 1.25193452835083,
+ "epoch": 2.937373737373737,
+ "grad_norm": 2.3499720096588135,
+ "learning_rate": 2.064646464646465e-06,
+ "loss": 1.2651758193969727,
+ "mean_token_accuracy": 0.6863092184066772,
+ "num_tokens": 23822336.0,
+ "step": 1454
+ },
+ {
+ "entropy": 1.355589747428894,
+ "epoch": 2.9393939393939394,
+ "grad_norm": 2.228123426437378,
+ "learning_rate": 2.062626262626263e-06,
+ "loss": 1.3623197078704834,
+ "mean_token_accuracy": 0.6623717546463013,
+ "num_tokens": 23838720.0,
+ "step": 1455
+ },
+ {
+ "entropy": 1.2293426990509033,
+ "epoch": 2.9414141414141413,
+ "grad_norm": 2.4237186908721924,
+ "learning_rate": 2.0606060606060607e-06,
+ "loss": 1.2226693630218506,
+ "mean_token_accuracy": 0.6903395056724548,
+ "num_tokens": 23855104.0,
+ "step": 1456
+ },
+ {
+ "entropy": 1.2312723398208618,
+ "epoch": 2.9434343434343435,
+ "grad_norm": 2.2350525856018066,
+ "learning_rate": 2.0585858585858586e-06,
+ "loss": 1.235978364944458,
+ "mean_token_accuracy": 0.696201741695404,
+ "num_tokens": 23871488.0,
+ "step": 1457
+ },
+ {
+ "entropy": 1.2731300592422485,
+ "epoch": 2.9454545454545453,
+ "grad_norm": 2.488929033279419,
+ "learning_rate": 2.0565656565656565e-06,
+ "loss": 1.2763257026672363,
+ "mean_token_accuracy": 0.6765388250350952,
+ "num_tokens": 23887872.0,
+ "step": 1458
+ },
+ {
+ "entropy": 1.1283669471740723,
+ "epoch": 2.9474747474747476,
+ "grad_norm": 2.116241693496704,
+ "learning_rate": 2.054545454545455e-06,
+ "loss": 1.1279038190841675,
+ "mean_token_accuracy": 0.7123228907585144,
+ "num_tokens": 23904256.0,
+ "step": 1459
+ },
+ {
+ "entropy": 1.4969894886016846,
+ "epoch": 2.9494949494949494,
+ "grad_norm": 2.17519474029541,
+ "learning_rate": 2.0525252525252527e-06,
+ "loss": 1.5177810192108154,
+ "mean_token_accuracy": 0.6544333100318909,
+ "num_tokens": 23920640.0,
+ "step": 1460
+ },
+ {
+ "entropy": 1.3133295774459839,
+ "epoch": 2.9515151515151516,
+ "grad_norm": 2.149486780166626,
+ "learning_rate": 2.0505050505050506e-06,
+ "loss": 1.2985190153121948,
+ "mean_token_accuracy": 0.6745237112045288,
+ "num_tokens": 23937024.0,
+ "step": 1461
+ },
+ {
+ "entropy": 1.2777348756790161,
+ "epoch": 2.9535353535353535,
+ "grad_norm": 2.124922752380371,
+ "learning_rate": 2.0484848484848485e-06,
+ "loss": 1.2731128931045532,
+ "mean_token_accuracy": 0.692415714263916,
+ "num_tokens": 23953408.0,
+ "step": 1462
+ },
+ {
+ "entropy": 1.0544995069503784,
+ "epoch": 2.9555555555555557,
+ "grad_norm": 2.177363872528076,
+ "learning_rate": 2.046464646464647e-06,
+ "loss": 1.0588994026184082,
+ "mean_token_accuracy": 0.7378480434417725,
+ "num_tokens": 23969792.0,
+ "step": 1463
+ },
+ {
+ "entropy": 1.441129446029663,
+ "epoch": 2.9575757575757575,
+ "grad_norm": 2.5355494022369385,
+ "learning_rate": 2.0444444444444447e-06,
+ "loss": 1.4684169292449951,
+ "mean_token_accuracy": 0.6495481133460999,
+ "num_tokens": 23986176.0,
+ "step": 1464
+ },
+ {
+ "entropy": 1.3359375,
+ "epoch": 2.9595959595959593,
+ "grad_norm": 2.309727191925049,
+ "learning_rate": 2.0424242424242426e-06,
+ "loss": 1.339503288269043,
+ "mean_token_accuracy": 0.6707376837730408,
+ "num_tokens": 24002560.0,
+ "step": 1465
+ },
+ {
+ "entropy": 1.5922675132751465,
+ "epoch": 2.9616161616161616,
+ "grad_norm": 2.191904067993164,
+ "learning_rate": 2.0404040404040405e-06,
+ "loss": 1.613649845123291,
+ "mean_token_accuracy": 0.635869562625885,
+ "num_tokens": 24018944.0,
+ "step": 1466
+ },
+ {
+ "entropy": 1.152869701385498,
+ "epoch": 2.963636363636364,
+ "grad_norm": 2.1656837463378906,
+ "learning_rate": 2.0383838383838388e-06,
+ "loss": 1.1791510581970215,
+ "mean_token_accuracy": 0.7013311982154846,
+ "num_tokens": 24035328.0,
+ "step": 1467
+ },
+ {
+ "entropy": 1.491494059562683,
+ "epoch": 2.9656565656565657,
+ "grad_norm": 2.397916555404663,
+ "learning_rate": 2.0363636363636367e-06,
+ "loss": 1.4803133010864258,
+ "mean_token_accuracy": 0.6395334601402283,
+ "num_tokens": 24051712.0,
+ "step": 1468
+ },
+ {
+ "entropy": 1.165432095527649,
+ "epoch": 2.9676767676767675,
+ "grad_norm": 2.0728659629821777,
+ "learning_rate": 2.0343434343434345e-06,
+ "loss": 1.1676859855651855,
+ "mean_token_accuracy": 0.7170249223709106,
+ "num_tokens": 24068096.0,
+ "step": 1469
+ },
+ {
+ "entropy": 0.7055315971374512,
+ "epoch": 2.9696969696969697,
+ "grad_norm": 1.949097752571106,
+ "learning_rate": 2.0323232323232324e-06,
+ "loss": 0.7094426155090332,
+ "mean_token_accuracy": 0.8047752976417542,
+ "num_tokens": 24084480.0,
+ "step": 1470
+ },
+ {
+ "entropy": 1.5079377889633179,
+ "epoch": 2.971717171717172,
+ "grad_norm": 2.282525062561035,
+ "learning_rate": 2.0303030303030303e-06,
+ "loss": 1.5309293270111084,
+ "mean_token_accuracy": 0.65486079454422,
+ "num_tokens": 24100864.0,
+ "step": 1471
+ },
+ {
+ "entropy": 1.6969871520996094,
+ "epoch": 2.973737373737374,
+ "grad_norm": 2.2211408615112305,
+ "learning_rate": 2.0282828282828286e-06,
+ "loss": 1.6919132471084595,
+ "mean_token_accuracy": 0.6259770393371582,
+ "num_tokens": 24117248.0,
+ "step": 1472
+ },
+ {
+ "entropy": 1.3981496095657349,
+ "epoch": 2.9757575757575756,
+ "grad_norm": 2.27559232711792,
+ "learning_rate": 2.0262626262626265e-06,
+ "loss": 1.4188807010650635,
+ "mean_token_accuracy": 0.6502198576927185,
+ "num_tokens": 24133632.0,
+ "step": 1473
+ },
+ {
+ "entropy": 1.2966978549957275,
+ "epoch": 2.977777777777778,
+ "grad_norm": 2.2315874099731445,
+ "learning_rate": 2.0242424242424244e-06,
+ "loss": 1.3034720420837402,
+ "mean_token_accuracy": 0.6849047541618347,
+ "num_tokens": 24150016.0,
+ "step": 1474
+ },
+ {
+ "entropy": 1.4312105178833008,
+ "epoch": 2.9797979797979797,
+ "grad_norm": 2.3630731105804443,
+ "learning_rate": 2.0222222222222223e-06,
+ "loss": 1.412591814994812,
+ "mean_token_accuracy": 0.6673180460929871,
+ "num_tokens": 24166400.0,
+ "step": 1475
+ },
+ {
+ "entropy": 1.246411681175232,
+ "epoch": 2.981818181818182,
+ "grad_norm": 2.101142644882202,
+ "learning_rate": 2.02020202020202e-06,
+ "loss": 1.2674810886383057,
+ "mean_token_accuracy": 0.691255509853363,
+ "num_tokens": 24182784.0,
+ "step": 1476
+ },
+ {
+ "entropy": 1.2367678880691528,
+ "epoch": 2.9838383838383837,
+ "grad_norm": 2.3414289951324463,
+ "learning_rate": 2.0181818181818185e-06,
+ "loss": 1.254493236541748,
+ "mean_token_accuracy": 0.6811797618865967,
+ "num_tokens": 24199168.0,
+ "step": 1477
+ },
+ {
+ "entropy": 1.3566304445266724,
+ "epoch": 2.985858585858586,
+ "grad_norm": 2.3050434589385986,
+ "learning_rate": 2.0161616161616164e-06,
+ "loss": 1.3336361646652222,
+ "mean_token_accuracy": 0.6761724352836609,
+ "num_tokens": 24215552.0,
+ "step": 1478
+ },
+ {
+ "entropy": 1.3928223848342896,
+ "epoch": 2.987878787878788,
+ "grad_norm": 2.2100086212158203,
+ "learning_rate": 2.0141414141414143e-06,
+ "loss": 1.4050222635269165,
+ "mean_token_accuracy": 0.6573033928871155,
+ "num_tokens": 24231936.0,
+ "step": 1479
+ },
+ {
+ "entropy": 1.067913293838501,
+ "epoch": 2.98989898989899,
+ "grad_norm": 2.0196874141693115,
+ "learning_rate": 2.012121212121212e-06,
+ "loss": 1.0836842060089111,
+ "mean_token_accuracy": 0.7427943348884583,
+ "num_tokens": 24248320.0,
+ "step": 1480
+ },
+ {
+ "entropy": 1.077475905418396,
+ "epoch": 2.991919191919192,
+ "grad_norm": 2.117917776107788,
+ "learning_rate": 2.01010101010101e-06,
+ "loss": 1.0825674533843994,
+ "mean_token_accuracy": 0.7288104295730591,
+ "num_tokens": 24264704.0,
+ "step": 1481
+ },
+ {
+ "entropy": 1.092706561088562,
+ "epoch": 2.993939393939394,
+ "grad_norm": 2.143916130065918,
+ "learning_rate": 2.008080808080808e-06,
+ "loss": 1.1073365211486816,
+ "mean_token_accuracy": 0.7271006107330322,
+ "num_tokens": 24281088.0,
+ "step": 1482
+ },
+ {
+ "entropy": 1.3734595775604248,
+ "epoch": 2.995959595959596,
+ "grad_norm": 2.3651299476623535,
+ "learning_rate": 2.0060606060606062e-06,
+ "loss": 1.3319274187088013,
+ "mean_token_accuracy": 0.6729360222816467,
+ "num_tokens": 24297472.0,
+ "step": 1483
+ },
+ {
+ "entropy": 1.5602202415466309,
+ "epoch": 2.9979797979797977,
+ "grad_norm": 2.4872968196868896,
+ "learning_rate": 2.004040404040404e-06,
+ "loss": 1.5969634056091309,
+ "mean_token_accuracy": 0.6352589130401611,
+ "num_tokens": 24313856.0,
+ "step": 1484
+ },
+ {
+ "entropy": 1.4174549579620361,
+ "epoch": 3.0,
+ "grad_norm": 2.1410610675811768,
+ "learning_rate": 2.0020202020202024e-06,
+ "loss": 1.4155133962631226,
+ "mean_token_accuracy": 0.6683561205863953,
+ "num_tokens": 24330240.0,
+ "step": 1485
+ },
+ {
+ "entropy": 1.5710192918777466,
+ "epoch": 3.002020202020202,
+ "grad_norm": 2.1068179607391357,
+ "learning_rate": 2.0000000000000003e-06,
+ "loss": 1.5049738883972168,
+ "mean_token_accuracy": 0.6563873887062073,
+ "num_tokens": 24346624.0,
+ "step": 1486
+ },
+ {
+ "entropy": 1.2311826944351196,
+ "epoch": 3.004040404040404,
+ "grad_norm": 2.238032579421997,
+ "learning_rate": 1.9979797979797982e-06,
+ "loss": 1.1740195751190186,
+ "mean_token_accuracy": 0.7018197178840637,
+ "num_tokens": 24363008.0,
+ "step": 1487
+ },
+ {
+ "entropy": 1.0681109428405762,
+ "epoch": 3.006060606060606,
+ "grad_norm": 2.0808138847351074,
+ "learning_rate": 1.995959595959596e-06,
+ "loss": 1.0269291400909424,
+ "mean_token_accuracy": 0.739130437374115,
+ "num_tokens": 24379392.0,
+ "step": 1488
+ },
+ {
+ "entropy": 1.2553848028182983,
+ "epoch": 3.008080808080808,
+ "grad_norm": 2.155320644378662,
+ "learning_rate": 1.993939393939394e-06,
+ "loss": 1.209822654724121,
+ "mean_token_accuracy": 0.7072545289993286,
+ "num_tokens": 24395776.0,
+ "step": 1489
+ },
+ {
+ "entropy": 1.5376615524291992,
+ "epoch": 3.01010101010101,
+ "grad_norm": 2.376460075378418,
+ "learning_rate": 1.9919191919191923e-06,
+ "loss": 1.485018014907837,
+ "mean_token_accuracy": 0.6547996997833252,
+ "num_tokens": 24412160.0,
+ "step": 1490
+ },
+ {
+ "entropy": 1.0649843215942383,
+ "epoch": 3.012121212121212,
+ "grad_norm": 2.1056692600250244,
+ "learning_rate": 1.98989898989899e-06,
+ "loss": 1.037630558013916,
+ "mean_token_accuracy": 0.7341841459274292,
+ "num_tokens": 24428544.0,
+ "step": 1491
+ },
+ {
+ "entropy": 1.3711966276168823,
+ "epoch": 3.014141414141414,
+ "grad_norm": 2.238981008529663,
+ "learning_rate": 1.987878787878788e-06,
+ "loss": 1.3336507081985474,
+ "mean_token_accuracy": 0.6914386749267578,
+ "num_tokens": 24444928.0,
+ "step": 1492
+ },
+ {
+ "entropy": 1.3955618143081665,
+ "epoch": 3.0161616161616163,
+ "grad_norm": 2.183464765548706,
+ "learning_rate": 1.985858585858586e-06,
+ "loss": 1.3517842292785645,
+ "mean_token_accuracy": 0.6735466718673706,
+ "num_tokens": 24461312.0,
+ "step": 1493
+ },
+ {
+ "entropy": 1.0063103437423706,
+ "epoch": 3.018181818181818,
+ "grad_norm": 2.1911609172821045,
+ "learning_rate": 1.983838383838384e-06,
+ "loss": 0.9973118305206299,
+ "mean_token_accuracy": 0.7280166149139404,
+ "num_tokens": 24477696.0,
+ "step": 1494
+ },
+ {
+ "entropy": 1.3080904483795166,
+ "epoch": 3.0202020202020203,
+ "grad_norm": 2.1129884719848633,
+ "learning_rate": 1.981818181818182e-06,
+ "loss": 1.2758756875991821,
+ "mean_token_accuracy": 0.7027357220649719,
+ "num_tokens": 24494080.0,
+ "step": 1495
+ },
+ {
+ "entropy": 1.5414087772369385,
+ "epoch": 3.022222222222222,
+ "grad_norm": 2.1628341674804688,
+ "learning_rate": 1.97979797979798e-06,
+ "loss": 1.527054786682129,
+ "mean_token_accuracy": 0.6569370031356812,
+ "num_tokens": 24510464.0,
+ "step": 1496
+ },
+ {
+ "entropy": 1.44732666015625,
+ "epoch": 3.0242424242424244,
+ "grad_norm": 2.2525594234466553,
+ "learning_rate": 1.977777777777778e-06,
+ "loss": 1.3968576192855835,
+ "mean_token_accuracy": 0.6604787707328796,
+ "num_tokens": 24526848.0,
+ "step": 1497
+ },
+ {
+ "entropy": 1.522241234779358,
+ "epoch": 3.026262626262626,
+ "grad_norm": 2.1826529502868652,
+ "learning_rate": 1.975757575757576e-06,
+ "loss": 1.529256820678711,
+ "mean_token_accuracy": 0.6493649482727051,
+ "num_tokens": 24543232.0,
+ "step": 1498
+ },
+ {
+ "entropy": 0.8186452984809875,
+ "epoch": 3.0282828282828285,
+ "grad_norm": 2.1004087924957275,
+ "learning_rate": 1.9737373737373737e-06,
+ "loss": 0.8136417865753174,
+ "mean_token_accuracy": 0.7819370031356812,
+ "num_tokens": 24559616.0,
+ "step": 1499
+ },
+ {
+ "entropy": 1.2122654914855957,
+ "epoch": 3.0303030303030303,
+ "grad_norm": 2.4567720890045166,
+ "learning_rate": 1.9717171717171716e-06,
+ "loss": 1.236933946609497,
+ "mean_token_accuracy": 0.6828895807266235,
+ "num_tokens": 24576000.0,
+ "step": 1500
+ },
+ {
+ "epoch": 3.0303030303030303,
+ "eval_entropy": 1.3221501962792488,
+ "eval_loss": 1.5418624877929688,
+ "eval_mean_token_accuracy": 0.6455981039231823,
+ "eval_num_tokens": 24576000.0,
+ "eval_runtime": 43.8085,
+ "eval_samples_per_second": 22.598,
+ "eval_steps_per_second": 2.831,
+ "step": 1500
+ }
+ ],
+ "logging_steps": 1,
+ "max_steps": 2475,
+ "num_input_tokens_seen": 0,
+ "num_train_epochs": 5,
+ "save_steps": 500,
+ "stateful_callbacks": {
+ "TrainerControl": {
+ "args": {
+ "should_epoch_stop": false,
+ "should_evaluate": false,
+ "should_log": false,
+ "should_save": true,
+ "should_training_stop": false
+ },
+ "attributes": {}
+ }
+ },
+ "total_flos": 2.07825968037888e+17,
+ "train_batch_size": 4,
+ "trial_name": null,
+ "trial_params": null
+}
diff --git a/qwen3-1.7b-teutonic-5e6/checkpoint-1500/training_args.bin b/qwen3-1.7b-teutonic-5e6/checkpoint-1500/training_args.bin
new file mode 100644
index 0000000000000000000000000000000000000000..37649d4bae96a1df88666daf20b4e5b6e092d976
--- /dev/null
+++ b/qwen3-1.7b-teutonic-5e6/checkpoint-1500/training_args.bin
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:58bbcabfe99a1cf24f2e76aab66b507c2b720d9271dc9f17c8208d741a9c3a65
+size 7121
diff --git a/qwen3-1.7b-teutonic-5e6/checkpoint-1500/zero_to_fp32.py b/qwen3-1.7b-teutonic-5e6/checkpoint-1500/zero_to_fp32.py
new file mode 100644
index 0000000000000000000000000000000000000000..3970548c400fd4361bd923b763db90e963ddaf8c
--- /dev/null
+++ b/qwen3-1.7b-teutonic-5e6/checkpoint-1500/zero_to_fp32.py
@@ -0,0 +1,854 @@
+#!/usr/bin/env python
+
+# Copyright (c) Microsoft Corporation.
+# SPDX-License-Identifier: Apache-2.0
+
+# DeepSpeed Team
+
+# This script extracts fp32 consolidated weights from a zero 1, 2 and 3 DeepSpeed checkpoints. It gets
+# copied into the top level checkpoint dir, so the user can easily do the conversion at any point in
+# the future. Once extracted, the weights don't require DeepSpeed and can be used in any
+# application.
+#
+# example:
+# python zero_to_fp32.py . output_dir/
+# or
+# python zero_to_fp32.py . output_dir/ --safe_serialization
+
+import argparse
+import torch
+import glob
+import math
+import os
+import re
+import gc
+import json
+import numpy as np
+from tqdm import tqdm
+from collections import OrderedDict
+from dataclasses import dataclass
+
+# while this script doesn't use deepspeed to recover data, since the checkpoints are pickled with
+# DeepSpeed data structures it has to be available in the current python environment.
+from deepspeed.utils import logger
+from deepspeed.checkpoint.constants import (DS_VERSION, OPTIMIZER_STATE_DICT, SINGLE_PARTITION_OF_FP32_GROUPS,
+ FP32_FLAT_GROUPS, ZERO_STAGE, PARTITION_COUNT, PARAM_SHAPES, BUFFER_NAMES,
+ FROZEN_PARAM_SHAPES, FROZEN_PARAM_FRAGMENTS, AUTOEP_LAYERS_KEY,
+ AUTOEP_LAYERS_KEY_LEGACY, AUTOEP_ZERO3_EXPERT_STATE_FORMAT_KEY,
+ AUTOEP_ZERO3_PARTITIONED_EXPERT_STATE_FORMAT, PARAM_ALIGNMENT_PADDINGS)
+
+
+@dataclass
+class zero_model_state:
+ buffers: dict()
+ param_shapes: dict()
+ shared_params: list
+ ds_version: int
+ frozen_param_shapes: dict()
+ frozen_param_fragments: dict()
+
+
+debug = 0
+
+# load to cpu
+device = torch.device('cpu')
+
+OUTPUT_DTYPE_NAMES = {
+ 'float32': torch.float32,
+ 'fp32': torch.float32,
+ 'float16': torch.float16,
+ 'fp16': torch.float16,
+ 'bfloat16': torch.bfloat16,
+ 'bf16': torch.bfloat16,
+}
+
+
+def _resolve_output_dtype(dtype):
+ requested_dtype = dtype
+ if isinstance(dtype, str):
+ dtype_name = dtype[6:] if dtype.startswith('torch.') else dtype
+ dtype = OUTPUT_DTYPE_NAMES.get(dtype_name.lower())
+ if dtype not in set(OUTPUT_DTYPE_NAMES.values()):
+ supported = ', '.join(sorted(OUTPUT_DTYPE_NAMES))
+ raise ValueError(f"Unsupported output dtype {requested_dtype!r}. Choose one of: {supported}")
+ return dtype
+
+
+def atoi(text):
+ return int(text) if text.isdigit() else text
+
+
+def natural_keys(text):
+ '''
+ alist.sort(key=natural_keys) sorts in human order
+ http://nedbatchelder.com/blog/200712/human_sorting.html
+ (See Toothy's implementation in the comments)
+ '''
+ return [atoi(c) for c in re.split(r'(\d+)', text)]
+
+
+def get_model_state_file(checkpoint_dir, zero_stage):
+ if not os.path.isdir(checkpoint_dir):
+ raise FileNotFoundError(f"Directory '{checkpoint_dir}' doesn't exist")
+
+ # there should be only one file
+ if zero_stage <= 2:
+ file = os.path.join(checkpoint_dir, "mp_rank_00_model_states.pt")
+ elif zero_stage == 3:
+ file = os.path.join(checkpoint_dir, "zero_pp_rank_0_mp_rank_00_model_states.pt")
+
+ if not os.path.exists(file):
+ raise FileNotFoundError(f"can't find model states file at '{file}'")
+
+ return file
+
+
+def get_checkpoint_files(checkpoint_dir, glob_pattern):
+ # XXX: need to test that this simple glob rule works for multi-node setup too
+ ckpt_files = sorted(glob.glob(os.path.join(checkpoint_dir, glob_pattern)), key=natural_keys)
+
+ if len(ckpt_files) == 0:
+ raise FileNotFoundError(f"can't find {glob_pattern} files in directory '{checkpoint_dir}'")
+
+ return ckpt_files
+
+
+def get_optim_files(checkpoint_dir):
+ return get_checkpoint_files(checkpoint_dir, "*_optim_states.pt")
+
+
+def get_model_state_files(checkpoint_dir):
+ return get_checkpoint_files(checkpoint_dir, "*_model_states.pt")
+
+
+def _has_autoep_zero3_partitioned_metadata(state_dict):
+ autoep_layers = state_dict.get(AUTOEP_LAYERS_KEY)
+ if autoep_layers is None:
+ autoep_layers = state_dict.get(AUTOEP_LAYERS_KEY_LEGACY)
+ if not isinstance(autoep_layers, list):
+ return False
+ return any(
+ isinstance(entry, dict)
+ and entry.get(AUTOEP_ZERO3_EXPERT_STATE_FORMAT_KEY) == AUTOEP_ZERO3_PARTITIONED_EXPERT_STATE_FORMAT
+ for entry in autoep_layers)
+
+
+def _raise_if_autoep_zero3_partitioned_state(state_dict):
+ if _has_autoep_zero3_partitioned_metadata(state_dict):
+ raise NotImplementedError("zero_to_fp32 does not support AutoEP ZeRO-3 partition-native checkpoints. "
+ "AutoEP expert parameters are partitioned over expert replica groups, so "
+ "global data-parallel consolidation would produce incomplete expert tensors. "
+ "Use ds_to_universal.py for expert-aware conversion.")
+
+
+def _raise_if_autoep_zero3_partitioned_checkpoint(model_files):
+ for file in model_files:
+ state_dict = torch.load(file, map_location=device, weights_only=False)
+ _raise_if_autoep_zero3_partitioned_state(state_dict)
+
+
+def parse_model_states(files):
+ zero_model_states = []
+ for file in files:
+ state_dict = torch.load(file, map_location=device, weights_only=False)
+ _raise_if_autoep_zero3_partitioned_state(state_dict)
+
+ if BUFFER_NAMES not in state_dict:
+ raise ValueError(f"{file} is not a model state checkpoint")
+ buffer_names = state_dict[BUFFER_NAMES]
+ if debug:
+ print("Found buffers:", buffer_names)
+
+ # recover just the buffers while restoring them to fp32 if they were saved in fp16
+ buffers = {k: v.float() for k, v in state_dict["module"].items() if k in buffer_names}
+ param_shapes = state_dict[PARAM_SHAPES]
+
+ # collect parameters that are included in param_shapes
+ param_names = []
+ for s in param_shapes:
+ for name in s.keys():
+ param_names.append(name)
+
+ # update with frozen parameters
+ frozen_param_shapes = state_dict.get(FROZEN_PARAM_SHAPES, None)
+ if frozen_param_shapes is not None:
+ if debug:
+ print(f"Found frozen_param_shapes: {frozen_param_shapes}")
+ param_names += list(frozen_param_shapes.keys())
+
+ # handle shared params
+ shared_params = [[k, v] for k, v in state_dict["shared_params"].items()]
+
+ ds_version = state_dict.get(DS_VERSION, None)
+
+ frozen_param_fragments = state_dict.get(FROZEN_PARAM_FRAGMENTS, None)
+
+ z_model_state = zero_model_state(buffers=buffers,
+ param_shapes=param_shapes,
+ shared_params=shared_params,
+ ds_version=ds_version,
+ frozen_param_shapes=frozen_param_shapes,
+ frozen_param_fragments=frozen_param_fragments)
+ zero_model_states.append(z_model_state)
+
+ return zero_model_states
+
+
+def parse_optim_states(files, ds_checkpoint_dir):
+ total_files = len(files)
+ state_dicts = []
+ for f in tqdm(files, desc='Loading checkpoint shards'):
+ state_dict = torch.load(f, map_location=device, mmap=True, weights_only=False)
+ # immediately discard the potentially huge 2 optimizer states as we only care for fp32 master weights
+ # and also handle the case where it was already removed by another helper script
+ state_dict["optimizer_state_dict"].pop("optimizer_state_dict", None)
+ state_dicts.append(state_dict)
+
+ if ZERO_STAGE not in state_dicts[0][OPTIMIZER_STATE_DICT]:
+ raise ValueError(f"{files[0]} is not a zero checkpoint")
+ zero_stage = state_dicts[0][OPTIMIZER_STATE_DICT][ZERO_STAGE]
+ world_size = state_dicts[0][OPTIMIZER_STATE_DICT][PARTITION_COUNT]
+
+ # For ZeRO-2 each param group can have different partition_count as data parallelism for expert
+ # parameters can be different from data parallelism for non-expert parameters. So we can just
+ # use the max of the partition_count to get the dp world_size.
+
+ if type(world_size) is list:
+ world_size = max(world_size)
+
+ if world_size != total_files:
+ raise ValueError(
+ f"Expected {world_size} of '*_optim_states.pt' under '{ds_checkpoint_dir}' but found {total_files} files. "
+ "Possibly due to an overwrite of an old checkpoint, or a checkpoint didn't get saved by one or more processes."
+ )
+
+ # the groups are named differently in each stage
+ if zero_stage <= 2:
+ fp32_groups_key = SINGLE_PARTITION_OF_FP32_GROUPS
+ elif zero_stage == 3:
+ fp32_groups_key = FP32_FLAT_GROUPS
+ else:
+ raise ValueError(f"unknown zero stage {zero_stage}")
+
+ fp32_flat_groups = [state_dicts[i][OPTIMIZER_STATE_DICT][fp32_groups_key] for i in range(len(state_dicts))]
+ param_alignment_paddings = state_dicts[0][OPTIMIZER_STATE_DICT].get(PARAM_ALIGNMENT_PADDINGS)
+ return zero_stage, world_size, fp32_flat_groups, param_alignment_paddings
+
+
+def _get_fp32_state_dict_from_zero_checkpoint(ds_checkpoint_dir, exclude_frozen_parameters):
+ """
+ Returns fp32 state_dict reconstructed from ds checkpoint
+
+ Args:
+ - ``ds_checkpoint_dir``: path to the deepspeed checkpoint folder (where the optimizer files are)
+
+ """
+ print(f"Processing zero checkpoint '{ds_checkpoint_dir}'")
+
+ # parse_model_states rejects AutoEP ZeRO-3 partition-native checkpoints
+ # before the expensive optimizer-shard load below.
+ model_files = get_model_state_files(ds_checkpoint_dir)
+ zero_model_states = parse_model_states(model_files)
+ print(f'Parsing checkpoint created by deepspeed=={zero_model_states[0].ds_version}')
+
+ optim_files = get_optim_files(ds_checkpoint_dir)
+ zero_stage, world_size, fp32_flat_groups, param_alignment_paddings = parse_optim_states(
+ optim_files, ds_checkpoint_dir)
+ print(f"Detected checkpoint of type zero stage {zero_stage}, world_size: {world_size}")
+
+ if zero_stage <= 2:
+ return _get_fp32_state_dict_from_zero2_checkpoint(world_size, fp32_flat_groups, zero_model_states,
+ exclude_frozen_parameters, param_alignment_paddings)
+ elif zero_stage == 3:
+ return _get_fp32_state_dict_from_zero3_checkpoint(world_size, fp32_flat_groups, zero_model_states,
+ exclude_frozen_parameters)
+
+
+def _zero2_merge_frozen_params(state_dict, zero_model_states):
+ if zero_model_states[0].frozen_param_shapes is None or len(zero_model_states[0].frozen_param_shapes) == 0:
+ return
+
+ frozen_param_shapes = zero_model_states[0].frozen_param_shapes
+ frozen_param_fragments = zero_model_states[0].frozen_param_fragments
+
+ if debug:
+ num_elem = sum(s.numel() for s in frozen_param_shapes.values())
+ print(f'rank 0: {FROZEN_PARAM_SHAPES}.numel = {num_elem}')
+
+ wanted_params = len(frozen_param_shapes)
+ wanted_numel = sum(s.numel() for s in frozen_param_shapes.values())
+ avail_numel = sum([p.numel() for p in frozen_param_fragments.values()])
+ print(f'Frozen params: Have {avail_numel} numels to process.')
+ print(f'Frozen params: Need {wanted_numel} numels in {wanted_params} params')
+
+ total_params = 0
+ total_numel = 0
+ for name, shape in frozen_param_shapes.items():
+ total_params += 1
+ unpartitioned_numel = shape.numel()
+ total_numel += unpartitioned_numel
+
+ state_dict[name] = frozen_param_fragments[name]
+
+ if debug:
+ print(f"{name} full shape: {shape} unpartitioned numel {unpartitioned_numel} ")
+
+ print(f"Reconstructed Frozen fp32 state dict with {total_params} params {total_numel} elements")
+
+
+def _has_callable(obj, fn):
+ attr = getattr(obj, fn, None)
+ return callable(attr)
+
+
+def _zero2_merge_trainable_params(state_dict,
+ world_size,
+ fp32_flat_groups,
+ zero_model_states,
+ param_alignment_paddings=None):
+ param_shapes = zero_model_states[0].param_shapes
+
+ # Reconstruction protocol:
+ #
+ # XXX: document this
+
+ if debug:
+ for i in range(world_size):
+ for j in range(len(fp32_flat_groups[0])):
+ print(f"{FP32_FLAT_GROUPS}[{i}][{j}].shape={fp32_flat_groups[i][j].shape}")
+
+ # XXX: memory usage doubles here (zero2)
+ num_param_groups = len(fp32_flat_groups[0])
+ merged_single_partition_of_fp32_groups = []
+ for i in range(num_param_groups):
+ merged_partitions = [sd[i] for sd in fp32_flat_groups]
+ full_single_fp32_vector = torch.cat(merged_partitions, 0)
+ merged_single_partition_of_fp32_groups.append(full_single_fp32_vector)
+ avail_numel = sum(
+ [full_single_fp32_vector.numel() for full_single_fp32_vector in merged_single_partition_of_fp32_groups])
+
+ if debug:
+ wanted_params = sum([len(shapes) for shapes in param_shapes])
+ wanted_numel = sum([sum(shape.numel() for shape in shapes.values()) for shapes in param_shapes])
+ # not asserting if there is a mismatch due to possible padding
+ print(f"Have {avail_numel} numels to process.")
+ print(f"Need {wanted_numel} numels in {wanted_params} params.")
+
+ # params
+ # XXX: for huge models that can't fit into the host's RAM we will have to recode this to support
+ # out-of-core computing solution
+ total_numel = 0
+ total_params = 0
+ for group_idx, (shapes,
+ full_single_fp32_vector) in enumerate(zip(param_shapes, merged_single_partition_of_fp32_groups)):
+ offset = 0
+ avail_numel = full_single_fp32_vector.numel()
+ group_alignment_paddings = None
+ if param_alignment_paddings is not None:
+ group_alignment_paddings = param_alignment_paddings[group_idx]
+ if len(group_alignment_paddings) != len(shapes):
+ raise ValueError(f"Expected {len(shapes)} parameter alignment paddings for group {group_idx}, "
+ f"but found {len(group_alignment_paddings)}")
+
+ for param_idx, (name, shape) in enumerate(shapes.items()):
+
+ unpartitioned_numel = shape.numel() if _has_callable(shape, 'numel') else math.prod(shape)
+ total_numel += unpartitioned_numel
+ total_params += 1
+
+ if debug:
+ print(f"{name} full shape: {shape} unpartitioned numel {unpartitioned_numel} ")
+ state_dict[name] = full_single_fp32_vector.narrow(0, offset, unpartitioned_numel).view(shape)
+ offset += unpartitioned_numel
+ if group_alignment_paddings is not None:
+ offset += group_alignment_paddings[param_idx]
+
+ # Z2 started to align to 2*world_size to improve nccl performance. Therefore both offset and
+ # avail_numel can differ by anywhere between 0..2*world_size. Due to two unrelated complex
+ # paddings performed in the code it's almost impossible to predict the exact numbers w/o the
+ # live optimizer object, so we are checking that the numbers are within the right range
+ align_to = 2 * world_size
+
+ def zero2_align(x):
+ return align_to * math.ceil(x / align_to)
+
+ if debug:
+ print(f"original offset={offset}, avail_numel={avail_numel}")
+
+ offset = zero2_align(offset)
+ avail_numel = zero2_align(avail_numel)
+
+ if debug:
+ print(f"aligned offset={offset}, avail_numel={avail_numel}")
+
+ # Sanity check
+ if offset != avail_numel:
+ raise ValueError(f"consumed {offset} numels out of {avail_numel} - something is wrong")
+
+ print(f"Reconstructed fp32 state dict with {total_params} params {total_numel} elements")
+
+
+def _get_fp32_state_dict_from_zero2_checkpoint(world_size,
+ fp32_flat_groups,
+ zero_model_states,
+ exclude_frozen_parameters,
+ param_alignment_paddings=None):
+ state_dict = OrderedDict()
+
+ # buffers
+ buffers = zero_model_states[0].buffers
+ state_dict.update(buffers)
+ if debug:
+ print(f"added {len(buffers)} buffers")
+
+ if not exclude_frozen_parameters:
+ _zero2_merge_frozen_params(state_dict, zero_model_states)
+
+ _zero2_merge_trainable_params(state_dict, world_size, fp32_flat_groups, zero_model_states,
+ param_alignment_paddings)
+
+ # recover shared parameters
+ for pair in zero_model_states[0].shared_params:
+ if pair[1] in state_dict:
+ state_dict[pair[0]] = state_dict[pair[1]]
+
+ return state_dict
+
+
+def zero3_partitioned_param_info(unpartitioned_numel, world_size):
+ remainder = unpartitioned_numel % world_size
+ padding_numel = (world_size - remainder) if remainder else 0
+ partitioned_numel = math.ceil(unpartitioned_numel / world_size)
+ return partitioned_numel, padding_numel
+
+
+def _zero3_merge_frozen_params(state_dict, world_size, zero_model_states):
+ if zero_model_states[0].frozen_param_shapes is None or len(zero_model_states[0].frozen_param_shapes) == 0:
+ return
+
+ if debug:
+ for i in range(world_size):
+ num_elem = sum(s.numel() for s in zero_model_states[i].frozen_param_fragments.values())
+ print(f'rank {i}: {FROZEN_PARAM_SHAPES}.numel = {num_elem}')
+
+ frozen_param_shapes = zero_model_states[0].frozen_param_shapes
+ wanted_params = len(frozen_param_shapes)
+ wanted_numel = sum(s.numel() for s in frozen_param_shapes.values())
+ avail_numel = sum([p.numel() for p in zero_model_states[0].frozen_param_fragments.values()]) * world_size
+ print(f'Frozen params: Have {avail_numel} numels to process.')
+ print(f'Frozen params: Need {wanted_numel} numels in {wanted_params} params')
+
+ total_params = 0
+ total_numel = 0
+ for name, shape in zero_model_states[0].frozen_param_shapes.items():
+ total_params += 1
+ unpartitioned_numel = shape.numel()
+ total_numel += unpartitioned_numel
+
+ param_frags = tuple(model_state.frozen_param_fragments[name] for model_state in zero_model_states)
+ state_dict[name] = torch.cat(param_frags, 0).narrow(0, 0, unpartitioned_numel).view(shape)
+
+ partitioned_numel, partitioned_padding_numel = zero3_partitioned_param_info(unpartitioned_numel, world_size)
+
+ if debug:
+ print(
+ f"Frozen params: {total_params} {name} full shape: {shape} partition0 numel={partitioned_numel} partitioned_padding_numel={partitioned_padding_numel}"
+ )
+
+ print(f"Reconstructed Frozen fp32 state dict with {total_params} params {total_numel} elements")
+
+
+class GatheredTensor:
+ """
+ A pseudo tensor that collects partitioned weights.
+ It is more memory efficient when there are multiple groups.
+ """
+
+ def __init__(self, flat_groups, flat_groups_offset, offset, partitioned_numel, shape):
+ self.flat_groups = flat_groups
+ self.flat_groups_offset = flat_groups_offset
+ self.offset = offset
+ self.partitioned_numel = partitioned_numel
+ self.shape = shape
+ self.dtype = self.flat_groups[0][0].dtype
+
+ def contiguous(self):
+ """
+ Merge partitioned weights from flat_groups into a single tensor.
+ """
+ end_idx = self.offset + self.partitioned_numel
+ world_size = len(self.flat_groups)
+ pad_flat_param_chunks = []
+
+ for rank_i in range(world_size):
+ # for each rank, we need to collect weights from related group/groups
+ flat_groups_at_rank_i = self.flat_groups[rank_i]
+ start_group_id = None
+ end_group_id = None
+ for group_id in range(len(self.flat_groups_offset)):
+ if self.flat_groups_offset[group_id] <= self.offset < self.flat_groups_offset[group_id + 1]:
+ start_group_id = group_id
+ if self.flat_groups_offset[group_id] < end_idx <= self.flat_groups_offset[group_id + 1]:
+ end_group_id = group_id
+ break
+ # collect weights from related group/groups
+ for group_id in range(start_group_id, end_group_id + 1):
+ flat_tensor = flat_groups_at_rank_i[group_id]
+ start_offset = self.offset - self.flat_groups_offset[group_id]
+ end_offset = min(end_idx, self.flat_groups_offset[group_id + 1]) - self.flat_groups_offset[group_id]
+ pad_flat_param_chunks.append(flat_tensor[start_offset:end_offset])
+
+ # collect weights from all ranks
+ pad_flat_param = torch.cat(pad_flat_param_chunks, dim=0)
+ param = pad_flat_param[:self.shape.numel()].view(self.shape).contiguous()
+ return param
+
+
+def _zero3_merge_trainable_params(state_dict, world_size, fp32_flat_groups, zero_model_states):
+ param_shapes = zero_model_states[0].param_shapes
+ avail_numel = sum([flat_group.numel() for flat_group in fp32_flat_groups[0]]) * world_size
+
+ # Reconstruction protocol: For zero3 we need to zip the partitions together at boundary of each
+ # param, re-consolidating each param, while dealing with padding if any
+
+ # merge list of dicts, preserving order
+ param_shapes = {k: v for d in param_shapes for k, v in d.items()}
+
+ if debug:
+ for i in range(world_size):
+ print(f"{FP32_FLAT_GROUPS}[{i}].shape={fp32_flat_groups[i].shape}")
+
+ wanted_params = len(param_shapes)
+ wanted_numel = sum(shape.numel() for shape in param_shapes.values())
+ # not asserting if there is a mismatch due to possible padding
+ avail_numel = fp32_flat_groups[0].numel() * world_size
+ print(f"Trainable params: Have {avail_numel} numels to process.")
+ print(f"Trainable params: Need {wanted_numel} numels in {wanted_params} params.")
+
+ # params
+ # XXX: for huge models that can't fit into the host's RAM we will have to recode this to support
+ # out-of-core computing solution
+ offset = 0
+ total_numel = 0
+ total_params = 0
+ flat_groups_offset = [0] + list(np.cumsum([flat_tensor.numel() for flat_tensor in fp32_flat_groups[0]]))
+ for name, shape in tqdm(param_shapes.items(), desc='Gathering sharded weights'):
+ unpartitioned_numel = shape.numel()
+ total_numel += unpartitioned_numel
+ total_params += 1
+ partitioned_numel, partitioned_padding_numel = zero3_partitioned_param_info(unpartitioned_numel, world_size)
+
+ if debug:
+ print(
+ f"Trainable params: {total_params} {name} full shape: {shape} partition0 numel={partitioned_numel} partitioned_padding_numel={partitioned_padding_numel}"
+ )
+
+ # memory efficient tensor
+ tensor = GatheredTensor(fp32_flat_groups, flat_groups_offset, offset, partitioned_numel, shape)
+ state_dict[name] = tensor
+ offset += partitioned_numel
+
+ offset *= world_size
+
+ # Sanity check
+ if offset != avail_numel:
+ raise ValueError(f"consumed {offset} numels out of {avail_numel} - something is wrong")
+
+ print(f"Reconstructed Trainable fp32 state dict with {total_params} params {total_numel} elements")
+
+
+def _get_fp32_state_dict_from_zero3_checkpoint(world_size, fp32_flat_groups, zero_model_states,
+ exclude_frozen_parameters):
+ state_dict = OrderedDict()
+
+ # buffers
+ buffers = zero_model_states[0].buffers
+ state_dict.update(buffers)
+ if debug:
+ print(f"added {len(buffers)} buffers")
+
+ if not exclude_frozen_parameters:
+ _zero3_merge_frozen_params(state_dict, world_size, zero_model_states)
+
+ _zero3_merge_trainable_params(state_dict, world_size, fp32_flat_groups, zero_model_states)
+
+ # recover shared parameters
+ for pair in zero_model_states[0].shared_params:
+ if pair[1] in state_dict:
+ state_dict[pair[0]] = state_dict[pair[1]]
+
+ return state_dict
+
+
+def to_torch_tensor(state_dict, return_empty_tensor=False, dtype=None):
+ """
+ Convert state_dict of GatheredTensor to torch tensor
+ """
+ if dtype is not None:
+ dtype = _resolve_output_dtype(dtype)
+
+ torch_state_dict = {}
+ converted_tensors = {}
+ for name, tensor in state_dict.items():
+ tensor_id = id(tensor)
+ if tensor_id in converted_tensors: # shared tensors
+ shared_tensor = torch_state_dict[converted_tensors[tensor_id]]
+ torch_state_dict[name] = shared_tensor
+ else:
+ converted_tensors[tensor_id] = name
+ if return_empty_tensor:
+ torch_state_dict[name] = torch.empty(tensor.shape, dtype=dtype or tensor.dtype)
+ else:
+ contiguous_tensor = tensor.contiguous()
+ torch_state_dict[name] = contiguous_tensor.to(dtype=dtype) if dtype else contiguous_tensor
+ return torch_state_dict
+
+
+def get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir,
+ tag=None,
+ exclude_frozen_parameters=False,
+ lazy_mode=False):
+ """
+ Convert ZeRO 2 or 3 checkpoint into a single fp32 consolidated state_dict that can be loaded with
+ ``load_state_dict()`` and used for training without DeepSpeed or shared with others, for example
+ via a model hub.
+
+ Args:
+ - ``checkpoint_dir``: path to the desired checkpoint folder
+ - ``tag``: checkpoint tag used as a unique identifier for checkpoint. If not provided will attempt to load tag in 'latest' file. e.g., ``global_step14``
+ - ``exclude_frozen_parameters``: exclude frozen parameters
+ - ``lazy_mode``: get state_dict in lazy mode. It returns a dict of pesduo tensor instead of torch tensor, which is more memory efficient.
+ Convert the pesduo tensor to torch tensor by ``.contiguous()``
+
+ Returns:
+ - pytorch ``state_dict``
+
+ A typical usage might be ::
+
+ from deepspeed.utils.zero_to_fp32 import get_fp32_state_dict_from_zero_checkpoint
+ # do the training and checkpoint saving
+ state_dict = get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir) # already on cpu
+ model = model.cpu() # move to cpu
+ model.load_state_dict(state_dict)
+ # submit to model hub or save the model to share with others
+
+ In this example the ``model`` will no longer be usable in the deepspeed context of the same
+ application. i.e. you will need to re-initialize the deepspeed engine, since
+ ``model.load_state_dict(state_dict)`` will remove all the deepspeed magic from it.
+
+ If you want it all done for you, use ``load_state_dict_from_zero_checkpoint`` instead.
+
+ Note: the above usage may not work if your application doesn't have sufficient free CPU memory.
+ You may need to use the offline approach using the ``zero_to_fp32.py`` script that is saved with
+ the checkpoint. Or you can load state_dict in lazy mode ::
+
+ from deepspeed.utils.zero_to_fp32 import get_fp32_state_dict_from_zero_checkpoint
+ state_dict = get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir, lazy_mode=True) # not on cpu
+ for name, lazy_tensor in state_dict.item():
+ tensor = lazy_tensor.contiguous() # to cpu
+ print(name, tensor)
+ # del tensor to release memory if it no longer in use
+ """
+ if tag is None:
+ latest_path = os.path.join(checkpoint_dir, 'latest')
+ if os.path.isfile(latest_path):
+ with open(latest_path, 'r') as fd:
+ tag = fd.read().strip()
+ else:
+ raise ValueError(f"Unable to find 'latest' file at {latest_path}")
+
+ ds_checkpoint_dir = os.path.join(checkpoint_dir, tag)
+
+ if not os.path.isdir(ds_checkpoint_dir):
+ raise FileNotFoundError(f"Directory '{ds_checkpoint_dir}' doesn't exist")
+
+ state_dict = _get_fp32_state_dict_from_zero_checkpoint(ds_checkpoint_dir, exclude_frozen_parameters)
+ if lazy_mode:
+ return state_dict
+ else:
+ return to_torch_tensor(state_dict)
+
+
+def convert_zero_checkpoint_to_state_dict(checkpoint_dir,
+ output_dir,
+ dtype=torch.float32,
+ max_shard_size="5GB",
+ safe_serialization=False,
+ tag=None,
+ exclude_frozen_parameters=False):
+ """
+ Convert ZeRO 2 or 3 checkpoint into a consolidated ``state_dict`` file that can be
+ loaded with ``torch.load(file)`` + ``load_state_dict()`` and used for training without DeepSpeed.
+
+ Args:
+ - ``checkpoint_dir``: path to the desired checkpoint folder. (one that contains the tag-folder, like ``global_step14``)
+ - ``output_dir``: directory for the PyTorch state_dict output files
+ - ``dtype``: output tensor dtype. Supports float32, float16, and bfloat16 as strings or torch dtypes.
+ - ``max_shard_size``: the maximum size for a checkpoint before being sharded, default value is 5GB
+ - ``safe_serialization``: whether to save the model using `safetensors` or the traditional PyTorch way (that uses `pickle`).
+ - ``tag``: checkpoint tag used as a unique identifier for checkpoint. If not provided will attempt to load tag in the file named ``latest`` in the checkpoint folder, e.g., ``global_step14``
+ - ``exclude_frozen_parameters``: exclude frozen parameters
+ """
+
+ dtype = _resolve_output_dtype(dtype)
+
+ # Dependency pre-check
+ if safe_serialization:
+ try:
+ from safetensors.torch import save_file
+ except ImportError:
+ print('If you want to use `safe_serialization`, please `pip install safetensors`')
+ raise
+ if max_shard_size is not None:
+ try:
+ from huggingface_hub import split_torch_state_dict_into_shards
+ except ImportError:
+ print('If you want to use `max_shard_size`, please `pip install huggingface_hub`')
+ raise
+
+ # Convert zero checkpoint to state_dict
+ state_dict = get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir,
+ tag,
+ exclude_frozen_parameters,
+ lazy_mode=True)
+
+ # Shard the model if it is too big.
+ weights_name = "model.safetensors" if safe_serialization else "pytorch_model.bin"
+ if max_shard_size is not None:
+ filename_pattern = weights_name.replace(".bin", "{suffix}.bin").replace(".safetensors", "{suffix}.safetensors")
+ # an memory-efficient approach for sharding
+ empty_state_dict = to_torch_tensor(state_dict, return_empty_tensor=True, dtype=dtype)
+ state_dict_split = split_torch_state_dict_into_shards(empty_state_dict,
+ filename_pattern=filename_pattern,
+ max_shard_size=max_shard_size)
+ else:
+ from collections import namedtuple
+ StateDictSplit = namedtuple("StateDictSplit", ["is_sharded", "filename_to_tensors"])
+ state_dict_split = StateDictSplit(is_sharded=False,
+ filename_to_tensors={weights_name: list(state_dict.keys())})
+
+ # Save the model by shard
+ os.makedirs(output_dir, exist_ok=True)
+ filename_to_tensors = state_dict_split.filename_to_tensors.items()
+ for shard_file, tensors in tqdm(filename_to_tensors, desc="Saving checkpoint shards"):
+ shard_state_dict = {tensor_name: state_dict[tensor_name] for tensor_name in tensors}
+ shard_state_dict = to_torch_tensor(shard_state_dict, dtype=dtype)
+ output_path = os.path.join(output_dir, shard_file)
+ if safe_serialization:
+ save_file(shard_state_dict, output_path, metadata={"format": "pt"})
+ else:
+ torch.save(shard_state_dict, output_path)
+ # release the memory of current shard
+ for tensor_name in list(shard_state_dict.keys()):
+ del state_dict[tensor_name]
+ del shard_state_dict[tensor_name]
+ del shard_state_dict
+ gc.collect()
+
+ # Save index if sharded
+ if state_dict_split.is_sharded:
+ index = {
+ "metadata": state_dict_split.metadata,
+ "weight_map": state_dict_split.tensor_to_filename,
+ }
+ save_index_file = "model.safetensors.index.json" if safe_serialization else "pytorch_model.bin.index.json"
+ save_index_file = os.path.join(output_dir, save_index_file)
+ with open(save_index_file, "w", encoding="utf-8") as f:
+ content = json.dumps(index, indent=2, sort_keys=True) + "\n"
+ f.write(content)
+
+
+def convert_zero_checkpoint_to_fp32_state_dict(checkpoint_dir,
+ output_dir,
+ max_shard_size="5GB",
+ safe_serialization=False,
+ tag=None,
+ exclude_frozen_parameters=False):
+ """Backward-compatible fp32 checkpoint conversion."""
+ return convert_zero_checkpoint_to_state_dict(checkpoint_dir,
+ output_dir,
+ dtype=torch.float32,
+ max_shard_size=max_shard_size,
+ safe_serialization=safe_serialization,
+ tag=tag,
+ exclude_frozen_parameters=exclude_frozen_parameters)
+
+
+def load_state_dict_from_zero_checkpoint(model, checkpoint_dir, tag=None):
+ """
+ 1. Put the provided model to cpu
+ 2. Convert ZeRO 2 or 3 checkpoint into a single fp32 consolidated ``state_dict``
+ 3. Load it into the provided model
+
+ Args:
+ - ``model``: the model object to update
+ - ``checkpoint_dir``: path to the desired checkpoint folder. (one that contains the tag-folder, like ``global_step14``)
+ - ``tag``: checkpoint tag used as a unique identifier for checkpoint. If not provided will attempt to load tag in the file named ``latest`` in the checkpoint folder, e.g., ``global_step14``
+
+ Returns:
+ - ``model`: modified model
+
+ Make sure you have plenty of CPU memory available before you call this function. If you don't
+ have enough use the ``zero_to_fp32.py`` utility to do the conversion. You will find it
+ conveniently placed for you in the checkpoint folder.
+
+ A typical usage might be ::
+
+ from deepspeed.utils.zero_to_fp32 import load_state_dict_from_zero_checkpoint
+ model = load_state_dict_from_zero_checkpoint(trainer.model, checkpoint_dir)
+ # submit to model hub or save the model to share with others
+
+ Note, that once this was run, the ``model`` will no longer be usable in the deepspeed context
+ of the same application. i.e. you will need to re-initialize the deepspeed engine, since
+ ``model.load_state_dict(state_dict)`` will remove all the deepspeed magic from it.
+
+ """
+ logger.info("Extracting fp32 weights")
+ state_dict = get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir, tag)
+
+ logger.info("Overwriting model with fp32 weights")
+ model = model.cpu()
+ model.load_state_dict(state_dict, strict=False)
+
+ return model
+
+
+if __name__ == "__main__":
+ parser = argparse.ArgumentParser()
+ parser.add_argument("checkpoint_dir",
+ type=str,
+ help="path to the desired checkpoint folder, e.g., path/checkpoint-12")
+ parser.add_argument("output_dir",
+ type=str,
+ help="directory to the pytorch fp32 state_dict output files"
+ "(e.g. path/checkpoint-12-output/)")
+ parser.add_argument(
+ "--max_shard_size",
+ type=str,
+ default="5GB",
+ help="The maximum size for a checkpoint before being sharded. Checkpoints shard will then be each of size"
+ "lower than this size. If expressed as a string, needs to be digits followed by a unit (like `5MB`"
+ "We default it to 5GB in order for models to be able to run easily on free-tier google colab instances"
+ "without CPU OOM issues.")
+ parser.add_argument(
+ "--safe_serialization",
+ default=False,
+ action='store_true',
+ help="Whether to save the model using `safetensors` or the traditional PyTorch way (that uses `pickle`).")
+ parser.add_argument("-t",
+ "--tag",
+ type=str,
+ default=None,
+ help="checkpoint tag used as a unique identifier for checkpoint. e.g., global_step1")
+ parser.add_argument("--exclude_frozen_parameters", action='store_true', help="exclude frozen parameters")
+ parser.add_argument("-d", "--debug", action='store_true', help="enable debug")
+ args = parser.parse_args()
+
+ debug = args.debug
+
+ convert_zero_checkpoint_to_fp32_state_dict(args.checkpoint_dir,
+ args.output_dir,
+ max_shard_size=args.max_shard_size,
+ safe_serialization=args.safe_serialization,
+ tag=args.tag,
+ exclude_frozen_parameters=args.exclude_frozen_parameters)
diff --git a/qwen3-1.7b-teutonic-5e6/checkpoint-1500/zero_to_torch.py b/qwen3-1.7b-teutonic-5e6/checkpoint-1500/zero_to_torch.py
new file mode 100644
index 0000000000000000000000000000000000000000..81312e252400d77f03cc1a88522f8f18ebe70ee7
--- /dev/null
+++ b/qwen3-1.7b-teutonic-5e6/checkpoint-1500/zero_to_torch.py
@@ -0,0 +1,54 @@
+#!/usr/bin/env python
+
+# SPDX-License-Identifier: Apache-2.0
+# DeepSpeed Team
+
+import argparse
+
+if __package__:
+ from . import zero_to_fp32
+else:
+ import zero_to_fp32
+
+
+def main(args=None):
+ parser = argparse.ArgumentParser(
+ description="Convert a DeepSpeed ZeRO checkpoint to float32, float16, or bfloat16 PyTorch weights.")
+ parser.add_argument("checkpoint_dir",
+ type=str,
+ help="path to the desired checkpoint folder, e.g., path/checkpoint-12")
+ parser.add_argument("output_dir", type=str, help="directory for the converted PyTorch state_dict files")
+ parser.add_argument("--dtype",
+ type=str,
+ choices=sorted(zero_to_fp32.OUTPUT_DTYPE_NAMES),
+ required=True,
+ help="output tensor dtype")
+ parser.add_argument("--max_shard_size",
+ type=str,
+ default="5GB",
+ help="maximum size of each checkpoint shard, such as 5GB or 500MB")
+ parser.add_argument("--safe_serialization",
+ default=False,
+ action='store_true',
+ help="save with safetensors instead of PyTorch pickle serialization")
+ parser.add_argument("-t",
+ "--tag",
+ type=str,
+ default=None,
+ help="checkpoint tag used as a unique identifier, e.g., global_step1")
+ parser.add_argument("--exclude_frozen_parameters", action='store_true', help="exclude frozen parameters")
+ parser.add_argument("-d", "--debug", action='store_true', help="enable debug output")
+ parsed_args = parser.parse_args(args)
+
+ zero_to_fp32.debug = parsed_args.debug
+ zero_to_fp32.convert_zero_checkpoint_to_state_dict(parsed_args.checkpoint_dir,
+ parsed_args.output_dir,
+ dtype=parsed_args.dtype,
+ max_shard_size=parsed_args.max_shard_size,
+ safe_serialization=parsed_args.safe_serialization,
+ tag=parsed_args.tag,
+ exclude_frozen_parameters=parsed_args.exclude_frozen_parameters)
+
+
+if __name__ == "__main__":
+ main()
diff --git a/qwen3-1.7b-teutonic-5e6/checkpoint-2000/chat_template.jinja b/qwen3-1.7b-teutonic-5e6/checkpoint-2000/chat_template.jinja
new file mode 100644
index 0000000000000000000000000000000000000000..01be9b307daa2d425f7c168c9fb145a286e0afb4
--- /dev/null
+++ b/qwen3-1.7b-teutonic-5e6/checkpoint-2000/chat_template.jinja
@@ -0,0 +1,89 @@
+{%- if tools %}
+ {{- '<|im_start|>system\n' }}
+ {%- if messages[0].role == 'system' %}
+ {{- messages[0].content + '\n\n' }}
+ {%- endif %}
+ {{- "# Tools\n\nYou may call one or more functions to assist with the user query.\n\nYou are provided with function signatures within XML tags:\n" }}
+ {%- for tool in tools %}
+ {{- "\n" }}
+ {{- tool | tojson }}
+ {%- endfor %}
+ {{- "\n\n\nFor each function call, return a json object with function name and arguments within XML tags:\n\n{\"name\": , \"arguments\": }\n<|im_end|>\n" }}
+{%- else %}
+ {%- if messages[0].role == 'system' %}
+ {{- '<|im_start|>system\n' + messages[0].content + '<|im_end|>\n' }}
+ {%- endif %}
+{%- endif %}
+{%- set ns = namespace(multi_step_tool=true, last_query_index=messages|length - 1) %}
+{%- for message in messages[::-1] %}
+ {%- set index = (messages|length - 1) - loop.index0 %}
+ {%- if ns.multi_step_tool and message.role == "user" and message.content is string and not(message.content.startswith('') and message.content.endswith('')) %}
+ {%- set ns.multi_step_tool = false %}
+ {%- set ns.last_query_index = index %}
+ {%- endif %}
+{%- endfor %}
+{%- for message in messages %}
+ {%- if message.content is string %}
+ {%- set content = message.content %}
+ {%- else %}
+ {%- set content = '' %}
+ {%- endif %}
+ {%- if (message.role == "user") or (message.role == "system" and not loop.first) %}
+ {{- '<|im_start|>' + message.role + '\n' + content + '<|im_end|>' + '\n' }}
+ {%- elif message.role == "assistant" %}
+ {%- set reasoning_content = '' %}
+ {%- if message.reasoning_content is string %}
+ {%- set reasoning_content = message.reasoning_content %}
+ {%- else %}
+ {%- if '' in content %}
+ {%- set reasoning_content = content.split('')[0].rstrip('\n').split('')[-1].lstrip('\n') %}
+ {%- set content = content.split('')[-1].lstrip('\n') %}
+ {%- endif %}
+ {%- endif %}
+ {%- if loop.index0 > ns.last_query_index %}
+ {%- if loop.last or (not loop.last and reasoning_content) %}
+ {{- '<|im_start|>' + message.role + '\n\n' + reasoning_content.strip('\n') + '\n\n\n' + content.lstrip('\n') }}
+ {%- else %}
+ {{- '<|im_start|>' + message.role + '\n' + content }}
+ {%- endif %}
+ {%- else %}
+ {{- '<|im_start|>' + message.role + '\n' + content }}
+ {%- endif %}
+ {%- if message.tool_calls %}
+ {%- for tool_call in message.tool_calls %}
+ {%- if (loop.first and content) or (not loop.first) %}
+ {{- '\n' }}
+ {%- endif %}
+ {%- if tool_call.function %}
+ {%- set tool_call = tool_call.function %}
+ {%- endif %}
+ {{- '\n{"name": "' }}
+ {{- tool_call.name }}
+ {{- '", "arguments": ' }}
+ {%- if tool_call.arguments is string %}
+ {{- tool_call.arguments }}
+ {%- else %}
+ {{- tool_call.arguments | tojson }}
+ {%- endif %}
+ {{- '}\n' }}
+ {%- endfor %}
+ {%- endif %}
+ {{- '<|im_end|>\n' }}
+ {%- elif message.role == "tool" %}
+ {%- if loop.first or (messages[loop.index0 - 1].role != "tool") %}
+ {{- '<|im_start|>user' }}
+ {%- endif %}
+ {{- '\n\n' }}
+ {{- content }}
+ {{- '\n' }}
+ {%- if loop.last or (messages[loop.index0 + 1].role != "tool") %}
+ {{- '<|im_end|>\n' }}
+ {%- endif %}
+ {%- endif %}
+{%- endfor %}
+{%- if add_generation_prompt %}
+ {{- '<|im_start|>assistant\n' }}
+ {%- if enable_thinking is defined and enable_thinking is false %}
+ {{- '\n\n\n\n' }}
+ {%- endif %}
+{%- endif %}
\ No newline at end of file
diff --git a/qwen3-1.7b-teutonic-5e6/checkpoint-2000/config.json b/qwen3-1.7b-teutonic-5e6/checkpoint-2000/config.json
new file mode 100644
index 0000000000000000000000000000000000000000..5d9cef07396baadff5390e4508eb33025967a029
--- /dev/null
+++ b/qwen3-1.7b-teutonic-5e6/checkpoint-2000/config.json
@@ -0,0 +1,63 @@
+{
+ "architectures": [
+ "Qwen3ForCausalLM"
+ ],
+ "attention_bias": false,
+ "attention_dropout": 0.0,
+ "bos_token_id": null,
+ "dtype": "bfloat16",
+ "eos_token_id": 151645,
+ "head_dim": 128,
+ "hidden_act": "silu",
+ "hidden_size": 2048,
+ "initializer_range": 0.02,
+ "intermediate_size": 6144,
+ "layer_types": [
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention"
+ ],
+ "max_position_embeddings": 40960,
+ "max_window_layers": 28,
+ "model_type": "qwen3",
+ "num_attention_heads": 16,
+ "num_hidden_layers": 28,
+ "num_key_value_heads": 8,
+ "pad_token_id": 151643,
+ "rms_norm_eps": 1e-06,
+ "rope_parameters": {
+ "rope_theta": 1000000,
+ "rope_type": "default"
+ },
+ "sliding_window": null,
+ "tie_word_embeddings": true,
+ "transformers_version": "5.17.0",
+ "use_cache": false,
+ "use_sliding_window": false,
+ "vocab_size": 151936
+}
diff --git a/qwen3-1.7b-teutonic-5e6/checkpoint-2000/generation_config.json b/qwen3-1.7b-teutonic-5e6/checkpoint-2000/generation_config.json
new file mode 100644
index 0000000000000000000000000000000000000000..e6941fe4b04f26113d6eef6255d005c4d7090bda
--- /dev/null
+++ b/qwen3-1.7b-teutonic-5e6/checkpoint-2000/generation_config.json
@@ -0,0 +1,12 @@
+{
+ "do_sample": true,
+ "eos_token_id": [
+ 151645,
+ 151643
+ ],
+ "pad_token_id": 151643,
+ "temperature": 0.6,
+ "top_k": 20,
+ "top_p": 0.95,
+ "transformers_version": "5.17.0"
+}
diff --git a/qwen3-1.7b-teutonic-5e6/checkpoint-2000/global_step2000/bf16_zero_pp_rank_0_mp_rank_00_optim_states.pt b/qwen3-1.7b-teutonic-5e6/checkpoint-2000/global_step2000/bf16_zero_pp_rank_0_mp_rank_00_optim_states.pt
new file mode 100644
index 0000000000000000000000000000000000000000..834cd976d9e71cf7b099bd0efa63507b555ac495
--- /dev/null
+++ b/qwen3-1.7b-teutonic-5e6/checkpoint-2000/global_step2000/bf16_zero_pp_rank_0_mp_rank_00_optim_states.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a0d779bffcda80802a00b348c93b525e847aeccc7cbcafda7ed86b1a88ec1069
+size 10323466465
diff --git a/qwen3-1.7b-teutonic-5e6/checkpoint-2000/global_step2000/bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt b/qwen3-1.7b-teutonic-5e6/checkpoint-2000/global_step2000/bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt
new file mode 100644
index 0000000000000000000000000000000000000000..dd7215effa406a87d884bd3af9b0f85b7c43b90d
--- /dev/null
+++ b/qwen3-1.7b-teutonic-5e6/checkpoint-2000/global_step2000/bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:daecf4566e66c88fcd309ce3aba5da089ab37abdc6891f3b6ba8d8df0d48ba8f
+size 10323469601
diff --git a/qwen3-1.7b-teutonic-5e6/checkpoint-2000/global_step2000/mp_rank_00_model_states.pt b/qwen3-1.7b-teutonic-5e6/checkpoint-2000/global_step2000/mp_rank_00_model_states.pt
new file mode 100644
index 0000000000000000000000000000000000000000..dc74033440963f3e77ac017a965f3a45030b0f28
--- /dev/null
+++ b/qwen3-1.7b-teutonic-5e6/checkpoint-2000/global_step2000/mp_rank_00_model_states.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:45c09d755ed01b0ea4fe35f32b10bd63db0e8086769ac84fd5049bdba762a059
+size 3441239653
diff --git a/qwen3-1.7b-teutonic-5e6/checkpoint-2000/latest b/qwen3-1.7b-teutonic-5e6/checkpoint-2000/latest
new file mode 100644
index 0000000000000000000000000000000000000000..2a79fdc19587e6bc9de060e90633f3a151b04516
--- /dev/null
+++ b/qwen3-1.7b-teutonic-5e6/checkpoint-2000/latest
@@ -0,0 +1 @@
+global_step2000
\ No newline at end of file
diff --git a/qwen3-1.7b-teutonic-5e6/checkpoint-2000/model.safetensors b/qwen3-1.7b-teutonic-5e6/checkpoint-2000/model.safetensors
new file mode 100644
index 0000000000000000000000000000000000000000..eb107f5e76a61c6ec0ddd81a8c5648813cb3aedb
--- /dev/null
+++ b/qwen3-1.7b-teutonic-5e6/checkpoint-2000/model.safetensors
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e9ad2ae2eccd2d25725cbab98630d5cf046772ec49e9ae29babb873050effdd0
+size 4063515640
diff --git a/qwen3-1.7b-teutonic-5e6/checkpoint-2000/rng_state_0.pth b/qwen3-1.7b-teutonic-5e6/checkpoint-2000/rng_state_0.pth
new file mode 100644
index 0000000000000000000000000000000000000000..04513840b62d3e018ff37182c0f54f8ed73c4e0a
--- /dev/null
+++ b/qwen3-1.7b-teutonic-5e6/checkpoint-2000/rng_state_0.pth
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:65b696b81e6048057ec73ebbecce9e099e8e7e9def2b9e40427c243c14361122
+size 14917
diff --git a/qwen3-1.7b-teutonic-5e6/checkpoint-2000/rng_state_1.pth b/qwen3-1.7b-teutonic-5e6/checkpoint-2000/rng_state_1.pth
new file mode 100644
index 0000000000000000000000000000000000000000..6688ff31bee1ef557de16309352a444d38022bdd
--- /dev/null
+++ b/qwen3-1.7b-teutonic-5e6/checkpoint-2000/rng_state_1.pth
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d593efd097dd20c875d98b61c5076e822e2b0d0f0d46d6585d7cf1ad3dce6064
+size 14917
diff --git a/qwen3-1.7b-teutonic-5e6/checkpoint-2000/scheduler.pt b/qwen3-1.7b-teutonic-5e6/checkpoint-2000/scheduler.pt
new file mode 100644
index 0000000000000000000000000000000000000000..2467ac2cfd96c61f28a002b407da7209db76d322
--- /dev/null
+++ b/qwen3-1.7b-teutonic-5e6/checkpoint-2000/scheduler.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:20171101247c2634d0eeb994a96067416f05cf852a206d8c12e9dfa66c1485fe
+size 1465
diff --git a/qwen3-1.7b-teutonic-5e6/checkpoint-2000/tokenizer.json b/qwen3-1.7b-teutonic-5e6/checkpoint-2000/tokenizer.json
new file mode 100644
index 0000000000000000000000000000000000000000..c7afbed2efcdf019f88ab0572ec29d3bf595dfe2
--- /dev/null
+++ b/qwen3-1.7b-teutonic-5e6/checkpoint-2000/tokenizer.json
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:be75606093db2094d7cd20f3c2f385c212750648bd6ea4fb2bf507a6a4c55506
+size 11422650
diff --git a/qwen3-1.7b-teutonic-5e6/checkpoint-2000/tokenizer_config.json b/qwen3-1.7b-teutonic-5e6/checkpoint-2000/tokenizer_config.json
new file mode 100644
index 0000000000000000000000000000000000000000..770e41d6c92519d525eede4cbcf3ba27f6425311
--- /dev/null
+++ b/qwen3-1.7b-teutonic-5e6/checkpoint-2000/tokenizer_config.json
@@ -0,0 +1,30 @@
+{
+ "add_prefix_space": false,
+ "backend": "tokenizers",
+ "bos_token": null,
+ "clean_up_tokenization_spaces": false,
+ "eos_token": "<|im_end|>",
+ "errors": "replace",
+ "extra_special_tokens": [
+ "<|im_start|>",
+ "<|im_end|>",
+ "<|object_ref_start|>",
+ "<|object_ref_end|>",
+ "<|box_start|>",
+ "<|box_end|>",
+ "<|quad_start|>",
+ "<|quad_end|>",
+ "<|vision_start|>",
+ "<|vision_end|>",
+ "<|vision_pad|>",
+ "<|image_pad|>",
+ "<|video_pad|>"
+ ],
+ "is_local": false,
+ "local_files_only": false,
+ "model_max_length": 131072,
+ "pad_token": "<|endoftext|>",
+ "split_special_tokens": false,
+ "tokenizer_class": "Qwen2Tokenizer",
+ "unk_token": null
+}
diff --git a/qwen3-1.7b-teutonic-5e6/checkpoint-2000/trainer_state.json b/qwen3-1.7b-teutonic-5e6/checkpoint-2000/trainer_state.json
new file mode 100644
index 0000000000000000000000000000000000000000..1ac7d399edb6edefe641329287dff76d57f4441a
--- /dev/null
+++ b/qwen3-1.7b-teutonic-5e6/checkpoint-2000/trainer_state.json
@@ -0,0 +1,20474 @@
+{
+ "best_global_step": 950,
+ "best_metric": 1.5272752046585083,
+ "best_model_checkpoint": "./checkpoints/qwen3-1.7b-teutonic-5e6/checkpoint-500",
+ "epoch": 4.040404040404041,
+ "eval_steps": 50,
+ "global_step": 2000,
+ "is_hyper_param_search": false,
+ "is_local_process_zero": true,
+ "is_world_process_zero": true,
+ "log_history": [
+ {
+ "entropy": 0.8295683860778809,
+ "epoch": 0.00202020202020202,
+ "grad_norm": 10.886772155761719,
+ "learning_rate": 5e-06,
+ "loss": 1.5185801982879639,
+ "mean_token_accuracy": 0.672874927520752,
+ "num_tokens": 16384.0,
+ "step": 1
+ },
+ {
+ "entropy": 1.1781222820281982,
+ "epoch": 0.00404040404040404,
+ "grad_norm": 13.80163288116455,
+ "learning_rate": 4.997979797979798e-06,
+ "loss": 2.18377685546875,
+ "mean_token_accuracy": 0.5700415372848511,
+ "num_tokens": 32768.0,
+ "step": 2
+ },
+ {
+ "entropy": 0.7972303032875061,
+ "epoch": 0.006060606060606061,
+ "grad_norm": 13.71261215209961,
+ "learning_rate": 4.995959595959596e-06,
+ "loss": 1.5810277462005615,
+ "mean_token_accuracy": 0.6782486438751221,
+ "num_tokens": 49152.0,
+ "step": 3
+ },
+ {
+ "entropy": 0.9709298610687256,
+ "epoch": 0.00808080808080808,
+ "grad_norm": 9.18797779083252,
+ "learning_rate": 4.993939393939394e-06,
+ "loss": 1.6923656463623047,
+ "mean_token_accuracy": 0.6447850465774536,
+ "num_tokens": 65536.0,
+ "step": 4
+ },
+ {
+ "entropy": 1.2161898612976074,
+ "epoch": 0.010101010101010102,
+ "grad_norm": 8.97508430480957,
+ "learning_rate": 4.991919191919192e-06,
+ "loss": 1.9361608028411865,
+ "mean_token_accuracy": 0.5922691822052002,
+ "num_tokens": 81920.0,
+ "step": 5
+ },
+ {
+ "entropy": 0.9950039982795715,
+ "epoch": 0.012121212121212121,
+ "grad_norm": 6.453213214874268,
+ "learning_rate": 4.98989898989899e-06,
+ "loss": 1.5012480020523071,
+ "mean_token_accuracy": 0.6671959161758423,
+ "num_tokens": 98304.0,
+ "step": 6
+ },
+ {
+ "entropy": 1.452682614326477,
+ "epoch": 0.014141414141414142,
+ "grad_norm": 8.50411605834961,
+ "learning_rate": 4.987878787878789e-06,
+ "loss": 2.1297783851623535,
+ "mean_token_accuracy": 0.5807889699935913,
+ "num_tokens": 114688.0,
+ "step": 7
+ },
+ {
+ "entropy": 1.4661014080047607,
+ "epoch": 0.01616161616161616,
+ "grad_norm": 4.290146827697754,
+ "learning_rate": 4.9858585858585865e-06,
+ "loss": 1.7391963005065918,
+ "mean_token_accuracy": 0.611993134021759,
+ "num_tokens": 131072.0,
+ "step": 8
+ },
+ {
+ "entropy": 1.9030903577804565,
+ "epoch": 0.01818181818181818,
+ "grad_norm": 4.503584384918213,
+ "learning_rate": 4.983838383838384e-06,
+ "loss": 2.283596992492676,
+ "mean_token_accuracy": 0.5261968970298767,
+ "num_tokens": 147456.0,
+ "step": 9
+ },
+ {
+ "entropy": 1.2784137725830078,
+ "epoch": 0.020202020202020204,
+ "grad_norm": 3.427410840988159,
+ "learning_rate": 4.981818181818182e-06,
+ "loss": 1.5032392740249634,
+ "mean_token_accuracy": 0.6578529477119446,
+ "num_tokens": 163840.0,
+ "step": 10
+ },
+ {
+ "entropy": 1.7487188577651978,
+ "epoch": 0.022222222222222223,
+ "grad_norm": 3.20536470413208,
+ "learning_rate": 4.97979797979798e-06,
+ "loss": 1.9184643030166626,
+ "mean_token_accuracy": 0.6093673706054688,
+ "num_tokens": 180224.0,
+ "step": 11
+ },
+ {
+ "entropy": 1.6585394144058228,
+ "epoch": 0.024242424242424242,
+ "grad_norm": 3.2463598251342773,
+ "learning_rate": 4.977777777777778e-06,
+ "loss": 1.8364639282226562,
+ "mean_token_accuracy": 0.6014899611473083,
+ "num_tokens": 196608.0,
+ "step": 12
+ },
+ {
+ "entropy": 1.4869519472122192,
+ "epoch": 0.026262626262626262,
+ "grad_norm": 2.8681719303131104,
+ "learning_rate": 4.975757575757576e-06,
+ "loss": 1.6356265544891357,
+ "mean_token_accuracy": 0.6361138224601746,
+ "num_tokens": 212992.0,
+ "step": 13
+ },
+ {
+ "entropy": 1.5179094076156616,
+ "epoch": 0.028282828282828285,
+ "grad_norm": 3.0470707416534424,
+ "learning_rate": 4.973737373737374e-06,
+ "loss": 1.6317660808563232,
+ "mean_token_accuracy": 0.6334269642829895,
+ "num_tokens": 229376.0,
+ "step": 14
+ },
+ {
+ "entropy": 1.7384330034255981,
+ "epoch": 0.030303030303030304,
+ "grad_norm": 2.6149742603302,
+ "learning_rate": 4.971717171717172e-06,
+ "loss": 1.67923903465271,
+ "mean_token_accuracy": 0.6469223499298096,
+ "num_tokens": 245760.0,
+ "step": 15
+ },
+ {
+ "entropy": 2.108799457550049,
+ "epoch": 0.03232323232323232,
+ "grad_norm": 3.694795608520508,
+ "learning_rate": 4.9696969696969696e-06,
+ "loss": 1.9052371978759766,
+ "mean_token_accuracy": 0.5927577018737793,
+ "num_tokens": 262144.0,
+ "step": 16
+ },
+ {
+ "entropy": 1.7532848119735718,
+ "epoch": 0.03434343434343434,
+ "grad_norm": 3.1510567665100098,
+ "learning_rate": 4.9676767676767675e-06,
+ "loss": 1.6382691860198975,
+ "mean_token_accuracy": 0.6311675906181335,
+ "num_tokens": 278528.0,
+ "step": 17
+ },
+ {
+ "entropy": 1.723548412322998,
+ "epoch": 0.03636363636363636,
+ "grad_norm": 3.093825340270996,
+ "learning_rate": 4.965656565656566e-06,
+ "loss": 1.6039624214172363,
+ "mean_token_accuracy": 0.6313507556915283,
+ "num_tokens": 294912.0,
+ "step": 18
+ },
+ {
+ "entropy": 1.4790765047073364,
+ "epoch": 0.03838383838383838,
+ "grad_norm": 2.7150535583496094,
+ "learning_rate": 4.963636363636364e-06,
+ "loss": 1.4173667430877686,
+ "mean_token_accuracy": 0.6627381443977356,
+ "num_tokens": 311296.0,
+ "step": 19
+ },
+ {
+ "entropy": 1.5827536582946777,
+ "epoch": 0.04040404040404041,
+ "grad_norm": 2.610966444015503,
+ "learning_rate": 4.961616161616162e-06,
+ "loss": 1.4604644775390625,
+ "mean_token_accuracy": 0.656509518623352,
+ "num_tokens": 327680.0,
+ "step": 20
+ },
+ {
+ "entropy": 2.054673194885254,
+ "epoch": 0.04242424242424243,
+ "grad_norm": 2.612940788269043,
+ "learning_rate": 4.95959595959596e-06,
+ "loss": 1.962029218673706,
+ "mean_token_accuracy": 0.5719956159591675,
+ "num_tokens": 344064.0,
+ "step": 21
+ },
+ {
+ "entropy": 1.6684198379516602,
+ "epoch": 0.044444444444444446,
+ "grad_norm": 2.5426013469696045,
+ "learning_rate": 4.957575757575758e-06,
+ "loss": 1.591176986694336,
+ "mean_token_accuracy": 0.6519907116889954,
+ "num_tokens": 360448.0,
+ "step": 22
+ },
+ {
+ "entropy": 1.9070855379104614,
+ "epoch": 0.046464646464646465,
+ "grad_norm": 2.5853357315063477,
+ "learning_rate": 4.9555555555555565e-06,
+ "loss": 1.930276870727539,
+ "mean_token_accuracy": 0.5895823240280151,
+ "num_tokens": 376832.0,
+ "step": 23
+ },
+ {
+ "entropy": 1.716689109802246,
+ "epoch": 0.048484848484848485,
+ "grad_norm": 2.7608494758605957,
+ "learning_rate": 4.953535353535354e-06,
+ "loss": 1.739159345626831,
+ "mean_token_accuracy": 0.6028944849967957,
+ "num_tokens": 393216.0,
+ "step": 24
+ },
+ {
+ "entropy": 1.5666695833206177,
+ "epoch": 0.050505050505050504,
+ "grad_norm": 2.5973074436187744,
+ "learning_rate": 4.951515151515152e-06,
+ "loss": 1.6016302108764648,
+ "mean_token_accuracy": 0.626099169254303,
+ "num_tokens": 409600.0,
+ "step": 25
+ },
+ {
+ "entropy": 1.4267652034759521,
+ "epoch": 0.052525252525252523,
+ "grad_norm": 2.163451910018921,
+ "learning_rate": 4.94949494949495e-06,
+ "loss": 1.5093598365783691,
+ "mean_token_accuracy": 0.6530288457870483,
+ "num_tokens": 425984.0,
+ "step": 26
+ },
+ {
+ "entropy": 1.8537150621414185,
+ "epoch": 0.05454545454545454,
+ "grad_norm": 2.6983258724212646,
+ "learning_rate": 4.947474747474748e-06,
+ "loss": 1.8831868171691895,
+ "mean_token_accuracy": 0.5802393555641174,
+ "num_tokens": 442368.0,
+ "step": 27
+ },
+ {
+ "entropy": 1.410015344619751,
+ "epoch": 0.05656565656565657,
+ "grad_norm": 2.4777796268463135,
+ "learning_rate": 4.945454545454546e-06,
+ "loss": 1.4835734367370605,
+ "mean_token_accuracy": 0.6519907116889954,
+ "num_tokens": 458752.0,
+ "step": 28
+ },
+ {
+ "entropy": 1.2399017810821533,
+ "epoch": 0.05858585858585859,
+ "grad_norm": 2.1676478385925293,
+ "learning_rate": 4.943434343434344e-06,
+ "loss": 1.330633282661438,
+ "mean_token_accuracy": 0.6802638173103333,
+ "num_tokens": 475136.0,
+ "step": 29
+ },
+ {
+ "entropy": 1.6478898525238037,
+ "epoch": 0.06060606060606061,
+ "grad_norm": 2.5325536727905273,
+ "learning_rate": 4.941414141414142e-06,
+ "loss": 1.771777868270874,
+ "mean_token_accuracy": 0.6128480434417725,
+ "num_tokens": 491520.0,
+ "step": 30
+ },
+ {
+ "entropy": 1.7915360927581787,
+ "epoch": 0.06262626262626263,
+ "grad_norm": 2.2419557571411133,
+ "learning_rate": 4.93939393939394e-06,
+ "loss": 1.870645523071289,
+ "mean_token_accuracy": 0.6074743270874023,
+ "num_tokens": 507904.0,
+ "step": 31
+ },
+ {
+ "entropy": 1.7638899087905884,
+ "epoch": 0.06464646464646465,
+ "grad_norm": 2.8063058853149414,
+ "learning_rate": 4.937373737373738e-06,
+ "loss": 1.83987557888031,
+ "mean_token_accuracy": 0.5836589932441711,
+ "num_tokens": 524288.0,
+ "step": 32
+ },
+ {
+ "entropy": 1.4097516536712646,
+ "epoch": 0.06666666666666667,
+ "grad_norm": 2.1857714653015137,
+ "learning_rate": 4.935353535353536e-06,
+ "loss": 1.477668285369873,
+ "mean_token_accuracy": 0.6507083773612976,
+ "num_tokens": 540672.0,
+ "step": 33
+ },
+ {
+ "entropy": 1.4447426795959473,
+ "epoch": 0.06868686868686869,
+ "grad_norm": 2.3771870136260986,
+ "learning_rate": 4.933333333333334e-06,
+ "loss": 1.536318063735962,
+ "mean_token_accuracy": 0.6383732557296753,
+ "num_tokens": 557056.0,
+ "step": 34
+ },
+ {
+ "entropy": 1.9658164978027344,
+ "epoch": 0.0707070707070707,
+ "grad_norm": 2.4784936904907227,
+ "learning_rate": 4.931313131313132e-06,
+ "loss": 2.0565035343170166,
+ "mean_token_accuracy": 0.5581949353218079,
+ "num_tokens": 573440.0,
+ "step": 35
+ },
+ {
+ "entropy": 1.6045317649841309,
+ "epoch": 0.07272727272727272,
+ "grad_norm": 2.1128504276275635,
+ "learning_rate": 4.92929292929293e-06,
+ "loss": 1.6234952211380005,
+ "mean_token_accuracy": 0.6293356418609619,
+ "num_tokens": 589824.0,
+ "step": 36
+ },
+ {
+ "entropy": 1.5491269826889038,
+ "epoch": 0.07474747474747474,
+ "grad_norm": 2.1672937870025635,
+ "learning_rate": 4.927272727272728e-06,
+ "loss": 1.5809822082519531,
+ "mean_token_accuracy": 0.6419760584831238,
+ "num_tokens": 606208.0,
+ "step": 37
+ },
+ {
+ "entropy": 1.545534610748291,
+ "epoch": 0.07676767676767676,
+ "grad_norm": 2.2526934146881104,
+ "learning_rate": 4.925252525252526e-06,
+ "loss": 1.5707473754882812,
+ "mean_token_accuracy": 0.6344650983810425,
+ "num_tokens": 622592.0,
+ "step": 38
+ },
+ {
+ "entropy": 1.579596757888794,
+ "epoch": 0.07878787878787878,
+ "grad_norm": 2.3988492488861084,
+ "learning_rate": 4.9232323232323235e-06,
+ "loss": 1.6725983619689941,
+ "mean_token_accuracy": 0.6243282556533813,
+ "num_tokens": 638976.0,
+ "step": 39
+ },
+ {
+ "entropy": 2.0438785552978516,
+ "epoch": 0.08080808080808081,
+ "grad_norm": 2.3314359188079834,
+ "learning_rate": 4.9212121212121214e-06,
+ "loss": 2.0424888134002686,
+ "mean_token_accuracy": 0.570713222026825,
+ "num_tokens": 655360.0,
+ "step": 40
+ },
+ {
+ "entropy": 1.8690773248672485,
+ "epoch": 0.08282828282828283,
+ "grad_norm": 2.130401134490967,
+ "learning_rate": 4.919191919191919e-06,
+ "loss": 1.8796970844268799,
+ "mean_token_accuracy": 0.5882999300956726,
+ "num_tokens": 671744.0,
+ "step": 41
+ },
+ {
+ "entropy": 1.756626009941101,
+ "epoch": 0.08484848484848485,
+ "grad_norm": 2.342318534851074,
+ "learning_rate": 4.917171717171717e-06,
+ "loss": 1.71901535987854,
+ "mean_token_accuracy": 0.6173668503761292,
+ "num_tokens": 688128.0,
+ "step": 42
+ },
+ {
+ "entropy": 1.4642508029937744,
+ "epoch": 0.08686868686868687,
+ "grad_norm": 1.993338704109192,
+ "learning_rate": 4.915151515151516e-06,
+ "loss": 1.4660165309906006,
+ "mean_token_accuracy": 0.6687225103378296,
+ "num_tokens": 704512.0,
+ "step": 43
+ },
+ {
+ "entropy": 1.790807843208313,
+ "epoch": 0.08888888888888889,
+ "grad_norm": 2.3186943531036377,
+ "learning_rate": 4.913131313131314e-06,
+ "loss": 1.7102060317993164,
+ "mean_token_accuracy": 0.6238397359848022,
+ "num_tokens": 720896.0,
+ "step": 44
+ },
+ {
+ "entropy": 1.5970485210418701,
+ "epoch": 0.09090909090909091,
+ "grad_norm": 2.299307346343994,
+ "learning_rate": 4.911111111111112e-06,
+ "loss": 1.5170013904571533,
+ "mean_token_accuracy": 0.652662456035614,
+ "num_tokens": 737280.0,
+ "step": 45
+ },
+ {
+ "entropy": 1.3955466747283936,
+ "epoch": 0.09292929292929293,
+ "grad_norm": 2.171952962875366,
+ "learning_rate": 4.90909090909091e-06,
+ "loss": 1.4059661626815796,
+ "mean_token_accuracy": 0.6654250025749207,
+ "num_tokens": 753664.0,
+ "step": 46
+ },
+ {
+ "entropy": 1.7198575735092163,
+ "epoch": 0.09494949494949495,
+ "grad_norm": 2.385092258453369,
+ "learning_rate": 4.9070707070707075e-06,
+ "loss": 1.730346918106079,
+ "mean_token_accuracy": 0.6191987991333008,
+ "num_tokens": 770048.0,
+ "step": 47
+ },
+ {
+ "entropy": 1.3542555570602417,
+ "epoch": 0.09696969696969697,
+ "grad_norm": 2.1463189125061035,
+ "learning_rate": 4.905050505050505e-06,
+ "loss": 1.346085786819458,
+ "mean_token_accuracy": 0.680446982383728,
+ "num_tokens": 786432.0,
+ "step": 48
+ },
+ {
+ "entropy": 1.8084443807601929,
+ "epoch": 0.09898989898989899,
+ "grad_norm": 2.1505849361419678,
+ "learning_rate": 4.903030303030303e-06,
+ "loss": 1.847151756286621,
+ "mean_token_accuracy": 0.5926355719566345,
+ "num_tokens": 802816.0,
+ "step": 49
+ },
+ {
+ "entropy": 1.751160979270935,
+ "epoch": 0.10101010101010101,
+ "grad_norm": 2.1436259746551514,
+ "learning_rate": 4.901010101010101e-06,
+ "loss": 1.7802404165267944,
+ "mean_token_accuracy": 0.5996580123901367,
+ "num_tokens": 819200.0,
+ "step": 50
+ },
+ {
+ "epoch": 0.10101010101010101,
+ "eval_entropy": 1.6292865045609013,
+ "eval_loss": 1.6725393533706665,
+ "eval_mean_token_accuracy": 0.6230050469598463,
+ "eval_num_tokens": 819200.0,
+ "eval_runtime": 43.9148,
+ "eval_samples_per_second": 22.544,
+ "eval_steps_per_second": 2.824,
+ "step": 50
+ },
+ {
+ "entropy": 1.4428319931030273,
+ "epoch": 0.10303030303030303,
+ "grad_norm": 2.0954954624176025,
+ "learning_rate": 4.898989898989899e-06,
+ "loss": 1.4927232265472412,
+ "mean_token_accuracy": 0.6522349715232849,
+ "num_tokens": 835584.0,
+ "step": 51
+ },
+ {
+ "entropy": 1.7493115663528442,
+ "epoch": 0.10505050505050505,
+ "grad_norm": 2.300030469894409,
+ "learning_rate": 4.896969696969697e-06,
+ "loss": 1.737417221069336,
+ "mean_token_accuracy": 0.6213361024856567,
+ "num_tokens": 851968.0,
+ "step": 52
+ },
+ {
+ "entropy": 1.7801647186279297,
+ "epoch": 0.10707070707070707,
+ "grad_norm": 2.3947081565856934,
+ "learning_rate": 4.894949494949495e-06,
+ "loss": 1.8372564315795898,
+ "mean_token_accuracy": 0.5931240916252136,
+ "num_tokens": 868352.0,
+ "step": 53
+ },
+ {
+ "entropy": 1.6934887170791626,
+ "epoch": 0.10909090909090909,
+ "grad_norm": 2.1981089115142822,
+ "learning_rate": 4.8929292929292936e-06,
+ "loss": 1.734646201133728,
+ "mean_token_accuracy": 0.611932098865509,
+ "num_tokens": 884736.0,
+ "step": 54
+ },
+ {
+ "entropy": 1.3401941061019897,
+ "epoch": 0.1111111111111111,
+ "grad_norm": 2.193511724472046,
+ "learning_rate": 4.8909090909090914e-06,
+ "loss": 1.399888038635254,
+ "mean_token_accuracy": 0.6745847463607788,
+ "num_tokens": 901120.0,
+ "step": 55
+ },
+ {
+ "entropy": 1.8244661092758179,
+ "epoch": 0.11313131313131314,
+ "grad_norm": 2.4358489513397217,
+ "learning_rate": 4.888888888888889e-06,
+ "loss": 1.8716282844543457,
+ "mean_token_accuracy": 0.5867122411727905,
+ "num_tokens": 917504.0,
+ "step": 56
+ },
+ {
+ "entropy": 1.5019619464874268,
+ "epoch": 0.11515151515151516,
+ "grad_norm": 2.1135261058807373,
+ "learning_rate": 4.886868686868687e-06,
+ "loss": 1.523103952407837,
+ "mean_token_accuracy": 0.6414265036582947,
+ "num_tokens": 933888.0,
+ "step": 57
+ },
+ {
+ "entropy": 1.6132855415344238,
+ "epoch": 0.11717171717171718,
+ "grad_norm": 2.026301145553589,
+ "learning_rate": 4.884848484848485e-06,
+ "loss": 1.6233609914779663,
+ "mean_token_accuracy": 0.6279311180114746,
+ "num_tokens": 950272.0,
+ "step": 58
+ },
+ {
+ "entropy": 1.738538384437561,
+ "epoch": 0.1191919191919192,
+ "grad_norm": 2.1539394855499268,
+ "learning_rate": 4.882828282828283e-06,
+ "loss": 1.8147332668304443,
+ "mean_token_accuracy": 0.6124816536903381,
+ "num_tokens": 966656.0,
+ "step": 59
+ },
+ {
+ "entropy": 1.5533764362335205,
+ "epoch": 0.12121212121212122,
+ "grad_norm": 2.2603628635406494,
+ "learning_rate": 4.880808080808081e-06,
+ "loss": 1.562873125076294,
+ "mean_token_accuracy": 0.6398388147354126,
+ "num_tokens": 983040.0,
+ "step": 60
+ },
+ {
+ "entropy": 1.722406268119812,
+ "epoch": 0.12323232323232323,
+ "grad_norm": 2.3630757331848145,
+ "learning_rate": 4.878787878787879e-06,
+ "loss": 1.7461689710617065,
+ "mean_token_accuracy": 0.6023448705673218,
+ "num_tokens": 999424.0,
+ "step": 61
+ },
+ {
+ "entropy": 1.6533517837524414,
+ "epoch": 0.12525252525252525,
+ "grad_norm": 2.2165415287017822,
+ "learning_rate": 4.876767676767677e-06,
+ "loss": 1.676560640335083,
+ "mean_token_accuracy": 0.6437469720840454,
+ "num_tokens": 1015808.0,
+ "step": 62
+ },
+ {
+ "entropy": 1.749995470046997,
+ "epoch": 0.12727272727272726,
+ "grad_norm": 2.1572678089141846,
+ "learning_rate": 4.8747474747474745e-06,
+ "loss": 1.784087896347046,
+ "mean_token_accuracy": 0.5884831547737122,
+ "num_tokens": 1032192.0,
+ "step": 63
+ },
+ {
+ "entropy": 1.442152738571167,
+ "epoch": 0.1292929292929293,
+ "grad_norm": 2.163490056991577,
+ "learning_rate": 4.872727272727273e-06,
+ "loss": 1.4307503700256348,
+ "mean_token_accuracy": 0.6618832349777222,
+ "num_tokens": 1048576.0,
+ "step": 64
+ },
+ {
+ "entropy": 1.2657029628753662,
+ "epoch": 0.13131313131313133,
+ "grad_norm": 2.1225337982177734,
+ "learning_rate": 4.870707070707071e-06,
+ "loss": 1.2731966972351074,
+ "mean_token_accuracy": 0.688568651676178,
+ "num_tokens": 1064960.0,
+ "step": 65
+ },
+ {
+ "entropy": 1.1613880395889282,
+ "epoch": 0.13333333333333333,
+ "grad_norm": 1.9527196884155273,
+ "learning_rate": 4.868686868686869e-06,
+ "loss": 1.13922119140625,
+ "mean_token_accuracy": 0.7389472126960754,
+ "num_tokens": 1081344.0,
+ "step": 66
+ },
+ {
+ "entropy": 1.6936380863189697,
+ "epoch": 0.13535353535353536,
+ "grad_norm": 2.004284620285034,
+ "learning_rate": 4.866666666666667e-06,
+ "loss": 1.6982238292694092,
+ "mean_token_accuracy": 0.6191987991333008,
+ "num_tokens": 1097728.0,
+ "step": 67
+ },
+ {
+ "entropy": 1.750565528869629,
+ "epoch": 0.13737373737373737,
+ "grad_norm": 2.225955009460449,
+ "learning_rate": 4.864646464646466e-06,
+ "loss": 1.796521782875061,
+ "mean_token_accuracy": 0.5934293866157532,
+ "num_tokens": 1114112.0,
+ "step": 68
+ },
+ {
+ "entropy": 1.6608220338821411,
+ "epoch": 0.1393939393939394,
+ "grad_norm": 2.127035617828369,
+ "learning_rate": 4.8626262626262636e-06,
+ "loss": 1.6633095741271973,
+ "mean_token_accuracy": 0.6356253027915955,
+ "num_tokens": 1130496.0,
+ "step": 69
+ },
+ {
+ "entropy": 1.4848661422729492,
+ "epoch": 0.1414141414141414,
+ "grad_norm": 2.0338215827941895,
+ "learning_rate": 4.8606060606060615e-06,
+ "loss": 1.4789674282073975,
+ "mean_token_accuracy": 0.6680508255958557,
+ "num_tokens": 1146880.0,
+ "step": 70
+ },
+ {
+ "entropy": 1.2310466766357422,
+ "epoch": 0.14343434343434344,
+ "grad_norm": 2.105898141860962,
+ "learning_rate": 4.858585858585859e-06,
+ "loss": 1.2301937341690063,
+ "mean_token_accuracy": 0.7040791511535645,
+ "num_tokens": 1163264.0,
+ "step": 71
+ },
+ {
+ "entropy": 1.6310514211654663,
+ "epoch": 0.14545454545454545,
+ "grad_norm": 2.152125358581543,
+ "learning_rate": 4.856565656565657e-06,
+ "loss": 1.643636703491211,
+ "mean_token_accuracy": 0.6221299171447754,
+ "num_tokens": 1179648.0,
+ "step": 72
+ },
+ {
+ "entropy": 1.4747940301895142,
+ "epoch": 0.14747474747474748,
+ "grad_norm": 2.096461296081543,
+ "learning_rate": 4.854545454545455e-06,
+ "loss": 1.42953360080719,
+ "mean_token_accuracy": 0.6651807427406311,
+ "num_tokens": 1196032.0,
+ "step": 73
+ },
+ {
+ "entropy": 1.819259524345398,
+ "epoch": 0.1494949494949495,
+ "grad_norm": 2.2852063179016113,
+ "learning_rate": 4.852525252525253e-06,
+ "loss": 1.845325231552124,
+ "mean_token_accuracy": 0.6036272644996643,
+ "num_tokens": 1212416.0,
+ "step": 74
+ },
+ {
+ "entropy": 1.4233598709106445,
+ "epoch": 0.15151515151515152,
+ "grad_norm": 2.1157381534576416,
+ "learning_rate": 4.850505050505051e-06,
+ "loss": 1.4565438032150269,
+ "mean_token_accuracy": 0.6607230305671692,
+ "num_tokens": 1228800.0,
+ "step": 75
+ },
+ {
+ "entropy": 1.6441459655761719,
+ "epoch": 0.15353535353535352,
+ "grad_norm": 2.2092180252075195,
+ "learning_rate": 4.848484848484849e-06,
+ "loss": 1.6467639207839966,
+ "mean_token_accuracy": 0.6285417675971985,
+ "num_tokens": 1245184.0,
+ "step": 76
+ },
+ {
+ "entropy": 1.6124308109283447,
+ "epoch": 0.15555555555555556,
+ "grad_norm": 2.231876850128174,
+ "learning_rate": 4.846464646464647e-06,
+ "loss": 1.617384672164917,
+ "mean_token_accuracy": 0.6340987086296082,
+ "num_tokens": 1261568.0,
+ "step": 77
+ },
+ {
+ "entropy": 1.828405499458313,
+ "epoch": 0.15757575757575756,
+ "grad_norm": 2.1372313499450684,
+ "learning_rate": 4.8444444444444446e-06,
+ "loss": 1.8333203792572021,
+ "mean_token_accuracy": 0.5979481935501099,
+ "num_tokens": 1277952.0,
+ "step": 78
+ },
+ {
+ "entropy": 1.855564832687378,
+ "epoch": 0.1595959595959596,
+ "grad_norm": 2.1422762870788574,
+ "learning_rate": 4.842424242424243e-06,
+ "loss": 1.9133609533309937,
+ "mean_token_accuracy": 0.570652186870575,
+ "num_tokens": 1294336.0,
+ "step": 79
+ },
+ {
+ "entropy": 1.7908183336257935,
+ "epoch": 0.16161616161616163,
+ "grad_norm": 2.172368049621582,
+ "learning_rate": 4.840404040404041e-06,
+ "loss": 1.8109419345855713,
+ "mean_token_accuracy": 0.60332190990448,
+ "num_tokens": 1310720.0,
+ "step": 80
+ },
+ {
+ "entropy": 1.9537721872329712,
+ "epoch": 0.16363636363636364,
+ "grad_norm": 2.199505090713501,
+ "learning_rate": 4.838383838383839e-06,
+ "loss": 2.0001401901245117,
+ "mean_token_accuracy": 0.5585613250732422,
+ "num_tokens": 1327104.0,
+ "step": 81
+ },
+ {
+ "entropy": 1.5365772247314453,
+ "epoch": 0.16565656565656567,
+ "grad_norm": 2.0778913497924805,
+ "learning_rate": 4.836363636363637e-06,
+ "loss": 1.575089931488037,
+ "mean_token_accuracy": 0.6375183463096619,
+ "num_tokens": 1343488.0,
+ "step": 82
+ },
+ {
+ "entropy": 1.7177143096923828,
+ "epoch": 0.16767676767676767,
+ "grad_norm": 2.1010894775390625,
+ "learning_rate": 4.834343434343435e-06,
+ "loss": 1.7428388595581055,
+ "mean_token_accuracy": 0.6113824844360352,
+ "num_tokens": 1359872.0,
+ "step": 83
+ },
+ {
+ "entropy": 1.5080527067184448,
+ "epoch": 0.1696969696969697,
+ "grad_norm": 2.021969795227051,
+ "learning_rate": 4.832323232323233e-06,
+ "loss": 1.5361201763153076,
+ "mean_token_accuracy": 0.656509518623352,
+ "num_tokens": 1376256.0,
+ "step": 84
+ },
+ {
+ "entropy": 1.5042120218276978,
+ "epoch": 0.1717171717171717,
+ "grad_norm": 2.121314287185669,
+ "learning_rate": 4.830303030303031e-06,
+ "loss": 1.4912033081054688,
+ "mean_token_accuracy": 0.6621274948120117,
+ "num_tokens": 1392640.0,
+ "step": 85
+ },
+ {
+ "entropy": 1.9523948431015015,
+ "epoch": 0.17373737373737375,
+ "grad_norm": 2.1702609062194824,
+ "learning_rate": 4.8282828282828285e-06,
+ "loss": 1.9369449615478516,
+ "mean_token_accuracy": 0.5727283954620361,
+ "num_tokens": 1409024.0,
+ "step": 86
+ },
+ {
+ "entropy": 1.6174770593643188,
+ "epoch": 0.17575757575757575,
+ "grad_norm": 2.222412109375,
+ "learning_rate": 4.826262626262626e-06,
+ "loss": 1.6794973611831665,
+ "mean_token_accuracy": 0.6151685118675232,
+ "num_tokens": 1425408.0,
+ "step": 87
+ },
+ {
+ "entropy": 1.1480307579040527,
+ "epoch": 0.17777777777777778,
+ "grad_norm": 1.98936128616333,
+ "learning_rate": 4.824242424242424e-06,
+ "loss": 1.1324063539505005,
+ "mean_token_accuracy": 0.7193453907966614,
+ "num_tokens": 1441792.0,
+ "step": 88
+ },
+ {
+ "entropy": 1.5290263891220093,
+ "epoch": 0.1797979797979798,
+ "grad_norm": 2.098860263824463,
+ "learning_rate": 4.822222222222222e-06,
+ "loss": 1.556044340133667,
+ "mean_token_accuracy": 0.6415486335754395,
+ "num_tokens": 1458176.0,
+ "step": 89
+ },
+ {
+ "entropy": 1.1366127729415894,
+ "epoch": 0.18181818181818182,
+ "grad_norm": 1.9387420415878296,
+ "learning_rate": 4.820202020202021e-06,
+ "loss": 1.1635141372680664,
+ "mean_token_accuracy": 0.7168416976928711,
+ "num_tokens": 1474560.0,
+ "step": 90
+ },
+ {
+ "entropy": 1.5142875909805298,
+ "epoch": 0.18383838383838383,
+ "grad_norm": 2.259131908416748,
+ "learning_rate": 4.818181818181819e-06,
+ "loss": 1.5329954624176025,
+ "mean_token_accuracy": 0.6471666097640991,
+ "num_tokens": 1490944.0,
+ "step": 91
+ },
+ {
+ "entropy": 1.5197135210037231,
+ "epoch": 0.18585858585858586,
+ "grad_norm": 2.1230807304382324,
+ "learning_rate": 4.816161616161617e-06,
+ "loss": 1.541062593460083,
+ "mean_token_accuracy": 0.6398388147354126,
+ "num_tokens": 1507328.0,
+ "step": 92
+ },
+ {
+ "entropy": 1.5469954013824463,
+ "epoch": 0.18787878787878787,
+ "grad_norm": 2.0568583011627197,
+ "learning_rate": 4.8141414141414146e-06,
+ "loss": 1.5564078092575073,
+ "mean_token_accuracy": 0.6384953856468201,
+ "num_tokens": 1523712.0,
+ "step": 93
+ },
+ {
+ "entropy": 1.5429692268371582,
+ "epoch": 0.1898989898989899,
+ "grad_norm": 2.200144052505493,
+ "learning_rate": 4.8121212121212125e-06,
+ "loss": 1.5292989015579224,
+ "mean_token_accuracy": 0.6373351216316223,
+ "num_tokens": 1540096.0,
+ "step": 94
+ },
+ {
+ "entropy": 1.4556076526641846,
+ "epoch": 0.1919191919191919,
+ "grad_norm": 2.250291585922241,
+ "learning_rate": 4.81010101010101e-06,
+ "loss": 1.426419734954834,
+ "mean_token_accuracy": 0.6604176759719849,
+ "num_tokens": 1556480.0,
+ "step": 95
+ },
+ {
+ "entropy": 1.4021440744400024,
+ "epoch": 0.19393939393939394,
+ "grad_norm": 2.287038803100586,
+ "learning_rate": 4.808080808080808e-06,
+ "loss": 1.4377000331878662,
+ "mean_token_accuracy": 0.6572422981262207,
+ "num_tokens": 1572864.0,
+ "step": 96
+ },
+ {
+ "entropy": 1.658683180809021,
+ "epoch": 0.19595959595959597,
+ "grad_norm": 2.1817963123321533,
+ "learning_rate": 4.806060606060606e-06,
+ "loss": 1.6514620780944824,
+ "mean_token_accuracy": 0.6278700828552246,
+ "num_tokens": 1589248.0,
+ "step": 97
+ },
+ {
+ "entropy": 1.5182844400405884,
+ "epoch": 0.19797979797979798,
+ "grad_norm": 2.144071340560913,
+ "learning_rate": 4.804040404040404e-06,
+ "loss": 1.5554628372192383,
+ "mean_token_accuracy": 0.6311064958572388,
+ "num_tokens": 1605632.0,
+ "step": 98
+ },
+ {
+ "entropy": 1.5997719764709473,
+ "epoch": 0.2,
+ "grad_norm": 2.0921664237976074,
+ "learning_rate": 4.802020202020202e-06,
+ "loss": 1.6185517311096191,
+ "mean_token_accuracy": 0.6286028623580933,
+ "num_tokens": 1622016.0,
+ "step": 99
+ },
+ {
+ "entropy": 1.4011279344558716,
+ "epoch": 0.20202020202020202,
+ "grad_norm": 2.2240700721740723,
+ "learning_rate": 4.800000000000001e-06,
+ "loss": 1.4112927913665771,
+ "mean_token_accuracy": 0.6696995496749878,
+ "num_tokens": 1638400.0,
+ "step": 100
+ },
+ {
+ "epoch": 0.20202020202020202,
+ "eval_entropy": 1.606662715634992,
+ "eval_loss": 1.6283859014511108,
+ "eval_mean_token_accuracy": 0.6293055717983553,
+ "eval_num_tokens": 1638400.0,
+ "eval_runtime": 43.7782,
+ "eval_samples_per_second": 22.614,
+ "eval_steps_per_second": 2.832,
+ "step": 100
+ },
+ {
+ "entropy": 1.6743865013122559,
+ "epoch": 0.20404040404040405,
+ "grad_norm": 2.025153398513794,
+ "learning_rate": 4.7979797979797985e-06,
+ "loss": 1.6404725313186646,
+ "mean_token_accuracy": 0.656509518623352,
+ "num_tokens": 1654784.0,
+ "step": 101
+ },
+ {
+ "entropy": 1.6196308135986328,
+ "epoch": 0.20606060606060606,
+ "grad_norm": 2.037229299545288,
+ "learning_rate": 4.795959595959596e-06,
+ "loss": 1.647303581237793,
+ "mean_token_accuracy": 0.6203590631484985,
+ "num_tokens": 1671168.0,
+ "step": 102
+ },
+ {
+ "entropy": 1.5600125789642334,
+ "epoch": 0.2080808080808081,
+ "grad_norm": 2.144221782684326,
+ "learning_rate": 4.793939393939394e-06,
+ "loss": 1.6199731826782227,
+ "mean_token_accuracy": 0.6249389052391052,
+ "num_tokens": 1687552.0,
+ "step": 103
+ },
+ {
+ "entropy": 2.064497947692871,
+ "epoch": 0.2101010101010101,
+ "grad_norm": 2.3956470489501953,
+ "learning_rate": 4.791919191919192e-06,
+ "loss": 2.048987627029419,
+ "mean_token_accuracy": 0.5507450103759766,
+ "num_tokens": 1703936.0,
+ "step": 104
+ },
+ {
+ "entropy": 1.7934812307357788,
+ "epoch": 0.21212121212121213,
+ "grad_norm": 2.487302780151367,
+ "learning_rate": 4.78989898989899e-06,
+ "loss": 1.8285956382751465,
+ "mean_token_accuracy": 0.5749877691268921,
+ "num_tokens": 1720320.0,
+ "step": 105
+ },
+ {
+ "entropy": 1.365216851234436,
+ "epoch": 0.21414141414141413,
+ "grad_norm": 2.0714964866638184,
+ "learning_rate": 4.787878787878788e-06,
+ "loss": 1.3870220184326172,
+ "mean_token_accuracy": 0.6692721247673035,
+ "num_tokens": 1736704.0,
+ "step": 106
+ },
+ {
+ "entropy": 1.4983874559402466,
+ "epoch": 0.21616161616161617,
+ "grad_norm": 1.9995853900909424,
+ "learning_rate": 4.785858585858586e-06,
+ "loss": 1.4949266910552979,
+ "mean_token_accuracy": 0.6554103493690491,
+ "num_tokens": 1753088.0,
+ "step": 107
+ },
+ {
+ "entropy": 2.0503509044647217,
+ "epoch": 0.21818181818181817,
+ "grad_norm": 2.190884590148926,
+ "learning_rate": 4.783838383838385e-06,
+ "loss": 2.079286813735962,
+ "mean_token_accuracy": 0.5657669901847839,
+ "num_tokens": 1769472.0,
+ "step": 108
+ },
+ {
+ "entropy": 1.5557374954223633,
+ "epoch": 0.2202020202020202,
+ "grad_norm": 2.044100761413574,
+ "learning_rate": 4.7818181818181825e-06,
+ "loss": 1.5735318660736084,
+ "mean_token_accuracy": 0.6359916925430298,
+ "num_tokens": 1785856.0,
+ "step": 109
+ },
+ {
+ "entropy": 1.5567635297775269,
+ "epoch": 0.2222222222222222,
+ "grad_norm": 2.3714160919189453,
+ "learning_rate": 4.77979797979798e-06,
+ "loss": 1.6026921272277832,
+ "mean_token_accuracy": 0.6290302872657776,
+ "num_tokens": 1802240.0,
+ "step": 110
+ },
+ {
+ "entropy": 1.8457536697387695,
+ "epoch": 0.22424242424242424,
+ "grad_norm": 2.202939987182617,
+ "learning_rate": 4.777777777777778e-06,
+ "loss": 1.7986081838607788,
+ "mean_token_accuracy": 0.5952613353729248,
+ "num_tokens": 1818624.0,
+ "step": 111
+ },
+ {
+ "entropy": 1.8822020292282104,
+ "epoch": 0.22626262626262628,
+ "grad_norm": 2.2674992084503174,
+ "learning_rate": 4.775757575757576e-06,
+ "loss": 1.9095954895019531,
+ "mean_token_accuracy": 0.5769418478012085,
+ "num_tokens": 1835008.0,
+ "step": 112
+ },
+ {
+ "entropy": 1.5445998907089233,
+ "epoch": 0.22828282828282828,
+ "grad_norm": 2.2848100662231445,
+ "learning_rate": 4.773737373737374e-06,
+ "loss": 1.5832195281982422,
+ "mean_token_accuracy": 0.6337933540344238,
+ "num_tokens": 1851392.0,
+ "step": 113
+ },
+ {
+ "entropy": 1.934509515762329,
+ "epoch": 0.23030303030303031,
+ "grad_norm": 2.3585174083709717,
+ "learning_rate": 4.771717171717172e-06,
+ "loss": 1.9809319972991943,
+ "mean_token_accuracy": 0.5789570212364197,
+ "num_tokens": 1867776.0,
+ "step": 114
+ },
+ {
+ "entropy": 1.7354214191436768,
+ "epoch": 0.23232323232323232,
+ "grad_norm": 2.13913631439209,
+ "learning_rate": 4.769696969696971e-06,
+ "loss": 1.7527806758880615,
+ "mean_token_accuracy": 0.6061308979988098,
+ "num_tokens": 1884160.0,
+ "step": 115
+ },
+ {
+ "entropy": 1.7714784145355225,
+ "epoch": 0.23434343434343435,
+ "grad_norm": 1.9915403127670288,
+ "learning_rate": 4.7676767676767685e-06,
+ "loss": 1.8065431118011475,
+ "mean_token_accuracy": 0.5986199378967285,
+ "num_tokens": 1900544.0,
+ "step": 116
+ },
+ {
+ "entropy": 1.5272125005722046,
+ "epoch": 0.23636363636363636,
+ "grad_norm": 2.004565715789795,
+ "learning_rate": 4.765656565656566e-06,
+ "loss": 1.5074517726898193,
+ "mean_token_accuracy": 0.6601123809814453,
+ "num_tokens": 1916928.0,
+ "step": 117
+ },
+ {
+ "entropy": 1.5393218994140625,
+ "epoch": 0.2383838383838384,
+ "grad_norm": 2.018089532852173,
+ "learning_rate": 4.763636363636364e-06,
+ "loss": 1.5607573986053467,
+ "mean_token_accuracy": 0.6540058851242065,
+ "num_tokens": 1933312.0,
+ "step": 118
+ },
+ {
+ "entropy": 1.7827534675598145,
+ "epoch": 0.2404040404040404,
+ "grad_norm": 2.062368392944336,
+ "learning_rate": 4.761616161616162e-06,
+ "loss": 1.7844001054763794,
+ "mean_token_accuracy": 0.6187102794647217,
+ "num_tokens": 1949696.0,
+ "step": 119
+ },
+ {
+ "entropy": 1.416417121887207,
+ "epoch": 0.24242424242424243,
+ "grad_norm": 2.0980024337768555,
+ "learning_rate": 4.75959595959596e-06,
+ "loss": 1.4041050672531128,
+ "mean_token_accuracy": 0.6561431288719177,
+ "num_tokens": 1966080.0,
+ "step": 120
+ },
+ {
+ "entropy": 1.3325153589248657,
+ "epoch": 0.24444444444444444,
+ "grad_norm": 1.9985002279281616,
+ "learning_rate": 4.757575757575758e-06,
+ "loss": 1.3197274208068848,
+ "mean_token_accuracy": 0.6806912422180176,
+ "num_tokens": 1982464.0,
+ "step": 121
+ },
+ {
+ "entropy": 1.7306798696517944,
+ "epoch": 0.24646464646464647,
+ "grad_norm": 2.2098441123962402,
+ "learning_rate": 4.755555555555556e-06,
+ "loss": 1.7595295906066895,
+ "mean_token_accuracy": 0.6110160946846008,
+ "num_tokens": 1998848.0,
+ "step": 122
+ },
+ {
+ "entropy": 1.3733264207839966,
+ "epoch": 0.24848484848484848,
+ "grad_norm": 1.9827327728271484,
+ "learning_rate": 4.753535353535354e-06,
+ "loss": 1.4026854038238525,
+ "mean_token_accuracy": 0.65486079454422,
+ "num_tokens": 2015232.0,
+ "step": 123
+ },
+ {
+ "entropy": 1.5910528898239136,
+ "epoch": 0.2505050505050505,
+ "grad_norm": 1.9615319967269897,
+ "learning_rate": 4.751515151515152e-06,
+ "loss": 1.6071114540100098,
+ "mean_token_accuracy": 0.648937463760376,
+ "num_tokens": 2031616.0,
+ "step": 124
+ },
+ {
+ "entropy": 1.3300938606262207,
+ "epoch": 0.25252525252525254,
+ "grad_norm": 1.9878504276275635,
+ "learning_rate": 4.7494949494949495e-06,
+ "loss": 1.3623383045196533,
+ "mean_token_accuracy": 0.6873473525047302,
+ "num_tokens": 2048000.0,
+ "step": 125
+ },
+ {
+ "entropy": 1.6087368726730347,
+ "epoch": 0.2545454545454545,
+ "grad_norm": 2.264647960662842,
+ "learning_rate": 4.747474747474748e-06,
+ "loss": 1.634058952331543,
+ "mean_token_accuracy": 0.6211528778076172,
+ "num_tokens": 2064384.0,
+ "step": 126
+ },
+ {
+ "entropy": 1.536401391029358,
+ "epoch": 0.25656565656565655,
+ "grad_norm": 1.9561539888381958,
+ "learning_rate": 4.745454545454546e-06,
+ "loss": 1.5699501037597656,
+ "mean_token_accuracy": 0.6248167753219604,
+ "num_tokens": 2080768.0,
+ "step": 127
+ },
+ {
+ "entropy": 1.656266212463379,
+ "epoch": 0.2585858585858586,
+ "grad_norm": 2.1124353408813477,
+ "learning_rate": 4.743434343434344e-06,
+ "loss": 1.6612167358398438,
+ "mean_token_accuracy": 0.623900830745697,
+ "num_tokens": 2097152.0,
+ "step": 128
+ },
+ {
+ "entropy": 1.170629620552063,
+ "epoch": 0.2606060606060606,
+ "grad_norm": 1.917840600013733,
+ "learning_rate": 4.741414141414142e-06,
+ "loss": 1.1992300748825073,
+ "mean_token_accuracy": 0.6995603442192078,
+ "num_tokens": 2113536.0,
+ "step": 129
+ },
+ {
+ "entropy": 1.4126694202423096,
+ "epoch": 0.26262626262626265,
+ "grad_norm": 1.9041539430618286,
+ "learning_rate": 4.73939393939394e-06,
+ "loss": 1.4334447383880615,
+ "mean_token_accuracy": 0.6637151837348938,
+ "num_tokens": 2129920.0,
+ "step": 130
+ },
+ {
+ "entropy": 1.8490277528762817,
+ "epoch": 0.26464646464646463,
+ "grad_norm": 2.1440138816833496,
+ "learning_rate": 4.737373737373738e-06,
+ "loss": 1.9226353168487549,
+ "mean_token_accuracy": 0.5805447101593018,
+ "num_tokens": 2146304.0,
+ "step": 131
+ },
+ {
+ "entropy": 1.5000964403152466,
+ "epoch": 0.26666666666666666,
+ "grad_norm": 1.9615508317947388,
+ "learning_rate": 4.735353535353536e-06,
+ "loss": 1.495596170425415,
+ "mean_token_accuracy": 0.6542501449584961,
+ "num_tokens": 2162688.0,
+ "step": 132
+ },
+ {
+ "entropy": 1.9371142387390137,
+ "epoch": 0.2686868686868687,
+ "grad_norm": 2.202200412750244,
+ "learning_rate": 4.7333333333333335e-06,
+ "loss": 1.94151771068573,
+ "mean_token_accuracy": 0.5810332298278809,
+ "num_tokens": 2179072.0,
+ "step": 133
+ },
+ {
+ "entropy": 1.4535126686096191,
+ "epoch": 0.27070707070707073,
+ "grad_norm": 1.9804027080535889,
+ "learning_rate": 4.731313131313131e-06,
+ "loss": 1.4401545524597168,
+ "mean_token_accuracy": 0.6585246920585632,
+ "num_tokens": 2195456.0,
+ "step": 134
+ },
+ {
+ "entropy": 2.0871424674987793,
+ "epoch": 0.2727272727272727,
+ "grad_norm": 2.1417081356048584,
+ "learning_rate": 4.729292929292929e-06,
+ "loss": 2.117375373840332,
+ "mean_token_accuracy": 0.5600268840789795,
+ "num_tokens": 2211840.0,
+ "step": 135
+ },
+ {
+ "entropy": 1.2986469268798828,
+ "epoch": 0.27474747474747474,
+ "grad_norm": 2.0299136638641357,
+ "learning_rate": 4.727272727272728e-06,
+ "loss": 1.3631991147994995,
+ "mean_token_accuracy": 0.6750732660293579,
+ "num_tokens": 2228224.0,
+ "step": 136
+ },
+ {
+ "entropy": 1.163429617881775,
+ "epoch": 0.2767676767676768,
+ "grad_norm": 1.8368210792541504,
+ "learning_rate": 4.725252525252526e-06,
+ "loss": 1.1603795289993286,
+ "mean_token_accuracy": 0.7150708436965942,
+ "num_tokens": 2244608.0,
+ "step": 137
+ },
+ {
+ "entropy": 1.4301519393920898,
+ "epoch": 0.2787878787878788,
+ "grad_norm": 1.996767520904541,
+ "learning_rate": 4.723232323232324e-06,
+ "loss": 1.422170639038086,
+ "mean_token_accuracy": 0.6816682815551758,
+ "num_tokens": 2260992.0,
+ "step": 138
+ },
+ {
+ "entropy": 1.5160106420516968,
+ "epoch": 0.2808080808080808,
+ "grad_norm": 2.004770278930664,
+ "learning_rate": 4.721212121212122e-06,
+ "loss": 1.5385751724243164,
+ "mean_token_accuracy": 0.640693724155426,
+ "num_tokens": 2277376.0,
+ "step": 139
+ },
+ {
+ "entropy": 1.2483867406845093,
+ "epoch": 0.2828282828282828,
+ "grad_norm": 1.8488364219665527,
+ "learning_rate": 4.7191919191919195e-06,
+ "loss": 1.2563843727111816,
+ "mean_token_accuracy": 0.6998046040534973,
+ "num_tokens": 2293760.0,
+ "step": 140
+ },
+ {
+ "entropy": 1.3214129209518433,
+ "epoch": 0.28484848484848485,
+ "grad_norm": 2.1334660053253174,
+ "learning_rate": 4.717171717171717e-06,
+ "loss": 1.3392664194107056,
+ "mean_token_accuracy": 0.6822789311408997,
+ "num_tokens": 2310144.0,
+ "step": 141
+ },
+ {
+ "entropy": 1.611849069595337,
+ "epoch": 0.2868686868686869,
+ "grad_norm": 2.0539803504943848,
+ "learning_rate": 4.715151515151515e-06,
+ "loss": 1.6335396766662598,
+ "mean_token_accuracy": 0.6235954761505127,
+ "num_tokens": 2326528.0,
+ "step": 142
+ },
+ {
+ "entropy": 1.5847896337509155,
+ "epoch": 0.28888888888888886,
+ "grad_norm": 2.1658759117126465,
+ "learning_rate": 4.713131313131313e-06,
+ "loss": 1.603764533996582,
+ "mean_token_accuracy": 0.633671224117279,
+ "num_tokens": 2342912.0,
+ "step": 143
+ },
+ {
+ "entropy": 1.5731406211853027,
+ "epoch": 0.2909090909090909,
+ "grad_norm": 2.0830390453338623,
+ "learning_rate": 4.711111111111111e-06,
+ "loss": 1.558933973312378,
+ "mean_token_accuracy": 0.6392892003059387,
+ "num_tokens": 2359296.0,
+ "step": 144
+ },
+ {
+ "entropy": 1.8901628255844116,
+ "epoch": 0.29292929292929293,
+ "grad_norm": 2.1436922550201416,
+ "learning_rate": 4.709090909090909e-06,
+ "loss": 1.878631353378296,
+ "mean_token_accuracy": 0.57333904504776,
+ "num_tokens": 2375680.0,
+ "step": 145
+ },
+ {
+ "entropy": 1.9344228506088257,
+ "epoch": 0.29494949494949496,
+ "grad_norm": 2.1245176792144775,
+ "learning_rate": 4.707070707070707e-06,
+ "loss": 1.9546704292297363,
+ "mean_token_accuracy": 0.5987420678138733,
+ "num_tokens": 2392064.0,
+ "step": 146
+ },
+ {
+ "entropy": 1.2455096244812012,
+ "epoch": 0.296969696969697,
+ "grad_norm": 1.896581768989563,
+ "learning_rate": 4.705050505050506e-06,
+ "loss": 1.2738728523254395,
+ "mean_token_accuracy": 0.7025524973869324,
+ "num_tokens": 2408448.0,
+ "step": 147
+ },
+ {
+ "entropy": 1.4235948324203491,
+ "epoch": 0.298989898989899,
+ "grad_norm": 2.522636890411377,
+ "learning_rate": 4.7030303030303035e-06,
+ "loss": 1.4524480104446411,
+ "mean_token_accuracy": 0.6441133618354797,
+ "num_tokens": 2424832.0,
+ "step": 148
+ },
+ {
+ "entropy": 1.764552116394043,
+ "epoch": 0.301010101010101,
+ "grad_norm": 2.0232255458831787,
+ "learning_rate": 4.701010101010101e-06,
+ "loss": 1.7811740636825562,
+ "mean_token_accuracy": 0.6229848265647888,
+ "num_tokens": 2441216.0,
+ "step": 149
+ },
+ {
+ "entropy": 1.74843168258667,
+ "epoch": 0.30303030303030304,
+ "grad_norm": 2.1589369773864746,
+ "learning_rate": 4.698989898989899e-06,
+ "loss": 1.7337679862976074,
+ "mean_token_accuracy": 0.5975207686424255,
+ "num_tokens": 2457600.0,
+ "step": 150
+ },
+ {
+ "epoch": 0.30303030303030304,
+ "eval_entropy": 1.6033287221385586,
+ "eval_loss": 1.6036633253097534,
+ "eval_mean_token_accuracy": 0.6329842450157288,
+ "eval_num_tokens": 2457600.0,
+ "eval_runtime": 43.7655,
+ "eval_samples_per_second": 22.621,
+ "eval_steps_per_second": 2.833,
+ "step": 150
+ },
+ {
+ "entropy": 2.0572290420532227,
+ "epoch": 0.30505050505050507,
+ "grad_norm": 2.0511579513549805,
+ "learning_rate": 4.696969696969698e-06,
+ "loss": 2.0196030139923096,
+ "mean_token_accuracy": 0.5519052147865295,
+ "num_tokens": 2473984.0,
+ "step": 151
+ },
+ {
+ "entropy": 1.5792397260665894,
+ "epoch": 0.30707070707070705,
+ "grad_norm": 2.048396587371826,
+ "learning_rate": 4.694949494949496e-06,
+ "loss": 1.575985074043274,
+ "mean_token_accuracy": 0.623961865901947,
+ "num_tokens": 2490368.0,
+ "step": 152
+ },
+ {
+ "entropy": 1.487528681755066,
+ "epoch": 0.3090909090909091,
+ "grad_norm": 2.1903791427612305,
+ "learning_rate": 4.692929292929294e-06,
+ "loss": 1.479973316192627,
+ "mean_token_accuracy": 0.6595017313957214,
+ "num_tokens": 2506752.0,
+ "step": 153
+ },
+ {
+ "entropy": 1.766797423362732,
+ "epoch": 0.3111111111111111,
+ "grad_norm": 2.1843011379241943,
+ "learning_rate": 4.690909090909092e-06,
+ "loss": 1.7993412017822266,
+ "mean_token_accuracy": 0.5965437293052673,
+ "num_tokens": 2523136.0,
+ "step": 154
+ },
+ {
+ "entropy": 1.7058254480361938,
+ "epoch": 0.31313131313131315,
+ "grad_norm": 2.204461097717285,
+ "learning_rate": 4.6888888888888895e-06,
+ "loss": 1.7346007823944092,
+ "mean_token_accuracy": 0.6077185869216919,
+ "num_tokens": 2539520.0,
+ "step": 155
+ },
+ {
+ "entropy": 1.4929591417312622,
+ "epoch": 0.3151515151515151,
+ "grad_norm": 2.1739237308502197,
+ "learning_rate": 4.6868686868686874e-06,
+ "loss": 1.4708621501922607,
+ "mean_token_accuracy": 0.6427088379859924,
+ "num_tokens": 2555904.0,
+ "step": 156
+ },
+ {
+ "entropy": 1.5797587633132935,
+ "epoch": 0.31717171717171716,
+ "grad_norm": 2.150561571121216,
+ "learning_rate": 4.684848484848485e-06,
+ "loss": 1.5921857357025146,
+ "mean_token_accuracy": 0.6262823939323425,
+ "num_tokens": 2572288.0,
+ "step": 157
+ },
+ {
+ "entropy": 1.6198230981826782,
+ "epoch": 0.3191919191919192,
+ "grad_norm": 2.061169385910034,
+ "learning_rate": 4.682828282828283e-06,
+ "loss": 1.6601009368896484,
+ "mean_token_accuracy": 0.619015634059906,
+ "num_tokens": 2588672.0,
+ "step": 158
+ },
+ {
+ "entropy": 1.4026126861572266,
+ "epoch": 0.3212121212121212,
+ "grad_norm": 2.066208839416504,
+ "learning_rate": 4.680808080808081e-06,
+ "loss": 1.4062483310699463,
+ "mean_token_accuracy": 0.6681729555130005,
+ "num_tokens": 2605056.0,
+ "step": 159
+ },
+ {
+ "entropy": 1.4608066082000732,
+ "epoch": 0.32323232323232326,
+ "grad_norm": 2.022627353668213,
+ "learning_rate": 4.678787878787879e-06,
+ "loss": 1.4644970893859863,
+ "mean_token_accuracy": 0.6533341407775879,
+ "num_tokens": 2621440.0,
+ "step": 160
+ },
+ {
+ "entropy": 1.222448468208313,
+ "epoch": 0.32525252525252524,
+ "grad_norm": 2.0399329662323,
+ "learning_rate": 4.676767676767677e-06,
+ "loss": 1.2547852993011475,
+ "mean_token_accuracy": 0.6925378441810608,
+ "num_tokens": 2637824.0,
+ "step": 161
+ },
+ {
+ "entropy": 1.8702255487442017,
+ "epoch": 0.32727272727272727,
+ "grad_norm": 2.177307367324829,
+ "learning_rate": 4.674747474747476e-06,
+ "loss": 1.9041712284088135,
+ "mean_token_accuracy": 0.5754152536392212,
+ "num_tokens": 2654208.0,
+ "step": 162
+ },
+ {
+ "entropy": 1.6861947774887085,
+ "epoch": 0.3292929292929293,
+ "grad_norm": 2.009544610977173,
+ "learning_rate": 4.6727272727272735e-06,
+ "loss": 1.7079355716705322,
+ "mean_token_accuracy": 0.615290641784668,
+ "num_tokens": 2670592.0,
+ "step": 163
+ },
+ {
+ "entropy": 1.3880327939987183,
+ "epoch": 0.33131313131313134,
+ "grad_norm": 2.16359543800354,
+ "learning_rate": 4.670707070707071e-06,
+ "loss": 1.4220571517944336,
+ "mean_token_accuracy": 0.6604787707328796,
+ "num_tokens": 2686976.0,
+ "step": 164
+ },
+ {
+ "entropy": 1.5760643482208252,
+ "epoch": 0.3333333333333333,
+ "grad_norm": 2.09773325920105,
+ "learning_rate": 4.668686868686869e-06,
+ "loss": 1.6036784648895264,
+ "mean_token_accuracy": 0.6267098188400269,
+ "num_tokens": 2703360.0,
+ "step": 165
+ },
+ {
+ "entropy": 1.078303575515747,
+ "epoch": 0.33535353535353535,
+ "grad_norm": 1.7760599851608276,
+ "learning_rate": 4.666666666666667e-06,
+ "loss": 1.0612695217132568,
+ "mean_token_accuracy": 0.7349169254302979,
+ "num_tokens": 2719744.0,
+ "step": 166
+ },
+ {
+ "entropy": 1.3722130060195923,
+ "epoch": 0.3373737373737374,
+ "grad_norm": 2.0816409587860107,
+ "learning_rate": 4.664646464646465e-06,
+ "loss": 1.3693504333496094,
+ "mean_token_accuracy": 0.6682950854301453,
+ "num_tokens": 2736128.0,
+ "step": 167
+ },
+ {
+ "entropy": 1.8752760887145996,
+ "epoch": 0.3393939393939394,
+ "grad_norm": 2.1670358180999756,
+ "learning_rate": 4.662626262626263e-06,
+ "loss": 1.8874578475952148,
+ "mean_token_accuracy": 0.5881778001785278,
+ "num_tokens": 2752512.0,
+ "step": 168
+ },
+ {
+ "entropy": 1.7384581565856934,
+ "epoch": 0.3414141414141414,
+ "grad_norm": 2.0072429180145264,
+ "learning_rate": 4.660606060606061e-06,
+ "loss": 1.7491583824157715,
+ "mean_token_accuracy": 0.6041157841682434,
+ "num_tokens": 2768896.0,
+ "step": 169
+ },
+ {
+ "entropy": 1.6651691198349,
+ "epoch": 0.3434343434343434,
+ "grad_norm": 2.0812578201293945,
+ "learning_rate": 4.658585858585859e-06,
+ "loss": 1.6808438301086426,
+ "mean_token_accuracy": 0.6273204684257507,
+ "num_tokens": 2785280.0,
+ "step": 170
+ },
+ {
+ "entropy": 1.3426616191864014,
+ "epoch": 0.34545454545454546,
+ "grad_norm": 1.9108822345733643,
+ "learning_rate": 4.656565656565657e-06,
+ "loss": 1.3404264450073242,
+ "mean_token_accuracy": 0.6703101992607117,
+ "num_tokens": 2801664.0,
+ "step": 171
+ },
+ {
+ "entropy": 1.631722092628479,
+ "epoch": 0.3474747474747475,
+ "grad_norm": 2.07317852973938,
+ "learning_rate": 4.654545454545455e-06,
+ "loss": 1.6545813083648682,
+ "mean_token_accuracy": 0.6143136024475098,
+ "num_tokens": 2818048.0,
+ "step": 172
+ },
+ {
+ "entropy": 1.3678289651870728,
+ "epoch": 0.34949494949494947,
+ "grad_norm": 1.8517454862594604,
+ "learning_rate": 4.652525252525253e-06,
+ "loss": 1.3759769201278687,
+ "mean_token_accuracy": 0.6693942546844482,
+ "num_tokens": 2834432.0,
+ "step": 173
+ },
+ {
+ "entropy": 1.3927773237228394,
+ "epoch": 0.3515151515151515,
+ "grad_norm": 2.1304867267608643,
+ "learning_rate": 4.650505050505051e-06,
+ "loss": 1.4104211330413818,
+ "mean_token_accuracy": 0.6612725853919983,
+ "num_tokens": 2850816.0,
+ "step": 174
+ },
+ {
+ "entropy": 1.3352863788604736,
+ "epoch": 0.35353535353535354,
+ "grad_norm": 2.2788777351379395,
+ "learning_rate": 4.648484848484849e-06,
+ "loss": 1.3665719032287598,
+ "mean_token_accuracy": 0.6640205383300781,
+ "num_tokens": 2867200.0,
+ "step": 175
+ },
+ {
+ "entropy": 1.6800754070281982,
+ "epoch": 0.35555555555555557,
+ "grad_norm": 2.030787944793701,
+ "learning_rate": 4.646464646464647e-06,
+ "loss": 1.7239181995391846,
+ "mean_token_accuracy": 0.6193209290504456,
+ "num_tokens": 2883584.0,
+ "step": 176
+ },
+ {
+ "entropy": 1.2852731943130493,
+ "epoch": 0.3575757575757576,
+ "grad_norm": 1.9148248434066772,
+ "learning_rate": 4.644444444444445e-06,
+ "loss": 1.2934880256652832,
+ "mean_token_accuracy": 0.6798363327980042,
+ "num_tokens": 2899968.0,
+ "step": 177
+ },
+ {
+ "entropy": 1.6366593837738037,
+ "epoch": 0.3595959595959596,
+ "grad_norm": 2.2264015674591064,
+ "learning_rate": 4.642424242424243e-06,
+ "loss": 1.632002592086792,
+ "mean_token_accuracy": 0.6180385947227478,
+ "num_tokens": 2916352.0,
+ "step": 178
+ },
+ {
+ "entropy": 1.3935565948486328,
+ "epoch": 0.3616161616161616,
+ "grad_norm": 2.0410475730895996,
+ "learning_rate": 4.6404040404040406e-06,
+ "loss": 1.407370686531067,
+ "mean_token_accuracy": 0.6722032427787781,
+ "num_tokens": 2932736.0,
+ "step": 179
+ },
+ {
+ "entropy": 1.0952510833740234,
+ "epoch": 0.36363636363636365,
+ "grad_norm": 1.937270164489746,
+ "learning_rate": 4.6383838383838384e-06,
+ "loss": 1.1101973056793213,
+ "mean_token_accuracy": 0.730947732925415,
+ "num_tokens": 2949120.0,
+ "step": 180
+ },
+ {
+ "entropy": 1.184926152229309,
+ "epoch": 0.3656565656565657,
+ "grad_norm": 1.9867528676986694,
+ "learning_rate": 4.636363636363636e-06,
+ "loss": 1.1932196617126465,
+ "mean_token_accuracy": 0.7085368633270264,
+ "num_tokens": 2965504.0,
+ "step": 181
+ },
+ {
+ "entropy": 1.4552072286605835,
+ "epoch": 0.36767676767676766,
+ "grad_norm": 1.9395766258239746,
+ "learning_rate": 4.634343434343434e-06,
+ "loss": 1.4559956789016724,
+ "mean_token_accuracy": 0.6534562706947327,
+ "num_tokens": 2981888.0,
+ "step": 182
+ },
+ {
+ "entropy": 1.6437443494796753,
+ "epoch": 0.3696969696969697,
+ "grad_norm": 2.1257567405700684,
+ "learning_rate": 4.632323232323233e-06,
+ "loss": 1.6260980367660522,
+ "mean_token_accuracy": 0.625,
+ "num_tokens": 2998272.0,
+ "step": 183
+ },
+ {
+ "entropy": 1.8264633417129517,
+ "epoch": 0.3717171717171717,
+ "grad_norm": 2.187041759490967,
+ "learning_rate": 4.630303030303031e-06,
+ "loss": 1.8431676626205444,
+ "mean_token_accuracy": 0.60332190990448,
+ "num_tokens": 3014656.0,
+ "step": 184
+ },
+ {
+ "entropy": 1.1121351718902588,
+ "epoch": 0.37373737373737376,
+ "grad_norm": 1.8460402488708496,
+ "learning_rate": 4.628282828282829e-06,
+ "loss": 1.1054425239562988,
+ "mean_token_accuracy": 0.7254518866539001,
+ "num_tokens": 3031040.0,
+ "step": 185
+ },
+ {
+ "entropy": 1.3531955480575562,
+ "epoch": 0.37575757575757573,
+ "grad_norm": 2.3463470935821533,
+ "learning_rate": 4.626262626262627e-06,
+ "loss": 1.359381914138794,
+ "mean_token_accuracy": 0.6610894203186035,
+ "num_tokens": 3047424.0,
+ "step": 186
+ },
+ {
+ "entropy": 1.2131577730178833,
+ "epoch": 0.37777777777777777,
+ "grad_norm": 1.9595171213150024,
+ "learning_rate": 4.6242424242424245e-06,
+ "loss": 1.2312005758285522,
+ "mean_token_accuracy": 0.6979115605354309,
+ "num_tokens": 3063808.0,
+ "step": 187
+ },
+ {
+ "entropy": 1.5441125631332397,
+ "epoch": 0.3797979797979798,
+ "grad_norm": 2.2971343994140625,
+ "learning_rate": 4.622222222222222e-06,
+ "loss": 1.5923339128494263,
+ "mean_token_accuracy": 0.6297020316123962,
+ "num_tokens": 3080192.0,
+ "step": 188
+ },
+ {
+ "entropy": 1.7998323440551758,
+ "epoch": 0.38181818181818183,
+ "grad_norm": 2.010552167892456,
+ "learning_rate": 4.62020202020202e-06,
+ "loss": 1.8257204294204712,
+ "mean_token_accuracy": 0.5997191071510315,
+ "num_tokens": 3096576.0,
+ "step": 189
+ },
+ {
+ "entropy": 1.389051914215088,
+ "epoch": 0.3838383838383838,
+ "grad_norm": 2.1183855533599854,
+ "learning_rate": 4.618181818181818e-06,
+ "loss": 1.431575059890747,
+ "mean_token_accuracy": 0.6537005305290222,
+ "num_tokens": 3112960.0,
+ "step": 190
+ },
+ {
+ "entropy": 2.0066142082214355,
+ "epoch": 0.38585858585858585,
+ "grad_norm": 2.061399221420288,
+ "learning_rate": 4.616161616161616e-06,
+ "loss": 2.0477406978607178,
+ "mean_token_accuracy": 0.5454934239387512,
+ "num_tokens": 3129344.0,
+ "step": 191
+ },
+ {
+ "entropy": 1.3063198328018188,
+ "epoch": 0.3878787878787879,
+ "grad_norm": 1.9490301609039307,
+ "learning_rate": 4.614141414141414e-06,
+ "loss": 1.3293564319610596,
+ "mean_token_accuracy": 0.6797142028808594,
+ "num_tokens": 3145728.0,
+ "step": 192
+ },
+ {
+ "entropy": 1.3636538982391357,
+ "epoch": 0.3898989898989899,
+ "grad_norm": 2.025458574295044,
+ "learning_rate": 4.612121212121212e-06,
+ "loss": 1.3736209869384766,
+ "mean_token_accuracy": 0.679286777973175,
+ "num_tokens": 3162112.0,
+ "step": 193
+ },
+ {
+ "entropy": 1.5669925212860107,
+ "epoch": 0.39191919191919194,
+ "grad_norm": 2.064033269882202,
+ "learning_rate": 4.6101010101010106e-06,
+ "loss": 1.5587083101272583,
+ "mean_token_accuracy": 0.6267709136009216,
+ "num_tokens": 3178496.0,
+ "step": 194
+ },
+ {
+ "entropy": 1.4489716291427612,
+ "epoch": 0.3939393939393939,
+ "grad_norm": 2.0174880027770996,
+ "learning_rate": 4.6080808080808085e-06,
+ "loss": 1.4331417083740234,
+ "mean_token_accuracy": 0.6571812629699707,
+ "num_tokens": 3194880.0,
+ "step": 195
+ },
+ {
+ "entropy": 2.026416778564453,
+ "epoch": 0.39595959595959596,
+ "grad_norm": 2.1820878982543945,
+ "learning_rate": 4.606060606060606e-06,
+ "loss": 2.029003143310547,
+ "mean_token_accuracy": 0.563568651676178,
+ "num_tokens": 3211264.0,
+ "step": 196
+ },
+ {
+ "entropy": 2.1844022274017334,
+ "epoch": 0.397979797979798,
+ "grad_norm": 2.175349473953247,
+ "learning_rate": 4.604040404040404e-06,
+ "loss": 2.2053825855255127,
+ "mean_token_accuracy": 0.5553248524665833,
+ "num_tokens": 3227648.0,
+ "step": 197
+ },
+ {
+ "entropy": 1.6750906705856323,
+ "epoch": 0.4,
+ "grad_norm": 1.8300689458847046,
+ "learning_rate": 4.602020202020203e-06,
+ "loss": 1.692289113998413,
+ "mean_token_accuracy": 0.6345261335372925,
+ "num_tokens": 3244032.0,
+ "step": 198
+ },
+ {
+ "entropy": 1.6192772388458252,
+ "epoch": 0.402020202020202,
+ "grad_norm": 1.9788341522216797,
+ "learning_rate": 4.600000000000001e-06,
+ "loss": 1.628842830657959,
+ "mean_token_accuracy": 0.615229606628418,
+ "num_tokens": 3260416.0,
+ "step": 199
+ },
+ {
+ "entropy": 1.444870948791504,
+ "epoch": 0.40404040404040403,
+ "grad_norm": 2.05141544342041,
+ "learning_rate": 4.597979797979799e-06,
+ "loss": 1.4365839958190918,
+ "mean_token_accuracy": 0.652723491191864,
+ "num_tokens": 3276800.0,
+ "step": 200
+ },
+ {
+ "epoch": 0.40404040404040403,
+ "eval_entropy": 1.5766179907706477,
+ "eval_loss": 1.5878442525863647,
+ "eval_mean_token_accuracy": 0.6355829551335304,
+ "eval_num_tokens": 3276800.0,
+ "eval_runtime": 43.7272,
+ "eval_samples_per_second": 22.64,
+ "eval_steps_per_second": 2.836,
+ "step": 200
+ },
+ {
+ "entropy": 1.3354029655456543,
+ "epoch": 0.40606060606060607,
+ "grad_norm": 2.012895345687866,
+ "learning_rate": 4.595959595959597e-06,
+ "loss": 1.3289990425109863,
+ "mean_token_accuracy": 0.685271143913269,
+ "num_tokens": 3293184.0,
+ "step": 201
+ },
+ {
+ "entropy": 1.1990011930465698,
+ "epoch": 0.4080808080808081,
+ "grad_norm": 1.9462409019470215,
+ "learning_rate": 4.5939393939393945e-06,
+ "loss": 1.224869728088379,
+ "mean_token_accuracy": 0.6976673007011414,
+ "num_tokens": 3309568.0,
+ "step": 202
+ },
+ {
+ "entropy": 1.6442710161209106,
+ "epoch": 0.4101010101010101,
+ "grad_norm": 2.064532995223999,
+ "learning_rate": 4.591919191919192e-06,
+ "loss": 1.6800963878631592,
+ "mean_token_accuracy": 0.6111993193626404,
+ "num_tokens": 3325952.0,
+ "step": 203
+ },
+ {
+ "entropy": 1.8336306810379028,
+ "epoch": 0.4121212121212121,
+ "grad_norm": 2.478731393814087,
+ "learning_rate": 4.58989898989899e-06,
+ "loss": 1.8518702983856201,
+ "mean_token_accuracy": 0.6021006107330322,
+ "num_tokens": 3342336.0,
+ "step": 204
+ },
+ {
+ "entropy": 1.6767220497131348,
+ "epoch": 0.41414141414141414,
+ "grad_norm": 2.0676512718200684,
+ "learning_rate": 4.587878787878788e-06,
+ "loss": 1.674522042274475,
+ "mean_token_accuracy": 0.6178553700447083,
+ "num_tokens": 3358720.0,
+ "step": 205
+ },
+ {
+ "entropy": 1.7236497402191162,
+ "epoch": 0.4161616161616162,
+ "grad_norm": 2.057074785232544,
+ "learning_rate": 4.585858585858586e-06,
+ "loss": 1.725832462310791,
+ "mean_token_accuracy": 0.620114803314209,
+ "num_tokens": 3375104.0,
+ "step": 206
+ },
+ {
+ "entropy": 1.6493065357208252,
+ "epoch": 0.41818181818181815,
+ "grad_norm": 2.0946099758148193,
+ "learning_rate": 4.583838383838384e-06,
+ "loss": 1.6826295852661133,
+ "mean_token_accuracy": 0.6251221299171448,
+ "num_tokens": 3391488.0,
+ "step": 207
+ },
+ {
+ "entropy": 1.3565926551818848,
+ "epoch": 0.4202020202020202,
+ "grad_norm": 1.9611284732818604,
+ "learning_rate": 4.581818181818183e-06,
+ "loss": 1.346787452697754,
+ "mean_token_accuracy": 0.685332179069519,
+ "num_tokens": 3407872.0,
+ "step": 208
+ },
+ {
+ "entropy": 1.4401211738586426,
+ "epoch": 0.4222222222222222,
+ "grad_norm": 1.8614766597747803,
+ "learning_rate": 4.579797979797981e-06,
+ "loss": 1.4717891216278076,
+ "mean_token_accuracy": 0.650036633014679,
+ "num_tokens": 3424256.0,
+ "step": 209
+ },
+ {
+ "entropy": 1.194276213645935,
+ "epoch": 0.42424242424242425,
+ "grad_norm": 2.0625810623168945,
+ "learning_rate": 4.5777777777777785e-06,
+ "loss": 1.2106804847717285,
+ "mean_token_accuracy": 0.700537383556366,
+ "num_tokens": 3440640.0,
+ "step": 210
+ },
+ {
+ "entropy": 1.7622313499450684,
+ "epoch": 0.4262626262626263,
+ "grad_norm": 2.330610752105713,
+ "learning_rate": 4.575757575757576e-06,
+ "loss": 1.7872710227966309,
+ "mean_token_accuracy": 0.598436713218689,
+ "num_tokens": 3457024.0,
+ "step": 211
+ },
+ {
+ "entropy": 1.409427523612976,
+ "epoch": 0.42828282828282827,
+ "grad_norm": 2.0988259315490723,
+ "learning_rate": 4.573737373737374e-06,
+ "loss": 1.416553020477295,
+ "mean_token_accuracy": 0.6682950854301453,
+ "num_tokens": 3473408.0,
+ "step": 212
+ },
+ {
+ "entropy": 1.6856107711791992,
+ "epoch": 0.4303030303030303,
+ "grad_norm": 2.341475009918213,
+ "learning_rate": 4.571717171717172e-06,
+ "loss": 1.692287802696228,
+ "mean_token_accuracy": 0.6024059653282166,
+ "num_tokens": 3489792.0,
+ "step": 213
+ },
+ {
+ "entropy": 1.997343897819519,
+ "epoch": 0.43232323232323233,
+ "grad_norm": 2.200498104095459,
+ "learning_rate": 4.56969696969697e-06,
+ "loss": 2.026289939880371,
+ "mean_token_accuracy": 0.5781631469726562,
+ "num_tokens": 3506176.0,
+ "step": 214
+ },
+ {
+ "entropy": 1.5175039768218994,
+ "epoch": 0.43434343434343436,
+ "grad_norm": 2.105257272720337,
+ "learning_rate": 4.567676767676768e-06,
+ "loss": 1.521841049194336,
+ "mean_token_accuracy": 0.6404494643211365,
+ "num_tokens": 3522560.0,
+ "step": 215
+ },
+ {
+ "entropy": 1.3616528511047363,
+ "epoch": 0.43636363636363634,
+ "grad_norm": 2.0035297870635986,
+ "learning_rate": 4.565656565656566e-06,
+ "loss": 1.3674360513687134,
+ "mean_token_accuracy": 0.6771494746208191,
+ "num_tokens": 3538944.0,
+ "step": 216
+ },
+ {
+ "entropy": 1.42499840259552,
+ "epoch": 0.4383838383838384,
+ "grad_norm": 1.9115629196166992,
+ "learning_rate": 4.563636363636364e-06,
+ "loss": 1.4129266738891602,
+ "mean_token_accuracy": 0.6686614751815796,
+ "num_tokens": 3555328.0,
+ "step": 217
+ },
+ {
+ "entropy": 1.2844709157943726,
+ "epoch": 0.4404040404040404,
+ "grad_norm": 2.3313121795654297,
+ "learning_rate": 4.5616161616161616e-06,
+ "loss": 1.315006971359253,
+ "mean_token_accuracy": 0.681546151638031,
+ "num_tokens": 3571712.0,
+ "step": 218
+ },
+ {
+ "entropy": 1.5227075815200806,
+ "epoch": 0.44242424242424244,
+ "grad_norm": 2.1849124431610107,
+ "learning_rate": 4.55959595959596e-06,
+ "loss": 1.517989158630371,
+ "mean_token_accuracy": 0.6466169953346252,
+ "num_tokens": 3588096.0,
+ "step": 219
+ },
+ {
+ "entropy": 1.494642734527588,
+ "epoch": 0.4444444444444444,
+ "grad_norm": 2.3109116554260254,
+ "learning_rate": 4.557575757575758e-06,
+ "loss": 1.525421142578125,
+ "mean_token_accuracy": 0.6392281651496887,
+ "num_tokens": 3604480.0,
+ "step": 220
+ },
+ {
+ "entropy": 1.6428948640823364,
+ "epoch": 0.44646464646464645,
+ "grad_norm": 2.1182680130004883,
+ "learning_rate": 4.555555555555556e-06,
+ "loss": 1.6634926795959473,
+ "mean_token_accuracy": 0.6207864880561829,
+ "num_tokens": 3620864.0,
+ "step": 221
+ },
+ {
+ "entropy": 1.7323675155639648,
+ "epoch": 0.4484848484848485,
+ "grad_norm": 2.2620837688446045,
+ "learning_rate": 4.553535353535354e-06,
+ "loss": 1.7312631607055664,
+ "mean_token_accuracy": 0.6138250827789307,
+ "num_tokens": 3637248.0,
+ "step": 222
+ },
+ {
+ "entropy": 1.7863894701004028,
+ "epoch": 0.4505050505050505,
+ "grad_norm": 2.1416971683502197,
+ "learning_rate": 4.551515151515152e-06,
+ "loss": 1.799318552017212,
+ "mean_token_accuracy": 0.5906203985214233,
+ "num_tokens": 3653632.0,
+ "step": 223
+ },
+ {
+ "entropy": 1.7404330968856812,
+ "epoch": 0.45252525252525255,
+ "grad_norm": 2.0814504623413086,
+ "learning_rate": 4.54949494949495e-06,
+ "loss": 1.742197036743164,
+ "mean_token_accuracy": 0.6050317287445068,
+ "num_tokens": 3670016.0,
+ "step": 224
+ },
+ {
+ "entropy": 1.4387869834899902,
+ "epoch": 0.45454545454545453,
+ "grad_norm": 2.1386022567749023,
+ "learning_rate": 4.547474747474748e-06,
+ "loss": 1.4602317810058594,
+ "mean_token_accuracy": 0.6502198576927185,
+ "num_tokens": 3686400.0,
+ "step": 225
+ },
+ {
+ "entropy": 1.2733348608016968,
+ "epoch": 0.45656565656565656,
+ "grad_norm": 2.02687668800354,
+ "learning_rate": 4.5454545454545455e-06,
+ "loss": 1.2683714628219604,
+ "mean_token_accuracy": 0.6966902613639832,
+ "num_tokens": 3702784.0,
+ "step": 226
+ },
+ {
+ "entropy": 1.4554595947265625,
+ "epoch": 0.4585858585858586,
+ "grad_norm": 2.169268846511841,
+ "learning_rate": 4.543434343434343e-06,
+ "loss": 1.4541833400726318,
+ "mean_token_accuracy": 0.6522960662841797,
+ "num_tokens": 3719168.0,
+ "step": 227
+ },
+ {
+ "entropy": 1.487573266029358,
+ "epoch": 0.46060606060606063,
+ "grad_norm": 1.9726165533065796,
+ "learning_rate": 4.541414141414141e-06,
+ "loss": 1.5024409294128418,
+ "mean_token_accuracy": 0.6509526371955872,
+ "num_tokens": 3735552.0,
+ "step": 228
+ },
+ {
+ "entropy": 1.3991138935089111,
+ "epoch": 0.4626262626262626,
+ "grad_norm": 2.0992283821105957,
+ "learning_rate": 4.539393939393939e-06,
+ "loss": 1.4016860723495483,
+ "mean_token_accuracy": 0.6639594435691833,
+ "num_tokens": 3751936.0,
+ "step": 229
+ },
+ {
+ "entropy": 1.6564134359359741,
+ "epoch": 0.46464646464646464,
+ "grad_norm": 2.047656297683716,
+ "learning_rate": 4.537373737373738e-06,
+ "loss": 1.6598721742630005,
+ "mean_token_accuracy": 0.6166951656341553,
+ "num_tokens": 3768320.0,
+ "step": 230
+ },
+ {
+ "entropy": 1.5165014266967773,
+ "epoch": 0.4666666666666667,
+ "grad_norm": 2.079996109008789,
+ "learning_rate": 4.535353535353536e-06,
+ "loss": 1.4980010986328125,
+ "mean_token_accuracy": 0.6370908617973328,
+ "num_tokens": 3784704.0,
+ "step": 231
+ },
+ {
+ "entropy": 1.5409225225448608,
+ "epoch": 0.4686868686868687,
+ "grad_norm": 2.0996923446655273,
+ "learning_rate": 4.533333333333334e-06,
+ "loss": 1.5504257678985596,
+ "mean_token_accuracy": 0.6465559601783752,
+ "num_tokens": 3801088.0,
+ "step": 232
+ },
+ {
+ "entropy": 1.3590043783187866,
+ "epoch": 0.4707070707070707,
+ "grad_norm": 2.14617919921875,
+ "learning_rate": 4.531313131313132e-06,
+ "loss": 1.3581812381744385,
+ "mean_token_accuracy": 0.6683561205863953,
+ "num_tokens": 3817472.0,
+ "step": 233
+ },
+ {
+ "entropy": 1.641785740852356,
+ "epoch": 0.4727272727272727,
+ "grad_norm": 1.9499926567077637,
+ "learning_rate": 4.5292929292929295e-06,
+ "loss": 1.6653470993041992,
+ "mean_token_accuracy": 0.6319614052772522,
+ "num_tokens": 3833856.0,
+ "step": 234
+ },
+ {
+ "entropy": 1.4700758457183838,
+ "epoch": 0.47474747474747475,
+ "grad_norm": 1.9411437511444092,
+ "learning_rate": 4.527272727272727e-06,
+ "loss": 1.4762463569641113,
+ "mean_token_accuracy": 0.6631656289100647,
+ "num_tokens": 3850240.0,
+ "step": 235
+ },
+ {
+ "entropy": 1.5418941974639893,
+ "epoch": 0.4767676767676768,
+ "grad_norm": 2.3191940784454346,
+ "learning_rate": 4.525252525252526e-06,
+ "loss": 1.5151214599609375,
+ "mean_token_accuracy": 0.6326331496238708,
+ "num_tokens": 3866624.0,
+ "step": 236
+ },
+ {
+ "entropy": 2.1102776527404785,
+ "epoch": 0.47878787878787876,
+ "grad_norm": 2.2287707328796387,
+ "learning_rate": 4.523232323232324e-06,
+ "loss": 2.106567859649658,
+ "mean_token_accuracy": 0.5506839156150818,
+ "num_tokens": 3883008.0,
+ "step": 237
+ },
+ {
+ "entropy": 1.7448827028274536,
+ "epoch": 0.4808080808080808,
+ "grad_norm": 2.2352893352508545,
+ "learning_rate": 4.521212121212122e-06,
+ "loss": 1.7495017051696777,
+ "mean_token_accuracy": 0.60326087474823,
+ "num_tokens": 3899392.0,
+ "step": 238
+ },
+ {
+ "entropy": 1.315240740776062,
+ "epoch": 0.48282828282828283,
+ "grad_norm": 1.9933631420135498,
+ "learning_rate": 4.51919191919192e-06,
+ "loss": 1.315302848815918,
+ "mean_token_accuracy": 0.686431348323822,
+ "num_tokens": 3915776.0,
+ "step": 239
+ },
+ {
+ "entropy": 1.5851637125015259,
+ "epoch": 0.48484848484848486,
+ "grad_norm": 2.153303623199463,
+ "learning_rate": 4.517171717171718e-06,
+ "loss": 1.5947662591934204,
+ "mean_token_accuracy": 0.6297020316123962,
+ "num_tokens": 3932160.0,
+ "step": 240
+ },
+ {
+ "entropy": 1.8050944805145264,
+ "epoch": 0.4868686868686869,
+ "grad_norm": 2.025022506713867,
+ "learning_rate": 4.5151515151515155e-06,
+ "loss": 1.8051010370254517,
+ "mean_token_accuracy": 0.6055202484130859,
+ "num_tokens": 3948544.0,
+ "step": 241
+ },
+ {
+ "entropy": 1.518296718597412,
+ "epoch": 0.4888888888888889,
+ "grad_norm": 2.1021833419799805,
+ "learning_rate": 4.513131313131313e-06,
+ "loss": 1.5396809577941895,
+ "mean_token_accuracy": 0.6359916925430298,
+ "num_tokens": 3964928.0,
+ "step": 242
+ },
+ {
+ "entropy": 1.518903136253357,
+ "epoch": 0.4909090909090909,
+ "grad_norm": 2.0960140228271484,
+ "learning_rate": 4.511111111111111e-06,
+ "loss": 1.5528055429458618,
+ "mean_token_accuracy": 0.6224963068962097,
+ "num_tokens": 3981312.0,
+ "step": 243
+ },
+ {
+ "entropy": 1.6960705518722534,
+ "epoch": 0.49292929292929294,
+ "grad_norm": 1.8580718040466309,
+ "learning_rate": 4.50909090909091e-06,
+ "loss": 1.7192862033843994,
+ "mean_token_accuracy": 0.6259770393371582,
+ "num_tokens": 3997696.0,
+ "step": 244
+ },
+ {
+ "entropy": 1.4633033275604248,
+ "epoch": 0.494949494949495,
+ "grad_norm": 1.8983049392700195,
+ "learning_rate": 4.507070707070708e-06,
+ "loss": 1.451701283454895,
+ "mean_token_accuracy": 0.6669516563415527,
+ "num_tokens": 4014080.0,
+ "step": 245
+ },
+ {
+ "entropy": 1.6508095264434814,
+ "epoch": 0.49696969696969695,
+ "grad_norm": 2.298679828643799,
+ "learning_rate": 4.505050505050506e-06,
+ "loss": 1.6596897840499878,
+ "mean_token_accuracy": 0.6115046143531799,
+ "num_tokens": 4030464.0,
+ "step": 246
+ },
+ {
+ "entropy": 1.313779354095459,
+ "epoch": 0.498989898989899,
+ "grad_norm": 1.9894335269927979,
+ "learning_rate": 4.503030303030304e-06,
+ "loss": 1.3285409212112427,
+ "mean_token_accuracy": 0.6802027225494385,
+ "num_tokens": 4046848.0,
+ "step": 247
+ },
+ {
+ "entropy": 1.5139521360397339,
+ "epoch": 0.501010101010101,
+ "grad_norm": 2.028320789337158,
+ "learning_rate": 4.501010101010102e-06,
+ "loss": 1.5366487503051758,
+ "mean_token_accuracy": 0.6416707634925842,
+ "num_tokens": 4063232.0,
+ "step": 248
+ },
+ {
+ "entropy": 1.5277045965194702,
+ "epoch": 0.503030303030303,
+ "grad_norm": 2.3160979747772217,
+ "learning_rate": 4.4989898989898995e-06,
+ "loss": 1.5571036338806152,
+ "mean_token_accuracy": 0.6436858773231506,
+ "num_tokens": 4079616.0,
+ "step": 249
+ },
+ {
+ "entropy": 1.6084188222885132,
+ "epoch": 0.5050505050505051,
+ "grad_norm": 2.20550274848938,
+ "learning_rate": 4.496969696969697e-06,
+ "loss": 1.622127652168274,
+ "mean_token_accuracy": 0.6284196376800537,
+ "num_tokens": 4096000.0,
+ "step": 250
+ },
+ {
+ "epoch": 0.5050505050505051,
+ "eval_entropy": 1.5547234262189558,
+ "eval_loss": 1.5764786005020142,
+ "eval_mean_token_accuracy": 0.6375306306346771,
+ "eval_num_tokens": 4096000.0,
+ "eval_runtime": 43.7607,
+ "eval_samples_per_second": 22.623,
+ "eval_steps_per_second": 2.834,
+ "step": 250
+ },
+ {
+ "entropy": 1.4309135675430298,
+ "epoch": 0.5070707070707071,
+ "grad_norm": 2.080864906311035,
+ "learning_rate": 4.494949494949495e-06,
+ "loss": 1.4481091499328613,
+ "mean_token_accuracy": 0.6694552898406982,
+ "num_tokens": 4112384.0,
+ "step": 251
+ },
+ {
+ "entropy": 1.4067270755767822,
+ "epoch": 0.509090909090909,
+ "grad_norm": 2.036475419998169,
+ "learning_rate": 4.492929292929293e-06,
+ "loss": 1.4035298824310303,
+ "mean_token_accuracy": 0.6547996997833252,
+ "num_tokens": 4128768.0,
+ "step": 252
+ },
+ {
+ "entropy": 2.1751608848571777,
+ "epoch": 0.5111111111111111,
+ "grad_norm": 1.953995943069458,
+ "learning_rate": 4.490909090909091e-06,
+ "loss": 2.1616692543029785,
+ "mean_token_accuracy": 0.5592941045761108,
+ "num_tokens": 4145152.0,
+ "step": 253
+ },
+ {
+ "entropy": 1.45353102684021,
+ "epoch": 0.5131313131313131,
+ "grad_norm": 2.084437370300293,
+ "learning_rate": 4.488888888888889e-06,
+ "loss": 1.4633586406707764,
+ "mean_token_accuracy": 0.6620664596557617,
+ "num_tokens": 4161536.0,
+ "step": 254
+ },
+ {
+ "entropy": 1.64310884475708,
+ "epoch": 0.5151515151515151,
+ "grad_norm": 2.0700111389160156,
+ "learning_rate": 4.486868686868688e-06,
+ "loss": 1.6868078708648682,
+ "mean_token_accuracy": 0.616328775882721,
+ "num_tokens": 4177920.0,
+ "step": 255
+ },
+ {
+ "entropy": 1.5113472938537598,
+ "epoch": 0.5171717171717172,
+ "grad_norm": 2.102180242538452,
+ "learning_rate": 4.4848484848484855e-06,
+ "loss": 1.5240120887756348,
+ "mean_token_accuracy": 0.638067901134491,
+ "num_tokens": 4194304.0,
+ "step": 256
+ },
+ {
+ "entropy": 1.5752851963043213,
+ "epoch": 0.5191919191919192,
+ "grad_norm": 2.026737689971924,
+ "learning_rate": 4.4828282828282834e-06,
+ "loss": 1.6041791439056396,
+ "mean_token_accuracy": 0.6275647282600403,
+ "num_tokens": 4210688.0,
+ "step": 257
+ },
+ {
+ "entropy": 1.548423171043396,
+ "epoch": 0.5212121212121212,
+ "grad_norm": 2.0578935146331787,
+ "learning_rate": 4.480808080808081e-06,
+ "loss": 1.563529372215271,
+ "mean_token_accuracy": 0.6348925232887268,
+ "num_tokens": 4227072.0,
+ "step": 258
+ },
+ {
+ "entropy": 1.5324457883834839,
+ "epoch": 0.5232323232323233,
+ "grad_norm": 2.216235637664795,
+ "learning_rate": 4.478787878787879e-06,
+ "loss": 1.5559678077697754,
+ "mean_token_accuracy": 0.6300073266029358,
+ "num_tokens": 4243456.0,
+ "step": 259
+ },
+ {
+ "entropy": 1.5275540351867676,
+ "epoch": 0.5252525252525253,
+ "grad_norm": 2.037306070327759,
+ "learning_rate": 4.476767676767677e-06,
+ "loss": 1.541567087173462,
+ "mean_token_accuracy": 0.6424035429954529,
+ "num_tokens": 4259840.0,
+ "step": 260
+ },
+ {
+ "entropy": 1.6842185258865356,
+ "epoch": 0.5272727272727272,
+ "grad_norm": 2.0241005420684814,
+ "learning_rate": 4.474747474747475e-06,
+ "loss": 1.6819056272506714,
+ "mean_token_accuracy": 0.6102222800254822,
+ "num_tokens": 4276224.0,
+ "step": 261
+ },
+ {
+ "entropy": 1.5656248331069946,
+ "epoch": 0.5292929292929293,
+ "grad_norm": 1.993054986000061,
+ "learning_rate": 4.472727272727273e-06,
+ "loss": 1.6132277250289917,
+ "mean_token_accuracy": 0.634709358215332,
+ "num_tokens": 4292608.0,
+ "step": 262
+ },
+ {
+ "entropy": 1.6354466676712036,
+ "epoch": 0.5313131313131313,
+ "grad_norm": 2.06508731842041,
+ "learning_rate": 4.470707070707071e-06,
+ "loss": 1.6475149393081665,
+ "mean_token_accuracy": 0.6121763586997986,
+ "num_tokens": 4308992.0,
+ "step": 263
+ },
+ {
+ "entropy": 1.5267232656478882,
+ "epoch": 0.5333333333333333,
+ "grad_norm": 2.1227569580078125,
+ "learning_rate": 4.468686868686869e-06,
+ "loss": 1.5468671321868896,
+ "mean_token_accuracy": 0.6365413069725037,
+ "num_tokens": 4325376.0,
+ "step": 264
+ },
+ {
+ "entropy": 1.2252761125564575,
+ "epoch": 0.5353535353535354,
+ "grad_norm": 2.0596134662628174,
+ "learning_rate": 4.4666666666666665e-06,
+ "loss": 1.2242389917373657,
+ "mean_token_accuracy": 0.6973620057106018,
+ "num_tokens": 4341760.0,
+ "step": 265
+ },
+ {
+ "entropy": 1.6007431745529175,
+ "epoch": 0.5373737373737374,
+ "grad_norm": 1.9341918230056763,
+ "learning_rate": 4.464646464646465e-06,
+ "loss": 1.5989094972610474,
+ "mean_token_accuracy": 0.6284196376800537,
+ "num_tokens": 4358144.0,
+ "step": 266
+ },
+ {
+ "entropy": 1.2708780765533447,
+ "epoch": 0.5393939393939394,
+ "grad_norm": 4.594091415405273,
+ "learning_rate": 4.462626262626263e-06,
+ "loss": 1.3484312295913696,
+ "mean_token_accuracy": 0.6790425181388855,
+ "num_tokens": 4374528.0,
+ "step": 267
+ },
+ {
+ "entropy": 1.5423723459243774,
+ "epoch": 0.5414141414141415,
+ "grad_norm": 1.933602213859558,
+ "learning_rate": 4.460606060606061e-06,
+ "loss": 1.594527244567871,
+ "mean_token_accuracy": 0.6386785507202148,
+ "num_tokens": 4390912.0,
+ "step": 268
+ },
+ {
+ "entropy": 1.7213952541351318,
+ "epoch": 0.5434343434343434,
+ "grad_norm": 2.195904016494751,
+ "learning_rate": 4.458585858585859e-06,
+ "loss": 1.724353551864624,
+ "mean_token_accuracy": 0.6008182764053345,
+ "num_tokens": 4407296.0,
+ "step": 269
+ },
+ {
+ "entropy": 1.4772557020187378,
+ "epoch": 0.5454545454545454,
+ "grad_norm": 2.0990796089172363,
+ "learning_rate": 4.456565656565657e-06,
+ "loss": 1.486825942993164,
+ "mean_token_accuracy": 0.6507083773612976,
+ "num_tokens": 4423680.0,
+ "step": 270
+ },
+ {
+ "entropy": 1.6876367330551147,
+ "epoch": 0.5474747474747474,
+ "grad_norm": 2.1944479942321777,
+ "learning_rate": 4.454545454545455e-06,
+ "loss": 1.71107816696167,
+ "mean_token_accuracy": 0.6027112603187561,
+ "num_tokens": 4440064.0,
+ "step": 271
+ },
+ {
+ "entropy": 1.5480726957321167,
+ "epoch": 0.5494949494949495,
+ "grad_norm": 2.1579341888427734,
+ "learning_rate": 4.452525252525253e-06,
+ "loss": 1.544647216796875,
+ "mean_token_accuracy": 0.6345261335372925,
+ "num_tokens": 4456448.0,
+ "step": 272
+ },
+ {
+ "entropy": 1.6161645650863647,
+ "epoch": 0.5515151515151515,
+ "grad_norm": 1.981154203414917,
+ "learning_rate": 4.4505050505050505e-06,
+ "loss": 1.6109068393707275,
+ "mean_token_accuracy": 0.6318392753601074,
+ "num_tokens": 4472832.0,
+ "step": 273
+ },
+ {
+ "entropy": 1.2576326131820679,
+ "epoch": 0.5535353535353535,
+ "grad_norm": 1.95668625831604,
+ "learning_rate": 4.448484848484848e-06,
+ "loss": 1.2614656686782837,
+ "mean_token_accuracy": 0.7025524973869324,
+ "num_tokens": 4489216.0,
+ "step": 274
+ },
+ {
+ "entropy": 1.405206561088562,
+ "epoch": 0.5555555555555556,
+ "grad_norm": 2.0302884578704834,
+ "learning_rate": 4.446464646464646e-06,
+ "loss": 1.416546106338501,
+ "mean_token_accuracy": 0.6753786206245422,
+ "num_tokens": 4505600.0,
+ "step": 275
+ },
+ {
+ "entropy": 1.9038625955581665,
+ "epoch": 0.5575757575757576,
+ "grad_norm": 2.06475567817688,
+ "learning_rate": 4.444444444444444e-06,
+ "loss": 1.9273614883422852,
+ "mean_token_accuracy": 0.5813996195793152,
+ "num_tokens": 4521984.0,
+ "step": 276
+ },
+ {
+ "entropy": 1.7208465337753296,
+ "epoch": 0.5595959595959596,
+ "grad_norm": 2.0136189460754395,
+ "learning_rate": 4.442424242424243e-06,
+ "loss": 1.7260158061981201,
+ "mean_token_accuracy": 0.6006350517272949,
+ "num_tokens": 4538368.0,
+ "step": 277
+ },
+ {
+ "entropy": 1.7570570707321167,
+ "epoch": 0.5616161616161616,
+ "grad_norm": 2.148594379425049,
+ "learning_rate": 4.440404040404041e-06,
+ "loss": 1.7799286842346191,
+ "mean_token_accuracy": 0.5943453907966614,
+ "num_tokens": 4554752.0,
+ "step": 278
+ },
+ {
+ "entropy": 1.6388157606124878,
+ "epoch": 0.5636363636363636,
+ "grad_norm": 2.1301987171173096,
+ "learning_rate": 4.438383838383839e-06,
+ "loss": 1.63419508934021,
+ "mean_token_accuracy": 0.6405715942382812,
+ "num_tokens": 4571136.0,
+ "step": 279
+ },
+ {
+ "entropy": 1.5902295112609863,
+ "epoch": 0.5656565656565656,
+ "grad_norm": 2.0526790618896484,
+ "learning_rate": 4.436363636363637e-06,
+ "loss": 1.6278276443481445,
+ "mean_token_accuracy": 0.615229606628418,
+ "num_tokens": 4587520.0,
+ "step": 280
+ },
+ {
+ "entropy": 1.5494027137756348,
+ "epoch": 0.5676767676767677,
+ "grad_norm": 1.9863859415054321,
+ "learning_rate": 4.434343434343435e-06,
+ "loss": 1.5622975826263428,
+ "mean_token_accuracy": 0.6261602640151978,
+ "num_tokens": 4603904.0,
+ "step": 281
+ },
+ {
+ "entropy": 1.2140685319900513,
+ "epoch": 0.5696969696969697,
+ "grad_norm": 2.4206902980804443,
+ "learning_rate": 4.432323232323233e-06,
+ "loss": 1.2389612197875977,
+ "mean_token_accuracy": 0.6943697929382324,
+ "num_tokens": 4620288.0,
+ "step": 282
+ },
+ {
+ "entropy": 1.4347270727157593,
+ "epoch": 0.5717171717171717,
+ "grad_norm": 2.0198237895965576,
+ "learning_rate": 4.430303030303031e-06,
+ "loss": 1.4648659229278564,
+ "mean_token_accuracy": 0.6573033928871155,
+ "num_tokens": 4636672.0,
+ "step": 283
+ },
+ {
+ "entropy": 1.3239331245422363,
+ "epoch": 0.5737373737373738,
+ "grad_norm": 1.9862704277038574,
+ "learning_rate": 4.428282828282829e-06,
+ "loss": 1.3520365953445435,
+ "mean_token_accuracy": 0.6852100491523743,
+ "num_tokens": 4653056.0,
+ "step": 284
+ },
+ {
+ "entropy": 1.7534631490707397,
+ "epoch": 0.5757575757575758,
+ "grad_norm": 2.0964176654815674,
+ "learning_rate": 4.426262626262627e-06,
+ "loss": 1.7659757137298584,
+ "mean_token_accuracy": 0.5975207686424255,
+ "num_tokens": 4669440.0,
+ "step": 285
+ },
+ {
+ "entropy": 1.1573433876037598,
+ "epoch": 0.5777777777777777,
+ "grad_norm": 2.0671567916870117,
+ "learning_rate": 4.424242424242425e-06,
+ "loss": 1.1479461193084717,
+ "mean_token_accuracy": 0.7143380641937256,
+ "num_tokens": 4685824.0,
+ "step": 286
+ },
+ {
+ "entropy": 1.2439504861831665,
+ "epoch": 0.5797979797979798,
+ "grad_norm": 2.0317327976226807,
+ "learning_rate": 4.422222222222223e-06,
+ "loss": 1.255782961845398,
+ "mean_token_accuracy": 0.691255509853363,
+ "num_tokens": 4702208.0,
+ "step": 287
+ },
+ {
+ "entropy": 1.5569473505020142,
+ "epoch": 0.5818181818181818,
+ "grad_norm": 2.0759670734405518,
+ "learning_rate": 4.4202020202020205e-06,
+ "loss": 1.5564974546432495,
+ "mean_token_accuracy": 0.6284807324409485,
+ "num_tokens": 4718592.0,
+ "step": 288
+ },
+ {
+ "entropy": 1.478676199913025,
+ "epoch": 0.5838383838383838,
+ "grad_norm": 1.9528205394744873,
+ "learning_rate": 4.418181818181818e-06,
+ "loss": 1.491654396057129,
+ "mean_token_accuracy": 0.6520518064498901,
+ "num_tokens": 4734976.0,
+ "step": 289
+ },
+ {
+ "entropy": 1.2454503774642944,
+ "epoch": 0.5858585858585859,
+ "grad_norm": 1.9008079767227173,
+ "learning_rate": 4.416161616161616e-06,
+ "loss": 1.253904938697815,
+ "mean_token_accuracy": 0.6971787810325623,
+ "num_tokens": 4751360.0,
+ "step": 290
+ },
+ {
+ "entropy": 1.1740810871124268,
+ "epoch": 0.5878787878787879,
+ "grad_norm": 1.8888787031173706,
+ "learning_rate": 4.414141414141415e-06,
+ "loss": 1.195070505142212,
+ "mean_token_accuracy": 0.7048119306564331,
+ "num_tokens": 4767744.0,
+ "step": 291
+ },
+ {
+ "entropy": 1.6064486503601074,
+ "epoch": 0.5898989898989899,
+ "grad_norm": 2.073559284210205,
+ "learning_rate": 4.412121212121213e-06,
+ "loss": 1.6180689334869385,
+ "mean_token_accuracy": 0.6242061257362366,
+ "num_tokens": 4784128.0,
+ "step": 292
+ },
+ {
+ "entropy": 1.6119383573532104,
+ "epoch": 0.591919191919192,
+ "grad_norm": 1.8773425817489624,
+ "learning_rate": 4.410101010101011e-06,
+ "loss": 1.5875662565231323,
+ "mean_token_accuracy": 0.647838294506073,
+ "num_tokens": 4800512.0,
+ "step": 293
+ },
+ {
+ "entropy": 1.392905592918396,
+ "epoch": 0.593939393939394,
+ "grad_norm": 2.1699368953704834,
+ "learning_rate": 4.408080808080809e-06,
+ "loss": 1.3949127197265625,
+ "mean_token_accuracy": 0.6538837552070618,
+ "num_tokens": 4816896.0,
+ "step": 294
+ },
+ {
+ "entropy": 1.5057002305984497,
+ "epoch": 0.5959595959595959,
+ "grad_norm": 2.2730712890625,
+ "learning_rate": 4.4060606060606066e-06,
+ "loss": 1.4755051136016846,
+ "mean_token_accuracy": 0.6498534679412842,
+ "num_tokens": 4833280.0,
+ "step": 295
+ },
+ {
+ "entropy": 1.5468902587890625,
+ "epoch": 0.597979797979798,
+ "grad_norm": 1.9911075830459595,
+ "learning_rate": 4.4040404040404044e-06,
+ "loss": 1.539963722229004,
+ "mean_token_accuracy": 0.6520518064498901,
+ "num_tokens": 4849664.0,
+ "step": 296
+ },
+ {
+ "entropy": 1.5196901559829712,
+ "epoch": 0.6,
+ "grad_norm": 1.9859540462493896,
+ "learning_rate": 4.402020202020202e-06,
+ "loss": 1.523442029953003,
+ "mean_token_accuracy": 0.6417317986488342,
+ "num_tokens": 4866048.0,
+ "step": 297
+ },
+ {
+ "entropy": 1.8251866102218628,
+ "epoch": 0.602020202020202,
+ "grad_norm": 2.2566516399383545,
+ "learning_rate": 4.4e-06,
+ "loss": 1.8667771816253662,
+ "mean_token_accuracy": 0.5872007608413696,
+ "num_tokens": 4882432.0,
+ "step": 298
+ },
+ {
+ "entropy": 1.6816744804382324,
+ "epoch": 0.604040404040404,
+ "grad_norm": 2.0107715129852295,
+ "learning_rate": 4.397979797979798e-06,
+ "loss": 1.6907548904418945,
+ "mean_token_accuracy": 0.6176722049713135,
+ "num_tokens": 4898816.0,
+ "step": 299
+ },
+ {
+ "entropy": 1.4992223978042603,
+ "epoch": 0.6060606060606061,
+ "grad_norm": 2.1236329078674316,
+ "learning_rate": 4.395959595959596e-06,
+ "loss": 1.5015790462493896,
+ "mean_token_accuracy": 0.6372129917144775,
+ "num_tokens": 4915200.0,
+ "step": 300
+ },
+ {
+ "epoch": 0.6060606060606061,
+ "eval_entropy": 1.5544315297757425,
+ "eval_loss": 1.5676765441894531,
+ "eval_mean_token_accuracy": 0.6389380799185845,
+ "eval_num_tokens": 4915200.0,
+ "eval_runtime": 43.7857,
+ "eval_samples_per_second": 22.61,
+ "eval_steps_per_second": 2.832,
+ "step": 300
+ },
+ {
+ "entropy": 1.518556833267212,
+ "epoch": 0.6080808080808081,
+ "grad_norm": 2.88371205329895,
+ "learning_rate": 4.393939393939394e-06,
+ "loss": 1.5310916900634766,
+ "mean_token_accuracy": 0.6551660895347595,
+ "num_tokens": 4931584.0,
+ "step": 301
+ },
+ {
+ "entropy": 1.4650386571884155,
+ "epoch": 0.6101010101010101,
+ "grad_norm": 2.1849780082702637,
+ "learning_rate": 4.391919191919193e-06,
+ "loss": 1.4405598640441895,
+ "mean_token_accuracy": 0.6531509757041931,
+ "num_tokens": 4947968.0,
+ "step": 302
+ },
+ {
+ "entropy": 1.5209505558013916,
+ "epoch": 0.6121212121212121,
+ "grad_norm": 2.1135010719299316,
+ "learning_rate": 4.3898989898989905e-06,
+ "loss": 1.5324647426605225,
+ "mean_token_accuracy": 0.6561431288719177,
+ "num_tokens": 4964352.0,
+ "step": 303
+ },
+ {
+ "entropy": 1.448391318321228,
+ "epoch": 0.6141414141414141,
+ "grad_norm": 1.9959306716918945,
+ "learning_rate": 4.387878787878788e-06,
+ "loss": 1.4721965789794922,
+ "mean_token_accuracy": 0.6578529477119446,
+ "num_tokens": 4980736.0,
+ "step": 304
+ },
+ {
+ "entropy": 1.4609358310699463,
+ "epoch": 0.6161616161616161,
+ "grad_norm": 2.1308915615081787,
+ "learning_rate": 4.385858585858586e-06,
+ "loss": 1.4707520008087158,
+ "mean_token_accuracy": 0.6482046842575073,
+ "num_tokens": 4997120.0,
+ "step": 305
+ },
+ {
+ "entropy": 1.7125155925750732,
+ "epoch": 0.6181818181818182,
+ "grad_norm": 2.2170841693878174,
+ "learning_rate": 4.383838383838384e-06,
+ "loss": 1.7630269527435303,
+ "mean_token_accuracy": 0.5999022722244263,
+ "num_tokens": 5013504.0,
+ "step": 306
+ },
+ {
+ "entropy": 1.6740268468856812,
+ "epoch": 0.6202020202020202,
+ "grad_norm": 1.983644723892212,
+ "learning_rate": 4.381818181818182e-06,
+ "loss": 1.6849563121795654,
+ "mean_token_accuracy": 0.6237176060676575,
+ "num_tokens": 5029888.0,
+ "step": 307
+ },
+ {
+ "entropy": 1.3754971027374268,
+ "epoch": 0.6222222222222222,
+ "grad_norm": 2.005277395248413,
+ "learning_rate": 4.37979797979798e-06,
+ "loss": 1.385213851928711,
+ "mean_token_accuracy": 0.6638984084129333,
+ "num_tokens": 5046272.0,
+ "step": 308
+ },
+ {
+ "entropy": 1.5476733446121216,
+ "epoch": 0.6242424242424243,
+ "grad_norm": 2.4317610263824463,
+ "learning_rate": 4.377777777777778e-06,
+ "loss": 1.5417041778564453,
+ "mean_token_accuracy": 0.6323888897895813,
+ "num_tokens": 5062656.0,
+ "step": 309
+ },
+ {
+ "entropy": 1.4437031745910645,
+ "epoch": 0.6262626262626263,
+ "grad_norm": 2.1464109420776367,
+ "learning_rate": 4.375757575757576e-06,
+ "loss": 1.4704627990722656,
+ "mean_token_accuracy": 0.6772105693817139,
+ "num_tokens": 5079040.0,
+ "step": 310
+ },
+ {
+ "entropy": 1.4859641790390015,
+ "epoch": 0.6282828282828283,
+ "grad_norm": 2.0800492763519287,
+ "learning_rate": 4.373737373737374e-06,
+ "loss": 1.5072834491729736,
+ "mean_token_accuracy": 0.658707857131958,
+ "num_tokens": 5095424.0,
+ "step": 311
+ },
+ {
+ "entropy": 1.8733419179916382,
+ "epoch": 0.6303030303030303,
+ "grad_norm": 2.120965003967285,
+ "learning_rate": 4.3717171717171715e-06,
+ "loss": 1.8567254543304443,
+ "mean_token_accuracy": 0.5847581624984741,
+ "num_tokens": 5111808.0,
+ "step": 312
+ },
+ {
+ "entropy": 1.4578243494033813,
+ "epoch": 0.6323232323232323,
+ "grad_norm": 1.9577823877334595,
+ "learning_rate": 4.36969696969697e-06,
+ "loss": 1.4758052825927734,
+ "mean_token_accuracy": 0.6590742468833923,
+ "num_tokens": 5128192.0,
+ "step": 313
+ },
+ {
+ "entropy": 1.8680084943771362,
+ "epoch": 0.6343434343434343,
+ "grad_norm": 2.170994520187378,
+ "learning_rate": 4.367676767676768e-06,
+ "loss": 1.904400110244751,
+ "mean_token_accuracy": 0.5857962965965271,
+ "num_tokens": 5144576.0,
+ "step": 314
+ },
+ {
+ "entropy": 1.6488304138183594,
+ "epoch": 0.6363636363636364,
+ "grad_norm": 1.959490418434143,
+ "learning_rate": 4.365656565656566e-06,
+ "loss": 1.6570477485656738,
+ "mean_token_accuracy": 0.6257327795028687,
+ "num_tokens": 5160960.0,
+ "step": 315
+ },
+ {
+ "entropy": 1.5038025379180908,
+ "epoch": 0.6383838383838384,
+ "grad_norm": 2.072697401046753,
+ "learning_rate": 4.363636363636364e-06,
+ "loss": 1.5234860181808472,
+ "mean_token_accuracy": 0.6494259834289551,
+ "num_tokens": 5177344.0,
+ "step": 316
+ },
+ {
+ "entropy": 1.4154654741287231,
+ "epoch": 0.6404040404040404,
+ "grad_norm": 1.894587755203247,
+ "learning_rate": 4.361616161616162e-06,
+ "loss": 1.4131592512130737,
+ "mean_token_accuracy": 0.6667073965072632,
+ "num_tokens": 5193728.0,
+ "step": 317
+ },
+ {
+ "entropy": 1.5813250541687012,
+ "epoch": 0.6424242424242425,
+ "grad_norm": 2.4088737964630127,
+ "learning_rate": 4.35959595959596e-06,
+ "loss": 1.6236622333526611,
+ "mean_token_accuracy": 0.6313507556915283,
+ "num_tokens": 5210112.0,
+ "step": 318
+ },
+ {
+ "entropy": 1.4140045642852783,
+ "epoch": 0.6444444444444445,
+ "grad_norm": 1.991557240486145,
+ "learning_rate": 4.3575757575757576e-06,
+ "loss": 1.4242517948150635,
+ "mean_token_accuracy": 0.6569980382919312,
+ "num_tokens": 5226496.0,
+ "step": 319
+ },
+ {
+ "entropy": 1.615628957748413,
+ "epoch": 0.6464646464646465,
+ "grad_norm": 1.8819077014923096,
+ "learning_rate": 4.3555555555555555e-06,
+ "loss": 1.6331532001495361,
+ "mean_token_accuracy": 0.6299462914466858,
+ "num_tokens": 5242880.0,
+ "step": 320
+ },
+ {
+ "entropy": 1.3118394613265991,
+ "epoch": 0.6484848484848484,
+ "grad_norm": 2.0589284896850586,
+ "learning_rate": 4.353535353535353e-06,
+ "loss": 1.2880034446716309,
+ "mean_token_accuracy": 0.6842941045761108,
+ "num_tokens": 5259264.0,
+ "step": 321
+ },
+ {
+ "entropy": 1.2786612510681152,
+ "epoch": 0.6505050505050505,
+ "grad_norm": 2.033839225769043,
+ "learning_rate": 4.351515151515152e-06,
+ "loss": 1.2829055786132812,
+ "mean_token_accuracy": 0.6910112500190735,
+ "num_tokens": 5275648.0,
+ "step": 322
+ },
+ {
+ "entropy": 1.5632988214492798,
+ "epoch": 0.6525252525252525,
+ "grad_norm": 2.1970419883728027,
+ "learning_rate": 4.34949494949495e-06,
+ "loss": 1.5865097045898438,
+ "mean_token_accuracy": 0.642892062664032,
+ "num_tokens": 5292032.0,
+ "step": 323
+ },
+ {
+ "entropy": 1.1542725563049316,
+ "epoch": 0.6545454545454545,
+ "grad_norm": 1.9750654697418213,
+ "learning_rate": 4.347474747474748e-06,
+ "loss": 1.1401035785675049,
+ "mean_token_accuracy": 0.7061553597450256,
+ "num_tokens": 5308416.0,
+ "step": 324
+ },
+ {
+ "entropy": 1.6879942417144775,
+ "epoch": 0.6565656565656566,
+ "grad_norm": 2.0022354125976562,
+ "learning_rate": 4.345454545454546e-06,
+ "loss": 1.70950448513031,
+ "mean_token_accuracy": 0.6089398860931396,
+ "num_tokens": 5324800.0,
+ "step": 325
+ },
+ {
+ "entropy": 1.7589699029922485,
+ "epoch": 0.6585858585858586,
+ "grad_norm": 1.925520420074463,
+ "learning_rate": 4.343434343434344e-06,
+ "loss": 1.7749860286712646,
+ "mean_token_accuracy": 0.6015510559082031,
+ "num_tokens": 5341184.0,
+ "step": 326
+ },
+ {
+ "entropy": 1.4720325469970703,
+ "epoch": 0.6606060606060606,
+ "grad_norm": 1.9487124681472778,
+ "learning_rate": 4.341414141414142e-06,
+ "loss": 1.4983797073364258,
+ "mean_token_accuracy": 0.6537005305290222,
+ "num_tokens": 5357568.0,
+ "step": 327
+ },
+ {
+ "entropy": 1.676164150238037,
+ "epoch": 0.6626262626262627,
+ "grad_norm": 2.13053035736084,
+ "learning_rate": 4.33939393939394e-06,
+ "loss": 1.699425458908081,
+ "mean_token_accuracy": 0.6100390553474426,
+ "num_tokens": 5373952.0,
+ "step": 328
+ },
+ {
+ "entropy": 1.3424437046051025,
+ "epoch": 0.6646464646464646,
+ "grad_norm": 2.0245954990386963,
+ "learning_rate": 4.337373737373738e-06,
+ "loss": 1.3393046855926514,
+ "mean_token_accuracy": 0.677760124206543,
+ "num_tokens": 5390336.0,
+ "step": 329
+ },
+ {
+ "entropy": 1.4926583766937256,
+ "epoch": 0.6666666666666666,
+ "grad_norm": 1.893343448638916,
+ "learning_rate": 4.335353535353536e-06,
+ "loss": 1.4779269695281982,
+ "mean_token_accuracy": 0.6713483333587646,
+ "num_tokens": 5406720.0,
+ "step": 330
+ },
+ {
+ "entropy": 1.5753449201583862,
+ "epoch": 0.6686868686868687,
+ "grad_norm": 2.051647424697876,
+ "learning_rate": 4.333333333333334e-06,
+ "loss": 1.622597336769104,
+ "mean_token_accuracy": 0.6436248421669006,
+ "num_tokens": 5423104.0,
+ "step": 331
+ },
+ {
+ "entropy": 1.4573988914489746,
+ "epoch": 0.6707070707070707,
+ "grad_norm": 1.8709567785263062,
+ "learning_rate": 4.331313131313132e-06,
+ "loss": 1.4902422428131104,
+ "mean_token_accuracy": 0.6682950854301453,
+ "num_tokens": 5439488.0,
+ "step": 332
+ },
+ {
+ "entropy": 1.378867745399475,
+ "epoch": 0.6727272727272727,
+ "grad_norm": 2.235928773880005,
+ "learning_rate": 4.32929292929293e-06,
+ "loss": 1.4067103862762451,
+ "mean_token_accuracy": 0.662432849407196,
+ "num_tokens": 5455872.0,
+ "step": 333
+ },
+ {
+ "entropy": 1.7553350925445557,
+ "epoch": 0.6747474747474748,
+ "grad_norm": 2.0335066318511963,
+ "learning_rate": 4.327272727272728e-06,
+ "loss": 1.772943139076233,
+ "mean_token_accuracy": 0.5953834652900696,
+ "num_tokens": 5472256.0,
+ "step": 334
+ },
+ {
+ "entropy": 1.1587097644805908,
+ "epoch": 0.6767676767676768,
+ "grad_norm": 1.8764615058898926,
+ "learning_rate": 4.3252525252525255e-06,
+ "loss": 1.1396210193634033,
+ "mean_token_accuracy": 0.7197117805480957,
+ "num_tokens": 5488640.0,
+ "step": 335
+ },
+ {
+ "entropy": 1.4819872379302979,
+ "epoch": 0.6787878787878788,
+ "grad_norm": 2.0907511711120605,
+ "learning_rate": 4.323232323232323e-06,
+ "loss": 1.460550308227539,
+ "mean_token_accuracy": 0.6520518064498901,
+ "num_tokens": 5505024.0,
+ "step": 336
+ },
+ {
+ "entropy": 1.0744472742080688,
+ "epoch": 0.6808080808080809,
+ "grad_norm": 2.227606773376465,
+ "learning_rate": 4.321212121212121e-06,
+ "loss": 1.0909301042556763,
+ "mean_token_accuracy": 0.7258182764053345,
+ "num_tokens": 5521408.0,
+ "step": 337
+ },
+ {
+ "entropy": 1.507999300956726,
+ "epoch": 0.6828282828282828,
+ "grad_norm": 2.0332415103912354,
+ "learning_rate": 4.31919191919192e-06,
+ "loss": 1.5173046588897705,
+ "mean_token_accuracy": 0.6374572515487671,
+ "num_tokens": 5537792.0,
+ "step": 338
+ },
+ {
+ "entropy": 1.6792720556259155,
+ "epoch": 0.6848484848484848,
+ "grad_norm": 2.0194997787475586,
+ "learning_rate": 4.317171717171718e-06,
+ "loss": 1.679445743560791,
+ "mean_token_accuracy": 0.6138250827789307,
+ "num_tokens": 5554176.0,
+ "step": 339
+ },
+ {
+ "entropy": 1.6470589637756348,
+ "epoch": 0.6868686868686869,
+ "grad_norm": 2.09116268157959,
+ "learning_rate": 4.315151515151516e-06,
+ "loss": 1.6386632919311523,
+ "mean_token_accuracy": 0.6146799921989441,
+ "num_tokens": 5570560.0,
+ "step": 340
+ },
+ {
+ "entropy": 1.3612488508224487,
+ "epoch": 0.6888888888888889,
+ "grad_norm": 2.1586217880249023,
+ "learning_rate": 4.313131313131314e-06,
+ "loss": 1.3698725700378418,
+ "mean_token_accuracy": 0.6696995496749878,
+ "num_tokens": 5586944.0,
+ "step": 341
+ },
+ {
+ "entropy": 1.400327205657959,
+ "epoch": 0.6909090909090909,
+ "grad_norm": 2.082094192504883,
+ "learning_rate": 4.3111111111111115e-06,
+ "loss": 1.396535873413086,
+ "mean_token_accuracy": 0.6780654788017273,
+ "num_tokens": 5603328.0,
+ "step": 342
+ },
+ {
+ "entropy": 1.389290690422058,
+ "epoch": 0.692929292929293,
+ "grad_norm": 2.0780303478240967,
+ "learning_rate": 4.309090909090909e-06,
+ "loss": 1.4275782108306885,
+ "mean_token_accuracy": 0.6656082272529602,
+ "num_tokens": 5619712.0,
+ "step": 343
+ },
+ {
+ "entropy": 1.5133870840072632,
+ "epoch": 0.694949494949495,
+ "grad_norm": 1.9819531440734863,
+ "learning_rate": 4.307070707070707e-06,
+ "loss": 1.5309596061706543,
+ "mean_token_accuracy": 0.6373351216316223,
+ "num_tokens": 5636096.0,
+ "step": 344
+ },
+ {
+ "entropy": 1.483219861984253,
+ "epoch": 0.696969696969697,
+ "grad_norm": 1.9794325828552246,
+ "learning_rate": 4.305050505050505e-06,
+ "loss": 1.4693087339401245,
+ "mean_token_accuracy": 0.6550439596176147,
+ "num_tokens": 5652480.0,
+ "step": 345
+ },
+ {
+ "entropy": 1.381489634513855,
+ "epoch": 0.6989898989898989,
+ "grad_norm": 2.0637686252593994,
+ "learning_rate": 4.303030303030303e-06,
+ "loss": 1.3862476348876953,
+ "mean_token_accuracy": 0.6631045341491699,
+ "num_tokens": 5668864.0,
+ "step": 346
+ },
+ {
+ "entropy": 1.9860024452209473,
+ "epoch": 0.701010101010101,
+ "grad_norm": 2.1626803874969482,
+ "learning_rate": 4.301010101010101e-06,
+ "loss": 1.9815950393676758,
+ "mean_token_accuracy": 0.5698583126068115,
+ "num_tokens": 5685248.0,
+ "step": 347
+ },
+ {
+ "entropy": 1.5044004917144775,
+ "epoch": 0.703030303030303,
+ "grad_norm": 2.060976266860962,
+ "learning_rate": 4.298989898989899e-06,
+ "loss": 1.4937127828598022,
+ "mean_token_accuracy": 0.6524792313575745,
+ "num_tokens": 5701632.0,
+ "step": 348
+ },
+ {
+ "entropy": 1.3397772312164307,
+ "epoch": 0.705050505050505,
+ "grad_norm": 2.0162901878356934,
+ "learning_rate": 4.296969696969698e-06,
+ "loss": 1.3358572721481323,
+ "mean_token_accuracy": 0.6949804425239563,
+ "num_tokens": 5718016.0,
+ "step": 349
+ },
+ {
+ "entropy": 1.145772099494934,
+ "epoch": 0.7070707070707071,
+ "grad_norm": 2.097634792327881,
+ "learning_rate": 4.2949494949494955e-06,
+ "loss": 1.1833416223526,
+ "mean_token_accuracy": 0.7104909420013428,
+ "num_tokens": 5734400.0,
+ "step": 350
+ },
+ {
+ "epoch": 0.7070707070707071,
+ "eval_entropy": 1.5476290602837839,
+ "eval_loss": 1.5603480339050293,
+ "eval_mean_token_accuracy": 0.640111118555069,
+ "eval_num_tokens": 5734400.0,
+ "eval_runtime": 43.7844,
+ "eval_samples_per_second": 22.611,
+ "eval_steps_per_second": 2.832,
+ "step": 350
+ },
+ {
+ "entropy": 1.4311926364898682,
+ "epoch": 0.7090909090909091,
+ "grad_norm": 2.002321720123291,
+ "learning_rate": 4.292929292929293e-06,
+ "loss": 1.4534825086593628,
+ "mean_token_accuracy": 0.6614558100700378,
+ "num_tokens": 5750784.0,
+ "step": 351
+ },
+ {
+ "entropy": 1.3983922004699707,
+ "epoch": 0.7111111111111111,
+ "grad_norm": 2.1724867820739746,
+ "learning_rate": 4.290909090909091e-06,
+ "loss": 1.3969402313232422,
+ "mean_token_accuracy": 0.6652418375015259,
+ "num_tokens": 5767168.0,
+ "step": 352
+ },
+ {
+ "entropy": 1.8218920230865479,
+ "epoch": 0.7131313131313132,
+ "grad_norm": 2.1629281044006348,
+ "learning_rate": 4.288888888888889e-06,
+ "loss": 1.8196980953216553,
+ "mean_token_accuracy": 0.6027112603187561,
+ "num_tokens": 5783552.0,
+ "step": 353
+ },
+ {
+ "entropy": 1.5322320461273193,
+ "epoch": 0.7151515151515152,
+ "grad_norm": 1.8486647605895996,
+ "learning_rate": 4.286868686868687e-06,
+ "loss": 1.5504610538482666,
+ "mean_token_accuracy": 0.6533341407775879,
+ "num_tokens": 5799936.0,
+ "step": 354
+ },
+ {
+ "entropy": 1.935966968536377,
+ "epoch": 0.7171717171717171,
+ "grad_norm": 2.252814292907715,
+ "learning_rate": 4.284848484848485e-06,
+ "loss": 1.9300384521484375,
+ "mean_token_accuracy": 0.5656448602676392,
+ "num_tokens": 5816320.0,
+ "step": 355
+ },
+ {
+ "entropy": 1.6711398363113403,
+ "epoch": 0.7191919191919192,
+ "grad_norm": 2.023379325866699,
+ "learning_rate": 4.282828282828283e-06,
+ "loss": 1.6584455966949463,
+ "mean_token_accuracy": 0.6235954761505127,
+ "num_tokens": 5832704.0,
+ "step": 356
+ },
+ {
+ "entropy": 1.3922803401947021,
+ "epoch": 0.7212121212121212,
+ "grad_norm": 2.0487611293792725,
+ "learning_rate": 4.280808080808081e-06,
+ "loss": 1.4220250844955444,
+ "mean_token_accuracy": 0.6591963768005371,
+ "num_tokens": 5849088.0,
+ "step": 357
+ },
+ {
+ "entropy": 1.59521484375,
+ "epoch": 0.7232323232323232,
+ "grad_norm": 1.8598614931106567,
+ "learning_rate": 4.278787878787879e-06,
+ "loss": 1.5979329347610474,
+ "mean_token_accuracy": 0.6237176060676575,
+ "num_tokens": 5865472.0,
+ "step": 358
+ },
+ {
+ "entropy": 1.5810115337371826,
+ "epoch": 0.7252525252525253,
+ "grad_norm": 2.140115737915039,
+ "learning_rate": 4.276767676767677e-06,
+ "loss": 1.5774705410003662,
+ "mean_token_accuracy": 0.6274425983428955,
+ "num_tokens": 5881856.0,
+ "step": 359
+ },
+ {
+ "entropy": 1.5839109420776367,
+ "epoch": 0.7272727272727273,
+ "grad_norm": 2.0947749614715576,
+ "learning_rate": 4.274747474747475e-06,
+ "loss": 1.601511001586914,
+ "mean_token_accuracy": 0.630984365940094,
+ "num_tokens": 5898240.0,
+ "step": 360
+ },
+ {
+ "entropy": 1.195770263671875,
+ "epoch": 0.7292929292929293,
+ "grad_norm": 1.8740363121032715,
+ "learning_rate": 4.272727272727273e-06,
+ "loss": 1.195652961730957,
+ "mean_token_accuracy": 0.7077430486679077,
+ "num_tokens": 5914624.0,
+ "step": 361
+ },
+ {
+ "entropy": 1.375698208808899,
+ "epoch": 0.7313131313131314,
+ "grad_norm": 1.9580703973770142,
+ "learning_rate": 4.270707070707071e-06,
+ "loss": 1.3746864795684814,
+ "mean_token_accuracy": 0.6775158643722534,
+ "num_tokens": 5931008.0,
+ "step": 362
+ },
+ {
+ "entropy": 1.7259451150894165,
+ "epoch": 0.7333333333333333,
+ "grad_norm": 2.3127365112304688,
+ "learning_rate": 4.268686868686869e-06,
+ "loss": 1.745998501777649,
+ "mean_token_accuracy": 0.6040546894073486,
+ "num_tokens": 5947392.0,
+ "step": 363
+ },
+ {
+ "entropy": 1.209228754043579,
+ "epoch": 0.7353535353535353,
+ "grad_norm": 1.9004793167114258,
+ "learning_rate": 4.266666666666668e-06,
+ "loss": 1.232427716255188,
+ "mean_token_accuracy": 0.6925989389419556,
+ "num_tokens": 5963776.0,
+ "step": 364
+ },
+ {
+ "entropy": 1.2811263799667358,
+ "epoch": 0.7373737373737373,
+ "grad_norm": 1.9568246603012085,
+ "learning_rate": 4.2646464646464655e-06,
+ "loss": 1.291853427886963,
+ "mean_token_accuracy": 0.6884465217590332,
+ "num_tokens": 5980160.0,
+ "step": 365
+ },
+ {
+ "entropy": 1.843044638633728,
+ "epoch": 0.7393939393939394,
+ "grad_norm": 2.132735252380371,
+ "learning_rate": 4.262626262626263e-06,
+ "loss": 1.8818857669830322,
+ "mean_token_accuracy": 0.5785295367240906,
+ "num_tokens": 5996544.0,
+ "step": 366
+ },
+ {
+ "entropy": 1.3353440761566162,
+ "epoch": 0.7414141414141414,
+ "grad_norm": 1.8893013000488281,
+ "learning_rate": 4.260606060606061e-06,
+ "loss": 1.354011058807373,
+ "mean_token_accuracy": 0.6797752976417542,
+ "num_tokens": 6012928.0,
+ "step": 367
+ },
+ {
+ "entropy": 1.3889728784561157,
+ "epoch": 0.7434343434343434,
+ "grad_norm": 1.980757236480713,
+ "learning_rate": 4.258585858585859e-06,
+ "loss": 1.3671875,
+ "mean_token_accuracy": 0.6627381443977356,
+ "num_tokens": 6029312.0,
+ "step": 368
+ },
+ {
+ "entropy": 1.4605348110198975,
+ "epoch": 0.7454545454545455,
+ "grad_norm": 2.1033780574798584,
+ "learning_rate": 4.256565656565657e-06,
+ "loss": 1.455024003982544,
+ "mean_token_accuracy": 0.6451514363288879,
+ "num_tokens": 6045696.0,
+ "step": 369
+ },
+ {
+ "entropy": 1.4382058382034302,
+ "epoch": 0.7474747474747475,
+ "grad_norm": 2.1068074703216553,
+ "learning_rate": 4.254545454545455e-06,
+ "loss": 1.4422768354415894,
+ "mean_token_accuracy": 0.6630434989929199,
+ "num_tokens": 6062080.0,
+ "step": 370
+ },
+ {
+ "entropy": 1.8046759366989136,
+ "epoch": 0.7494949494949495,
+ "grad_norm": 2.214466094970703,
+ "learning_rate": 4.252525252525253e-06,
+ "loss": 1.8247106075286865,
+ "mean_token_accuracy": 0.6003297567367554,
+ "num_tokens": 6078464.0,
+ "step": 371
+ },
+ {
+ "entropy": 1.671690583229065,
+ "epoch": 0.7515151515151515,
+ "grad_norm": 1.9790785312652588,
+ "learning_rate": 4.250505050505051e-06,
+ "loss": 1.6907753944396973,
+ "mean_token_accuracy": 0.6232290863990784,
+ "num_tokens": 6094848.0,
+ "step": 372
+ },
+ {
+ "entropy": 1.6881897449493408,
+ "epoch": 0.7535353535353535,
+ "grad_norm": 1.8638463020324707,
+ "learning_rate": 4.248484848484849e-06,
+ "loss": 1.7132716178894043,
+ "mean_token_accuracy": 0.6357474327087402,
+ "num_tokens": 6111232.0,
+ "step": 373
+ },
+ {
+ "entropy": 1.2555122375488281,
+ "epoch": 0.7555555555555555,
+ "grad_norm": 2.082378387451172,
+ "learning_rate": 4.246464646464647e-06,
+ "loss": 1.240688681602478,
+ "mean_token_accuracy": 0.6965070962905884,
+ "num_tokens": 6127616.0,
+ "step": 374
+ },
+ {
+ "entropy": 1.847135305404663,
+ "epoch": 0.7575757575757576,
+ "grad_norm": 2.142962694168091,
+ "learning_rate": 4.244444444444445e-06,
+ "loss": 1.8840911388397217,
+ "mean_token_accuracy": 0.5822545289993286,
+ "num_tokens": 6144000.0,
+ "step": 375
+ },
+ {
+ "entropy": 1.239328384399414,
+ "epoch": 0.7595959595959596,
+ "grad_norm": 2.0589468479156494,
+ "learning_rate": 4.242424242424243e-06,
+ "loss": 1.256324052810669,
+ "mean_token_accuracy": 0.6845383644104004,
+ "num_tokens": 6160384.0,
+ "step": 376
+ },
+ {
+ "entropy": 1.5047202110290527,
+ "epoch": 0.7616161616161616,
+ "grad_norm": 1.7951308488845825,
+ "learning_rate": 4.240404040404041e-06,
+ "loss": 1.5063304901123047,
+ "mean_token_accuracy": 0.6583414673805237,
+ "num_tokens": 6176768.0,
+ "step": 377
+ },
+ {
+ "entropy": 1.4072566032409668,
+ "epoch": 0.7636363636363637,
+ "grad_norm": 2.1670594215393066,
+ "learning_rate": 4.238383838383839e-06,
+ "loss": 1.4205389022827148,
+ "mean_token_accuracy": 0.658707857131958,
+ "num_tokens": 6193152.0,
+ "step": 378
+ },
+ {
+ "entropy": 1.3207885026931763,
+ "epoch": 0.7656565656565657,
+ "grad_norm": 1.9017083644866943,
+ "learning_rate": 4.236363636363637e-06,
+ "loss": 1.3169896602630615,
+ "mean_token_accuracy": 0.6954079270362854,
+ "num_tokens": 6209536.0,
+ "step": 379
+ },
+ {
+ "entropy": 1.4762436151504517,
+ "epoch": 0.7676767676767676,
+ "grad_norm": 2.0023069381713867,
+ "learning_rate": 4.234343434343435e-06,
+ "loss": 1.4725041389465332,
+ "mean_token_accuracy": 0.6620053648948669,
+ "num_tokens": 6225920.0,
+ "step": 380
+ },
+ {
+ "entropy": 1.4895304441452026,
+ "epoch": 0.7696969696969697,
+ "grad_norm": 1.9528017044067383,
+ "learning_rate": 4.2323232323232325e-06,
+ "loss": 1.49650239944458,
+ "mean_token_accuracy": 0.6508915424346924,
+ "num_tokens": 6242304.0,
+ "step": 381
+ },
+ {
+ "entropy": 1.3024516105651855,
+ "epoch": 0.7717171717171717,
+ "grad_norm": 1.9855588674545288,
+ "learning_rate": 4.2303030303030304e-06,
+ "loss": 1.34218168258667,
+ "mean_token_accuracy": 0.6838055849075317,
+ "num_tokens": 6258688.0,
+ "step": 382
+ },
+ {
+ "entropy": 1.6005626916885376,
+ "epoch": 0.7737373737373737,
+ "grad_norm": 2.028286933898926,
+ "learning_rate": 4.228282828282828e-06,
+ "loss": 1.6299283504486084,
+ "mean_token_accuracy": 0.6315950155258179,
+ "num_tokens": 6275072.0,
+ "step": 383
+ },
+ {
+ "entropy": 1.7050484418869019,
+ "epoch": 0.7757575757575758,
+ "grad_norm": 2.474047899246216,
+ "learning_rate": 4.226262626262626e-06,
+ "loss": 1.777151346206665,
+ "mean_token_accuracy": 0.5919027924537659,
+ "num_tokens": 6291456.0,
+ "step": 384
+ },
+ {
+ "entropy": 1.5014543533325195,
+ "epoch": 0.7777777777777778,
+ "grad_norm": 1.9866594076156616,
+ "learning_rate": 4.224242424242425e-06,
+ "loss": 1.5308949947357178,
+ "mean_token_accuracy": 0.639106035232544,
+ "num_tokens": 6307840.0,
+ "step": 385
+ },
+ {
+ "entropy": 1.6957359313964844,
+ "epoch": 0.7797979797979798,
+ "grad_norm": 2.229820966720581,
+ "learning_rate": 4.222222222222223e-06,
+ "loss": 1.728761911392212,
+ "mean_token_accuracy": 0.6242061257362366,
+ "num_tokens": 6324224.0,
+ "step": 386
+ },
+ {
+ "entropy": 1.6823521852493286,
+ "epoch": 0.7818181818181819,
+ "grad_norm": 2.033383369445801,
+ "learning_rate": 4.220202020202021e-06,
+ "loss": 1.7310154438018799,
+ "mean_token_accuracy": 0.6257938742637634,
+ "num_tokens": 6340608.0,
+ "step": 387
+ },
+ {
+ "entropy": 1.710307240486145,
+ "epoch": 0.7838383838383839,
+ "grad_norm": 2.061861038208008,
+ "learning_rate": 4.218181818181819e-06,
+ "loss": 1.7066943645477295,
+ "mean_token_accuracy": 0.6123595237731934,
+ "num_tokens": 6356992.0,
+ "step": 388
+ },
+ {
+ "entropy": 1.241638422012329,
+ "epoch": 0.7858585858585858,
+ "grad_norm": 1.9743852615356445,
+ "learning_rate": 4.2161616161616165e-06,
+ "loss": 1.2520272731781006,
+ "mean_token_accuracy": 0.692415714263916,
+ "num_tokens": 6373376.0,
+ "step": 389
+ },
+ {
+ "entropy": 1.5918776988983154,
+ "epoch": 0.7878787878787878,
+ "grad_norm": 2.156675100326538,
+ "learning_rate": 4.214141414141414e-06,
+ "loss": 1.6060255765914917,
+ "mean_token_accuracy": 0.6178553700447083,
+ "num_tokens": 6389760.0,
+ "step": 390
+ },
+ {
+ "entropy": 1.7307862043380737,
+ "epoch": 0.7898989898989899,
+ "grad_norm": 2.171247959136963,
+ "learning_rate": 4.212121212121212e-06,
+ "loss": 1.7327581644058228,
+ "mean_token_accuracy": 0.6502808928489685,
+ "num_tokens": 6406144.0,
+ "step": 391
+ },
+ {
+ "entropy": 1.6725553274154663,
+ "epoch": 0.7919191919191919,
+ "grad_norm": 2.020228624343872,
+ "learning_rate": 4.21010101010101e-06,
+ "loss": 1.6803646087646484,
+ "mean_token_accuracy": 0.6383732557296753,
+ "num_tokens": 6422528.0,
+ "step": 392
+ },
+ {
+ "entropy": 1.4120928049087524,
+ "epoch": 0.793939393939394,
+ "grad_norm": 1.9896639585494995,
+ "learning_rate": 4.208080808080808e-06,
+ "loss": 1.3974279165267944,
+ "mean_token_accuracy": 0.6661577820777893,
+ "num_tokens": 6438912.0,
+ "step": 393
+ },
+ {
+ "entropy": 0.933545708656311,
+ "epoch": 0.795959595959596,
+ "grad_norm": 1.6884223222732544,
+ "learning_rate": 4.206060606060606e-06,
+ "loss": 0.9287986755371094,
+ "mean_token_accuracy": 0.7636175155639648,
+ "num_tokens": 6455296.0,
+ "step": 394
+ },
+ {
+ "entropy": 1.419004201889038,
+ "epoch": 0.797979797979798,
+ "grad_norm": 2.2590551376342773,
+ "learning_rate": 4.204040404040405e-06,
+ "loss": 1.4652538299560547,
+ "mean_token_accuracy": 0.6589521169662476,
+ "num_tokens": 6471680.0,
+ "step": 395
+ },
+ {
+ "entropy": 1.750221610069275,
+ "epoch": 0.8,
+ "grad_norm": 2.195030450820923,
+ "learning_rate": 4.2020202020202026e-06,
+ "loss": 1.7328863143920898,
+ "mean_token_accuracy": 0.6008182764053345,
+ "num_tokens": 6488064.0,
+ "step": 396
+ },
+ {
+ "entropy": 1.7440015077590942,
+ "epoch": 0.802020202020202,
+ "grad_norm": 1.9833927154541016,
+ "learning_rate": 4.2000000000000004e-06,
+ "loss": 1.7617835998535156,
+ "mean_token_accuracy": 0.5992916226387024,
+ "num_tokens": 6504448.0,
+ "step": 397
+ },
+ {
+ "entropy": 1.1636452674865723,
+ "epoch": 0.804040404040404,
+ "grad_norm": 1.9506802558898926,
+ "learning_rate": 4.197979797979798e-06,
+ "loss": 1.1344032287597656,
+ "mean_token_accuracy": 0.7122618556022644,
+ "num_tokens": 6520832.0,
+ "step": 398
+ },
+ {
+ "entropy": 1.6173542737960815,
+ "epoch": 0.806060606060606,
+ "grad_norm": 2.0997231006622314,
+ "learning_rate": 4.195959595959596e-06,
+ "loss": 1.6303956508636475,
+ "mean_token_accuracy": 0.6215192675590515,
+ "num_tokens": 6537216.0,
+ "step": 399
+ },
+ {
+ "entropy": 1.4391542673110962,
+ "epoch": 0.8080808080808081,
+ "grad_norm": 2.356828212738037,
+ "learning_rate": 4.193939393939394e-06,
+ "loss": 1.4465923309326172,
+ "mean_token_accuracy": 0.6594406366348267,
+ "num_tokens": 6553600.0,
+ "step": 400
+ },
+ {
+ "epoch": 0.8080808080808081,
+ "eval_entropy": 1.5504949785047961,
+ "eval_loss": 1.5544542074203491,
+ "eval_mean_token_accuracy": 0.6409837569921247,
+ "eval_num_tokens": 6553600.0,
+ "eval_runtime": 43.7986,
+ "eval_samples_per_second": 22.603,
+ "eval_steps_per_second": 2.831,
+ "step": 400
+ },
+ {
+ "entropy": 1.60548996925354,
+ "epoch": 0.8101010101010101,
+ "grad_norm": 2.1894404888153076,
+ "learning_rate": 4.191919191919192e-06,
+ "loss": 1.6116085052490234,
+ "mean_token_accuracy": 0.6276868581771851,
+ "num_tokens": 6569984.0,
+ "step": 401
+ },
+ {
+ "entropy": 1.436041235923767,
+ "epoch": 0.8121212121212121,
+ "grad_norm": 2.1180264949798584,
+ "learning_rate": 4.18989898989899e-06,
+ "loss": 1.4203698635101318,
+ "mean_token_accuracy": 0.6554714441299438,
+ "num_tokens": 6586368.0,
+ "step": 402
+ },
+ {
+ "entropy": 1.1348106861114502,
+ "epoch": 0.8141414141414142,
+ "grad_norm": 2.0420851707458496,
+ "learning_rate": 4.187878787878788e-06,
+ "loss": 1.1424527168273926,
+ "mean_token_accuracy": 0.7154372334480286,
+ "num_tokens": 6602752.0,
+ "step": 403
+ },
+ {
+ "entropy": 1.4039727449417114,
+ "epoch": 0.8161616161616162,
+ "grad_norm": 2.148340940475464,
+ "learning_rate": 4.185858585858586e-06,
+ "loss": 1.4246132373809814,
+ "mean_token_accuracy": 0.6723864078521729,
+ "num_tokens": 6619136.0,
+ "step": 404
+ },
+ {
+ "entropy": 1.4222626686096191,
+ "epoch": 0.8181818181818182,
+ "grad_norm": 1.9436827898025513,
+ "learning_rate": 4.1838383838383835e-06,
+ "loss": 1.4235990047454834,
+ "mean_token_accuracy": 0.6759281754493713,
+ "num_tokens": 6635520.0,
+ "step": 405
+ },
+ {
+ "entropy": 1.4771310091018677,
+ "epoch": 0.8202020202020202,
+ "grad_norm": 2.0953726768493652,
+ "learning_rate": 4.181818181818182e-06,
+ "loss": 1.501934289932251,
+ "mean_token_accuracy": 0.6535173654556274,
+ "num_tokens": 6651904.0,
+ "step": 406
+ },
+ {
+ "entropy": 1.526256799697876,
+ "epoch": 0.8222222222222222,
+ "grad_norm": 2.245994806289673,
+ "learning_rate": 4.17979797979798e-06,
+ "loss": 1.539383888244629,
+ "mean_token_accuracy": 0.6417317986488342,
+ "num_tokens": 6668288.0,
+ "step": 407
+ },
+ {
+ "entropy": 1.1716097593307495,
+ "epoch": 0.8242424242424242,
+ "grad_norm": 1.8283652067184448,
+ "learning_rate": 4.177777777777778e-06,
+ "loss": 1.1798359155654907,
+ "mean_token_accuracy": 0.7123839855194092,
+ "num_tokens": 6684672.0,
+ "step": 408
+ },
+ {
+ "entropy": 1.4581540822982788,
+ "epoch": 0.8262626262626263,
+ "grad_norm": 1.9325168132781982,
+ "learning_rate": 4.175757575757576e-06,
+ "loss": 1.4526585340499878,
+ "mean_token_accuracy": 0.662432849407196,
+ "num_tokens": 6701056.0,
+ "step": 409
+ },
+ {
+ "entropy": 1.4073102474212646,
+ "epoch": 0.8282828282828283,
+ "grad_norm": 2.1543467044830322,
+ "learning_rate": 4.173737373737375e-06,
+ "loss": 1.437713861465454,
+ "mean_token_accuracy": 0.6725696325302124,
+ "num_tokens": 6717440.0,
+ "step": 410
+ },
+ {
+ "entropy": 1.5970062017440796,
+ "epoch": 0.8303030303030303,
+ "grad_norm": 2.1714792251586914,
+ "learning_rate": 4.1717171717171726e-06,
+ "loss": 1.613295078277588,
+ "mean_token_accuracy": 0.6229848265647888,
+ "num_tokens": 6733824.0,
+ "step": 411
+ },
+ {
+ "entropy": 1.449837327003479,
+ "epoch": 0.8323232323232324,
+ "grad_norm": 2.2499871253967285,
+ "learning_rate": 4.1696969696969705e-06,
+ "loss": 1.489758014678955,
+ "mean_token_accuracy": 0.645639955997467,
+ "num_tokens": 6750208.0,
+ "step": 412
+ },
+ {
+ "entropy": 1.7696173191070557,
+ "epoch": 0.8343434343434344,
+ "grad_norm": 2.3637073040008545,
+ "learning_rate": 4.167676767676768e-06,
+ "loss": 1.8147599697113037,
+ "mean_token_accuracy": 0.5821323990821838,
+ "num_tokens": 6766592.0,
+ "step": 413
+ },
+ {
+ "entropy": 1.99376380443573,
+ "epoch": 0.8363636363636363,
+ "grad_norm": 2.265683174133301,
+ "learning_rate": 4.165656565656566e-06,
+ "loss": 2.009024143218994,
+ "mean_token_accuracy": 0.5591108798980713,
+ "num_tokens": 6782976.0,
+ "step": 414
+ },
+ {
+ "entropy": 1.7276980876922607,
+ "epoch": 0.8383838383838383,
+ "grad_norm": 1.9407477378845215,
+ "learning_rate": 4.163636363636364e-06,
+ "loss": 1.7130229473114014,
+ "mean_token_accuracy": 0.6202979683876038,
+ "num_tokens": 6799360.0,
+ "step": 415
+ },
+ {
+ "entropy": 1.5418223142623901,
+ "epoch": 0.8404040404040404,
+ "grad_norm": 1.9765743017196655,
+ "learning_rate": 4.161616161616162e-06,
+ "loss": 1.5627632141113281,
+ "mean_token_accuracy": 0.6433194875717163,
+ "num_tokens": 6815744.0,
+ "step": 416
+ },
+ {
+ "entropy": 1.6040345430374146,
+ "epoch": 0.8424242424242424,
+ "grad_norm": 2.1942877769470215,
+ "learning_rate": 4.15959595959596e-06,
+ "loss": 1.6044622659683228,
+ "mean_token_accuracy": 0.6278700828552246,
+ "num_tokens": 6832128.0,
+ "step": 417
+ },
+ {
+ "entropy": 1.8461577892303467,
+ "epoch": 0.8444444444444444,
+ "grad_norm": 2.289196729660034,
+ "learning_rate": 4.157575757575758e-06,
+ "loss": 1.8679372072219849,
+ "mean_token_accuracy": 0.5887884497642517,
+ "num_tokens": 6848512.0,
+ "step": 418
+ },
+ {
+ "entropy": 1.4079350233078003,
+ "epoch": 0.8464646464646465,
+ "grad_norm": 1.9526047706604004,
+ "learning_rate": 4.155555555555556e-06,
+ "loss": 1.4145114421844482,
+ "mean_token_accuracy": 0.6731802821159363,
+ "num_tokens": 6864896.0,
+ "step": 419
+ },
+ {
+ "entropy": 1.1973973512649536,
+ "epoch": 0.8484848484848485,
+ "grad_norm": 2.0716679096221924,
+ "learning_rate": 4.1535353535353536e-06,
+ "loss": 1.260810136795044,
+ "mean_token_accuracy": 0.701697587966919,
+ "num_tokens": 6881280.0,
+ "step": 420
+ },
+ {
+ "entropy": 1.507702350616455,
+ "epoch": 0.8505050505050505,
+ "grad_norm": 2.0233314037323,
+ "learning_rate": 4.151515151515152e-06,
+ "loss": 1.513720154762268,
+ "mean_token_accuracy": 0.6497313380241394,
+ "num_tokens": 6897664.0,
+ "step": 421
+ },
+ {
+ "entropy": 1.54402756690979,
+ "epoch": 0.8525252525252526,
+ "grad_norm": 2.23366379737854,
+ "learning_rate": 4.14949494949495e-06,
+ "loss": 1.5434964895248413,
+ "mean_token_accuracy": 0.6502198576927185,
+ "num_tokens": 6914048.0,
+ "step": 422
+ },
+ {
+ "entropy": 1.820296049118042,
+ "epoch": 0.8545454545454545,
+ "grad_norm": 2.066905975341797,
+ "learning_rate": 4.147474747474748e-06,
+ "loss": 1.8356924057006836,
+ "mean_token_accuracy": 0.5886663198471069,
+ "num_tokens": 6930432.0,
+ "step": 423
+ },
+ {
+ "entropy": 1.4747720956802368,
+ "epoch": 0.8565656565656565,
+ "grad_norm": 2.040022850036621,
+ "learning_rate": 4.145454545454546e-06,
+ "loss": 1.4495999813079834,
+ "mean_token_accuracy": 0.6532731056213379,
+ "num_tokens": 6946816.0,
+ "step": 424
+ },
+ {
+ "entropy": 1.7536696195602417,
+ "epoch": 0.8585858585858586,
+ "grad_norm": 2.0884320735931396,
+ "learning_rate": 4.143434343434344e-06,
+ "loss": 1.7520158290863037,
+ "mean_token_accuracy": 0.6083903312683105,
+ "num_tokens": 6963200.0,
+ "step": 425
+ },
+ {
+ "entropy": 1.715582251548767,
+ "epoch": 0.8606060606060606,
+ "grad_norm": 2.2991514205932617,
+ "learning_rate": 4.141414141414142e-06,
+ "loss": 1.7138090133666992,
+ "mean_token_accuracy": 0.607107937335968,
+ "num_tokens": 6979584.0,
+ "step": 426
+ },
+ {
+ "entropy": 1.8116382360458374,
+ "epoch": 0.8626262626262626,
+ "grad_norm": 2.289909839630127,
+ "learning_rate": 4.13939393939394e-06,
+ "loss": 1.7755894660949707,
+ "mean_token_accuracy": 0.5776746273040771,
+ "num_tokens": 6995968.0,
+ "step": 427
+ },
+ {
+ "entropy": 1.487213373184204,
+ "epoch": 0.8646464646464647,
+ "grad_norm": 1.8834803104400635,
+ "learning_rate": 4.1373737373737375e-06,
+ "loss": 1.511157751083374,
+ "mean_token_accuracy": 0.6546165347099304,
+ "num_tokens": 7012352.0,
+ "step": 428
+ },
+ {
+ "entropy": 1.2769891023635864,
+ "epoch": 0.8666666666666667,
+ "grad_norm": 1.892616629600525,
+ "learning_rate": 4.135353535353535e-06,
+ "loss": 1.2772598266601562,
+ "mean_token_accuracy": 0.6984611749649048,
+ "num_tokens": 7028736.0,
+ "step": 429
+ },
+ {
+ "entropy": 1.579519271850586,
+ "epoch": 0.8686868686868687,
+ "grad_norm": 1.9801563024520874,
+ "learning_rate": 4.133333333333333e-06,
+ "loss": 1.5647528171539307,
+ "mean_token_accuracy": 0.6482046842575073,
+ "num_tokens": 7045120.0,
+ "step": 430
+ },
+ {
+ "entropy": 1.407600998878479,
+ "epoch": 0.8707070707070707,
+ "grad_norm": 2.1737446784973145,
+ "learning_rate": 4.131313131313132e-06,
+ "loss": 1.4184494018554688,
+ "mean_token_accuracy": 0.6640815734863281,
+ "num_tokens": 7061504.0,
+ "step": 431
+ },
+ {
+ "entropy": 1.9118441343307495,
+ "epoch": 0.8727272727272727,
+ "grad_norm": 1.9541205167770386,
+ "learning_rate": 4.12929292929293e-06,
+ "loss": 1.9581422805786133,
+ "mean_token_accuracy": 0.564667820930481,
+ "num_tokens": 7077888.0,
+ "step": 432
+ },
+ {
+ "entropy": 1.925604224205017,
+ "epoch": 0.8747474747474747,
+ "grad_norm": 1.9696223735809326,
+ "learning_rate": 4.127272727272728e-06,
+ "loss": 1.9226163625717163,
+ "mean_token_accuracy": 0.5870786309242249,
+ "num_tokens": 7094272.0,
+ "step": 433
+ },
+ {
+ "entropy": 1.3994735479354858,
+ "epoch": 0.8767676767676768,
+ "grad_norm": 2.003532886505127,
+ "learning_rate": 4.125252525252526e-06,
+ "loss": 1.4291752576828003,
+ "mean_token_accuracy": 0.669516384601593,
+ "num_tokens": 7110656.0,
+ "step": 434
+ },
+ {
+ "entropy": 1.5550929307937622,
+ "epoch": 0.8787878787878788,
+ "grad_norm": 2.2658586502075195,
+ "learning_rate": 4.1232323232323236e-06,
+ "loss": 1.5533335208892822,
+ "mean_token_accuracy": 0.6242672204971313,
+ "num_tokens": 7127040.0,
+ "step": 435
+ },
+ {
+ "entropy": 1.6622785329818726,
+ "epoch": 0.8808080808080808,
+ "grad_norm": 2.0748393535614014,
+ "learning_rate": 4.1212121212121215e-06,
+ "loss": 1.700000524520874,
+ "mean_token_accuracy": 0.6221299171447754,
+ "num_tokens": 7143424.0,
+ "step": 436
+ },
+ {
+ "entropy": 1.1223996877670288,
+ "epoch": 0.8828282828282829,
+ "grad_norm": 2.0348756313323975,
+ "learning_rate": 4.119191919191919e-06,
+ "loss": 1.143293857574463,
+ "mean_token_accuracy": 0.7045676708221436,
+ "num_tokens": 7159808.0,
+ "step": 437
+ },
+ {
+ "entropy": 1.7156380414962769,
+ "epoch": 0.8848484848484849,
+ "grad_norm": 2.306549549102783,
+ "learning_rate": 4.117171717171717e-06,
+ "loss": 1.7625794410705566,
+ "mean_token_accuracy": 0.6049706935882568,
+ "num_tokens": 7176192.0,
+ "step": 438
+ },
+ {
+ "entropy": 1.8507202863693237,
+ "epoch": 0.8868686868686869,
+ "grad_norm": 2.2955853939056396,
+ "learning_rate": 4.115151515151515e-06,
+ "loss": 1.8923052549362183,
+ "mean_token_accuracy": 0.5938568711280823,
+ "num_tokens": 7192576.0,
+ "step": 439
+ },
+ {
+ "entropy": 1.813754916191101,
+ "epoch": 0.8888888888888888,
+ "grad_norm": 2.095700263977051,
+ "learning_rate": 4.113131313131313e-06,
+ "loss": 1.8375307321548462,
+ "mean_token_accuracy": 0.5848192572593689,
+ "num_tokens": 7208960.0,
+ "step": 440
+ },
+ {
+ "entropy": 1.4185007810592651,
+ "epoch": 0.8909090909090909,
+ "grad_norm": 2.118213176727295,
+ "learning_rate": 4.111111111111111e-06,
+ "loss": 1.4548171758651733,
+ "mean_token_accuracy": 0.657608687877655,
+ "num_tokens": 7225344.0,
+ "step": 441
+ },
+ {
+ "entropy": 1.1041690111160278,
+ "epoch": 0.8929292929292929,
+ "grad_norm": 1.9638988971710205,
+ "learning_rate": 4.10909090909091e-06,
+ "loss": 1.1232322454452515,
+ "mean_token_accuracy": 0.7068881392478943,
+ "num_tokens": 7241728.0,
+ "step": 442
+ },
+ {
+ "entropy": 1.6423180103302002,
+ "epoch": 0.8949494949494949,
+ "grad_norm": 2.32987117767334,
+ "learning_rate": 4.1070707070707075e-06,
+ "loss": 1.665462613105774,
+ "mean_token_accuracy": 0.6133365631103516,
+ "num_tokens": 7258112.0,
+ "step": 443
+ },
+ {
+ "entropy": 1.4778310060501099,
+ "epoch": 0.896969696969697,
+ "grad_norm": 2.1247661113739014,
+ "learning_rate": 4.105050505050505e-06,
+ "loss": 1.4554922580718994,
+ "mean_token_accuracy": 0.6474108695983887,
+ "num_tokens": 7274496.0,
+ "step": 444
+ },
+ {
+ "entropy": 1.3777755498886108,
+ "epoch": 0.898989898989899,
+ "grad_norm": 1.9243263006210327,
+ "learning_rate": 4.103030303030303e-06,
+ "loss": 1.3942883014678955,
+ "mean_token_accuracy": 0.6656692624092102,
+ "num_tokens": 7290880.0,
+ "step": 445
+ },
+ {
+ "entropy": 1.1036415100097656,
+ "epoch": 0.901010101010101,
+ "grad_norm": 1.9711178541183472,
+ "learning_rate": 4.101010101010101e-06,
+ "loss": 1.1070351600646973,
+ "mean_token_accuracy": 0.7213605046272278,
+ "num_tokens": 7307264.0,
+ "step": 446
+ },
+ {
+ "entropy": 1.695487380027771,
+ "epoch": 0.9030303030303031,
+ "grad_norm": 2.1113150119781494,
+ "learning_rate": 4.098989898989899e-06,
+ "loss": 1.6993653774261475,
+ "mean_token_accuracy": 0.5992305874824524,
+ "num_tokens": 7323648.0,
+ "step": 447
+ },
+ {
+ "entropy": 1.496254801750183,
+ "epoch": 0.9050505050505051,
+ "grad_norm": 2.072986602783203,
+ "learning_rate": 4.096969696969697e-06,
+ "loss": 1.5046567916870117,
+ "mean_token_accuracy": 0.6558378338813782,
+ "num_tokens": 7340032.0,
+ "step": 448
+ },
+ {
+ "entropy": 2.1845033168792725,
+ "epoch": 0.907070707070707,
+ "grad_norm": 2.3246262073516846,
+ "learning_rate": 4.094949494949495e-06,
+ "loss": 2.1508708000183105,
+ "mean_token_accuracy": 0.5376160144805908,
+ "num_tokens": 7356416.0,
+ "step": 449
+ },
+ {
+ "entropy": 1.6329164505004883,
+ "epoch": 0.9090909090909091,
+ "grad_norm": 1.997310757637024,
+ "learning_rate": 4.092929292929294e-06,
+ "loss": 1.6422264575958252,
+ "mean_token_accuracy": 0.6232290863990784,
+ "num_tokens": 7372800.0,
+ "step": 450
+ },
+ {
+ "epoch": 0.9090909090909091,
+ "eval_entropy": 1.5515337480652718,
+ "eval_loss": 1.5485161542892456,
+ "eval_mean_token_accuracy": 0.6417716929028111,
+ "eval_num_tokens": 7372800.0,
+ "eval_runtime": 43.7901,
+ "eval_samples_per_second": 22.608,
+ "eval_steps_per_second": 2.832,
+ "step": 450
+ },
+ {
+ "entropy": 1.7854291200637817,
+ "epoch": 0.9111111111111111,
+ "grad_norm": 2.0212173461914062,
+ "learning_rate": 4.0909090909090915e-06,
+ "loss": 1.7920666933059692,
+ "mean_token_accuracy": 0.5950170755386353,
+ "num_tokens": 7389184.0,
+ "step": 451
+ },
+ {
+ "entropy": 1.216304898262024,
+ "epoch": 0.9131313131313131,
+ "grad_norm": 1.8800705671310425,
+ "learning_rate": 4.088888888888889e-06,
+ "loss": 1.2014267444610596,
+ "mean_token_accuracy": 0.7076819539070129,
+ "num_tokens": 7405568.0,
+ "step": 452
+ },
+ {
+ "entropy": 1.5593703985214233,
+ "epoch": 0.9151515151515152,
+ "grad_norm": 2.101381301879883,
+ "learning_rate": 4.086868686868687e-06,
+ "loss": 1.559610366821289,
+ "mean_token_accuracy": 0.6337933540344238,
+ "num_tokens": 7421952.0,
+ "step": 453
+ },
+ {
+ "entropy": 1.4199841022491455,
+ "epoch": 0.9171717171717172,
+ "grad_norm": 2.0038692951202393,
+ "learning_rate": 4.084848484848485e-06,
+ "loss": 1.4040653705596924,
+ "mean_token_accuracy": 0.660845160484314,
+ "num_tokens": 7438336.0,
+ "step": 454
+ },
+ {
+ "entropy": 1.2010453939437866,
+ "epoch": 0.9191919191919192,
+ "grad_norm": 2.004110336303711,
+ "learning_rate": 4.082828282828283e-06,
+ "loss": 1.194115400314331,
+ "mean_token_accuracy": 0.6955910921096802,
+ "num_tokens": 7454720.0,
+ "step": 455
+ },
+ {
+ "entropy": 1.2295804023742676,
+ "epoch": 0.9212121212121213,
+ "grad_norm": 2.130387544631958,
+ "learning_rate": 4.080808080808081e-06,
+ "loss": 1.2125787734985352,
+ "mean_token_accuracy": 0.6993771195411682,
+ "num_tokens": 7471104.0,
+ "step": 456
+ },
+ {
+ "entropy": 1.3706485033035278,
+ "epoch": 0.9232323232323232,
+ "grad_norm": 2.1667706966400146,
+ "learning_rate": 4.07878787878788e-06,
+ "loss": 1.3886957168579102,
+ "mean_token_accuracy": 0.6686614751815796,
+ "num_tokens": 7487488.0,
+ "step": 457
+ },
+ {
+ "entropy": 1.714059591293335,
+ "epoch": 0.9252525252525252,
+ "grad_norm": 2.0084264278411865,
+ "learning_rate": 4.0767676767676775e-06,
+ "loss": 1.724153757095337,
+ "mean_token_accuracy": 0.602161705493927,
+ "num_tokens": 7503872.0,
+ "step": 458
+ },
+ {
+ "entropy": 1.3170617818832397,
+ "epoch": 0.9272727272727272,
+ "grad_norm": 1.9923994541168213,
+ "learning_rate": 4.0747474747474754e-06,
+ "loss": 1.3518096208572388,
+ "mean_token_accuracy": 0.6801416873931885,
+ "num_tokens": 7520256.0,
+ "step": 459
+ },
+ {
+ "entropy": 1.7946380376815796,
+ "epoch": 0.9292929292929293,
+ "grad_norm": 2.3475260734558105,
+ "learning_rate": 4.072727272727273e-06,
+ "loss": 1.824514389038086,
+ "mean_token_accuracy": 0.6005740165710449,
+ "num_tokens": 7536640.0,
+ "step": 460
+ },
+ {
+ "entropy": 1.3695449829101562,
+ "epoch": 0.9313131313131313,
+ "grad_norm": 2.0983943939208984,
+ "learning_rate": 4.070707070707071e-06,
+ "loss": 1.3934273719787598,
+ "mean_token_accuracy": 0.6723253726959229,
+ "num_tokens": 7553024.0,
+ "step": 461
+ },
+ {
+ "entropy": 1.5890663862228394,
+ "epoch": 0.9333333333333333,
+ "grad_norm": 2.1465413570404053,
+ "learning_rate": 4.068686868686869e-06,
+ "loss": 1.611652135848999,
+ "mean_token_accuracy": 0.6408158540725708,
+ "num_tokens": 7569408.0,
+ "step": 462
+ },
+ {
+ "entropy": 1.2593727111816406,
+ "epoch": 0.9353535353535354,
+ "grad_norm": 1.7885241508483887,
+ "learning_rate": 4.066666666666667e-06,
+ "loss": 1.2781705856323242,
+ "mean_token_accuracy": 0.6946751475334167,
+ "num_tokens": 7585792.0,
+ "step": 463
+ },
+ {
+ "entropy": 1.5680429935455322,
+ "epoch": 0.9373737373737374,
+ "grad_norm": 2.1466636657714844,
+ "learning_rate": 4.064646464646465e-06,
+ "loss": 1.60218346118927,
+ "mean_token_accuracy": 0.6334269642829895,
+ "num_tokens": 7602176.0,
+ "step": 464
+ },
+ {
+ "entropy": 1.681670069694519,
+ "epoch": 0.9393939393939394,
+ "grad_norm": 2.2211875915527344,
+ "learning_rate": 4.062626262626263e-06,
+ "loss": 1.6819908618927002,
+ "mean_token_accuracy": 0.6177943348884583,
+ "num_tokens": 7618560.0,
+ "step": 465
+ },
+ {
+ "entropy": 1.2239924669265747,
+ "epoch": 0.9414141414141414,
+ "grad_norm": 1.9997022151947021,
+ "learning_rate": 4.060606060606061e-06,
+ "loss": 1.2383348941802979,
+ "mean_token_accuracy": 0.6922325491905212,
+ "num_tokens": 7634944.0,
+ "step": 466
+ },
+ {
+ "entropy": 1.291772723197937,
+ "epoch": 0.9434343434343434,
+ "grad_norm": 2.0868117809295654,
+ "learning_rate": 4.058585858585859e-06,
+ "loss": 1.2879221439361572,
+ "mean_token_accuracy": 0.6883854269981384,
+ "num_tokens": 7651328.0,
+ "step": 467
+ },
+ {
+ "entropy": 1.2860186100006104,
+ "epoch": 0.9454545454545454,
+ "grad_norm": 1.8586393594741821,
+ "learning_rate": 4.056565656565657e-06,
+ "loss": 1.2820203304290771,
+ "mean_token_accuracy": 0.6802638173103333,
+ "num_tokens": 7667712.0,
+ "step": 468
+ },
+ {
+ "entropy": 1.860795259475708,
+ "epoch": 0.9474747474747475,
+ "grad_norm": 2.364405393600464,
+ "learning_rate": 4.054545454545455e-06,
+ "loss": 1.8880727291107178,
+ "mean_token_accuracy": 0.5821323990821838,
+ "num_tokens": 7684096.0,
+ "step": 469
+ },
+ {
+ "entropy": 1.6017589569091797,
+ "epoch": 0.9494949494949495,
+ "grad_norm": 2.023054599761963,
+ "learning_rate": 4.052525252525253e-06,
+ "loss": 1.5993852615356445,
+ "mean_token_accuracy": 0.6366634368896484,
+ "num_tokens": 7700480.0,
+ "step": 470
+ },
+ {
+ "entropy": 1.8498003482818604,
+ "epoch": 0.9515151515151515,
+ "grad_norm": 2.4474003314971924,
+ "learning_rate": 4.050505050505051e-06,
+ "loss": 1.868700623512268,
+ "mean_token_accuracy": 0.5826209187507629,
+ "num_tokens": 7716864.0,
+ "step": 471
+ },
+ {
+ "entropy": 1.5625540018081665,
+ "epoch": 0.9535353535353536,
+ "grad_norm": 2.1018362045288086,
+ "learning_rate": 4.048484848484849e-06,
+ "loss": 1.571077585220337,
+ "mean_token_accuracy": 0.6433805823326111,
+ "num_tokens": 7733248.0,
+ "step": 472
+ },
+ {
+ "entropy": 1.8534579277038574,
+ "epoch": 0.9555555555555556,
+ "grad_norm": 2.407588243484497,
+ "learning_rate": 4.046464646464647e-06,
+ "loss": 1.8789153099060059,
+ "mean_token_accuracy": 0.5859794616699219,
+ "num_tokens": 7749632.0,
+ "step": 473
+ },
+ {
+ "entropy": 1.3264559507369995,
+ "epoch": 0.9575757575757575,
+ "grad_norm": 2.007199764251709,
+ "learning_rate": 4.044444444444445e-06,
+ "loss": 1.3489338159561157,
+ "mean_token_accuracy": 0.6650586128234863,
+ "num_tokens": 7766016.0,
+ "step": 474
+ },
+ {
+ "entropy": 1.1361761093139648,
+ "epoch": 0.9595959595959596,
+ "grad_norm": 1.9226998090744019,
+ "learning_rate": 4.0424242424242425e-06,
+ "loss": 1.156738519668579,
+ "mean_token_accuracy": 0.7226428985595703,
+ "num_tokens": 7782400.0,
+ "step": 475
+ },
+ {
+ "entropy": 1.550872802734375,
+ "epoch": 0.9616161616161616,
+ "grad_norm": 2.0082473754882812,
+ "learning_rate": 4.04040404040404e-06,
+ "loss": 1.5657379627227783,
+ "mean_token_accuracy": 0.642953097820282,
+ "num_tokens": 7798784.0,
+ "step": 476
+ },
+ {
+ "entropy": 1.0122721195220947,
+ "epoch": 0.9636363636363636,
+ "grad_norm": 1.9946776628494263,
+ "learning_rate": 4.038383838383838e-06,
+ "loss": 1.0301119089126587,
+ "mean_token_accuracy": 0.7361993193626404,
+ "num_tokens": 7815168.0,
+ "step": 477
+ },
+ {
+ "entropy": 1.61614990234375,
+ "epoch": 0.9656565656565657,
+ "grad_norm": 2.5009427070617676,
+ "learning_rate": 4.036363636363637e-06,
+ "loss": 1.6194994449615479,
+ "mean_token_accuracy": 0.620175838470459,
+ "num_tokens": 7831552.0,
+ "step": 478
+ },
+ {
+ "entropy": 1.5723443031311035,
+ "epoch": 0.9676767676767677,
+ "grad_norm": 1.9101917743682861,
+ "learning_rate": 4.034343434343435e-06,
+ "loss": 1.5586073398590088,
+ "mean_token_accuracy": 0.6223741769790649,
+ "num_tokens": 7847936.0,
+ "step": 479
+ },
+ {
+ "entropy": 1.271834373474121,
+ "epoch": 0.9696969696969697,
+ "grad_norm": 1.768438458442688,
+ "learning_rate": 4.032323232323233e-06,
+ "loss": 1.2466087341308594,
+ "mean_token_accuracy": 0.692354679107666,
+ "num_tokens": 7864320.0,
+ "step": 480
+ },
+ {
+ "entropy": 1.4550020694732666,
+ "epoch": 0.9717171717171718,
+ "grad_norm": 1.9753917455673218,
+ "learning_rate": 4.030303030303031e-06,
+ "loss": 1.4689980745315552,
+ "mean_token_accuracy": 0.6530288457870483,
+ "num_tokens": 7880704.0,
+ "step": 481
+ },
+ {
+ "entropy": 2.023750066757202,
+ "epoch": 0.9737373737373738,
+ "grad_norm": 2.3014674186706543,
+ "learning_rate": 4.0282828282828285e-06,
+ "loss": 2.0601019859313965,
+ "mean_token_accuracy": 0.5528212189674377,
+ "num_tokens": 7897088.0,
+ "step": 482
+ },
+ {
+ "entropy": 1.3225218057632446,
+ "epoch": 0.9757575757575757,
+ "grad_norm": 1.9937688112258911,
+ "learning_rate": 4.0262626262626264e-06,
+ "loss": 1.3186583518981934,
+ "mean_token_accuracy": 0.6751343607902527,
+ "num_tokens": 7913472.0,
+ "step": 483
+ },
+ {
+ "entropy": 1.4357911348342896,
+ "epoch": 0.9777777777777777,
+ "grad_norm": 1.9591492414474487,
+ "learning_rate": 4.024242424242424e-06,
+ "loss": 1.4157384634017944,
+ "mean_token_accuracy": 0.6618832349777222,
+ "num_tokens": 7929856.0,
+ "step": 484
+ },
+ {
+ "entropy": 1.470274567604065,
+ "epoch": 0.9797979797979798,
+ "grad_norm": 1.9318779706954956,
+ "learning_rate": 4.022222222222222e-06,
+ "loss": 1.4754853248596191,
+ "mean_token_accuracy": 0.6497923731803894,
+ "num_tokens": 7946240.0,
+ "step": 485
+ },
+ {
+ "entropy": 1.345750331878662,
+ "epoch": 0.9818181818181818,
+ "grad_norm": 1.9079619646072388,
+ "learning_rate": 4.02020202020202e-06,
+ "loss": 1.336526870727539,
+ "mean_token_accuracy": 0.6800805926322937,
+ "num_tokens": 7962624.0,
+ "step": 486
+ },
+ {
+ "entropy": 1.4641839265823364,
+ "epoch": 0.9838383838383838,
+ "grad_norm": 1.8413265943527222,
+ "learning_rate": 4.018181818181818e-06,
+ "loss": 1.4556326866149902,
+ "mean_token_accuracy": 0.6591963768005371,
+ "num_tokens": 7979008.0,
+ "step": 487
+ },
+ {
+ "entropy": 1.7966911792755127,
+ "epoch": 0.9858585858585859,
+ "grad_norm": 2.138899564743042,
+ "learning_rate": 4.016161616161616e-06,
+ "loss": 1.822898507118225,
+ "mean_token_accuracy": 0.5959941148757935,
+ "num_tokens": 7995392.0,
+ "step": 488
+ },
+ {
+ "entropy": 1.4334503412246704,
+ "epoch": 0.9878787878787879,
+ "grad_norm": 2.206122636795044,
+ "learning_rate": 4.014141414141415e-06,
+ "loss": 1.4127681255340576,
+ "mean_token_accuracy": 0.6609672904014587,
+ "num_tokens": 8011776.0,
+ "step": 489
+ },
+ {
+ "entropy": 1.684112548828125,
+ "epoch": 0.98989898989899,
+ "grad_norm": 2.0637400150299072,
+ "learning_rate": 4.0121212121212125e-06,
+ "loss": 1.6790317296981812,
+ "mean_token_accuracy": 0.6159012913703918,
+ "num_tokens": 8028160.0,
+ "step": 490
+ },
+ {
+ "entropy": 2.0716898441314697,
+ "epoch": 0.9919191919191919,
+ "grad_norm": 3.3912627696990967,
+ "learning_rate": 4.01010101010101e-06,
+ "loss": 2.0814826488494873,
+ "mean_token_accuracy": 0.5522105693817139,
+ "num_tokens": 8044544.0,
+ "step": 491
+ },
+ {
+ "entropy": 1.8764609098434448,
+ "epoch": 0.9939393939393939,
+ "grad_norm": 2.00569748878479,
+ "learning_rate": 4.008080808080808e-06,
+ "loss": 1.9150068759918213,
+ "mean_token_accuracy": 0.5823766589164734,
+ "num_tokens": 8060928.0,
+ "step": 492
+ },
+ {
+ "entropy": 1.630242943763733,
+ "epoch": 0.9959595959595959,
+ "grad_norm": 2.0648858547210693,
+ "learning_rate": 4.006060606060607e-06,
+ "loss": 1.6354784965515137,
+ "mean_token_accuracy": 0.621213972568512,
+ "num_tokens": 8077312.0,
+ "step": 493
+ },
+ {
+ "entropy": 1.7174078226089478,
+ "epoch": 0.997979797979798,
+ "grad_norm": 2.1376583576202393,
+ "learning_rate": 4.004040404040405e-06,
+ "loss": 1.7092773914337158,
+ "mean_token_accuracy": 0.6129701733589172,
+ "num_tokens": 8093696.0,
+ "step": 494
+ },
+ {
+ "entropy": 1.2112717628479004,
+ "epoch": 1.0,
+ "grad_norm": 1.9164371490478516,
+ "learning_rate": 4.002020202020203e-06,
+ "loss": 1.2106354236602783,
+ "mean_token_accuracy": 0.7040180563926697,
+ "num_tokens": 8110080.0,
+ "step": 495
+ },
+ {
+ "entropy": 1.2168891429901123,
+ "epoch": 1.002020202020202,
+ "grad_norm": 1.9845075607299805,
+ "learning_rate": 4.000000000000001e-06,
+ "loss": 1.1398870944976807,
+ "mean_token_accuracy": 0.7053004503250122,
+ "num_tokens": 8126464.0,
+ "step": 496
+ },
+ {
+ "entropy": 1.4670923948287964,
+ "epoch": 1.004040404040404,
+ "grad_norm": 2.0864014625549316,
+ "learning_rate": 3.9979797979797986e-06,
+ "loss": 1.4241242408752441,
+ "mean_token_accuracy": 0.6605398058891296,
+ "num_tokens": 8142848.0,
+ "step": 497
+ },
+ {
+ "entropy": 1.560593843460083,
+ "epoch": 1.006060606060606,
+ "grad_norm": 1.8882137537002563,
+ "learning_rate": 3.9959595959595964e-06,
+ "loss": 1.5164835453033447,
+ "mean_token_accuracy": 0.6610894203186035,
+ "num_tokens": 8159232.0,
+ "step": 498
+ },
+ {
+ "entropy": 1.2573609352111816,
+ "epoch": 1.0080808080808081,
+ "grad_norm": 2.011486530303955,
+ "learning_rate": 3.993939393939394e-06,
+ "loss": 1.2119636535644531,
+ "mean_token_accuracy": 0.6832559704780579,
+ "num_tokens": 8175616.0,
+ "step": 499
+ },
+ {
+ "entropy": 1.6119122505187988,
+ "epoch": 1.0101010101010102,
+ "grad_norm": 2.0089032649993896,
+ "learning_rate": 3.991919191919192e-06,
+ "loss": 1.5869622230529785,
+ "mean_token_accuracy": 0.6301905512809753,
+ "num_tokens": 8192000.0,
+ "step": 500
+ },
+ {
+ "epoch": 1.0101010101010102,
+ "eval_entropy": 1.4907484121860997,
+ "eval_loss": 1.5444872379302979,
+ "eval_mean_token_accuracy": 0.6426531960887294,
+ "eval_num_tokens": 8192000.0,
+ "eval_runtime": 43.729,
+ "eval_samples_per_second": 22.639,
+ "eval_steps_per_second": 2.836,
+ "step": 500
+ },
+ {
+ "entropy": 1.5645418167114258,
+ "epoch": 1.0121212121212122,
+ "grad_norm": 2.171133279800415,
+ "learning_rate": 3.98989898989899e-06,
+ "loss": 1.5252666473388672,
+ "mean_token_accuracy": 0.639594554901123,
+ "num_tokens": 8208384.0,
+ "step": 501
+ },
+ {
+ "entropy": 1.2985379695892334,
+ "epoch": 1.0141414141414142,
+ "grad_norm": 1.9075100421905518,
+ "learning_rate": 3.987878787878788e-06,
+ "loss": 1.2863168716430664,
+ "mean_token_accuracy": 0.6797142028808594,
+ "num_tokens": 8224768.0,
+ "step": 502
+ },
+ {
+ "entropy": 1.4055086374282837,
+ "epoch": 1.0161616161616163,
+ "grad_norm": 1.8511557579040527,
+ "learning_rate": 3.985858585858587e-06,
+ "loss": 1.440205454826355,
+ "mean_token_accuracy": 0.6671348214149475,
+ "num_tokens": 8241152.0,
+ "step": 503
+ },
+ {
+ "entropy": 1.0818580389022827,
+ "epoch": 1.018181818181818,
+ "grad_norm": 1.8912067413330078,
+ "learning_rate": 3.983838383838385e-06,
+ "loss": 1.0614542961120605,
+ "mean_token_accuracy": 0.7345505356788635,
+ "num_tokens": 8257536.0,
+ "step": 504
+ },
+ {
+ "entropy": 0.9176992177963257,
+ "epoch": 1.02020202020202,
+ "grad_norm": 1.7737077474594116,
+ "learning_rate": 3.9818181818181825e-06,
+ "loss": 0.9281337261199951,
+ "mean_token_accuracy": 0.7594040036201477,
+ "num_tokens": 8273920.0,
+ "step": 505
+ },
+ {
+ "entropy": 1.8801840543746948,
+ "epoch": 1.0222222222222221,
+ "grad_norm": 2.191689968109131,
+ "learning_rate": 3.97979797979798e-06,
+ "loss": 1.9127342700958252,
+ "mean_token_accuracy": 0.5732169151306152,
+ "num_tokens": 8290304.0,
+ "step": 506
+ },
+ {
+ "entropy": 1.2336418628692627,
+ "epoch": 1.0242424242424242,
+ "grad_norm": 2.0369555950164795,
+ "learning_rate": 3.977777777777778e-06,
+ "loss": 1.2513363361358643,
+ "mean_token_accuracy": 0.6852100491523743,
+ "num_tokens": 8306688.0,
+ "step": 507
+ },
+ {
+ "entropy": 1.8379974365234375,
+ "epoch": 1.0262626262626262,
+ "grad_norm": 2.176361083984375,
+ "learning_rate": 3.975757575757576e-06,
+ "loss": 1.8472888469696045,
+ "mean_token_accuracy": 0.5863458514213562,
+ "num_tokens": 8323072.0,
+ "step": 508
+ },
+ {
+ "entropy": 1.4421337842941284,
+ "epoch": 1.0282828282828282,
+ "grad_norm": 2.305441379547119,
+ "learning_rate": 3.973737373737374e-06,
+ "loss": 1.4835591316223145,
+ "mean_token_accuracy": 0.6530288457870483,
+ "num_tokens": 8339456.0,
+ "step": 509
+ },
+ {
+ "entropy": 1.535184621810913,
+ "epoch": 1.0303030303030303,
+ "grad_norm": 2.0329015254974365,
+ "learning_rate": 3.971717171717172e-06,
+ "loss": 1.547795057296753,
+ "mean_token_accuracy": 0.6268319487571716,
+ "num_tokens": 8355840.0,
+ "step": 510
+ },
+ {
+ "entropy": 1.295592188835144,
+ "epoch": 1.0323232323232323,
+ "grad_norm": 2.0321359634399414,
+ "learning_rate": 3.96969696969697e-06,
+ "loss": 1.2885974645614624,
+ "mean_token_accuracy": 0.681485116481781,
+ "num_tokens": 8372224.0,
+ "step": 511
+ },
+ {
+ "entropy": 1.345553994178772,
+ "epoch": 1.0343434343434343,
+ "grad_norm": 1.9151290655136108,
+ "learning_rate": 3.967676767676768e-06,
+ "loss": 1.3364320993423462,
+ "mean_token_accuracy": 0.6849657893180847,
+ "num_tokens": 8388608.0,
+ "step": 512
+ },
+ {
+ "entropy": 1.3150815963745117,
+ "epoch": 1.0363636363636364,
+ "grad_norm": 1.9379258155822754,
+ "learning_rate": 3.965656565656566e-06,
+ "loss": 1.317258596420288,
+ "mean_token_accuracy": 0.6873473525047302,
+ "num_tokens": 8404992.0,
+ "step": 513
+ },
+ {
+ "entropy": 1.46832275390625,
+ "epoch": 1.0383838383838384,
+ "grad_norm": 2.1009161472320557,
+ "learning_rate": 3.963636363636364e-06,
+ "loss": 1.4794366359710693,
+ "mean_token_accuracy": 0.6513800621032715,
+ "num_tokens": 8421376.0,
+ "step": 514
+ },
+ {
+ "entropy": 1.389290452003479,
+ "epoch": 1.0404040404040404,
+ "grad_norm": 1.8813941478729248,
+ "learning_rate": 3.961616161616162e-06,
+ "loss": 1.3930776119232178,
+ "mean_token_accuracy": 0.6799584627151489,
+ "num_tokens": 8437760.0,
+ "step": 515
+ },
+ {
+ "entropy": 1.2919796705245972,
+ "epoch": 1.0424242424242425,
+ "grad_norm": 2.0765745639801025,
+ "learning_rate": 3.95959595959596e-06,
+ "loss": 1.2943801879882812,
+ "mean_token_accuracy": 0.6929653286933899,
+ "num_tokens": 8454144.0,
+ "step": 516
+ },
+ {
+ "entropy": 1.172440767288208,
+ "epoch": 1.0444444444444445,
+ "grad_norm": 2.126800298690796,
+ "learning_rate": 3.957575757575758e-06,
+ "loss": 1.1923961639404297,
+ "mean_token_accuracy": 0.7098192572593689,
+ "num_tokens": 8470528.0,
+ "step": 517
+ },
+ {
+ "entropy": 1.8239458799362183,
+ "epoch": 1.0464646464646465,
+ "grad_norm": 1.96744966506958,
+ "learning_rate": 3.955555555555556e-06,
+ "loss": 1.840916633605957,
+ "mean_token_accuracy": 0.6033830046653748,
+ "num_tokens": 8486912.0,
+ "step": 518
+ },
+ {
+ "entropy": 1.4117076396942139,
+ "epoch": 1.0484848484848486,
+ "grad_norm": 2.0560126304626465,
+ "learning_rate": 3.953535353535354e-06,
+ "loss": 1.421530842781067,
+ "mean_token_accuracy": 0.6581583023071289,
+ "num_tokens": 8503296.0,
+ "step": 519
+ },
+ {
+ "entropy": 1.7085175514221191,
+ "epoch": 1.0505050505050506,
+ "grad_norm": 2.1386539936065674,
+ "learning_rate": 3.951515151515152e-06,
+ "loss": 1.7334365844726562,
+ "mean_token_accuracy": 0.6282975077629089,
+ "num_tokens": 8519680.0,
+ "step": 520
+ },
+ {
+ "entropy": 1.0607928037643433,
+ "epoch": 1.0525252525252524,
+ "grad_norm": 1.981020212173462,
+ "learning_rate": 3.9494949494949496e-06,
+ "loss": 1.0639690160751343,
+ "mean_token_accuracy": 0.722337543964386,
+ "num_tokens": 8536064.0,
+ "step": 521
+ },
+ {
+ "entropy": 1.2227518558502197,
+ "epoch": 1.0545454545454545,
+ "grad_norm": 1.9116015434265137,
+ "learning_rate": 3.9474747474747474e-06,
+ "loss": 1.2139748334884644,
+ "mean_token_accuracy": 0.704384446144104,
+ "num_tokens": 8552448.0,
+ "step": 522
+ },
+ {
+ "entropy": 1.2655551433563232,
+ "epoch": 1.0565656565656565,
+ "grad_norm": 2.176706075668335,
+ "learning_rate": 3.945454545454545e-06,
+ "loss": 1.2869982719421387,
+ "mean_token_accuracy": 0.678798258304596,
+ "num_tokens": 8568832.0,
+ "step": 523
+ },
+ {
+ "entropy": 1.436476707458496,
+ "epoch": 1.0585858585858585,
+ "grad_norm": 2.034846544265747,
+ "learning_rate": 3.943434343434343e-06,
+ "loss": 1.431445837020874,
+ "mean_token_accuracy": 0.6650586128234863,
+ "num_tokens": 8585216.0,
+ "step": 524
+ },
+ {
+ "entropy": 1.4563549757003784,
+ "epoch": 1.0606060606060606,
+ "grad_norm": 2.0301241874694824,
+ "learning_rate": 3.941414141414142e-06,
+ "loss": 1.468353033065796,
+ "mean_token_accuracy": 0.6550439596176147,
+ "num_tokens": 8601600.0,
+ "step": 525
+ },
+ {
+ "entropy": 1.4636993408203125,
+ "epoch": 1.0626262626262626,
+ "grad_norm": 2.092679023742676,
+ "learning_rate": 3.93939393939394e-06,
+ "loss": 1.4488987922668457,
+ "mean_token_accuracy": 0.652723491191864,
+ "num_tokens": 8617984.0,
+ "step": 526
+ },
+ {
+ "entropy": 1.1217654943466187,
+ "epoch": 1.0646464646464646,
+ "grad_norm": 1.953627347946167,
+ "learning_rate": 3.937373737373738e-06,
+ "loss": 1.1403257846832275,
+ "mean_token_accuracy": 0.7172080874443054,
+ "num_tokens": 8634368.0,
+ "step": 527
+ },
+ {
+ "entropy": 1.7785910367965698,
+ "epoch": 1.0666666666666667,
+ "grad_norm": 1.8496789932250977,
+ "learning_rate": 3.935353535353536e-06,
+ "loss": 1.7961615324020386,
+ "mean_token_accuracy": 0.6036272644996643,
+ "num_tokens": 8650752.0,
+ "step": 528
+ },
+ {
+ "entropy": 1.0998575687408447,
+ "epoch": 1.0686868686868687,
+ "grad_norm": 1.8732962608337402,
+ "learning_rate": 3.9333333333333335e-06,
+ "loss": 1.0960031747817993,
+ "mean_token_accuracy": 0.7223986387252808,
+ "num_tokens": 8667136.0,
+ "step": 529
+ },
+ {
+ "entropy": 1.6707724332809448,
+ "epoch": 1.0707070707070707,
+ "grad_norm": 2.220453977584839,
+ "learning_rate": 3.931313131313131e-06,
+ "loss": 1.6837453842163086,
+ "mean_token_accuracy": 0.6148021221160889,
+ "num_tokens": 8683520.0,
+ "step": 530
+ },
+ {
+ "entropy": 1.6108022928237915,
+ "epoch": 1.0727272727272728,
+ "grad_norm": 2.2464118003845215,
+ "learning_rate": 3.929292929292929e-06,
+ "loss": 1.633517861366272,
+ "mean_token_accuracy": 0.6195651888847351,
+ "num_tokens": 8699904.0,
+ "step": 531
+ },
+ {
+ "entropy": 1.1888173818588257,
+ "epoch": 1.0747474747474748,
+ "grad_norm": 1.8803986310958862,
+ "learning_rate": 3.927272727272727e-06,
+ "loss": 1.1904420852661133,
+ "mean_token_accuracy": 0.7134220600128174,
+ "num_tokens": 8716288.0,
+ "step": 532
+ },
+ {
+ "entropy": 1.0656445026397705,
+ "epoch": 1.0767676767676768,
+ "grad_norm": 2.038243532180786,
+ "learning_rate": 3.925252525252525e-06,
+ "loss": 1.0505216121673584,
+ "mean_token_accuracy": 0.7284440398216248,
+ "num_tokens": 8732672.0,
+ "step": 533
+ },
+ {
+ "entropy": 1.1473655700683594,
+ "epoch": 1.0787878787878789,
+ "grad_norm": 1.981107234954834,
+ "learning_rate": 3.923232323232323e-06,
+ "loss": 1.136179804801941,
+ "mean_token_accuracy": 0.7085368633270264,
+ "num_tokens": 8749056.0,
+ "step": 534
+ },
+ {
+ "entropy": 1.340027928352356,
+ "epoch": 1.0808080808080809,
+ "grad_norm": 2.2797436714172363,
+ "learning_rate": 3.921212121212122e-06,
+ "loss": 1.334214210510254,
+ "mean_token_accuracy": 0.6749511361122131,
+ "num_tokens": 8765440.0,
+ "step": 535
+ },
+ {
+ "entropy": 1.1464864015579224,
+ "epoch": 1.082828282828283,
+ "grad_norm": 1.9428092241287231,
+ "learning_rate": 3.9191919191919196e-06,
+ "loss": 1.1649314165115356,
+ "mean_token_accuracy": 0.72013920545578,
+ "num_tokens": 8781824.0,
+ "step": 536
+ },
+ {
+ "entropy": 1.784925103187561,
+ "epoch": 1.084848484848485,
+ "grad_norm": 2.157468795776367,
+ "learning_rate": 3.9171717171717175e-06,
+ "loss": 1.7725508213043213,
+ "mean_token_accuracy": 0.6014899611473083,
+ "num_tokens": 8798208.0,
+ "step": 537
+ },
+ {
+ "entropy": 1.362166166305542,
+ "epoch": 1.0868686868686868,
+ "grad_norm": 2.109646797180176,
+ "learning_rate": 3.915151515151515e-06,
+ "loss": 1.331969976425171,
+ "mean_token_accuracy": 0.6720200181007385,
+ "num_tokens": 8814592.0,
+ "step": 538
+ },
+ {
+ "entropy": 1.5614176988601685,
+ "epoch": 1.0888888888888888,
+ "grad_norm": 2.1047098636627197,
+ "learning_rate": 3.913131313131314e-06,
+ "loss": 1.534292459487915,
+ "mean_token_accuracy": 0.6446629166603088,
+ "num_tokens": 8830976.0,
+ "step": 539
+ },
+ {
+ "entropy": 1.5551490783691406,
+ "epoch": 1.0909090909090908,
+ "grad_norm": 2.10304594039917,
+ "learning_rate": 3.911111111111112e-06,
+ "loss": 1.5706363916397095,
+ "mean_token_accuracy": 0.6322056651115417,
+ "num_tokens": 8847360.0,
+ "step": 540
+ },
+ {
+ "entropy": 1.7433172464370728,
+ "epoch": 1.0929292929292929,
+ "grad_norm": 1.8621257543563843,
+ "learning_rate": 3.90909090909091e-06,
+ "loss": 1.7367552518844604,
+ "mean_token_accuracy": 0.6162066459655762,
+ "num_tokens": 8863744.0,
+ "step": 541
+ },
+ {
+ "entropy": 1.4073128700256348,
+ "epoch": 1.094949494949495,
+ "grad_norm": 1.939588189125061,
+ "learning_rate": 3.907070707070708e-06,
+ "loss": 1.3844857215881348,
+ "mean_token_accuracy": 0.6634709239006042,
+ "num_tokens": 8880128.0,
+ "step": 542
+ },
+ {
+ "entropy": 1.479836344718933,
+ "epoch": 1.096969696969697,
+ "grad_norm": 2.06921648979187,
+ "learning_rate": 3.905050505050506e-06,
+ "loss": 1.4933744668960571,
+ "mean_token_accuracy": 0.6510136723518372,
+ "num_tokens": 8896512.0,
+ "step": 543
+ },
+ {
+ "entropy": 1.3934576511383057,
+ "epoch": 1.098989898989899,
+ "grad_norm": 2.0697920322418213,
+ "learning_rate": 3.9030303030303035e-06,
+ "loss": 1.3886666297912598,
+ "mean_token_accuracy": 0.6775158643722534,
+ "num_tokens": 8912896.0,
+ "step": 544
+ },
+ {
+ "entropy": 1.5223709344863892,
+ "epoch": 1.101010101010101,
+ "grad_norm": 2.1627066135406494,
+ "learning_rate": 3.901010101010101e-06,
+ "loss": 1.5601061582565308,
+ "mean_token_accuracy": 0.6392892003059387,
+ "num_tokens": 8929280.0,
+ "step": 545
+ },
+ {
+ "entropy": 1.4462933540344238,
+ "epoch": 1.103030303030303,
+ "grad_norm": 1.9871046543121338,
+ "learning_rate": 3.898989898989899e-06,
+ "loss": 1.463792085647583,
+ "mean_token_accuracy": 0.6518075466156006,
+ "num_tokens": 8945664.0,
+ "step": 546
+ },
+ {
+ "entropy": 1.325921654701233,
+ "epoch": 1.105050505050505,
+ "grad_norm": 1.9081087112426758,
+ "learning_rate": 3.896969696969697e-06,
+ "loss": 1.3303362131118774,
+ "mean_token_accuracy": 0.7009648084640503,
+ "num_tokens": 8962048.0,
+ "step": 547
+ },
+ {
+ "entropy": 1.415048360824585,
+ "epoch": 1.107070707070707,
+ "grad_norm": 2.3306756019592285,
+ "learning_rate": 3.894949494949495e-06,
+ "loss": 1.4139145612716675,
+ "mean_token_accuracy": 0.6849047541618347,
+ "num_tokens": 8978432.0,
+ "step": 548
+ },
+ {
+ "entropy": 1.5863295793533325,
+ "epoch": 1.1090909090909091,
+ "grad_norm": 1.993159294128418,
+ "learning_rate": 3.892929292929293e-06,
+ "loss": 1.6010534763336182,
+ "mean_token_accuracy": 0.6424035429954529,
+ "num_tokens": 8994816.0,
+ "step": 549
+ },
+ {
+ "entropy": 1.5526721477508545,
+ "epoch": 1.1111111111111112,
+ "grad_norm": 2.0528876781463623,
+ "learning_rate": 3.890909090909092e-06,
+ "loss": 1.5465538501739502,
+ "mean_token_accuracy": 0.659807026386261,
+ "num_tokens": 9011200.0,
+ "step": 550
+ },
+ {
+ "epoch": 1.1111111111111112,
+ "eval_entropy": 1.4479231247978825,
+ "eval_loss": 1.5459802150726318,
+ "eval_mean_token_accuracy": 0.6425906530311031,
+ "eval_num_tokens": 9011200.0,
+ "eval_runtime": 43.7828,
+ "eval_samples_per_second": 22.612,
+ "eval_steps_per_second": 2.832,
+ "step": 550
+ },
+ {
+ "entropy": 1.4322376251220703,
+ "epoch": 1.1131313131313132,
+ "grad_norm": 1.933549404144287,
+ "learning_rate": 3.88888888888889e-06,
+ "loss": 1.4371025562286377,
+ "mean_token_accuracy": 0.670676589012146,
+ "num_tokens": 9027584.0,
+ "step": 551
+ },
+ {
+ "entropy": 1.5050671100616455,
+ "epoch": 1.1151515151515152,
+ "grad_norm": 1.9736099243164062,
+ "learning_rate": 3.8868686868686875e-06,
+ "loss": 1.5362370014190674,
+ "mean_token_accuracy": 0.651624321937561,
+ "num_tokens": 9043968.0,
+ "step": 552
+ },
+ {
+ "entropy": 1.4335887432098389,
+ "epoch": 1.1171717171717173,
+ "grad_norm": 1.8896421194076538,
+ "learning_rate": 3.884848484848485e-06,
+ "loss": 1.4439888000488281,
+ "mean_token_accuracy": 0.670615553855896,
+ "num_tokens": 9060352.0,
+ "step": 553
+ },
+ {
+ "entropy": 1.5979355573654175,
+ "epoch": 1.1191919191919193,
+ "grad_norm": 2.1710526943206787,
+ "learning_rate": 3.882828282828283e-06,
+ "loss": 1.629098653793335,
+ "mean_token_accuracy": 0.6226184368133545,
+ "num_tokens": 9076736.0,
+ "step": 554
+ },
+ {
+ "entropy": 1.2901698350906372,
+ "epoch": 1.121212121212121,
+ "grad_norm": 2.129443407058716,
+ "learning_rate": 3.880808080808081e-06,
+ "loss": 1.2824913263320923,
+ "mean_token_accuracy": 0.6816682815551758,
+ "num_tokens": 9093120.0,
+ "step": 555
+ },
+ {
+ "entropy": 1.403212308883667,
+ "epoch": 1.1232323232323231,
+ "grad_norm": 1.883240818977356,
+ "learning_rate": 3.878787878787879e-06,
+ "loss": 1.3960213661193848,
+ "mean_token_accuracy": 0.6651197075843811,
+ "num_tokens": 9109504.0,
+ "step": 556
+ },
+ {
+ "entropy": 1.3959479331970215,
+ "epoch": 1.1252525252525252,
+ "grad_norm": 2.1145691871643066,
+ "learning_rate": 3.876767676767677e-06,
+ "loss": 1.4046590328216553,
+ "mean_token_accuracy": 0.6607230305671692,
+ "num_tokens": 9125888.0,
+ "step": 557
+ },
+ {
+ "entropy": 1.0899873971939087,
+ "epoch": 1.1272727272727272,
+ "grad_norm": 1.8862192630767822,
+ "learning_rate": 3.874747474747475e-06,
+ "loss": 1.0965509414672852,
+ "mean_token_accuracy": 0.7245969772338867,
+ "num_tokens": 9142272.0,
+ "step": 558
+ },
+ {
+ "entropy": 1.0380184650421143,
+ "epoch": 1.1292929292929292,
+ "grad_norm": 1.946702241897583,
+ "learning_rate": 3.872727272727273e-06,
+ "loss": 1.0539875030517578,
+ "mean_token_accuracy": 0.7423058152198792,
+ "num_tokens": 9158656.0,
+ "step": 559
+ },
+ {
+ "entropy": 1.1173628568649292,
+ "epoch": 1.1313131313131313,
+ "grad_norm": 2.002847909927368,
+ "learning_rate": 3.8707070707070706e-06,
+ "loss": 1.128816843032837,
+ "mean_token_accuracy": 0.7112848162651062,
+ "num_tokens": 9175040.0,
+ "step": 560
+ },
+ {
+ "entropy": 1.6536206007003784,
+ "epoch": 1.1333333333333333,
+ "grad_norm": 2.212761640548706,
+ "learning_rate": 3.868686868686869e-06,
+ "loss": 1.6286437511444092,
+ "mean_token_accuracy": 0.6277479529380798,
+ "num_tokens": 9191424.0,
+ "step": 561
+ },
+ {
+ "entropy": 1.7187970876693726,
+ "epoch": 1.1353535353535353,
+ "grad_norm": 2.207310199737549,
+ "learning_rate": 3.866666666666667e-06,
+ "loss": 1.7072829008102417,
+ "mean_token_accuracy": 0.6180996298789978,
+ "num_tokens": 9207808.0,
+ "step": 562
+ },
+ {
+ "entropy": 1.7057682275772095,
+ "epoch": 1.1373737373737374,
+ "grad_norm": 2.1582961082458496,
+ "learning_rate": 3.864646464646465e-06,
+ "loss": 1.7216498851776123,
+ "mean_token_accuracy": 0.6105886697769165,
+ "num_tokens": 9224192.0,
+ "step": 563
+ },
+ {
+ "entropy": 1.8016951084136963,
+ "epoch": 1.1393939393939394,
+ "grad_norm": 2.035249948501587,
+ "learning_rate": 3.862626262626263e-06,
+ "loss": 1.8101240396499634,
+ "mean_token_accuracy": 0.6199315786361694,
+ "num_tokens": 9240576.0,
+ "step": 564
+ },
+ {
+ "entropy": 1.094889760017395,
+ "epoch": 1.1414141414141414,
+ "grad_norm": 2.119861364364624,
+ "learning_rate": 3.860606060606061e-06,
+ "loss": 1.0990344285964966,
+ "mean_token_accuracy": 0.7071323990821838,
+ "num_tokens": 9256960.0,
+ "step": 565
+ },
+ {
+ "entropy": 1.537480115890503,
+ "epoch": 1.1434343434343435,
+ "grad_norm": 2.167386770248413,
+ "learning_rate": 3.858585858585859e-06,
+ "loss": 1.5262541770935059,
+ "mean_token_accuracy": 0.6343429684638977,
+ "num_tokens": 9273344.0,
+ "step": 566
+ },
+ {
+ "entropy": 1.2940728664398193,
+ "epoch": 1.1454545454545455,
+ "grad_norm": 1.941097617149353,
+ "learning_rate": 3.856565656565657e-06,
+ "loss": 1.3145123720169067,
+ "mean_token_accuracy": 0.6822789311408997,
+ "num_tokens": 9289728.0,
+ "step": 567
+ },
+ {
+ "entropy": 1.3154000043869019,
+ "epoch": 1.1474747474747475,
+ "grad_norm": 2.088576078414917,
+ "learning_rate": 3.8545454545454545e-06,
+ "loss": 1.3154902458190918,
+ "mean_token_accuracy": 0.6703101992607117,
+ "num_tokens": 9306112.0,
+ "step": 568
+ },
+ {
+ "entropy": 1.3576427698135376,
+ "epoch": 1.1494949494949496,
+ "grad_norm": 1.9895246028900146,
+ "learning_rate": 3.852525252525252e-06,
+ "loss": 1.3295378684997559,
+ "mean_token_accuracy": 0.6762335300445557,
+ "num_tokens": 9322496.0,
+ "step": 569
+ },
+ {
+ "entropy": 1.6067878007888794,
+ "epoch": 1.1515151515151516,
+ "grad_norm": 2.021191358566284,
+ "learning_rate": 3.85050505050505e-06,
+ "loss": 1.6273870468139648,
+ "mean_token_accuracy": 0.6329995393753052,
+ "num_tokens": 9338880.0,
+ "step": 570
+ },
+ {
+ "entropy": 1.4831246137619019,
+ "epoch": 1.1535353535353536,
+ "grad_norm": 2.114612340927124,
+ "learning_rate": 3.848484848484848e-06,
+ "loss": 1.4925904273986816,
+ "mean_token_accuracy": 0.6566316485404968,
+ "num_tokens": 9355264.0,
+ "step": 571
+ },
+ {
+ "entropy": 1.4476258754730225,
+ "epoch": 1.1555555555555554,
+ "grad_norm": 2.3435230255126953,
+ "learning_rate": 3.846464646464647e-06,
+ "loss": 1.438629388809204,
+ "mean_token_accuracy": 0.656020998954773,
+ "num_tokens": 9371648.0,
+ "step": 572
+ },
+ {
+ "entropy": 1.7930315732955933,
+ "epoch": 1.1575757575757575,
+ "grad_norm": 2.1700010299682617,
+ "learning_rate": 3.844444444444445e-06,
+ "loss": 1.786928653717041,
+ "mean_token_accuracy": 0.6028333902359009,
+ "num_tokens": 9388032.0,
+ "step": 573
+ },
+ {
+ "entropy": 1.3800512552261353,
+ "epoch": 1.1595959595959595,
+ "grad_norm": 2.2036688327789307,
+ "learning_rate": 3.842424242424243e-06,
+ "loss": 1.3767224550247192,
+ "mean_token_accuracy": 0.6692110300064087,
+ "num_tokens": 9404416.0,
+ "step": 574
+ },
+ {
+ "entropy": 0.8562329411506653,
+ "epoch": 1.1616161616161615,
+ "grad_norm": 1.765388011932373,
+ "learning_rate": 3.840404040404041e-06,
+ "loss": 0.8366047143936157,
+ "mean_token_accuracy": 0.783707857131958,
+ "num_tokens": 9420800.0,
+ "step": 575
+ },
+ {
+ "entropy": 1.4267486333847046,
+ "epoch": 1.1636363636363636,
+ "grad_norm": 2.0768239498138428,
+ "learning_rate": 3.8383838383838385e-06,
+ "loss": 1.4165449142456055,
+ "mean_token_accuracy": 0.656020998954773,
+ "num_tokens": 9437184.0,
+ "step": 576
+ },
+ {
+ "entropy": 1.4123388528823853,
+ "epoch": 1.1656565656565656,
+ "grad_norm": 2.028716802597046,
+ "learning_rate": 3.836363636363636e-06,
+ "loss": 1.420403003692627,
+ "mean_token_accuracy": 0.6729360222816467,
+ "num_tokens": 9453568.0,
+ "step": 577
+ },
+ {
+ "entropy": 1.1549286842346191,
+ "epoch": 1.1676767676767676,
+ "grad_norm": 2.1142919063568115,
+ "learning_rate": 3.834343434343435e-06,
+ "loss": 1.1487064361572266,
+ "mean_token_accuracy": 0.7040180563926697,
+ "num_tokens": 9469952.0,
+ "step": 578
+ },
+ {
+ "entropy": 1.4779953956604004,
+ "epoch": 1.1696969696969697,
+ "grad_norm": 2.166046142578125,
+ "learning_rate": 3.832323232323233e-06,
+ "loss": 1.4788683652877808,
+ "mean_token_accuracy": 0.6436858773231506,
+ "num_tokens": 9486336.0,
+ "step": 579
+ },
+ {
+ "entropy": 1.568039059638977,
+ "epoch": 1.1717171717171717,
+ "grad_norm": 2.067704200744629,
+ "learning_rate": 3.830303030303031e-06,
+ "loss": 1.5830929279327393,
+ "mean_token_accuracy": 0.6461895704269409,
+ "num_tokens": 9502720.0,
+ "step": 580
+ },
+ {
+ "entropy": 1.1785298585891724,
+ "epoch": 1.1737373737373737,
+ "grad_norm": 1.9789953231811523,
+ "learning_rate": 3.828282828282829e-06,
+ "loss": 1.2083481550216675,
+ "mean_token_accuracy": 0.7120786309242249,
+ "num_tokens": 9519104.0,
+ "step": 581
+ },
+ {
+ "entropy": 1.3783847093582153,
+ "epoch": 1.1757575757575758,
+ "grad_norm": 2.1503992080688477,
+ "learning_rate": 3.826262626262627e-06,
+ "loss": 1.3721052408218384,
+ "mean_token_accuracy": 0.6669516563415527,
+ "num_tokens": 9535488.0,
+ "step": 582
+ },
+ {
+ "entropy": 1.120093584060669,
+ "epoch": 1.1777777777777778,
+ "grad_norm": 1.9822942018508911,
+ "learning_rate": 3.8242424242424245e-06,
+ "loss": 1.1215628385543823,
+ "mean_token_accuracy": 0.7122618556022644,
+ "num_tokens": 9551872.0,
+ "step": 583
+ },
+ {
+ "entropy": 1.4621506929397583,
+ "epoch": 1.1797979797979798,
+ "grad_norm": 2.159489631652832,
+ "learning_rate": 3.8222222222222224e-06,
+ "loss": 1.4614722728729248,
+ "mean_token_accuracy": 0.6535173654556274,
+ "num_tokens": 9568256.0,
+ "step": 584
+ },
+ {
+ "entropy": 1.015797734260559,
+ "epoch": 1.1818181818181819,
+ "grad_norm": 1.846848726272583,
+ "learning_rate": 3.82020202020202e-06,
+ "loss": 1.0082862377166748,
+ "mean_token_accuracy": 0.7394968271255493,
+ "num_tokens": 9584640.0,
+ "step": 585
+ },
+ {
+ "entropy": 1.364890217781067,
+ "epoch": 1.183838383838384,
+ "grad_norm": 2.175546646118164,
+ "learning_rate": 3.818181818181819e-06,
+ "loss": 1.3496818542480469,
+ "mean_token_accuracy": 0.6665241718292236,
+ "num_tokens": 9601024.0,
+ "step": 586
+ },
+ {
+ "entropy": 1.6056870222091675,
+ "epoch": 1.185858585858586,
+ "grad_norm": 2.1069748401641846,
+ "learning_rate": 3.816161616161617e-06,
+ "loss": 1.6048622131347656,
+ "mean_token_accuracy": 0.642892062664032,
+ "num_tokens": 9617408.0,
+ "step": 587
+ },
+ {
+ "entropy": 1.415107250213623,
+ "epoch": 1.187878787878788,
+ "grad_norm": 2.184544324874878,
+ "learning_rate": 3.8141414141414144e-06,
+ "loss": 1.4305826425552368,
+ "mean_token_accuracy": 0.6619443297386169,
+ "num_tokens": 9633792.0,
+ "step": 588
+ },
+ {
+ "entropy": 1.6554296016693115,
+ "epoch": 1.1898989898989898,
+ "grad_norm": 2.109793186187744,
+ "learning_rate": 3.8121212121212127e-06,
+ "loss": 1.6850776672363281,
+ "mean_token_accuracy": 0.6116267442703247,
+ "num_tokens": 9650176.0,
+ "step": 589
+ },
+ {
+ "entropy": 1.681472659111023,
+ "epoch": 1.1919191919191918,
+ "grad_norm": 2.3179714679718018,
+ "learning_rate": 3.8101010101010106e-06,
+ "loss": 1.699328064918518,
+ "mean_token_accuracy": 0.6105275750160217,
+ "num_tokens": 9666560.0,
+ "step": 590
+ },
+ {
+ "entropy": 1.113366961479187,
+ "epoch": 1.1939393939393939,
+ "grad_norm": 1.9885324239730835,
+ "learning_rate": 3.8080808080808085e-06,
+ "loss": 1.1357836723327637,
+ "mean_token_accuracy": 0.7157425284385681,
+ "num_tokens": 9682944.0,
+ "step": 591
+ },
+ {
+ "entropy": 1.1010088920593262,
+ "epoch": 1.195959595959596,
+ "grad_norm": 1.9745922088623047,
+ "learning_rate": 3.8060606060606064e-06,
+ "loss": 1.094293475151062,
+ "mean_token_accuracy": 0.7214215993881226,
+ "num_tokens": 9699328.0,
+ "step": 592
+ },
+ {
+ "entropy": 1.562911033630371,
+ "epoch": 1.197979797979798,
+ "grad_norm": 2.0526134967803955,
+ "learning_rate": 3.8040404040404043e-06,
+ "loss": 1.5747783184051514,
+ "mean_token_accuracy": 0.6342818737030029,
+ "num_tokens": 9715712.0,
+ "step": 593
+ },
+ {
+ "entropy": 1.6541608572006226,
+ "epoch": 1.2,
+ "grad_norm": 2.1268227100372314,
+ "learning_rate": 3.8020202020202026e-06,
+ "loss": 1.696983814239502,
+ "mean_token_accuracy": 0.6127259135246277,
+ "num_tokens": 9732096.0,
+ "step": 594
+ },
+ {
+ "entropy": 1.5259482860565186,
+ "epoch": 1.202020202020202,
+ "grad_norm": 2.240692377090454,
+ "learning_rate": 3.8000000000000005e-06,
+ "loss": 1.5582221746444702,
+ "mean_token_accuracy": 0.644052267074585,
+ "num_tokens": 9748480.0,
+ "step": 595
+ },
+ {
+ "entropy": 1.455372929573059,
+ "epoch": 1.204040404040404,
+ "grad_norm": 2.0934698581695557,
+ "learning_rate": 3.7979797979797984e-06,
+ "loss": 1.464172124862671,
+ "mean_token_accuracy": 0.6555935740470886,
+ "num_tokens": 9764864.0,
+ "step": 596
+ },
+ {
+ "entropy": 1.5074836015701294,
+ "epoch": 1.206060606060606,
+ "grad_norm": 1.9779568910598755,
+ "learning_rate": 3.7959595959595962e-06,
+ "loss": 1.5280466079711914,
+ "mean_token_accuracy": 0.6460063457489014,
+ "num_tokens": 9781248.0,
+ "step": 597
+ },
+ {
+ "entropy": 1.3286548852920532,
+ "epoch": 1.208080808080808,
+ "grad_norm": 1.9357717037200928,
+ "learning_rate": 3.793939393939394e-06,
+ "loss": 1.3448848724365234,
+ "mean_token_accuracy": 0.6816072463989258,
+ "num_tokens": 9797632.0,
+ "step": 598
+ },
+ {
+ "entropy": 1.5161375999450684,
+ "epoch": 1.2101010101010101,
+ "grad_norm": 2.224217414855957,
+ "learning_rate": 3.791919191919192e-06,
+ "loss": 1.5168235301971436,
+ "mean_token_accuracy": 0.6453346610069275,
+ "num_tokens": 9814016.0,
+ "step": 599
+ },
+ {
+ "entropy": 1.3337587118148804,
+ "epoch": 1.2121212121212122,
+ "grad_norm": 1.9308290481567383,
+ "learning_rate": 3.7898989898989903e-06,
+ "loss": 1.3429791927337646,
+ "mean_token_accuracy": 0.6827064156532288,
+ "num_tokens": 9830400.0,
+ "step": 600
+ },
+ {
+ "epoch": 1.2121212121212122,
+ "eval_entropy": 1.4592116455877981,
+ "eval_loss": 1.5424447059631348,
+ "eval_mean_token_accuracy": 0.6431668299821115,
+ "eval_num_tokens": 9830400.0,
+ "eval_runtime": 43.8289,
+ "eval_samples_per_second": 22.588,
+ "eval_steps_per_second": 2.829,
+ "step": 600
+ },
+ {
+ "entropy": 1.6697322130203247,
+ "epoch": 1.2141414141414142,
+ "grad_norm": 1.9462894201278687,
+ "learning_rate": 3.7878787878787882e-06,
+ "loss": 1.6434354782104492,
+ "mean_token_accuracy": 0.635808527469635,
+ "num_tokens": 9846784.0,
+ "step": 601
+ },
+ {
+ "entropy": 1.427184820175171,
+ "epoch": 1.2161616161616162,
+ "grad_norm": 2.196017026901245,
+ "learning_rate": 3.785858585858586e-06,
+ "loss": 1.4024749994277954,
+ "mean_token_accuracy": 0.6720200181007385,
+ "num_tokens": 9863168.0,
+ "step": 602
+ },
+ {
+ "entropy": 1.4623222351074219,
+ "epoch": 1.2181818181818183,
+ "grad_norm": 2.160585403442383,
+ "learning_rate": 3.783838383838384e-06,
+ "loss": 1.4455416202545166,
+ "mean_token_accuracy": 0.6557767391204834,
+ "num_tokens": 9879552.0,
+ "step": 603
+ },
+ {
+ "entropy": 1.348331093788147,
+ "epoch": 1.2202020202020203,
+ "grad_norm": 2.079458713531494,
+ "learning_rate": 3.781818181818182e-06,
+ "loss": 1.3672473430633545,
+ "mean_token_accuracy": 0.6834391951560974,
+ "num_tokens": 9895936.0,
+ "step": 604
+ },
+ {
+ "entropy": 1.362277865409851,
+ "epoch": 1.2222222222222223,
+ "grad_norm": 2.10718035697937,
+ "learning_rate": 3.77979797979798e-06,
+ "loss": 1.3567216396331787,
+ "mean_token_accuracy": 0.6659135222434998,
+ "num_tokens": 9912320.0,
+ "step": 605
+ },
+ {
+ "entropy": 1.2743226289749146,
+ "epoch": 1.2242424242424241,
+ "grad_norm": 2.116549491882324,
+ "learning_rate": 3.777777777777778e-06,
+ "loss": 1.303470253944397,
+ "mean_token_accuracy": 0.6791035532951355,
+ "num_tokens": 9928704.0,
+ "step": 606
+ },
+ {
+ "entropy": 1.079649567604065,
+ "epoch": 1.2262626262626264,
+ "grad_norm": 1.8429855108261108,
+ "learning_rate": 3.775757575757576e-06,
+ "loss": 1.0616915225982666,
+ "mean_token_accuracy": 0.7321690320968628,
+ "num_tokens": 9945088.0,
+ "step": 607
+ },
+ {
+ "entropy": 1.363136649131775,
+ "epoch": 1.2282828282828282,
+ "grad_norm": 2.171295166015625,
+ "learning_rate": 3.773737373737374e-06,
+ "loss": 1.3525331020355225,
+ "mean_token_accuracy": 0.6732413172721863,
+ "num_tokens": 9961472.0,
+ "step": 608
+ },
+ {
+ "entropy": 1.324924349784851,
+ "epoch": 1.2303030303030302,
+ "grad_norm": 2.0872693061828613,
+ "learning_rate": 3.7717171717171717e-06,
+ "loss": 1.3282644748687744,
+ "mean_token_accuracy": 0.674462616443634,
+ "num_tokens": 9977856.0,
+ "step": 609
+ },
+ {
+ "entropy": 1.0672967433929443,
+ "epoch": 1.2323232323232323,
+ "grad_norm": 2.0819947719573975,
+ "learning_rate": 3.76969696969697e-06,
+ "loss": 1.0929367542266846,
+ "mean_token_accuracy": 0.715254008769989,
+ "num_tokens": 9994240.0,
+ "step": 610
+ },
+ {
+ "entropy": 1.4465869665145874,
+ "epoch": 1.2343434343434343,
+ "grad_norm": 2.1786868572235107,
+ "learning_rate": 3.767676767676768e-06,
+ "loss": 1.436307668685913,
+ "mean_token_accuracy": 0.6642037034034729,
+ "num_tokens": 10010624.0,
+ "step": 611
+ },
+ {
+ "entropy": 1.6424144506454468,
+ "epoch": 1.2363636363636363,
+ "grad_norm": 2.2582032680511475,
+ "learning_rate": 3.765656565656566e-06,
+ "loss": 1.6621832847595215,
+ "mean_token_accuracy": 0.6218246221542358,
+ "num_tokens": 10027008.0,
+ "step": 612
+ },
+ {
+ "entropy": 1.546712875366211,
+ "epoch": 1.2383838383838384,
+ "grad_norm": 2.1685879230499268,
+ "learning_rate": 3.7636363636363637e-06,
+ "loss": 1.5617464780807495,
+ "mean_token_accuracy": 0.6325110197067261,
+ "num_tokens": 10043392.0,
+ "step": 613
+ },
+ {
+ "entropy": 1.5758557319641113,
+ "epoch": 1.2404040404040404,
+ "grad_norm": 2.0911128520965576,
+ "learning_rate": 3.7616161616161616e-06,
+ "loss": 1.5830271244049072,
+ "mean_token_accuracy": 0.6349536180496216,
+ "num_tokens": 10059776.0,
+ "step": 614
+ },
+ {
+ "entropy": 1.5223995447158813,
+ "epoch": 1.2424242424242424,
+ "grad_norm": 2.0333359241485596,
+ "learning_rate": 3.7595959595959595e-06,
+ "loss": 1.5132982730865479,
+ "mean_token_accuracy": 0.6518075466156006,
+ "num_tokens": 10076160.0,
+ "step": 615
+ },
+ {
+ "entropy": 1.0950185060501099,
+ "epoch": 1.2444444444444445,
+ "grad_norm": 2.0048129558563232,
+ "learning_rate": 3.757575757575758e-06,
+ "loss": 1.0901896953582764,
+ "mean_token_accuracy": 0.723375678062439,
+ "num_tokens": 10092544.0,
+ "step": 616
+ },
+ {
+ "entropy": 1.340641736984253,
+ "epoch": 1.2464646464646465,
+ "grad_norm": 2.079256534576416,
+ "learning_rate": 3.7555555555555557e-06,
+ "loss": 1.3583415746688843,
+ "mean_token_accuracy": 0.671775758266449,
+ "num_tokens": 10108928.0,
+ "step": 617
+ },
+ {
+ "entropy": 1.1836222410202026,
+ "epoch": 1.2484848484848485,
+ "grad_norm": 1.9266347885131836,
+ "learning_rate": 3.7535353535353536e-06,
+ "loss": 1.171565294265747,
+ "mean_token_accuracy": 0.7046287059783936,
+ "num_tokens": 10125312.0,
+ "step": 618
+ },
+ {
+ "entropy": 1.6007641553878784,
+ "epoch": 1.2505050505050506,
+ "grad_norm": 2.25252103805542,
+ "learning_rate": 3.7515151515151515e-06,
+ "loss": 1.5955660343170166,
+ "mean_token_accuracy": 0.6306790709495544,
+ "num_tokens": 10141696.0,
+ "step": 619
+ },
+ {
+ "entropy": 1.8159959316253662,
+ "epoch": 1.2525252525252526,
+ "grad_norm": 2.255959987640381,
+ "learning_rate": 3.74949494949495e-06,
+ "loss": 1.8223116397857666,
+ "mean_token_accuracy": 0.5867122411727905,
+ "num_tokens": 10158080.0,
+ "step": 620
+ },
+ {
+ "entropy": 1.6469950675964355,
+ "epoch": 1.2545454545454544,
+ "grad_norm": 1.9183777570724487,
+ "learning_rate": 3.747474747474748e-06,
+ "loss": 1.6623587608337402,
+ "mean_token_accuracy": 0.6283586025238037,
+ "num_tokens": 10174464.0,
+ "step": 621
+ },
+ {
+ "entropy": 1.3507136106491089,
+ "epoch": 1.2565656565656567,
+ "grad_norm": 1.9393805265426636,
+ "learning_rate": 3.745454545454546e-06,
+ "loss": 1.3371413946151733,
+ "mean_token_accuracy": 0.6808744668960571,
+ "num_tokens": 10190848.0,
+ "step": 622
+ },
+ {
+ "entropy": 1.5176761150360107,
+ "epoch": 1.2585858585858585,
+ "grad_norm": 2.0721757411956787,
+ "learning_rate": 3.743434343434344e-06,
+ "loss": 1.5010406970977783,
+ "mean_token_accuracy": 0.6603566408157349,
+ "num_tokens": 10207232.0,
+ "step": 623
+ },
+ {
+ "entropy": 1.1409586668014526,
+ "epoch": 1.2606060606060607,
+ "grad_norm": 1.90805983543396,
+ "learning_rate": 3.7414141414141418e-06,
+ "loss": 1.1339551210403442,
+ "mean_token_accuracy": 0.7151318788528442,
+ "num_tokens": 10223616.0,
+ "step": 624
+ },
+ {
+ "entropy": 1.2258127927780151,
+ "epoch": 1.2626262626262625,
+ "grad_norm": 1.9276432991027832,
+ "learning_rate": 3.73939393939394e-06,
+ "loss": 1.2232249975204468,
+ "mean_token_accuracy": 0.7053614854812622,
+ "num_tokens": 10240000.0,
+ "step": 625
+ },
+ {
+ "entropy": 1.236358642578125,
+ "epoch": 1.2646464646464646,
+ "grad_norm": 2.0792832374572754,
+ "learning_rate": 3.737373737373738e-06,
+ "loss": 1.2570738792419434,
+ "mean_token_accuracy": 0.6869809627532959,
+ "num_tokens": 10256384.0,
+ "step": 626
+ },
+ {
+ "entropy": 1.0995185375213623,
+ "epoch": 1.2666666666666666,
+ "grad_norm": 2.035024881362915,
+ "learning_rate": 3.735353535353536e-06,
+ "loss": 1.10158109664917,
+ "mean_token_accuracy": 0.7248412370681763,
+ "num_tokens": 10272768.0,
+ "step": 627
+ },
+ {
+ "entropy": 1.8980706930160522,
+ "epoch": 1.2686868686868686,
+ "grad_norm": 2.040414333343506,
+ "learning_rate": 3.7333333333333337e-06,
+ "loss": 1.9260807037353516,
+ "mean_token_accuracy": 0.5953834652900696,
+ "num_tokens": 10289152.0,
+ "step": 628
+ },
+ {
+ "entropy": 1.4265168905258179,
+ "epoch": 1.2707070707070707,
+ "grad_norm": 2.418179750442505,
+ "learning_rate": 3.7313131313131316e-06,
+ "loss": 1.4415385723114014,
+ "mean_token_accuracy": 0.6535784006118774,
+ "num_tokens": 10305536.0,
+ "step": 629
+ },
+ {
+ "entropy": 1.217471957206726,
+ "epoch": 1.2727272727272727,
+ "grad_norm": 2.072441577911377,
+ "learning_rate": 3.72929292929293e-06,
+ "loss": 1.2361960411071777,
+ "mean_token_accuracy": 0.6947972774505615,
+ "num_tokens": 10321920.0,
+ "step": 630
+ },
+ {
+ "entropy": 1.0714137554168701,
+ "epoch": 1.2747474747474747,
+ "grad_norm": 3.0428647994995117,
+ "learning_rate": 3.727272727272728e-06,
+ "loss": 1.112711787223816,
+ "mean_token_accuracy": 0.7243527173995972,
+ "num_tokens": 10338304.0,
+ "step": 631
+ },
+ {
+ "entropy": 1.5873851776123047,
+ "epoch": 1.2767676767676768,
+ "grad_norm": 1.945098638534546,
+ "learning_rate": 3.7252525252525257e-06,
+ "loss": 1.595362663269043,
+ "mean_token_accuracy": 0.639594554901123,
+ "num_tokens": 10354688.0,
+ "step": 632
+ },
+ {
+ "entropy": 1.9688408374786377,
+ "epoch": 1.2787878787878788,
+ "grad_norm": 2.0450353622436523,
+ "learning_rate": 3.7232323232323236e-06,
+ "loss": 1.9989259243011475,
+ "mean_token_accuracy": 0.5685759782791138,
+ "num_tokens": 10371072.0,
+ "step": 633
+ },
+ {
+ "entropy": 1.2927459478378296,
+ "epoch": 1.2808080808080808,
+ "grad_norm": 1.959108591079712,
+ "learning_rate": 3.7212121212121215e-06,
+ "loss": 1.3025646209716797,
+ "mean_token_accuracy": 0.6853932738304138,
+ "num_tokens": 10387456.0,
+ "step": 634
+ },
+ {
+ "entropy": 1.6645994186401367,
+ "epoch": 1.2828282828282829,
+ "grad_norm": 2.041923999786377,
+ "learning_rate": 3.7191919191919194e-06,
+ "loss": 1.6959854364395142,
+ "mean_token_accuracy": 0.6319003701210022,
+ "num_tokens": 10403840.0,
+ "step": 635
+ },
+ {
+ "entropy": 1.401423692703247,
+ "epoch": 1.284848484848485,
+ "grad_norm": 2.040456533432007,
+ "learning_rate": 3.7171717171717177e-06,
+ "loss": 1.3859784603118896,
+ "mean_token_accuracy": 0.6740962266921997,
+ "num_tokens": 10420224.0,
+ "step": 636
+ },
+ {
+ "entropy": 1.0597256422042847,
+ "epoch": 1.286868686868687,
+ "grad_norm": 2.053690195083618,
+ "learning_rate": 3.7151515151515156e-06,
+ "loss": 1.052672266960144,
+ "mean_token_accuracy": 0.7290546894073486,
+ "num_tokens": 10436608.0,
+ "step": 637
+ },
+ {
+ "entropy": 1.280874252319336,
+ "epoch": 1.2888888888888888,
+ "grad_norm": 2.0566959381103516,
+ "learning_rate": 3.7131313131313135e-06,
+ "loss": 1.2883412837982178,
+ "mean_token_accuracy": 0.6822178959846497,
+ "num_tokens": 10452992.0,
+ "step": 638
+ },
+ {
+ "entropy": 1.299896240234375,
+ "epoch": 1.290909090909091,
+ "grad_norm": 1.8896070718765259,
+ "learning_rate": 3.7111111111111113e-06,
+ "loss": 1.3038103580474854,
+ "mean_token_accuracy": 0.6712872385978699,
+ "num_tokens": 10469376.0,
+ "step": 639
+ },
+ {
+ "entropy": 1.1713769435882568,
+ "epoch": 1.2929292929292928,
+ "grad_norm": 2.1309149265289307,
+ "learning_rate": 3.7090909090909092e-06,
+ "loss": 1.1899120807647705,
+ "mean_token_accuracy": 0.700537383556366,
+ "num_tokens": 10485760.0,
+ "step": 640
+ },
+ {
+ "entropy": 1.3056974411010742,
+ "epoch": 1.294949494949495,
+ "grad_norm": 1.9990391731262207,
+ "learning_rate": 3.7070707070707075e-06,
+ "loss": 1.3287105560302734,
+ "mean_token_accuracy": 0.6773326992988586,
+ "num_tokens": 10502144.0,
+ "step": 641
+ },
+ {
+ "entropy": 1.7352585792541504,
+ "epoch": 1.2969696969696969,
+ "grad_norm": 2.100043535232544,
+ "learning_rate": 3.7050505050505054e-06,
+ "loss": 1.7806944847106934,
+ "mean_token_accuracy": 0.60332190990448,
+ "num_tokens": 10518528.0,
+ "step": 642
+ },
+ {
+ "entropy": 1.3748377561569214,
+ "epoch": 1.298989898989899,
+ "grad_norm": 2.1280689239501953,
+ "learning_rate": 3.7030303030303033e-06,
+ "loss": 1.379891037940979,
+ "mean_token_accuracy": 0.6617611050605774,
+ "num_tokens": 10534912.0,
+ "step": 643
+ },
+ {
+ "entropy": 0.9991571307182312,
+ "epoch": 1.301010101010101,
+ "grad_norm": 1.7343134880065918,
+ "learning_rate": 3.701010101010101e-06,
+ "loss": 1.0082780122756958,
+ "mean_token_accuracy": 0.740229606628418,
+ "num_tokens": 10551296.0,
+ "step": 644
+ },
+ {
+ "entropy": 1.6313605308532715,
+ "epoch": 1.303030303030303,
+ "grad_norm": 2.216167449951172,
+ "learning_rate": 3.698989898989899e-06,
+ "loss": 1.6183781623840332,
+ "mean_token_accuracy": 0.6164509057998657,
+ "num_tokens": 10567680.0,
+ "step": 645
+ },
+ {
+ "entropy": 1.4174962043762207,
+ "epoch": 1.305050505050505,
+ "grad_norm": 2.0921854972839355,
+ "learning_rate": 3.6969696969696974e-06,
+ "loss": 1.4087945222854614,
+ "mean_token_accuracy": 0.6705544590950012,
+ "num_tokens": 10584064.0,
+ "step": 646
+ },
+ {
+ "entropy": 1.8302873373031616,
+ "epoch": 1.307070707070707,
+ "grad_norm": 2.0742204189300537,
+ "learning_rate": 3.6949494949494953e-06,
+ "loss": 1.8456642627716064,
+ "mean_token_accuracy": 0.5914142727851868,
+ "num_tokens": 10600448.0,
+ "step": 647
+ },
+ {
+ "entropy": 1.29628324508667,
+ "epoch": 1.309090909090909,
+ "grad_norm": 2.1446709632873535,
+ "learning_rate": 3.692929292929293e-06,
+ "loss": 1.3066375255584717,
+ "mean_token_accuracy": 0.6798363327980042,
+ "num_tokens": 10616832.0,
+ "step": 648
+ },
+ {
+ "entropy": 1.396378993988037,
+ "epoch": 1.3111111111111111,
+ "grad_norm": 1.9578899145126343,
+ "learning_rate": 3.690909090909091e-06,
+ "loss": 1.3943686485290527,
+ "mean_token_accuracy": 0.6772105693817139,
+ "num_tokens": 10633216.0,
+ "step": 649
+ },
+ {
+ "entropy": 1.6457722187042236,
+ "epoch": 1.3131313131313131,
+ "grad_norm": 1.9999926090240479,
+ "learning_rate": 3.688888888888889e-06,
+ "loss": 1.6266872882843018,
+ "mean_token_accuracy": 0.630923330783844,
+ "num_tokens": 10649600.0,
+ "step": 650
+ },
+ {
+ "epoch": 1.3131313131313131,
+ "eval_entropy": 1.456459879875183,
+ "eval_loss": 1.539737582206726,
+ "eval_mean_token_accuracy": 0.643713459853203,
+ "eval_num_tokens": 10649600.0,
+ "eval_runtime": 43.7637,
+ "eval_samples_per_second": 22.621,
+ "eval_steps_per_second": 2.833,
+ "step": 650
+ },
+ {
+ "entropy": 1.1152859926223755,
+ "epoch": 1.3151515151515152,
+ "grad_norm": 2.9232723712921143,
+ "learning_rate": 3.686868686868687e-06,
+ "loss": 1.099653959274292,
+ "mean_token_accuracy": 0.7227039337158203,
+ "num_tokens": 10665984.0,
+ "step": 651
+ },
+ {
+ "entropy": 1.6141386032104492,
+ "epoch": 1.3171717171717172,
+ "grad_norm": 2.1557929515838623,
+ "learning_rate": 3.684848484848485e-06,
+ "loss": 1.6156055927276611,
+ "mean_token_accuracy": 0.6288471221923828,
+ "num_tokens": 10682368.0,
+ "step": 652
+ },
+ {
+ "entropy": 1.282158374786377,
+ "epoch": 1.3191919191919192,
+ "grad_norm": 2.061983346939087,
+ "learning_rate": 3.682828282828283e-06,
+ "loss": 1.2867733240127563,
+ "mean_token_accuracy": 0.6869198679924011,
+ "num_tokens": 10698752.0,
+ "step": 653
+ },
+ {
+ "entropy": 0.9986366033554077,
+ "epoch": 1.3212121212121213,
+ "grad_norm": 2.0446274280548096,
+ "learning_rate": 3.680808080808081e-06,
+ "loss": 1.019629716873169,
+ "mean_token_accuracy": 0.7427943348884583,
+ "num_tokens": 10715136.0,
+ "step": 654
+ },
+ {
+ "entropy": 1.4256011247634888,
+ "epoch": 1.3232323232323233,
+ "grad_norm": 2.1024749279022217,
+ "learning_rate": 3.678787878787879e-06,
+ "loss": 1.4269766807556152,
+ "mean_token_accuracy": 0.6674401760101318,
+ "num_tokens": 10731520.0,
+ "step": 655
+ },
+ {
+ "entropy": 1.4913274049758911,
+ "epoch": 1.3252525252525253,
+ "grad_norm": 2.068432569503784,
+ "learning_rate": 3.6767676767676767e-06,
+ "loss": 1.4930779933929443,
+ "mean_token_accuracy": 0.6474719047546387,
+ "num_tokens": 10747904.0,
+ "step": 656
+ },
+ {
+ "entropy": 1.2957285642623901,
+ "epoch": 1.3272727272727272,
+ "grad_norm": 2.2480008602142334,
+ "learning_rate": 3.674747474747475e-06,
+ "loss": 1.3001320362091064,
+ "mean_token_accuracy": 0.679286777973175,
+ "num_tokens": 10764288.0,
+ "step": 657
+ },
+ {
+ "entropy": 1.328946828842163,
+ "epoch": 1.3292929292929294,
+ "grad_norm": 2.311176300048828,
+ "learning_rate": 3.672727272727273e-06,
+ "loss": 1.3286024332046509,
+ "mean_token_accuracy": 0.662432849407196,
+ "num_tokens": 10780672.0,
+ "step": 658
+ },
+ {
+ "entropy": 1.458139419555664,
+ "epoch": 1.3313131313131312,
+ "grad_norm": 2.1967899799346924,
+ "learning_rate": 3.670707070707071e-06,
+ "loss": 1.4384279251098633,
+ "mean_token_accuracy": 0.6435637474060059,
+ "num_tokens": 10797056.0,
+ "step": 659
+ },
+ {
+ "entropy": 1.43666672706604,
+ "epoch": 1.3333333333333333,
+ "grad_norm": 2.163083076477051,
+ "learning_rate": 3.6686868686868687e-06,
+ "loss": 1.4396371841430664,
+ "mean_token_accuracy": 0.6531509757041931,
+ "num_tokens": 10813440.0,
+ "step": 660
+ },
+ {
+ "entropy": 1.6210501194000244,
+ "epoch": 1.3353535353535353,
+ "grad_norm": 2.3601770401000977,
+ "learning_rate": 3.6666666666666666e-06,
+ "loss": 1.6370227336883545,
+ "mean_token_accuracy": 0.6149242520332336,
+ "num_tokens": 10829824.0,
+ "step": 661
+ },
+ {
+ "entropy": 0.9602832794189453,
+ "epoch": 1.3373737373737373,
+ "grad_norm": 1.8452314138412476,
+ "learning_rate": 3.664646464646465e-06,
+ "loss": 0.9445997476577759,
+ "mean_token_accuracy": 0.759709358215332,
+ "num_tokens": 10846208.0,
+ "step": 662
+ },
+ {
+ "entropy": 1.2215200662612915,
+ "epoch": 1.3393939393939394,
+ "grad_norm": 2.019989252090454,
+ "learning_rate": 3.662626262626263e-06,
+ "loss": 1.2451637983322144,
+ "mean_token_accuracy": 0.7051172256469727,
+ "num_tokens": 10862592.0,
+ "step": 663
+ },
+ {
+ "entropy": 1.3253310918807983,
+ "epoch": 1.3414141414141414,
+ "grad_norm": 1.9700425863265991,
+ "learning_rate": 3.660606060606061e-06,
+ "loss": 1.3407385349273682,
+ "mean_token_accuracy": 0.6988885998725891,
+ "num_tokens": 10878976.0,
+ "step": 664
+ },
+ {
+ "entropy": 1.1397737264633179,
+ "epoch": 1.3434343434343434,
+ "grad_norm": 2.1194040775299072,
+ "learning_rate": 3.658585858585859e-06,
+ "loss": 1.1545679569244385,
+ "mean_token_accuracy": 0.7071323990821838,
+ "num_tokens": 10895360.0,
+ "step": 665
+ },
+ {
+ "entropy": 1.1714493036270142,
+ "epoch": 1.3454545454545455,
+ "grad_norm": 2.078051805496216,
+ "learning_rate": 3.6565656565656573e-06,
+ "loss": 1.1751537322998047,
+ "mean_token_accuracy": 0.6963238716125488,
+ "num_tokens": 10911744.0,
+ "step": 666
+ },
+ {
+ "entropy": 1.2954860925674438,
+ "epoch": 1.3474747474747475,
+ "grad_norm": 1.9348593950271606,
+ "learning_rate": 3.654545454545455e-06,
+ "loss": 1.2961801290512085,
+ "mean_token_accuracy": 0.6831338405609131,
+ "num_tokens": 10928128.0,
+ "step": 667
+ },
+ {
+ "entropy": 1.401628851890564,
+ "epoch": 1.3494949494949495,
+ "grad_norm": 2.1757614612579346,
+ "learning_rate": 3.652525252525253e-06,
+ "loss": 1.4025013446807861,
+ "mean_token_accuracy": 0.6590132117271423,
+ "num_tokens": 10944512.0,
+ "step": 668
+ },
+ {
+ "entropy": 1.4164557456970215,
+ "epoch": 1.3515151515151516,
+ "grad_norm": 2.0192055702209473,
+ "learning_rate": 3.650505050505051e-06,
+ "loss": 1.43379545211792,
+ "mean_token_accuracy": 0.6554714441299438,
+ "num_tokens": 10960896.0,
+ "step": 669
+ },
+ {
+ "entropy": 1.4888815879821777,
+ "epoch": 1.3535353535353536,
+ "grad_norm": 2.2166643142700195,
+ "learning_rate": 3.648484848484849e-06,
+ "loss": 1.494372844696045,
+ "mean_token_accuracy": 0.6402052044868469,
+ "num_tokens": 10977280.0,
+ "step": 670
+ },
+ {
+ "entropy": 1.215625286102295,
+ "epoch": 1.3555555555555556,
+ "grad_norm": 1.8059836626052856,
+ "learning_rate": 3.6464646464646467e-06,
+ "loss": 1.2074902057647705,
+ "mean_token_accuracy": 0.7123228907585144,
+ "num_tokens": 10993664.0,
+ "step": 671
+ },
+ {
+ "entropy": 0.9577685594558716,
+ "epoch": 1.3575757575757577,
+ "grad_norm": 1.8803631067276,
+ "learning_rate": 3.644444444444445e-06,
+ "loss": 0.956764817237854,
+ "mean_token_accuracy": 0.759709358215332,
+ "num_tokens": 11010048.0,
+ "step": 672
+ },
+ {
+ "entropy": 1.7215807437896729,
+ "epoch": 1.3595959595959597,
+ "grad_norm": 2.1421561241149902,
+ "learning_rate": 3.642424242424243e-06,
+ "loss": 1.7220706939697266,
+ "mean_token_accuracy": 0.6187102794647217,
+ "num_tokens": 11026432.0,
+ "step": 673
+ },
+ {
+ "entropy": 1.057277798652649,
+ "epoch": 1.3616161616161615,
+ "grad_norm": 1.9014254808425903,
+ "learning_rate": 3.640404040404041e-06,
+ "loss": 1.0582892894744873,
+ "mean_token_accuracy": 0.7327185869216919,
+ "num_tokens": 11042816.0,
+ "step": 674
+ },
+ {
+ "entropy": 1.220097541809082,
+ "epoch": 1.3636363636363638,
+ "grad_norm": 2.1406402587890625,
+ "learning_rate": 3.6383838383838387e-06,
+ "loss": 1.2152808904647827,
+ "mean_token_accuracy": 0.6976673007011414,
+ "num_tokens": 11059200.0,
+ "step": 675
+ },
+ {
+ "entropy": 1.4171489477157593,
+ "epoch": 1.3656565656565656,
+ "grad_norm": 2.1809890270233154,
+ "learning_rate": 3.6363636363636366e-06,
+ "loss": 1.4846035242080688,
+ "mean_token_accuracy": 0.6681118607521057,
+ "num_tokens": 11075584.0,
+ "step": 676
+ },
+ {
+ "entropy": 1.2915124893188477,
+ "epoch": 1.3676767676767676,
+ "grad_norm": 2.0533218383789062,
+ "learning_rate": 3.634343434343435e-06,
+ "loss": 1.3094089031219482,
+ "mean_token_accuracy": 0.6841719746589661,
+ "num_tokens": 11091968.0,
+ "step": 677
+ },
+ {
+ "entropy": 1.4717603921890259,
+ "epoch": 1.3696969696969696,
+ "grad_norm": 2.125450849533081,
+ "learning_rate": 3.6323232323232328e-06,
+ "loss": 1.479896903038025,
+ "mean_token_accuracy": 0.6601734161376953,
+ "num_tokens": 11108352.0,
+ "step": 678
+ },
+ {
+ "entropy": 1.0841981172561646,
+ "epoch": 1.3717171717171717,
+ "grad_norm": 1.936905026435852,
+ "learning_rate": 3.6303030303030307e-06,
+ "loss": 1.0895963907241821,
+ "mean_token_accuracy": 0.7307645082473755,
+ "num_tokens": 11124736.0,
+ "step": 679
+ },
+ {
+ "entropy": 0.85239177942276,
+ "epoch": 1.3737373737373737,
+ "grad_norm": 1.7117112874984741,
+ "learning_rate": 3.6282828282828286e-06,
+ "loss": 0.8716775178909302,
+ "mean_token_accuracy": 0.7721666097640991,
+ "num_tokens": 11141120.0,
+ "step": 680
+ },
+ {
+ "entropy": 1.4737738370895386,
+ "epoch": 1.3757575757575757,
+ "grad_norm": 2.0140671730041504,
+ "learning_rate": 3.6262626262626264e-06,
+ "loss": 1.5029723644256592,
+ "mean_token_accuracy": 0.653822660446167,
+ "num_tokens": 11157504.0,
+ "step": 681
+ },
+ {
+ "entropy": 1.260698914527893,
+ "epoch": 1.3777777777777778,
+ "grad_norm": 1.9868851900100708,
+ "learning_rate": 3.6242424242424248e-06,
+ "loss": 1.256290316581726,
+ "mean_token_accuracy": 0.6891182065010071,
+ "num_tokens": 11173888.0,
+ "step": 682
+ },
+ {
+ "entropy": 1.7418453693389893,
+ "epoch": 1.3797979797979798,
+ "grad_norm": 2.0905439853668213,
+ "learning_rate": 3.6222222222222226e-06,
+ "loss": 1.7568984031677246,
+ "mean_token_accuracy": 0.6291524171829224,
+ "num_tokens": 11190272.0,
+ "step": 683
+ },
+ {
+ "entropy": 1.5552523136138916,
+ "epoch": 1.3818181818181818,
+ "grad_norm": 2.2071683406829834,
+ "learning_rate": 3.6202020202020205e-06,
+ "loss": 1.555542230606079,
+ "mean_token_accuracy": 0.6403883695602417,
+ "num_tokens": 11206656.0,
+ "step": 684
+ },
+ {
+ "entropy": 1.4404902458190918,
+ "epoch": 1.3838383838383839,
+ "grad_norm": 2.276245594024658,
+ "learning_rate": 3.6181818181818184e-06,
+ "loss": 1.4155148267745972,
+ "mean_token_accuracy": 0.6630434989929199,
+ "num_tokens": 11223040.0,
+ "step": 685
+ },
+ {
+ "entropy": 1.1282018423080444,
+ "epoch": 1.385858585858586,
+ "grad_norm": 1.9824057817459106,
+ "learning_rate": 3.6161616161616163e-06,
+ "loss": 1.099952220916748,
+ "mean_token_accuracy": 0.7234978079795837,
+ "num_tokens": 11239424.0,
+ "step": 686
+ },
+ {
+ "entropy": 1.5559885501861572,
+ "epoch": 1.387878787878788,
+ "grad_norm": 2.243121862411499,
+ "learning_rate": 3.614141414141414e-06,
+ "loss": 1.520850658416748,
+ "mean_token_accuracy": 0.6340987086296082,
+ "num_tokens": 11255808.0,
+ "step": 687
+ },
+ {
+ "entropy": 1.322407841682434,
+ "epoch": 1.38989898989899,
+ "grad_norm": 2.049233913421631,
+ "learning_rate": 3.6121212121212125e-06,
+ "loss": 1.322263479232788,
+ "mean_token_accuracy": 0.6778212189674377,
+ "num_tokens": 11272192.0,
+ "step": 688
+ },
+ {
+ "entropy": 1.3112859725952148,
+ "epoch": 1.391919191919192,
+ "grad_norm": 2.0666465759277344,
+ "learning_rate": 3.6101010101010104e-06,
+ "loss": 1.3103243112564087,
+ "mean_token_accuracy": 0.6828285455703735,
+ "num_tokens": 11288576.0,
+ "step": 689
+ },
+ {
+ "entropy": 1.3321104049682617,
+ "epoch": 1.393939393939394,
+ "grad_norm": 2.5464279651641846,
+ "learning_rate": 3.6080808080808083e-06,
+ "loss": 1.3247697353363037,
+ "mean_token_accuracy": 0.6785539984703064,
+ "num_tokens": 11304960.0,
+ "step": 690
+ },
+ {
+ "entropy": 1.0779680013656616,
+ "epoch": 1.3959595959595958,
+ "grad_norm": 2.1386117935180664,
+ "learning_rate": 3.606060606060606e-06,
+ "loss": 1.0994013547897339,
+ "mean_token_accuracy": 0.7129946351051331,
+ "num_tokens": 11321344.0,
+ "step": 691
+ },
+ {
+ "entropy": 1.0701864957809448,
+ "epoch": 1.397979797979798,
+ "grad_norm": 1.9387527704238892,
+ "learning_rate": 3.604040404040404e-06,
+ "loss": 1.0633184909820557,
+ "mean_token_accuracy": 0.7378480434417725,
+ "num_tokens": 11337728.0,
+ "step": 692
+ },
+ {
+ "entropy": 1.2258752584457397,
+ "epoch": 1.4,
+ "grad_norm": 2.0486111640930176,
+ "learning_rate": 3.6020202020202024e-06,
+ "loss": 1.2098207473754883,
+ "mean_token_accuracy": 0.6999877691268921,
+ "num_tokens": 11354112.0,
+ "step": 693
+ },
+ {
+ "entropy": 1.4652774333953857,
+ "epoch": 1.402020202020202,
+ "grad_norm": 2.043079376220703,
+ "learning_rate": 3.6000000000000003e-06,
+ "loss": 1.4780974388122559,
+ "mean_token_accuracy": 0.6681729555130005,
+ "num_tokens": 11370496.0,
+ "step": 694
+ },
+ {
+ "entropy": 1.3215711116790771,
+ "epoch": 1.404040404040404,
+ "grad_norm": 2.1327438354492188,
+ "learning_rate": 3.597979797979798e-06,
+ "loss": 1.3116662502288818,
+ "mean_token_accuracy": 0.6757450103759766,
+ "num_tokens": 11386880.0,
+ "step": 695
+ },
+ {
+ "entropy": 1.624266266822815,
+ "epoch": 1.406060606060606,
+ "grad_norm": 2.114398717880249,
+ "learning_rate": 3.595959595959596e-06,
+ "loss": 1.628610372543335,
+ "mean_token_accuracy": 0.6487542986869812,
+ "num_tokens": 11403264.0,
+ "step": 696
+ },
+ {
+ "entropy": 1.6059489250183105,
+ "epoch": 1.408080808080808,
+ "grad_norm": 1.952991008758545,
+ "learning_rate": 3.593939393939394e-06,
+ "loss": 1.6145906448364258,
+ "mean_token_accuracy": 0.6257938742637634,
+ "num_tokens": 11419648.0,
+ "step": 697
+ },
+ {
+ "entropy": 1.0662391185760498,
+ "epoch": 1.41010101010101,
+ "grad_norm": 2.0804851055145264,
+ "learning_rate": 3.5919191919191922e-06,
+ "loss": 1.0772000551223755,
+ "mean_token_accuracy": 0.7187347412109375,
+ "num_tokens": 11436032.0,
+ "step": 698
+ },
+ {
+ "entropy": 1.221283197402954,
+ "epoch": 1.412121212121212,
+ "grad_norm": 1.9502744674682617,
+ "learning_rate": 3.58989898989899e-06,
+ "loss": 1.2352337837219238,
+ "mean_token_accuracy": 0.6918050646781921,
+ "num_tokens": 11452416.0,
+ "step": 699
+ },
+ {
+ "entropy": 1.4205158948898315,
+ "epoch": 1.4141414141414141,
+ "grad_norm": 2.2222518920898438,
+ "learning_rate": 3.587878787878788e-06,
+ "loss": 1.451093077659607,
+ "mean_token_accuracy": 0.6648754477500916,
+ "num_tokens": 11468800.0,
+ "step": 700
+ },
+ {
+ "epoch": 1.4141414141414141,
+ "eval_entropy": 1.4370074204860195,
+ "eval_loss": 1.5383458137512207,
+ "eval_mean_token_accuracy": 0.6441497793120723,
+ "eval_num_tokens": 11468800.0,
+ "eval_runtime": 43.744,
+ "eval_samples_per_second": 22.632,
+ "eval_steps_per_second": 2.835,
+ "step": 700
+ },
+ {
+ "entropy": 1.5432499647140503,
+ "epoch": 1.4161616161616162,
+ "grad_norm": 2.084463596343994,
+ "learning_rate": 3.585858585858586e-06,
+ "loss": 1.5958356857299805,
+ "mean_token_accuracy": 0.6377015113830566,
+ "num_tokens": 11485184.0,
+ "step": 701
+ },
+ {
+ "entropy": 1.297119379043579,
+ "epoch": 1.4181818181818182,
+ "grad_norm": 2.1996524333953857,
+ "learning_rate": 3.5838383838383838e-06,
+ "loss": 1.3346054553985596,
+ "mean_token_accuracy": 0.6801416873931885,
+ "num_tokens": 11501568.0,
+ "step": 702
+ },
+ {
+ "entropy": 1.3905766010284424,
+ "epoch": 1.4202020202020202,
+ "grad_norm": 2.1683545112609863,
+ "learning_rate": 3.5818181818181817e-06,
+ "loss": 1.40826416015625,
+ "mean_token_accuracy": 0.6653639674186707,
+ "num_tokens": 11517952.0,
+ "step": 703
+ },
+ {
+ "entropy": 0.9598695635795593,
+ "epoch": 1.4222222222222223,
+ "grad_norm": 1.9240537881851196,
+ "learning_rate": 3.57979797979798e-06,
+ "loss": 0.9487459659576416,
+ "mean_token_accuracy": 0.7556179761886597,
+ "num_tokens": 11534336.0,
+ "step": 704
+ },
+ {
+ "entropy": 1.2008687257766724,
+ "epoch": 1.4242424242424243,
+ "grad_norm": 1.9687083959579468,
+ "learning_rate": 3.577777777777778e-06,
+ "loss": 1.2234617471694946,
+ "mean_token_accuracy": 0.6982779502868652,
+ "num_tokens": 11550720.0,
+ "step": 705
+ },
+ {
+ "entropy": 1.1168856620788574,
+ "epoch": 1.4262626262626263,
+ "grad_norm": 2.1796388626098633,
+ "learning_rate": 3.575757575757576e-06,
+ "loss": 1.1320589780807495,
+ "mean_token_accuracy": 0.7057889699935913,
+ "num_tokens": 11567104.0,
+ "step": 706
+ },
+ {
+ "entropy": 1.3408682346343994,
+ "epoch": 1.4282828282828284,
+ "grad_norm": 1.9048689603805542,
+ "learning_rate": 3.573737373737374e-06,
+ "loss": 1.3476686477661133,
+ "mean_token_accuracy": 0.6839887499809265,
+ "num_tokens": 11583488.0,
+ "step": 707
+ },
+ {
+ "entropy": 1.4392427206039429,
+ "epoch": 1.4303030303030302,
+ "grad_norm": 2.183016777038574,
+ "learning_rate": 3.5717171717171724e-06,
+ "loss": 1.4443378448486328,
+ "mean_token_accuracy": 0.6546775698661804,
+ "num_tokens": 11599872.0,
+ "step": 708
+ },
+ {
+ "entropy": 1.1428641080856323,
+ "epoch": 1.4323232323232324,
+ "grad_norm": 2.0477187633514404,
+ "learning_rate": 3.5696969696969703e-06,
+ "loss": 1.1438124179840088,
+ "mean_token_accuracy": 0.7126282453536987,
+ "num_tokens": 11616256.0,
+ "step": 709
+ },
+ {
+ "entropy": 1.6266967058181763,
+ "epoch": 1.4343434343434343,
+ "grad_norm": 2.322098731994629,
+ "learning_rate": 3.567676767676768e-06,
+ "loss": 1.6161916255950928,
+ "mean_token_accuracy": 0.6202979683876038,
+ "num_tokens": 11632640.0,
+ "step": 710
+ },
+ {
+ "entropy": 1.349686622619629,
+ "epoch": 1.4363636363636363,
+ "grad_norm": 2.021871328353882,
+ "learning_rate": 3.565656565656566e-06,
+ "loss": 1.3713490962982178,
+ "mean_token_accuracy": 0.6596238613128662,
+ "num_tokens": 11649024.0,
+ "step": 711
+ },
+ {
+ "entropy": 1.4476866722106934,
+ "epoch": 1.4383838383838383,
+ "grad_norm": 2.194718837738037,
+ "learning_rate": 3.563636363636364e-06,
+ "loss": 1.4593651294708252,
+ "mean_token_accuracy": 0.6476551294326782,
+ "num_tokens": 11665408.0,
+ "step": 712
+ },
+ {
+ "entropy": 1.625157356262207,
+ "epoch": 1.4404040404040404,
+ "grad_norm": 2.0966169834136963,
+ "learning_rate": 3.5616161616161622e-06,
+ "loss": 1.6374845504760742,
+ "mean_token_accuracy": 0.6293356418609619,
+ "num_tokens": 11681792.0,
+ "step": 713
+ },
+ {
+ "entropy": 1.985729455947876,
+ "epoch": 1.4424242424242424,
+ "grad_norm": 2.129556655883789,
+ "learning_rate": 3.55959595959596e-06,
+ "loss": 2.0454561710357666,
+ "mean_token_accuracy": 0.5728505253791809,
+ "num_tokens": 11698176.0,
+ "step": 714
+ },
+ {
+ "entropy": 1.546321153640747,
+ "epoch": 1.4444444444444444,
+ "grad_norm": 2.1565628051757812,
+ "learning_rate": 3.557575757575758e-06,
+ "loss": 1.5467555522918701,
+ "mean_token_accuracy": 0.6351978778839111,
+ "num_tokens": 11714560.0,
+ "step": 715
+ },
+ {
+ "entropy": 1.3690776824951172,
+ "epoch": 1.4464646464646465,
+ "grad_norm": 2.206984519958496,
+ "learning_rate": 3.555555555555556e-06,
+ "loss": 1.3877894878387451,
+ "mean_token_accuracy": 0.6911333799362183,
+ "num_tokens": 11730944.0,
+ "step": 716
+ },
+ {
+ "entropy": 1.3643333911895752,
+ "epoch": 1.4484848484848485,
+ "grad_norm": 2.0049662590026855,
+ "learning_rate": 3.553535353535354e-06,
+ "loss": 1.364513635635376,
+ "mean_token_accuracy": 0.6781876087188721,
+ "num_tokens": 11747328.0,
+ "step": 717
+ },
+ {
+ "entropy": 1.337018370628357,
+ "epoch": 1.4505050505050505,
+ "grad_norm": 2.0919127464294434,
+ "learning_rate": 3.551515151515152e-06,
+ "loss": 1.3578065633773804,
+ "mean_token_accuracy": 0.672874927520752,
+ "num_tokens": 11763712.0,
+ "step": 718
+ },
+ {
+ "entropy": 1.3421508073806763,
+ "epoch": 1.4525252525252526,
+ "grad_norm": 2.0651159286499023,
+ "learning_rate": 3.54949494949495e-06,
+ "loss": 1.33928382396698,
+ "mean_token_accuracy": 0.6866145730018616,
+ "num_tokens": 11780096.0,
+ "step": 719
+ },
+ {
+ "entropy": 1.2841005325317383,
+ "epoch": 1.4545454545454546,
+ "grad_norm": 2.124370813369751,
+ "learning_rate": 3.547474747474748e-06,
+ "loss": 1.258314609527588,
+ "mean_token_accuracy": 0.6806302070617676,
+ "num_tokens": 11796480.0,
+ "step": 720
+ },
+ {
+ "entropy": 1.56289803981781,
+ "epoch": 1.4565656565656566,
+ "grad_norm": 1.9271100759506226,
+ "learning_rate": 3.5454545454545458e-06,
+ "loss": 1.5867946147918701,
+ "mean_token_accuracy": 0.6430752277374268,
+ "num_tokens": 11812864.0,
+ "step": 721
+ },
+ {
+ "entropy": 1.3374838829040527,
+ "epoch": 1.4585858585858587,
+ "grad_norm": 2.1113364696502686,
+ "learning_rate": 3.5434343434343437e-06,
+ "loss": 1.327169418334961,
+ "mean_token_accuracy": 0.6806912422180176,
+ "num_tokens": 11829248.0,
+ "step": 722
+ },
+ {
+ "entropy": 1.7233415842056274,
+ "epoch": 1.4606060606060607,
+ "grad_norm": 2.1862399578094482,
+ "learning_rate": 3.5414141414141416e-06,
+ "loss": 1.7290821075439453,
+ "mean_token_accuracy": 0.6056423783302307,
+ "num_tokens": 11845632.0,
+ "step": 723
+ },
+ {
+ "entropy": 1.28484046459198,
+ "epoch": 1.4626262626262627,
+ "grad_norm": 2.0059030055999756,
+ "learning_rate": 3.53939393939394e-06,
+ "loss": 1.2958555221557617,
+ "mean_token_accuracy": 0.6774548292160034,
+ "num_tokens": 11862016.0,
+ "step": 724
+ },
+ {
+ "entropy": 1.2592295408248901,
+ "epoch": 1.4646464646464645,
+ "grad_norm": 1.9422065019607544,
+ "learning_rate": 3.5373737373737378e-06,
+ "loss": 1.2501411437988281,
+ "mean_token_accuracy": 0.700537383556366,
+ "num_tokens": 11878400.0,
+ "step": 725
+ },
+ {
+ "entropy": 1.3708571195602417,
+ "epoch": 1.4666666666666668,
+ "grad_norm": 2.164499044418335,
+ "learning_rate": 3.5353535353535356e-06,
+ "loss": 1.3901662826538086,
+ "mean_token_accuracy": 0.6618832349777222,
+ "num_tokens": 11894784.0,
+ "step": 726
+ },
+ {
+ "entropy": 1.239045262336731,
+ "epoch": 1.4686868686868686,
+ "grad_norm": 1.9581060409545898,
+ "learning_rate": 3.5333333333333335e-06,
+ "loss": 1.2436164617538452,
+ "mean_token_accuracy": 0.6939423680305481,
+ "num_tokens": 11911168.0,
+ "step": 727
+ },
+ {
+ "entropy": 1.5965995788574219,
+ "epoch": 1.4707070707070706,
+ "grad_norm": 2.343669891357422,
+ "learning_rate": 3.5313131313131314e-06,
+ "loss": 1.5918347835540771,
+ "mean_token_accuracy": 0.6274425983428955,
+ "num_tokens": 11927552.0,
+ "step": 728
+ },
+ {
+ "entropy": 1.1196811199188232,
+ "epoch": 1.4727272727272727,
+ "grad_norm": 2.181840181350708,
+ "learning_rate": 3.5292929292929297e-06,
+ "loss": 1.1142621040344238,
+ "mean_token_accuracy": 0.7192842960357666,
+ "num_tokens": 11943936.0,
+ "step": 729
+ },
+ {
+ "entropy": 1.129361629486084,
+ "epoch": 1.4747474747474747,
+ "grad_norm": 2.065730571746826,
+ "learning_rate": 3.5272727272727276e-06,
+ "loss": 1.143613576889038,
+ "mean_token_accuracy": 0.7145212292671204,
+ "num_tokens": 11960320.0,
+ "step": 730
+ },
+ {
+ "entropy": 1.4940218925476074,
+ "epoch": 1.4767676767676767,
+ "grad_norm": 2.083453893661499,
+ "learning_rate": 3.5252525252525255e-06,
+ "loss": 1.4957321882247925,
+ "mean_token_accuracy": 0.6481436491012573,
+ "num_tokens": 11976704.0,
+ "step": 731
+ },
+ {
+ "entropy": 1.3427786827087402,
+ "epoch": 1.4787878787878788,
+ "grad_norm": 2.0897183418273926,
+ "learning_rate": 3.5232323232323234e-06,
+ "loss": 1.354066014289856,
+ "mean_token_accuracy": 0.6660356521606445,
+ "num_tokens": 11993088.0,
+ "step": 732
+ },
+ {
+ "entropy": 1.3300065994262695,
+ "epoch": 1.4808080808080808,
+ "grad_norm": 2.0537912845611572,
+ "learning_rate": 3.5212121212121213e-06,
+ "loss": 1.3332751989364624,
+ "mean_token_accuracy": 0.6817293763160706,
+ "num_tokens": 12009472.0,
+ "step": 733
+ },
+ {
+ "entropy": 1.0657827854156494,
+ "epoch": 1.4828282828282828,
+ "grad_norm": 1.9099130630493164,
+ "learning_rate": 3.5191919191919196e-06,
+ "loss": 1.067232608795166,
+ "mean_token_accuracy": 0.734245240688324,
+ "num_tokens": 12025856.0,
+ "step": 734
+ },
+ {
+ "entropy": 1.5265862941741943,
+ "epoch": 1.4848484848484849,
+ "grad_norm": 2.100236177444458,
+ "learning_rate": 3.5171717171717175e-06,
+ "loss": 1.5169599056243896,
+ "mean_token_accuracy": 0.6382511258125305,
+ "num_tokens": 12042240.0,
+ "step": 735
+ },
+ {
+ "entropy": 1.4428536891937256,
+ "epoch": 1.486868686868687,
+ "grad_norm": 1.9902187585830688,
+ "learning_rate": 3.5151515151515154e-06,
+ "loss": 1.4398434162139893,
+ "mean_token_accuracy": 0.675500750541687,
+ "num_tokens": 12058624.0,
+ "step": 736
+ },
+ {
+ "entropy": 1.528032660484314,
+ "epoch": 1.488888888888889,
+ "grad_norm": 2.043238878250122,
+ "learning_rate": 3.5131313131313133e-06,
+ "loss": 1.5280101299285889,
+ "mean_token_accuracy": 0.6431363224983215,
+ "num_tokens": 12075008.0,
+ "step": 737
+ },
+ {
+ "entropy": 1.4250329732894897,
+ "epoch": 1.490909090909091,
+ "grad_norm": 1.791640281677246,
+ "learning_rate": 3.511111111111111e-06,
+ "loss": 1.426466464996338,
+ "mean_token_accuracy": 0.6765388250350952,
+ "num_tokens": 12091392.0,
+ "step": 738
+ },
+ {
+ "entropy": 1.1264820098876953,
+ "epoch": 1.492929292929293,
+ "grad_norm": 2.0549607276916504,
+ "learning_rate": 3.509090909090909e-06,
+ "loss": 1.1271553039550781,
+ "mean_token_accuracy": 0.6996213793754578,
+ "num_tokens": 12107776.0,
+ "step": 739
+ },
+ {
+ "entropy": 1.538375735282898,
+ "epoch": 1.494949494949495,
+ "grad_norm": 2.22322940826416,
+ "learning_rate": 3.5070707070707073e-06,
+ "loss": 1.5588887929916382,
+ "mean_token_accuracy": 0.6384953856468201,
+ "num_tokens": 12124160.0,
+ "step": 740
+ },
+ {
+ "entropy": 1.2698255777359009,
+ "epoch": 1.496969696969697,
+ "grad_norm": 2.17710542678833,
+ "learning_rate": 3.5050505050505052e-06,
+ "loss": 1.29567289352417,
+ "mean_token_accuracy": 0.6762945652008057,
+ "num_tokens": 12140544.0,
+ "step": 741
+ },
+ {
+ "entropy": 1.523453950881958,
+ "epoch": 1.4989898989898989,
+ "grad_norm": 2.0350148677825928,
+ "learning_rate": 3.503030303030303e-06,
+ "loss": 1.5341038703918457,
+ "mean_token_accuracy": 0.6367855668067932,
+ "num_tokens": 12156928.0,
+ "step": 742
+ },
+ {
+ "entropy": 1.4250539541244507,
+ "epoch": 1.5010101010101011,
+ "grad_norm": 2.1306707859039307,
+ "learning_rate": 3.501010101010101e-06,
+ "loss": 1.4353611469268799,
+ "mean_token_accuracy": 0.6626160144805908,
+ "num_tokens": 12173312.0,
+ "step": 743
+ },
+ {
+ "entropy": 1.1440296173095703,
+ "epoch": 1.503030303030303,
+ "grad_norm": 2.053969383239746,
+ "learning_rate": 3.498989898989899e-06,
+ "loss": 1.1151821613311768,
+ "mean_token_accuracy": 0.7080483436584473,
+ "num_tokens": 12189696.0,
+ "step": 744
+ },
+ {
+ "entropy": 1.5290073156356812,
+ "epoch": 1.5050505050505052,
+ "grad_norm": 2.0550003051757812,
+ "learning_rate": 3.496969696969697e-06,
+ "loss": 1.5260624885559082,
+ "mean_token_accuracy": 0.6493038535118103,
+ "num_tokens": 12206080.0,
+ "step": 745
+ },
+ {
+ "entropy": 1.1096605062484741,
+ "epoch": 1.507070707070707,
+ "grad_norm": 1.8131766319274902,
+ "learning_rate": 3.494949494949495e-06,
+ "loss": 1.1221846342086792,
+ "mean_token_accuracy": 0.7263678312301636,
+ "num_tokens": 12222464.0,
+ "step": 746
+ },
+ {
+ "entropy": 1.282676339149475,
+ "epoch": 1.509090909090909,
+ "grad_norm": 2.3051960468292236,
+ "learning_rate": 3.492929292929293e-06,
+ "loss": 1.2863187789916992,
+ "mean_token_accuracy": 0.6765388250350952,
+ "num_tokens": 12238848.0,
+ "step": 747
+ },
+ {
+ "entropy": 1.2971303462982178,
+ "epoch": 1.511111111111111,
+ "grad_norm": 2.0721139907836914,
+ "learning_rate": 3.4909090909090913e-06,
+ "loss": 1.3010127544403076,
+ "mean_token_accuracy": 0.6753175258636475,
+ "num_tokens": 12255232.0,
+ "step": 748
+ },
+ {
+ "entropy": 1.3711751699447632,
+ "epoch": 1.513131313131313,
+ "grad_norm": 1.9657210111618042,
+ "learning_rate": 3.4888888888888896e-06,
+ "loss": 1.3703930377960205,
+ "mean_token_accuracy": 0.6686614751815796,
+ "num_tokens": 12271616.0,
+ "step": 749
+ },
+ {
+ "entropy": 1.512154459953308,
+ "epoch": 1.5151515151515151,
+ "grad_norm": 2.214172124862671,
+ "learning_rate": 3.4868686868686875e-06,
+ "loss": 1.5243256092071533,
+ "mean_token_accuracy": 0.6485100388526917,
+ "num_tokens": 12288000.0,
+ "step": 750
+ },
+ {
+ "epoch": 1.5151515151515151,
+ "eval_entropy": 1.4458443743567313,
+ "eval_loss": 1.535510540008545,
+ "eval_mean_token_accuracy": 0.6444821915318889,
+ "eval_num_tokens": 12288000.0,
+ "eval_runtime": 43.8043,
+ "eval_samples_per_second": 22.601,
+ "eval_steps_per_second": 2.831,
+ "step": 750
+ },
+ {
+ "entropy": 1.2052820920944214,
+ "epoch": 1.5171717171717172,
+ "grad_norm": 2.104870319366455,
+ "learning_rate": 3.4848484848484854e-06,
+ "loss": 1.2207554578781128,
+ "mean_token_accuracy": 0.695713222026825,
+ "num_tokens": 12304384.0,
+ "step": 751
+ },
+ {
+ "entropy": 1.732282280921936,
+ "epoch": 1.5191919191919192,
+ "grad_norm": 2.101760149002075,
+ "learning_rate": 3.4828282828282833e-06,
+ "loss": 1.7461241483688354,
+ "mean_token_accuracy": 0.6505251526832581,
+ "num_tokens": 12320768.0,
+ "step": 752
+ },
+ {
+ "entropy": 1.5805878639221191,
+ "epoch": 1.5212121212121212,
+ "grad_norm": 2.139244318008423,
+ "learning_rate": 3.480808080808081e-06,
+ "loss": 1.5813164710998535,
+ "mean_token_accuracy": 0.6281143426895142,
+ "num_tokens": 12337152.0,
+ "step": 753
+ },
+ {
+ "entropy": 1.1882269382476807,
+ "epoch": 1.5232323232323233,
+ "grad_norm": 2.187206745147705,
+ "learning_rate": 3.4787878787878795e-06,
+ "loss": 1.1732902526855469,
+ "mean_token_accuracy": 0.7063385248184204,
+ "num_tokens": 12353536.0,
+ "step": 754
+ },
+ {
+ "entropy": 1.4743424654006958,
+ "epoch": 1.5252525252525253,
+ "grad_norm": 2.12687349319458,
+ "learning_rate": 3.4767676767676774e-06,
+ "loss": 1.4681663513183594,
+ "mean_token_accuracy": 0.6558378338813782,
+ "num_tokens": 12369920.0,
+ "step": 755
+ },
+ {
+ "entropy": 1.5533041954040527,
+ "epoch": 1.5272727272727273,
+ "grad_norm": 2.1306610107421875,
+ "learning_rate": 3.4747474747474752e-06,
+ "loss": 1.5599431991577148,
+ "mean_token_accuracy": 0.6399609446525574,
+ "num_tokens": 12386304.0,
+ "step": 756
+ },
+ {
+ "entropy": 0.8933209180831909,
+ "epoch": 1.5292929292929291,
+ "grad_norm": 2.425602912902832,
+ "learning_rate": 3.472727272727273e-06,
+ "loss": 0.8840203285217285,
+ "mean_token_accuracy": 0.7638006806373596,
+ "num_tokens": 12402688.0,
+ "step": 757
+ },
+ {
+ "entropy": 1.6764189004898071,
+ "epoch": 1.5313131313131314,
+ "grad_norm": 2.0684356689453125,
+ "learning_rate": 3.470707070707071e-06,
+ "loss": 1.6948609352111816,
+ "mean_token_accuracy": 0.6087567210197449,
+ "num_tokens": 12419072.0,
+ "step": 758
+ },
+ {
+ "entropy": 1.1875263452529907,
+ "epoch": 1.5333333333333332,
+ "grad_norm": 2.0625953674316406,
+ "learning_rate": 3.468686868686869e-06,
+ "loss": 1.1840041875839233,
+ "mean_token_accuracy": 0.7067660093307495,
+ "num_tokens": 12435456.0,
+ "step": 759
+ },
+ {
+ "entropy": 1.1589192152023315,
+ "epoch": 1.5353535353535355,
+ "grad_norm": 1.9212846755981445,
+ "learning_rate": 3.4666666666666672e-06,
+ "loss": 1.1537678241729736,
+ "mean_token_accuracy": 0.7107962965965271,
+ "num_tokens": 12451840.0,
+ "step": 760
+ },
+ {
+ "entropy": 0.9687032699584961,
+ "epoch": 1.5373737373737373,
+ "grad_norm": 1.9402192831039429,
+ "learning_rate": 3.464646464646465e-06,
+ "loss": 0.9954442977905273,
+ "mean_token_accuracy": 0.7412676811218262,
+ "num_tokens": 12468224.0,
+ "step": 761
+ },
+ {
+ "entropy": 1.4129019975662231,
+ "epoch": 1.5393939393939395,
+ "grad_norm": 2.0903310775756836,
+ "learning_rate": 3.462626262626263e-06,
+ "loss": 1.4131944179534912,
+ "mean_token_accuracy": 0.6676844358444214,
+ "num_tokens": 12484608.0,
+ "step": 762
+ },
+ {
+ "entropy": 1.5170351266860962,
+ "epoch": 1.5414141414141413,
+ "grad_norm": 2.12807035446167,
+ "learning_rate": 3.460606060606061e-06,
+ "loss": 1.5342246294021606,
+ "mean_token_accuracy": 0.6621885895729065,
+ "num_tokens": 12500992.0,
+ "step": 763
+ },
+ {
+ "entropy": 1.1815718412399292,
+ "epoch": 1.5434343434343434,
+ "grad_norm": 2.2491719722747803,
+ "learning_rate": 3.4585858585858588e-06,
+ "loss": 1.1669261455535889,
+ "mean_token_accuracy": 0.694614052772522,
+ "num_tokens": 12517376.0,
+ "step": 764
+ },
+ {
+ "entropy": 1.547598958015442,
+ "epoch": 1.5454545454545454,
+ "grad_norm": 2.136096954345703,
+ "learning_rate": 3.456565656565657e-06,
+ "loss": 1.5515470504760742,
+ "mean_token_accuracy": 0.648876428604126,
+ "num_tokens": 12533760.0,
+ "step": 765
+ },
+ {
+ "entropy": 0.7156143188476562,
+ "epoch": 1.5474747474747474,
+ "grad_norm": 1.8139498233795166,
+ "learning_rate": 3.454545454545455e-06,
+ "loss": 0.7052675485610962,
+ "mean_token_accuracy": 0.8103932738304138,
+ "num_tokens": 12550144.0,
+ "step": 766
+ },
+ {
+ "entropy": 1.400156855583191,
+ "epoch": 1.5494949494949495,
+ "grad_norm": 2.14924693107605,
+ "learning_rate": 3.452525252525253e-06,
+ "loss": 1.4047740697860718,
+ "mean_token_accuracy": 0.6719589829444885,
+ "num_tokens": 12566528.0,
+ "step": 767
+ },
+ {
+ "entropy": 1.8254657983779907,
+ "epoch": 1.5515151515151515,
+ "grad_norm": 2.1882269382476807,
+ "learning_rate": 3.4505050505050507e-06,
+ "loss": 1.8615440130233765,
+ "mean_token_accuracy": 0.5776746273040771,
+ "num_tokens": 12582912.0,
+ "step": 768
+ },
+ {
+ "entropy": 1.5947710275650024,
+ "epoch": 1.5535353535353535,
+ "grad_norm": 2.3221426010131836,
+ "learning_rate": 3.4484848484848486e-06,
+ "loss": 1.6178712844848633,
+ "mean_token_accuracy": 0.6210918426513672,
+ "num_tokens": 12599296.0,
+ "step": 769
+ },
+ {
+ "entropy": 1.2678714990615845,
+ "epoch": 1.5555555555555556,
+ "grad_norm": 1.970064640045166,
+ "learning_rate": 3.446464646464647e-06,
+ "loss": 1.2646088600158691,
+ "mean_token_accuracy": 0.6990107297897339,
+ "num_tokens": 12615680.0,
+ "step": 770
+ },
+ {
+ "entropy": 1.3723604679107666,
+ "epoch": 1.5575757575757576,
+ "grad_norm": 2.171534776687622,
+ "learning_rate": 3.444444444444445e-06,
+ "loss": 1.3890197277069092,
+ "mean_token_accuracy": 0.6709819436073303,
+ "num_tokens": 12632064.0,
+ "step": 771
+ },
+ {
+ "entropy": 1.0273096561431885,
+ "epoch": 1.5595959595959596,
+ "grad_norm": 2.072798490524292,
+ "learning_rate": 3.4424242424242427e-06,
+ "loss": 1.029193639755249,
+ "mean_token_accuracy": 0.7348558902740479,
+ "num_tokens": 12648448.0,
+ "step": 772
+ },
+ {
+ "entropy": 1.4004037380218506,
+ "epoch": 1.5616161616161617,
+ "grad_norm": 2.111480474472046,
+ "learning_rate": 3.4404040404040406e-06,
+ "loss": 1.4213340282440186,
+ "mean_token_accuracy": 0.660906195640564,
+ "num_tokens": 12664832.0,
+ "step": 773
+ },
+ {
+ "entropy": 1.533424735069275,
+ "epoch": 1.5636363636363635,
+ "grad_norm": 2.0949435234069824,
+ "learning_rate": 3.4383838383838385e-06,
+ "loss": 1.5259253978729248,
+ "mean_token_accuracy": 0.6544333100318909,
+ "num_tokens": 12681216.0,
+ "step": 774
+ },
+ {
+ "entropy": 1.5620275735855103,
+ "epoch": 1.5656565656565657,
+ "grad_norm": 2.339731454849243,
+ "learning_rate": 3.4363636363636364e-06,
+ "loss": 1.5596071481704712,
+ "mean_token_accuracy": 0.6290302872657776,
+ "num_tokens": 12697600.0,
+ "step": 775
+ },
+ {
+ "entropy": 1.3272082805633545,
+ "epoch": 1.5676767676767676,
+ "grad_norm": 2.0871188640594482,
+ "learning_rate": 3.4343434343434347e-06,
+ "loss": 1.3413164615631104,
+ "mean_token_accuracy": 0.6809965968132019,
+ "num_tokens": 12713984.0,
+ "step": 776
+ },
+ {
+ "entropy": 1.202813982963562,
+ "epoch": 1.5696969696969698,
+ "grad_norm": 2.0538578033447266,
+ "learning_rate": 3.4323232323232326e-06,
+ "loss": 1.1947424411773682,
+ "mean_token_accuracy": 0.6969956159591675,
+ "num_tokens": 12730368.0,
+ "step": 777
+ },
+ {
+ "entropy": 1.3332773447036743,
+ "epoch": 1.5717171717171716,
+ "grad_norm": 1.9941202402114868,
+ "learning_rate": 3.4303030303030305e-06,
+ "loss": 1.3345956802368164,
+ "mean_token_accuracy": 0.6761724352836609,
+ "num_tokens": 12746752.0,
+ "step": 778
+ },
+ {
+ "entropy": 1.1724364757537842,
+ "epoch": 1.5737373737373739,
+ "grad_norm": 2.213365316390991,
+ "learning_rate": 3.4282828282828284e-06,
+ "loss": 1.1479787826538086,
+ "mean_token_accuracy": 0.7132999300956726,
+ "num_tokens": 12763136.0,
+ "step": 779
+ },
+ {
+ "entropy": 1.4718358516693115,
+ "epoch": 1.5757575757575757,
+ "grad_norm": 1.9743510484695435,
+ "learning_rate": 3.4262626262626262e-06,
+ "loss": 1.4844720363616943,
+ "mean_token_accuracy": 0.656448483467102,
+ "num_tokens": 12779520.0,
+ "step": 780
+ },
+ {
+ "entropy": 1.362882375717163,
+ "epoch": 1.5777777777777777,
+ "grad_norm": 2.370483875274658,
+ "learning_rate": 3.4242424242424246e-06,
+ "loss": 1.350801706314087,
+ "mean_token_accuracy": 0.6707376837730408,
+ "num_tokens": 12795904.0,
+ "step": 781
+ },
+ {
+ "entropy": 1.8302308320999146,
+ "epoch": 1.5797979797979798,
+ "grad_norm": 2.19045090675354,
+ "learning_rate": 3.4222222222222224e-06,
+ "loss": 1.8371320962905884,
+ "mean_token_accuracy": 0.5929408669471741,
+ "num_tokens": 12812288.0,
+ "step": 782
+ },
+ {
+ "entropy": 1.3435529470443726,
+ "epoch": 1.5818181818181818,
+ "grad_norm": 2.1254351139068604,
+ "learning_rate": 3.4202020202020203e-06,
+ "loss": 1.3621928691864014,
+ "mean_token_accuracy": 0.6867367029190063,
+ "num_tokens": 12828672.0,
+ "step": 783
+ },
+ {
+ "entropy": 1.8402602672576904,
+ "epoch": 1.5838383838383838,
+ "grad_norm": 2.123666524887085,
+ "learning_rate": 3.4181818181818182e-06,
+ "loss": 1.8564848899841309,
+ "mean_token_accuracy": 0.5871397256851196,
+ "num_tokens": 12845056.0,
+ "step": 784
+ },
+ {
+ "entropy": 1.441917896270752,
+ "epoch": 1.5858585858585859,
+ "grad_norm": 2.3286895751953125,
+ "learning_rate": 3.416161616161616e-06,
+ "loss": 1.4724184274673462,
+ "mean_token_accuracy": 0.6438080072402954,
+ "num_tokens": 12861440.0,
+ "step": 785
+ },
+ {
+ "entropy": 1.3071452379226685,
+ "epoch": 1.587878787878788,
+ "grad_norm": 2.149143934249878,
+ "learning_rate": 3.414141414141414e-06,
+ "loss": 1.331390142440796,
+ "mean_token_accuracy": 0.679347813129425,
+ "num_tokens": 12877824.0,
+ "step": 786
+ },
+ {
+ "entropy": 1.3547145128250122,
+ "epoch": 1.58989898989899,
+ "grad_norm": 2.2531180381774902,
+ "learning_rate": 3.4121212121212123e-06,
+ "loss": 1.361851453781128,
+ "mean_token_accuracy": 0.6767830848693848,
+ "num_tokens": 12894208.0,
+ "step": 787
+ },
+ {
+ "entropy": 0.9811291098594666,
+ "epoch": 1.591919191919192,
+ "grad_norm": 2.1053760051727295,
+ "learning_rate": 3.41010101010101e-06,
+ "loss": 0.9885507822036743,
+ "mean_token_accuracy": 0.7480459213256836,
+ "num_tokens": 12910592.0,
+ "step": 788
+ },
+ {
+ "entropy": 1.4940402507781982,
+ "epoch": 1.593939393939394,
+ "grad_norm": 2.240074634552002,
+ "learning_rate": 3.408080808080808e-06,
+ "loss": 1.5014076232910156,
+ "mean_token_accuracy": 0.6441743969917297,
+ "num_tokens": 12926976.0,
+ "step": 789
+ },
+ {
+ "entropy": 1.8134219646453857,
+ "epoch": 1.595959595959596,
+ "grad_norm": 2.040239095687866,
+ "learning_rate": 3.406060606060606e-06,
+ "loss": 1.8437519073486328,
+ "mean_token_accuracy": 0.5904372334480286,
+ "num_tokens": 12943360.0,
+ "step": 790
+ },
+ {
+ "entropy": 1.7233681678771973,
+ "epoch": 1.5979797979797978,
+ "grad_norm": 2.381786823272705,
+ "learning_rate": 3.4040404040404047e-06,
+ "loss": 1.7328217029571533,
+ "mean_token_accuracy": 0.6004518866539001,
+ "num_tokens": 12959744.0,
+ "step": 791
+ },
+ {
+ "entropy": 1.3966271877288818,
+ "epoch": 1.6,
+ "grad_norm": 2.17006778717041,
+ "learning_rate": 3.4020202020202026e-06,
+ "loss": 1.4065337181091309,
+ "mean_token_accuracy": 0.6629824042320251,
+ "num_tokens": 12976128.0,
+ "step": 792
+ },
+ {
+ "entropy": 1.5002557039260864,
+ "epoch": 1.602020202020202,
+ "grad_norm": 2.558037281036377,
+ "learning_rate": 3.4000000000000005e-06,
+ "loss": 1.4841307401657104,
+ "mean_token_accuracy": 0.647838294506073,
+ "num_tokens": 12992512.0,
+ "step": 793
+ },
+ {
+ "entropy": 1.7432687282562256,
+ "epoch": 1.6040404040404042,
+ "grad_norm": 2.134734630584717,
+ "learning_rate": 3.3979797979797984e-06,
+ "loss": 1.7539207935333252,
+ "mean_token_accuracy": 0.6028333902359009,
+ "num_tokens": 13008896.0,
+ "step": 794
+ },
+ {
+ "entropy": 1.4490517377853394,
+ "epoch": 1.606060606060606,
+ "grad_norm": 2.1590232849121094,
+ "learning_rate": 3.3959595959595963e-06,
+ "loss": 1.468411922454834,
+ "mean_token_accuracy": 0.656509518623352,
+ "num_tokens": 13025280.0,
+ "step": 795
+ },
+ {
+ "entropy": 1.3823497295379639,
+ "epoch": 1.6080808080808082,
+ "grad_norm": 2.1508285999298096,
+ "learning_rate": 3.3939393939393946e-06,
+ "loss": 1.4011330604553223,
+ "mean_token_accuracy": 0.6530898809432983,
+ "num_tokens": 13041664.0,
+ "step": 796
+ },
+ {
+ "entropy": 1.6982436180114746,
+ "epoch": 1.61010101010101,
+ "grad_norm": 2.077131986618042,
+ "learning_rate": 3.3919191919191925e-06,
+ "loss": 1.727889060974121,
+ "mean_token_accuracy": 0.6232290863990784,
+ "num_tokens": 13058048.0,
+ "step": 797
+ },
+ {
+ "entropy": 1.7268915176391602,
+ "epoch": 1.612121212121212,
+ "grad_norm": 2.0703542232513428,
+ "learning_rate": 3.3898989898989903e-06,
+ "loss": 1.7333730459213257,
+ "mean_token_accuracy": 0.6110160946846008,
+ "num_tokens": 13074432.0,
+ "step": 798
+ },
+ {
+ "entropy": 1.6253595352172852,
+ "epoch": 1.614141414141414,
+ "grad_norm": 2.241154432296753,
+ "learning_rate": 3.3878787878787882e-06,
+ "loss": 1.6275582313537598,
+ "mean_token_accuracy": 0.6198094487190247,
+ "num_tokens": 13090816.0,
+ "step": 799
+ },
+ {
+ "entropy": 1.8784692287445068,
+ "epoch": 1.6161616161616161,
+ "grad_norm": 2.505871057510376,
+ "learning_rate": 3.385858585858586e-06,
+ "loss": 1.8784008026123047,
+ "mean_token_accuracy": 0.5944064259529114,
+ "num_tokens": 13107200.0,
+ "step": 800
+ },
+ {
+ "epoch": 1.6161616161616161,
+ "eval_entropy": 1.4455043437980837,
+ "eval_loss": 1.5330992937088013,
+ "eval_mean_token_accuracy": 0.6448845310557273,
+ "eval_num_tokens": 13107200.0,
+ "eval_runtime": 43.766,
+ "eval_samples_per_second": 22.62,
+ "eval_steps_per_second": 2.833,
+ "step": 800
+ },
+ {
+ "entropy": 1.921440601348877,
+ "epoch": 1.6181818181818182,
+ "grad_norm": 2.1760783195495605,
+ "learning_rate": 3.3838383838383844e-06,
+ "loss": 1.9410110712051392,
+ "mean_token_accuracy": 0.5776746273040771,
+ "num_tokens": 13123584.0,
+ "step": 801
+ },
+ {
+ "entropy": 1.4233160018920898,
+ "epoch": 1.6202020202020202,
+ "grad_norm": 2.2687666416168213,
+ "learning_rate": 3.3818181818181823e-06,
+ "loss": 1.4465446472167969,
+ "mean_token_accuracy": 0.6488153338432312,
+ "num_tokens": 13139968.0,
+ "step": 802
+ },
+ {
+ "entropy": 1.697847843170166,
+ "epoch": 1.6222222222222222,
+ "grad_norm": 2.0649187564849854,
+ "learning_rate": 3.37979797979798e-06,
+ "loss": 1.7235385179519653,
+ "mean_token_accuracy": 0.6196262836456299,
+ "num_tokens": 13156352.0,
+ "step": 803
+ },
+ {
+ "entropy": 1.2027701139450073,
+ "epoch": 1.6242424242424243,
+ "grad_norm": 2.0115890502929688,
+ "learning_rate": 3.377777777777778e-06,
+ "loss": 1.1890676021575928,
+ "mean_token_accuracy": 0.6928431987762451,
+ "num_tokens": 13172736.0,
+ "step": 804
+ },
+ {
+ "entropy": 1.4847105741500854,
+ "epoch": 1.6262626262626263,
+ "grad_norm": 2.145098924636841,
+ "learning_rate": 3.375757575757576e-06,
+ "loss": 1.4817116260528564,
+ "mean_token_accuracy": 0.6415486335754395,
+ "num_tokens": 13189120.0,
+ "step": 805
+ },
+ {
+ "entropy": 1.3548893928527832,
+ "epoch": 1.6282828282828283,
+ "grad_norm": 2.3037521839141846,
+ "learning_rate": 3.3737373737373743e-06,
+ "loss": 1.3433952331542969,
+ "mean_token_accuracy": 0.6681729555130005,
+ "num_tokens": 13205504.0,
+ "step": 806
+ },
+ {
+ "entropy": 1.375793218612671,
+ "epoch": 1.6303030303030304,
+ "grad_norm": 2.0814852714538574,
+ "learning_rate": 3.371717171717172e-06,
+ "loss": 1.3506405353546143,
+ "mean_token_accuracy": 0.6634098887443542,
+ "num_tokens": 13221888.0,
+ "step": 807
+ },
+ {
+ "entropy": 1.1417323350906372,
+ "epoch": 1.6323232323232322,
+ "grad_norm": 2.0166866779327393,
+ "learning_rate": 3.36969696969697e-06,
+ "loss": 1.1731860637664795,
+ "mean_token_accuracy": 0.7123839855194092,
+ "num_tokens": 13238272.0,
+ "step": 808
+ },
+ {
+ "entropy": 1.3159022331237793,
+ "epoch": 1.6343434343434344,
+ "grad_norm": 1.9626281261444092,
+ "learning_rate": 3.367676767676768e-06,
+ "loss": 1.3286499977111816,
+ "mean_token_accuracy": 0.675561785697937,
+ "num_tokens": 13254656.0,
+ "step": 809
+ },
+ {
+ "entropy": 1.325775384902954,
+ "epoch": 1.6363636363636362,
+ "grad_norm": 2.1239242553710938,
+ "learning_rate": 3.365656565656566e-06,
+ "loss": 1.345514178276062,
+ "mean_token_accuracy": 0.6780043840408325,
+ "num_tokens": 13271040.0,
+ "step": 810
+ },
+ {
+ "entropy": 1.8307870626449585,
+ "epoch": 1.6383838383838385,
+ "grad_norm": 2.208296298980713,
+ "learning_rate": 3.3636363636363637e-06,
+ "loss": 1.8402234315872192,
+ "mean_token_accuracy": 0.5958719849586487,
+ "num_tokens": 13287424.0,
+ "step": 811
+ },
+ {
+ "entropy": 1.2596150636672974,
+ "epoch": 1.6404040404040403,
+ "grad_norm": 2.0936427116394043,
+ "learning_rate": 3.361616161616162e-06,
+ "loss": 1.2552804946899414,
+ "mean_token_accuracy": 0.6874083876609802,
+ "num_tokens": 13303808.0,
+ "step": 812
+ },
+ {
+ "entropy": 1.2575111389160156,
+ "epoch": 1.6424242424242426,
+ "grad_norm": 2.0232772827148438,
+ "learning_rate": 3.35959595959596e-06,
+ "loss": 1.2805989980697632,
+ "mean_token_accuracy": 0.6916829347610474,
+ "num_tokens": 13320192.0,
+ "step": 813
+ },
+ {
+ "entropy": 1.0270155668258667,
+ "epoch": 1.6444444444444444,
+ "grad_norm": 1.9090715646743774,
+ "learning_rate": 3.357575757575758e-06,
+ "loss": 1.0277824401855469,
+ "mean_token_accuracy": 0.7379091382026672,
+ "num_tokens": 13336576.0,
+ "step": 814
+ },
+ {
+ "entropy": 1.2992243766784668,
+ "epoch": 1.6464646464646466,
+ "grad_norm": 2.1718814373016357,
+ "learning_rate": 3.3555555555555557e-06,
+ "loss": 1.3278638124465942,
+ "mean_token_accuracy": 0.6764777898788452,
+ "num_tokens": 13352960.0,
+ "step": 815
+ },
+ {
+ "entropy": 1.5365362167358398,
+ "epoch": 1.6484848484848484,
+ "grad_norm": 2.0298495292663574,
+ "learning_rate": 3.3535353535353536e-06,
+ "loss": 1.5257508754730225,
+ "mean_token_accuracy": 0.6511358022689819,
+ "num_tokens": 13369344.0,
+ "step": 816
+ },
+ {
+ "entropy": 1.4384456872940063,
+ "epoch": 1.6505050505050505,
+ "grad_norm": 2.375277042388916,
+ "learning_rate": 3.351515151515152e-06,
+ "loss": 1.4287049770355225,
+ "mean_token_accuracy": 0.645639955997467,
+ "num_tokens": 13385728.0,
+ "step": 817
+ },
+ {
+ "entropy": 1.7104265689849854,
+ "epoch": 1.6525252525252525,
+ "grad_norm": 2.052535057067871,
+ "learning_rate": 3.34949494949495e-06,
+ "loss": 1.6971676349639893,
+ "mean_token_accuracy": 0.6159623861312866,
+ "num_tokens": 13402112.0,
+ "step": 818
+ },
+ {
+ "entropy": 1.5908539295196533,
+ "epoch": 1.6545454545454545,
+ "grad_norm": 2.047088861465454,
+ "learning_rate": 3.3474747474747477e-06,
+ "loss": 1.5640246868133545,
+ "mean_token_accuracy": 0.640754759311676,
+ "num_tokens": 13418496.0,
+ "step": 819
+ },
+ {
+ "entropy": 1.3660207986831665,
+ "epoch": 1.6565656565656566,
+ "grad_norm": 2.2711079120635986,
+ "learning_rate": 3.3454545454545456e-06,
+ "loss": 1.3601036071777344,
+ "mean_token_accuracy": 0.6577919125556946,
+ "num_tokens": 13434880.0,
+ "step": 820
+ },
+ {
+ "entropy": 1.5897634029388428,
+ "epoch": 1.6585858585858586,
+ "grad_norm": 2.0869712829589844,
+ "learning_rate": 3.3434343434343435e-06,
+ "loss": 1.5801420211791992,
+ "mean_token_accuracy": 0.6312897205352783,
+ "num_tokens": 13451264.0,
+ "step": 821
+ },
+ {
+ "entropy": 1.428406834602356,
+ "epoch": 1.6606060606060606,
+ "grad_norm": 2.049370765686035,
+ "learning_rate": 3.3414141414141413e-06,
+ "loss": 1.4145596027374268,
+ "mean_token_accuracy": 0.6649975776672363,
+ "num_tokens": 13467648.0,
+ "step": 822
+ },
+ {
+ "entropy": 1.5026675462722778,
+ "epoch": 1.6626262626262627,
+ "grad_norm": 2.157031297683716,
+ "learning_rate": 3.3393939393939397e-06,
+ "loss": 1.4902830123901367,
+ "mean_token_accuracy": 0.6524181962013245,
+ "num_tokens": 13484032.0,
+ "step": 823
+ },
+ {
+ "entropy": 1.4907052516937256,
+ "epoch": 1.6646464646464647,
+ "grad_norm": 2.1476478576660156,
+ "learning_rate": 3.3373737373737375e-06,
+ "loss": 1.5102436542510986,
+ "mean_token_accuracy": 0.6485100388526917,
+ "num_tokens": 13500416.0,
+ "step": 824
+ },
+ {
+ "entropy": 1.4277782440185547,
+ "epoch": 1.6666666666666665,
+ "grad_norm": 2.2880661487579346,
+ "learning_rate": 3.3353535353535354e-06,
+ "loss": 1.4358019828796387,
+ "mean_token_accuracy": 0.6522960662841797,
+ "num_tokens": 13516800.0,
+ "step": 825
+ },
+ {
+ "entropy": 1.5560128688812256,
+ "epoch": 1.6686868686868688,
+ "grad_norm": 2.048403024673462,
+ "learning_rate": 3.3333333333333333e-06,
+ "loss": 1.5472501516342163,
+ "mean_token_accuracy": 0.6488153338432312,
+ "num_tokens": 13533184.0,
+ "step": 826
+ },
+ {
+ "entropy": 1.665900707244873,
+ "epoch": 1.6707070707070706,
+ "grad_norm": 2.0828492641448975,
+ "learning_rate": 3.331313131313131e-06,
+ "loss": 1.6905372142791748,
+ "mean_token_accuracy": 0.629885196685791,
+ "num_tokens": 13549568.0,
+ "step": 827
+ },
+ {
+ "entropy": 1.3484426736831665,
+ "epoch": 1.6727272727272728,
+ "grad_norm": 2.1873531341552734,
+ "learning_rate": 3.3292929292929295e-06,
+ "loss": 1.3510258197784424,
+ "mean_token_accuracy": 0.6721421480178833,
+ "num_tokens": 13565952.0,
+ "step": 828
+ },
+ {
+ "entropy": 1.7577860355377197,
+ "epoch": 1.6747474747474747,
+ "grad_norm": 2.0467135906219482,
+ "learning_rate": 3.3272727272727274e-06,
+ "loss": 1.7791969776153564,
+ "mean_token_accuracy": 0.6028944849967957,
+ "num_tokens": 13582336.0,
+ "step": 829
+ },
+ {
+ "entropy": 1.4549124240875244,
+ "epoch": 1.676767676767677,
+ "grad_norm": 2.2004988193511963,
+ "learning_rate": 3.3252525252525253e-06,
+ "loss": 1.4512457847595215,
+ "mean_token_accuracy": 0.6521128416061401,
+ "num_tokens": 13598720.0,
+ "step": 830
+ },
+ {
+ "entropy": 1.2226758003234863,
+ "epoch": 1.6787878787878787,
+ "grad_norm": 2.0866682529449463,
+ "learning_rate": 3.323232323232323e-06,
+ "loss": 1.226635456085205,
+ "mean_token_accuracy": 0.6969345211982727,
+ "num_tokens": 13615104.0,
+ "step": 831
+ },
+ {
+ "entropy": 0.9826563000679016,
+ "epoch": 1.680808080808081,
+ "grad_norm": 1.9865771532058716,
+ "learning_rate": 3.321212121212121e-06,
+ "loss": 1.009648323059082,
+ "mean_token_accuracy": 0.7507327795028687,
+ "num_tokens": 13631488.0,
+ "step": 832
+ },
+ {
+ "entropy": 1.620038390159607,
+ "epoch": 1.6828282828282828,
+ "grad_norm": 2.1745760440826416,
+ "learning_rate": 3.3191919191919194e-06,
+ "loss": 1.615234136581421,
+ "mean_token_accuracy": 0.6305569410324097,
+ "num_tokens": 13647872.0,
+ "step": 833
+ },
+ {
+ "entropy": 1.4803876876831055,
+ "epoch": 1.6848484848484848,
+ "grad_norm": 2.328029155731201,
+ "learning_rate": 3.3171717171717177e-06,
+ "loss": 1.509049892425537,
+ "mean_token_accuracy": 0.6353810429573059,
+ "num_tokens": 13664256.0,
+ "step": 834
+ },
+ {
+ "entropy": 1.2290453910827637,
+ "epoch": 1.6868686868686869,
+ "grad_norm": 2.1777281761169434,
+ "learning_rate": 3.3151515151515156e-06,
+ "loss": 1.222726583480835,
+ "mean_token_accuracy": 0.6842330098152161,
+ "num_tokens": 13680640.0,
+ "step": 835
+ },
+ {
+ "entropy": 1.1965430974960327,
+ "epoch": 1.6888888888888889,
+ "grad_norm": 2.211397886276245,
+ "learning_rate": 3.3131313131313135e-06,
+ "loss": 1.2027852535247803,
+ "mean_token_accuracy": 0.7028578519821167,
+ "num_tokens": 13697024.0,
+ "step": 836
+ },
+ {
+ "entropy": 1.5326080322265625,
+ "epoch": 1.690909090909091,
+ "grad_norm": 2.8396589756011963,
+ "learning_rate": 3.3111111111111118e-06,
+ "loss": 1.5819613933563232,
+ "mean_token_accuracy": 0.625,
+ "num_tokens": 13713408.0,
+ "step": 837
+ },
+ {
+ "entropy": 1.2514865398406982,
+ "epoch": 1.692929292929293,
+ "grad_norm": 1.923771858215332,
+ "learning_rate": 3.3090909090909097e-06,
+ "loss": 1.2518126964569092,
+ "mean_token_accuracy": 0.6955300569534302,
+ "num_tokens": 13729792.0,
+ "step": 838
+ },
+ {
+ "entropy": 1.3717304468154907,
+ "epoch": 1.694949494949495,
+ "grad_norm": 2.0393056869506836,
+ "learning_rate": 3.3070707070707076e-06,
+ "loss": 1.400814175605774,
+ "mean_token_accuracy": 0.6618832349777222,
+ "num_tokens": 13746176.0,
+ "step": 839
+ },
+ {
+ "entropy": 1.2871394157409668,
+ "epoch": 1.696969696969697,
+ "grad_norm": 2.114823579788208,
+ "learning_rate": 3.3050505050505054e-06,
+ "loss": 1.302154541015625,
+ "mean_token_accuracy": 0.6790425181388855,
+ "num_tokens": 13762560.0,
+ "step": 840
+ },
+ {
+ "entropy": 1.518404483795166,
+ "epoch": 1.698989898989899,
+ "grad_norm": 2.186727523803711,
+ "learning_rate": 3.3030303030303033e-06,
+ "loss": 1.5067068338394165,
+ "mean_token_accuracy": 0.638006865978241,
+ "num_tokens": 13778944.0,
+ "step": 841
+ },
+ {
+ "entropy": 1.5698254108428955,
+ "epoch": 1.7010101010101009,
+ "grad_norm": 2.159721851348877,
+ "learning_rate": 3.3010101010101016e-06,
+ "loss": 1.5568101406097412,
+ "mean_token_accuracy": 0.6398998498916626,
+ "num_tokens": 13795328.0,
+ "step": 842
+ },
+ {
+ "entropy": 1.5159670114517212,
+ "epoch": 1.7030303030303031,
+ "grad_norm": 2.0655386447906494,
+ "learning_rate": 3.2989898989898995e-06,
+ "loss": 1.5152846574783325,
+ "mean_token_accuracy": 0.6441133618354797,
+ "num_tokens": 13811712.0,
+ "step": 843
+ },
+ {
+ "entropy": 1.231048822402954,
+ "epoch": 1.705050505050505,
+ "grad_norm": 1.99764084815979,
+ "learning_rate": 3.2969696969696974e-06,
+ "loss": 1.2419885396957397,
+ "mean_token_accuracy": 0.6936370134353638,
+ "num_tokens": 13828096.0,
+ "step": 844
+ },
+ {
+ "entropy": 1.1431403160095215,
+ "epoch": 1.7070707070707072,
+ "grad_norm": 2.0046544075012207,
+ "learning_rate": 3.2949494949494953e-06,
+ "loss": 1.147315502166748,
+ "mean_token_accuracy": 0.7155593633651733,
+ "num_tokens": 13844480.0,
+ "step": 845
+ },
+ {
+ "entropy": 1.903164267539978,
+ "epoch": 1.709090909090909,
+ "grad_norm": 2.068286418914795,
+ "learning_rate": 3.292929292929293e-06,
+ "loss": 1.9086098670959473,
+ "mean_token_accuracy": 0.5785295367240906,
+ "num_tokens": 13860864.0,
+ "step": 846
+ },
+ {
+ "entropy": 1.028931736946106,
+ "epoch": 1.7111111111111112,
+ "grad_norm": 2.008197546005249,
+ "learning_rate": 3.290909090909091e-06,
+ "loss": 1.0251559019088745,
+ "mean_token_accuracy": 0.7308866381645203,
+ "num_tokens": 13877248.0,
+ "step": 847
+ },
+ {
+ "entropy": 1.8438282012939453,
+ "epoch": 1.713131313131313,
+ "grad_norm": 2.180811882019043,
+ "learning_rate": 3.2888888888888894e-06,
+ "loss": 1.8572561740875244,
+ "mean_token_accuracy": 0.5788959264755249,
+ "num_tokens": 13893632.0,
+ "step": 848
+ },
+ {
+ "entropy": 1.437190294265747,
+ "epoch": 1.7151515151515153,
+ "grad_norm": 2.1013762950897217,
+ "learning_rate": 3.2868686868686873e-06,
+ "loss": 1.4337413311004639,
+ "mean_token_accuracy": 0.6489985585212708,
+ "num_tokens": 13910016.0,
+ "step": 849
+ },
+ {
+ "entropy": 1.482985496520996,
+ "epoch": 1.7171717171717171,
+ "grad_norm": 2.290910243988037,
+ "learning_rate": 3.284848484848485e-06,
+ "loss": 1.4959741830825806,
+ "mean_token_accuracy": 0.6447850465774536,
+ "num_tokens": 13926400.0,
+ "step": 850
+ },
+ {
+ "epoch": 1.7171717171717171,
+ "eval_entropy": 1.447427170411233,
+ "eval_loss": 1.5309957265853882,
+ "eval_mean_token_accuracy": 0.6452888370521607,
+ "eval_num_tokens": 13926400.0,
+ "eval_runtime": 43.7619,
+ "eval_samples_per_second": 22.622,
+ "eval_steps_per_second": 2.834,
+ "step": 850
+ },
+ {
+ "entropy": 1.4470106363296509,
+ "epoch": 1.7191919191919192,
+ "grad_norm": 2.2015624046325684,
+ "learning_rate": 3.282828282828283e-06,
+ "loss": 1.462576150894165,
+ "mean_token_accuracy": 0.643991231918335,
+ "num_tokens": 13942784.0,
+ "step": 851
+ },
+ {
+ "entropy": 1.4063832759857178,
+ "epoch": 1.7212121212121212,
+ "grad_norm": 2.1743109226226807,
+ "learning_rate": 3.280808080808081e-06,
+ "loss": 1.402491569519043,
+ "mean_token_accuracy": 0.6533952355384827,
+ "num_tokens": 13959168.0,
+ "step": 852
+ },
+ {
+ "entropy": 1.3735251426696777,
+ "epoch": 1.7232323232323232,
+ "grad_norm": 2.1350960731506348,
+ "learning_rate": 3.2787878787878793e-06,
+ "loss": 1.383405327796936,
+ "mean_token_accuracy": 0.6767830848693848,
+ "num_tokens": 13975552.0,
+ "step": 853
+ },
+ {
+ "entropy": 1.359702229499817,
+ "epoch": 1.7252525252525253,
+ "grad_norm": 2.1547393798828125,
+ "learning_rate": 3.276767676767677e-06,
+ "loss": 1.3454554080963135,
+ "mean_token_accuracy": 0.6712262034416199,
+ "num_tokens": 13991936.0,
+ "step": 854
+ },
+ {
+ "entropy": 1.0641778707504272,
+ "epoch": 1.7272727272727273,
+ "grad_norm": 2.0301826000213623,
+ "learning_rate": 3.274747474747475e-06,
+ "loss": 1.0626115798950195,
+ "mean_token_accuracy": 0.7188568711280823,
+ "num_tokens": 14008320.0,
+ "step": 855
+ },
+ {
+ "entropy": 1.598792552947998,
+ "epoch": 1.7292929292929293,
+ "grad_norm": 2.3324217796325684,
+ "learning_rate": 3.272727272727273e-06,
+ "loss": 1.6028941869735718,
+ "mean_token_accuracy": 0.6462506055831909,
+ "num_tokens": 14024704.0,
+ "step": 856
+ },
+ {
+ "entropy": 1.6304137706756592,
+ "epoch": 1.7313131313131314,
+ "grad_norm": 1.9968358278274536,
+ "learning_rate": 3.270707070707071e-06,
+ "loss": 1.6449415683746338,
+ "mean_token_accuracy": 0.6322056651115417,
+ "num_tokens": 14041088.0,
+ "step": 857
+ },
+ {
+ "entropy": 1.3219174146652222,
+ "epoch": 1.7333333333333334,
+ "grad_norm": 2.059927225112915,
+ "learning_rate": 3.2686868686868687e-06,
+ "loss": 1.3487168550491333,
+ "mean_token_accuracy": 0.6769052147865295,
+ "num_tokens": 14057472.0,
+ "step": 858
+ },
+ {
+ "entropy": 1.4645651578903198,
+ "epoch": 1.7353535353535352,
+ "grad_norm": 2.068962812423706,
+ "learning_rate": 3.266666666666667e-06,
+ "loss": 1.4480202198028564,
+ "mean_token_accuracy": 0.6521128416061401,
+ "num_tokens": 14073856.0,
+ "step": 859
+ },
+ {
+ "entropy": 1.5005300045013428,
+ "epoch": 1.7373737373737375,
+ "grad_norm": 2.228736162185669,
+ "learning_rate": 3.264646464646465e-06,
+ "loss": 1.5110201835632324,
+ "mean_token_accuracy": 0.6482657790184021,
+ "num_tokens": 14090240.0,
+ "step": 860
+ },
+ {
+ "entropy": 1.1641852855682373,
+ "epoch": 1.7393939393939393,
+ "grad_norm": 2.0611488819122314,
+ "learning_rate": 3.262626262626263e-06,
+ "loss": 1.1905086040496826,
+ "mean_token_accuracy": 0.7104909420013428,
+ "num_tokens": 14106624.0,
+ "step": 861
+ },
+ {
+ "entropy": 1.4315950870513916,
+ "epoch": 1.7414141414141415,
+ "grad_norm": 1.9914237260818481,
+ "learning_rate": 3.2606060606060607e-06,
+ "loss": 1.467012882232666,
+ "mean_token_accuracy": 0.657608687877655,
+ "num_tokens": 14123008.0,
+ "step": 862
+ },
+ {
+ "entropy": 1.5196973085403442,
+ "epoch": 1.7434343434343433,
+ "grad_norm": 2.3258447647094727,
+ "learning_rate": 3.2585858585858586e-06,
+ "loss": 1.5382301807403564,
+ "mean_token_accuracy": 0.6370298266410828,
+ "num_tokens": 14139392.0,
+ "step": 863
+ },
+ {
+ "entropy": 1.5098228454589844,
+ "epoch": 1.7454545454545456,
+ "grad_norm": 2.146327495574951,
+ "learning_rate": 3.256565656565657e-06,
+ "loss": 1.540649652481079,
+ "mean_token_accuracy": 0.6439301371574402,
+ "num_tokens": 14155776.0,
+ "step": 864
+ },
+ {
+ "entropy": 1.128011703491211,
+ "epoch": 1.7474747474747474,
+ "grad_norm": 1.967443823814392,
+ "learning_rate": 3.2545454545454548e-06,
+ "loss": 1.1707985401153564,
+ "mean_token_accuracy": 0.7298485636711121,
+ "num_tokens": 14172160.0,
+ "step": 865
+ },
+ {
+ "entropy": 1.9117931127548218,
+ "epoch": 1.7494949494949497,
+ "grad_norm": 2.097781181335449,
+ "learning_rate": 3.2525252525252527e-06,
+ "loss": 1.9291374683380127,
+ "mean_token_accuracy": 0.580483615398407,
+ "num_tokens": 14188544.0,
+ "step": 866
+ },
+ {
+ "entropy": 1.9053187370300293,
+ "epoch": 1.7515151515151515,
+ "grad_norm": 1.9865039587020874,
+ "learning_rate": 3.2505050505050505e-06,
+ "loss": 1.932790756225586,
+ "mean_token_accuracy": 0.5770639777183533,
+ "num_tokens": 14204928.0,
+ "step": 867
+ },
+ {
+ "entropy": 1.2647870779037476,
+ "epoch": 1.7535353535353535,
+ "grad_norm": 2.099891185760498,
+ "learning_rate": 3.2484848484848484e-06,
+ "loss": 1.281562328338623,
+ "mean_token_accuracy": 0.6938812732696533,
+ "num_tokens": 14221312.0,
+ "step": 868
+ },
+ {
+ "entropy": 1.5446428060531616,
+ "epoch": 1.7555555555555555,
+ "grad_norm": 1.9044189453125,
+ "learning_rate": 3.2464646464646467e-06,
+ "loss": 1.5440425872802734,
+ "mean_token_accuracy": 0.6547996997833252,
+ "num_tokens": 14237696.0,
+ "step": 869
+ },
+ {
+ "entropy": 1.179898977279663,
+ "epoch": 1.7575757575757576,
+ "grad_norm": 2.0257933139801025,
+ "learning_rate": 3.2444444444444446e-06,
+ "loss": 1.1607933044433594,
+ "mean_token_accuracy": 0.7053614854812622,
+ "num_tokens": 14254080.0,
+ "step": 870
+ },
+ {
+ "entropy": 1.0462855100631714,
+ "epoch": 1.7595959595959596,
+ "grad_norm": 1.8995661735534668,
+ "learning_rate": 3.2424242424242425e-06,
+ "loss": 1.0395057201385498,
+ "mean_token_accuracy": 0.7405959963798523,
+ "num_tokens": 14270464.0,
+ "step": 871
+ },
+ {
+ "entropy": 0.8866081833839417,
+ "epoch": 1.7616161616161616,
+ "grad_norm": 1.7807546854019165,
+ "learning_rate": 3.2404040404040404e-06,
+ "loss": 0.8795047998428345,
+ "mean_token_accuracy": 0.7721666097640991,
+ "num_tokens": 14286848.0,
+ "step": 872
+ },
+ {
+ "entropy": 1.9576365947723389,
+ "epoch": 1.7636363636363637,
+ "grad_norm": 2.1867618560791016,
+ "learning_rate": 3.2383838383838383e-06,
+ "loss": 1.9605103731155396,
+ "mean_token_accuracy": 0.5809110999107361,
+ "num_tokens": 14303232.0,
+ "step": 873
+ },
+ {
+ "entropy": 1.010751485824585,
+ "epoch": 1.7656565656565657,
+ "grad_norm": 1.9643300771713257,
+ "learning_rate": 3.236363636363636e-06,
+ "loss": 0.9990894794464111,
+ "mean_token_accuracy": 0.7386419177055359,
+ "num_tokens": 14319616.0,
+ "step": 874
+ },
+ {
+ "entropy": 1.3587232828140259,
+ "epoch": 1.7676767676767677,
+ "grad_norm": 2.2632040977478027,
+ "learning_rate": 3.2343434343434345e-06,
+ "loss": 1.3631021976470947,
+ "mean_token_accuracy": 0.665791392326355,
+ "num_tokens": 14336000.0,
+ "step": 875
+ },
+ {
+ "entropy": 1.6577962636947632,
+ "epoch": 1.7696969696969695,
+ "grad_norm": 2.3661980628967285,
+ "learning_rate": 3.232323232323233e-06,
+ "loss": 1.6802719831466675,
+ "mean_token_accuracy": 0.6024059653282166,
+ "num_tokens": 14352384.0,
+ "step": 876
+ },
+ {
+ "entropy": 1.4332902431488037,
+ "epoch": 1.7717171717171718,
+ "grad_norm": 2.2296342849731445,
+ "learning_rate": 3.2303030303030307e-06,
+ "loss": 1.4129014015197754,
+ "mean_token_accuracy": 0.6511358022689819,
+ "num_tokens": 14368768.0,
+ "step": 877
+ },
+ {
+ "entropy": 1.2260215282440186,
+ "epoch": 1.7737373737373736,
+ "grad_norm": 2.0659983158111572,
+ "learning_rate": 3.228282828282829e-06,
+ "loss": 1.2417192459106445,
+ "mean_token_accuracy": 0.693514883518219,
+ "num_tokens": 14385152.0,
+ "step": 878
+ },
+ {
+ "entropy": 1.5402687788009644,
+ "epoch": 1.7757575757575759,
+ "grad_norm": 2.2024054527282715,
+ "learning_rate": 3.226262626262627e-06,
+ "loss": 1.5535321235656738,
+ "mean_token_accuracy": 0.6331827044487,
+ "num_tokens": 14401536.0,
+ "step": 879
+ },
+ {
+ "entropy": 1.2205268144607544,
+ "epoch": 1.7777777777777777,
+ "grad_norm": 2.0056662559509277,
+ "learning_rate": 3.2242424242424248e-06,
+ "loss": 1.2244641780853271,
+ "mean_token_accuracy": 0.7037127614021301,
+ "num_tokens": 14417920.0,
+ "step": 880
+ },
+ {
+ "entropy": 1.542244791984558,
+ "epoch": 1.77979797979798,
+ "grad_norm": 2.0935254096984863,
+ "learning_rate": 3.2222222222222227e-06,
+ "loss": 1.5638453960418701,
+ "mean_token_accuracy": 0.648937463760376,
+ "num_tokens": 14434304.0,
+ "step": 881
+ },
+ {
+ "entropy": 1.4249905347824097,
+ "epoch": 1.7818181818181817,
+ "grad_norm": 1.9853826761245728,
+ "learning_rate": 3.2202020202020206e-06,
+ "loss": 1.4453399181365967,
+ "mean_token_accuracy": 0.6869809627532959,
+ "num_tokens": 14450688.0,
+ "step": 882
+ },
+ {
+ "entropy": 1.4777165651321411,
+ "epoch": 1.783838383838384,
+ "grad_norm": 2.064542055130005,
+ "learning_rate": 3.2181818181818184e-06,
+ "loss": 1.4756921529769897,
+ "mean_token_accuracy": 0.6640205383300781,
+ "num_tokens": 14467072.0,
+ "step": 883
+ },
+ {
+ "entropy": 1.3442282676696777,
+ "epoch": 1.7858585858585858,
+ "grad_norm": 2.043712854385376,
+ "learning_rate": 3.2161616161616168e-06,
+ "loss": 1.3412827253341675,
+ "mean_token_accuracy": 0.6821568012237549,
+ "num_tokens": 14483456.0,
+ "step": 884
+ },
+ {
+ "entropy": 1.5201579332351685,
+ "epoch": 1.7878787878787878,
+ "grad_norm": 2.060661792755127,
+ "learning_rate": 3.2141414141414146e-06,
+ "loss": 1.5367834568023682,
+ "mean_token_accuracy": 0.6442354917526245,
+ "num_tokens": 14499840.0,
+ "step": 885
+ },
+ {
+ "entropy": 1.1536649465560913,
+ "epoch": 1.7898989898989899,
+ "grad_norm": 2.1190037727355957,
+ "learning_rate": 3.2121212121212125e-06,
+ "loss": 1.1588165760040283,
+ "mean_token_accuracy": 0.7039570212364197,
+ "num_tokens": 14516224.0,
+ "step": 886
+ },
+ {
+ "entropy": 1.109966516494751,
+ "epoch": 1.791919191919192,
+ "grad_norm": 2.0320141315460205,
+ "learning_rate": 3.2101010101010104e-06,
+ "loss": 1.1113003492355347,
+ "mean_token_accuracy": 0.7123228907585144,
+ "num_tokens": 14532608.0,
+ "step": 887
+ },
+ {
+ "entropy": 1.5709099769592285,
+ "epoch": 1.793939393939394,
+ "grad_norm": 2.0467336177825928,
+ "learning_rate": 3.2080808080808083e-06,
+ "loss": 1.5823577642440796,
+ "mean_token_accuracy": 0.6404494643211365,
+ "num_tokens": 14548992.0,
+ "step": 888
+ },
+ {
+ "entropy": 1.4305545091629028,
+ "epoch": 1.795959595959596,
+ "grad_norm": 2.0764269828796387,
+ "learning_rate": 3.2060606060606066e-06,
+ "loss": 1.438415765762329,
+ "mean_token_accuracy": 0.6574255228042603,
+ "num_tokens": 14565376.0,
+ "step": 889
+ },
+ {
+ "entropy": 1.3523615598678589,
+ "epoch": 1.797979797979798,
+ "grad_norm": 2.167036294937134,
+ "learning_rate": 3.2040404040404045e-06,
+ "loss": 1.3782477378845215,
+ "mean_token_accuracy": 0.6645090579986572,
+ "num_tokens": 14581760.0,
+ "step": 890
+ },
+ {
+ "entropy": 1.491297721862793,
+ "epoch": 1.8,
+ "grad_norm": 2.3090412616729736,
+ "learning_rate": 3.2020202020202024e-06,
+ "loss": 1.4965641498565674,
+ "mean_token_accuracy": 0.636907696723938,
+ "num_tokens": 14598144.0,
+ "step": 891
+ },
+ {
+ "entropy": 1.2584501504898071,
+ "epoch": 1.802020202020202,
+ "grad_norm": 2.1699554920196533,
+ "learning_rate": 3.2000000000000003e-06,
+ "loss": 1.2622835636138916,
+ "mean_token_accuracy": 0.6901563405990601,
+ "num_tokens": 14614528.0,
+ "step": 892
+ },
+ {
+ "entropy": 2.149350643157959,
+ "epoch": 1.8040404040404039,
+ "grad_norm": 2.2669718265533447,
+ "learning_rate": 3.197979797979798e-06,
+ "loss": 2.167891263961792,
+ "mean_token_accuracy": 0.5497679710388184,
+ "num_tokens": 14630912.0,
+ "step": 893
+ },
+ {
+ "entropy": 1.1999561786651611,
+ "epoch": 1.8060606060606061,
+ "grad_norm": 2.163228988647461,
+ "learning_rate": 3.195959595959596e-06,
+ "loss": 1.2024329900741577,
+ "mean_token_accuracy": 0.7035906314849854,
+ "num_tokens": 14647296.0,
+ "step": 894
+ },
+ {
+ "entropy": 1.770652413368225,
+ "epoch": 1.808080808080808,
+ "grad_norm": 2.1631994247436523,
+ "learning_rate": 3.1939393939393944e-06,
+ "loss": 1.7568567991256714,
+ "mean_token_accuracy": 0.6030776500701904,
+ "num_tokens": 14663680.0,
+ "step": 895
+ },
+ {
+ "entropy": 1.2487529516220093,
+ "epoch": 1.8101010101010102,
+ "grad_norm": 1.987622857093811,
+ "learning_rate": 3.1919191919191923e-06,
+ "loss": 1.255242109298706,
+ "mean_token_accuracy": 0.6929042339324951,
+ "num_tokens": 14680064.0,
+ "step": 896
+ },
+ {
+ "entropy": 1.138524055480957,
+ "epoch": 1.812121212121212,
+ "grad_norm": 2.0367984771728516,
+ "learning_rate": 3.18989898989899e-06,
+ "loss": 1.1514946222305298,
+ "mean_token_accuracy": 0.7093917727470398,
+ "num_tokens": 14696448.0,
+ "step": 897
+ },
+ {
+ "entropy": 1.3337515592575073,
+ "epoch": 1.8141414141414143,
+ "grad_norm": 1.9483140707015991,
+ "learning_rate": 3.187878787878788e-06,
+ "loss": 1.3251439332962036,
+ "mean_token_accuracy": 0.6861260533332825,
+ "num_tokens": 14712832.0,
+ "step": 898
+ },
+ {
+ "entropy": 1.047120451927185,
+ "epoch": 1.816161616161616,
+ "grad_norm": 2.0481390953063965,
+ "learning_rate": 3.185858585858586e-06,
+ "loss": 1.071772575378418,
+ "mean_token_accuracy": 0.7275891304016113,
+ "num_tokens": 14729216.0,
+ "step": 899
+ },
+ {
+ "entropy": 1.3311363458633423,
+ "epoch": 1.8181818181818183,
+ "grad_norm": 2.3203413486480713,
+ "learning_rate": 3.1838383838383842e-06,
+ "loss": 1.334134817123413,
+ "mean_token_accuracy": 0.6643869280815125,
+ "num_tokens": 14745600.0,
+ "step": 900
+ },
+ {
+ "epoch": 1.8181818181818183,
+ "eval_entropy": 1.4476436400605786,
+ "eval_loss": 1.5290467739105225,
+ "eval_mean_token_accuracy": 0.6456143543604882,
+ "eval_num_tokens": 14745600.0,
+ "eval_runtime": 43.8113,
+ "eval_samples_per_second": 22.597,
+ "eval_steps_per_second": 2.83,
+ "step": 900
+ },
+ {
+ "entropy": 1.6842114925384521,
+ "epoch": 1.8202020202020202,
+ "grad_norm": 2.1709823608398438,
+ "learning_rate": 3.181818181818182e-06,
+ "loss": 1.6816682815551758,
+ "mean_token_accuracy": 0.6033830046653748,
+ "num_tokens": 14761984.0,
+ "step": 901
+ },
+ {
+ "entropy": 1.2661798000335693,
+ "epoch": 1.8222222222222222,
+ "grad_norm": 2.113783836364746,
+ "learning_rate": 3.17979797979798e-06,
+ "loss": 1.249183177947998,
+ "mean_token_accuracy": 0.6865534782409668,
+ "num_tokens": 14778368.0,
+ "step": 902
+ },
+ {
+ "entropy": 1.2666339874267578,
+ "epoch": 1.8242424242424242,
+ "grad_norm": 2.0210978984832764,
+ "learning_rate": 3.177777777777778e-06,
+ "loss": 1.2548515796661377,
+ "mean_token_accuracy": 0.686431348323822,
+ "num_tokens": 14794752.0,
+ "step": 903
+ },
+ {
+ "entropy": 1.1660066843032837,
+ "epoch": 1.8262626262626263,
+ "grad_norm": 2.0998034477233887,
+ "learning_rate": 3.1757575757575758e-06,
+ "loss": 1.1654324531555176,
+ "mean_token_accuracy": 0.695713222026825,
+ "num_tokens": 14811136.0,
+ "step": 904
+ },
+ {
+ "entropy": 1.2902735471725464,
+ "epoch": 1.8282828282828283,
+ "grad_norm": 2.1510303020477295,
+ "learning_rate": 3.173737373737374e-06,
+ "loss": 1.2879812717437744,
+ "mean_token_accuracy": 0.6693942546844482,
+ "num_tokens": 14827520.0,
+ "step": 905
+ },
+ {
+ "entropy": 1.1441799402236938,
+ "epoch": 1.8303030303030303,
+ "grad_norm": 1.989449143409729,
+ "learning_rate": 3.171717171717172e-06,
+ "loss": 1.1552281379699707,
+ "mean_token_accuracy": 0.7127503752708435,
+ "num_tokens": 14843904.0,
+ "step": 906
+ },
+ {
+ "entropy": 1.08464515209198,
+ "epoch": 1.8323232323232324,
+ "grad_norm": 2.094696044921875,
+ "learning_rate": 3.16969696969697e-06,
+ "loss": 1.0758531093597412,
+ "mean_token_accuracy": 0.7164142727851868,
+ "num_tokens": 14860288.0,
+ "step": 907
+ },
+ {
+ "entropy": 1.3847618103027344,
+ "epoch": 1.8343434343434344,
+ "grad_norm": 2.207976818084717,
+ "learning_rate": 3.1676767676767678e-06,
+ "loss": 1.3722931146621704,
+ "mean_token_accuracy": 0.6741573214530945,
+ "num_tokens": 14876672.0,
+ "step": 908
+ },
+ {
+ "entropy": 1.3558534383773804,
+ "epoch": 1.8363636363636364,
+ "grad_norm": 2.166674852371216,
+ "learning_rate": 3.1656565656565656e-06,
+ "loss": 1.3612618446350098,
+ "mean_token_accuracy": 0.6669516563415527,
+ "num_tokens": 14893056.0,
+ "step": 909
+ },
+ {
+ "entropy": 1.4160822629928589,
+ "epoch": 1.8383838383838382,
+ "grad_norm": 2.2242727279663086,
+ "learning_rate": 3.1636363636363635e-06,
+ "loss": 1.408278226852417,
+ "mean_token_accuracy": 0.6634098887443542,
+ "num_tokens": 14909440.0,
+ "step": 910
+ },
+ {
+ "entropy": 1.1337573528289795,
+ "epoch": 1.8404040404040405,
+ "grad_norm": 2.1882550716400146,
+ "learning_rate": 3.161616161616162e-06,
+ "loss": 1.1503205299377441,
+ "mean_token_accuracy": 0.7059721350669861,
+ "num_tokens": 14925824.0,
+ "step": 911
+ },
+ {
+ "entropy": 1.4373403787612915,
+ "epoch": 1.8424242424242423,
+ "grad_norm": 2.1595654487609863,
+ "learning_rate": 3.1595959595959597e-06,
+ "loss": 1.4344165325164795,
+ "mean_token_accuracy": 0.6599902510643005,
+ "num_tokens": 14942208.0,
+ "step": 912
+ },
+ {
+ "entropy": 1.0798827409744263,
+ "epoch": 1.8444444444444446,
+ "grad_norm": 2.101088047027588,
+ "learning_rate": 3.1575757575757576e-06,
+ "loss": 1.0873513221740723,
+ "mean_token_accuracy": 0.7169027924537659,
+ "num_tokens": 14958592.0,
+ "step": 913
+ },
+ {
+ "entropy": 1.7024599313735962,
+ "epoch": 1.8464646464646464,
+ "grad_norm": 2.291907548904419,
+ "learning_rate": 3.1555555555555555e-06,
+ "loss": 1.7014999389648438,
+ "mean_token_accuracy": 0.6105275750160217,
+ "num_tokens": 14974976.0,
+ "step": 914
+ },
+ {
+ "entropy": 1.7765953540802002,
+ "epoch": 1.8484848484848486,
+ "grad_norm": 2.1465439796447754,
+ "learning_rate": 3.1535353535353534e-06,
+ "loss": 1.788269281387329,
+ "mean_token_accuracy": 0.6066194176673889,
+ "num_tokens": 14991360.0,
+ "step": 915
+ },
+ {
+ "entropy": 1.5223884582519531,
+ "epoch": 1.8505050505050504,
+ "grad_norm": 2.313291549682617,
+ "learning_rate": 3.1515151515151517e-06,
+ "loss": 1.55397367477417,
+ "mean_token_accuracy": 0.6377626061439514,
+ "num_tokens": 15007744.0,
+ "step": 916
+ },
+ {
+ "entropy": 1.5820955038070679,
+ "epoch": 1.8525252525252527,
+ "grad_norm": 2.094886541366577,
+ "learning_rate": 3.1494949494949496e-06,
+ "loss": 1.58372962474823,
+ "mean_token_accuracy": 0.6339154839515686,
+ "num_tokens": 15024128.0,
+ "step": 917
+ },
+ {
+ "entropy": 1.7399076223373413,
+ "epoch": 1.8545454545454545,
+ "grad_norm": 2.0311105251312256,
+ "learning_rate": 3.1474747474747475e-06,
+ "loss": 1.7362079620361328,
+ "mean_token_accuracy": 0.604421079158783,
+ "num_tokens": 15040512.0,
+ "step": 918
+ },
+ {
+ "entropy": 1.4311537742614746,
+ "epoch": 1.8565656565656565,
+ "grad_norm": 2.13740873336792,
+ "learning_rate": 3.145454545454546e-06,
+ "loss": 1.4574000835418701,
+ "mean_token_accuracy": 0.648876428604126,
+ "num_tokens": 15056896.0,
+ "step": 919
+ },
+ {
+ "entropy": 1.6997709274291992,
+ "epoch": 1.8585858585858586,
+ "grad_norm": 2.1604959964752197,
+ "learning_rate": 3.143434343434344e-06,
+ "loss": 1.694093942642212,
+ "mean_token_accuracy": 0.6144357323646545,
+ "num_tokens": 15073280.0,
+ "step": 920
+ },
+ {
+ "entropy": 1.5796921253204346,
+ "epoch": 1.8606060606060606,
+ "grad_norm": 2.3852932453155518,
+ "learning_rate": 3.141414141414142e-06,
+ "loss": 1.5799579620361328,
+ "mean_token_accuracy": 0.6322667598724365,
+ "num_tokens": 15089664.0,
+ "step": 921
+ },
+ {
+ "entropy": 1.6185977458953857,
+ "epoch": 1.8626262626262626,
+ "grad_norm": 2.0631208419799805,
+ "learning_rate": 3.13939393939394e-06,
+ "loss": 1.6541715860366821,
+ "mean_token_accuracy": 0.632083535194397,
+ "num_tokens": 15106048.0,
+ "step": 922
+ },
+ {
+ "entropy": 1.5872372388839722,
+ "epoch": 1.8646464646464647,
+ "grad_norm": 2.154554843902588,
+ "learning_rate": 3.1373737373737378e-06,
+ "loss": 1.5946767330169678,
+ "mean_token_accuracy": 0.6294577717781067,
+ "num_tokens": 15122432.0,
+ "step": 923
+ },
+ {
+ "entropy": 1.6059986352920532,
+ "epoch": 1.8666666666666667,
+ "grad_norm": 1.9893742799758911,
+ "learning_rate": 3.1353535353535357e-06,
+ "loss": 1.6153039932250977,
+ "mean_token_accuracy": 0.6373962163925171,
+ "num_tokens": 15138816.0,
+ "step": 924
+ },
+ {
+ "entropy": 1.3143881559371948,
+ "epoch": 1.8686868686868687,
+ "grad_norm": 1.8989832401275635,
+ "learning_rate": 3.133333333333334e-06,
+ "loss": 1.3332319259643555,
+ "mean_token_accuracy": 0.6842330098152161,
+ "num_tokens": 15155200.0,
+ "step": 925
+ },
+ {
+ "entropy": 1.0474469661712646,
+ "epoch": 1.8707070707070708,
+ "grad_norm": 1.9129916429519653,
+ "learning_rate": 3.131313131313132e-06,
+ "loss": 1.0238264799118042,
+ "mean_token_accuracy": 0.7459697127342224,
+ "num_tokens": 15171584.0,
+ "step": 926
+ },
+ {
+ "entropy": 1.581508994102478,
+ "epoch": 1.8727272727272726,
+ "grad_norm": 2.0047285556793213,
+ "learning_rate": 3.1292929292929297e-06,
+ "loss": 1.6087100505828857,
+ "mean_token_accuracy": 0.6282364726066589,
+ "num_tokens": 15187968.0,
+ "step": 927
+ },
+ {
+ "entropy": 1.0985661745071411,
+ "epoch": 1.8747474747474748,
+ "grad_norm": 2.1207540035247803,
+ "learning_rate": 3.1272727272727276e-06,
+ "loss": 1.1158297061920166,
+ "mean_token_accuracy": 0.7111626863479614,
+ "num_tokens": 15204352.0,
+ "step": 928
+ },
+ {
+ "entropy": 1.4591490030288696,
+ "epoch": 1.8767676767676766,
+ "grad_norm": 2.2879691123962402,
+ "learning_rate": 3.1252525252525255e-06,
+ "loss": 1.4775316715240479,
+ "mean_token_accuracy": 0.6477161645889282,
+ "num_tokens": 15220736.0,
+ "step": 929
+ },
+ {
+ "entropy": 1.5469937324523926,
+ "epoch": 1.878787878787879,
+ "grad_norm": 2.0972726345062256,
+ "learning_rate": 3.1232323232323234e-06,
+ "loss": 1.5949021577835083,
+ "mean_token_accuracy": 0.6547996997833252,
+ "num_tokens": 15237120.0,
+ "step": 930
+ },
+ {
+ "entropy": 1.363472819328308,
+ "epoch": 1.8808080808080807,
+ "grad_norm": 2.149606943130493,
+ "learning_rate": 3.1212121212121217e-06,
+ "loss": 1.358612298965454,
+ "mean_token_accuracy": 0.6780654788017273,
+ "num_tokens": 15253504.0,
+ "step": 931
+ },
+ {
+ "entropy": 1.189648985862732,
+ "epoch": 1.882828282828283,
+ "grad_norm": 2.2575035095214844,
+ "learning_rate": 3.1191919191919196e-06,
+ "loss": 1.209303379058838,
+ "mean_token_accuracy": 0.7002320289611816,
+ "num_tokens": 15269888.0,
+ "step": 932
+ },
+ {
+ "entropy": 1.3054684400558472,
+ "epoch": 1.8848484848484848,
+ "grad_norm": 2.243915319442749,
+ "learning_rate": 3.1171717171717175e-06,
+ "loss": 1.333367109298706,
+ "mean_token_accuracy": 0.6703712940216064,
+ "num_tokens": 15286272.0,
+ "step": 933
+ },
+ {
+ "entropy": 1.6298167705535889,
+ "epoch": 1.886868686868687,
+ "grad_norm": 1.9654396772384644,
+ "learning_rate": 3.1151515151515154e-06,
+ "loss": 1.6496453285217285,
+ "mean_token_accuracy": 0.6248778700828552,
+ "num_tokens": 15302656.0,
+ "step": 934
+ },
+ {
+ "entropy": 1.1228219270706177,
+ "epoch": 1.8888888888888888,
+ "grad_norm": 2.2492377758026123,
+ "learning_rate": 3.1131313131313133e-06,
+ "loss": 1.1560063362121582,
+ "mean_token_accuracy": 0.699499249458313,
+ "num_tokens": 15319040.0,
+ "step": 935
+ },
+ {
+ "entropy": 1.7481759786605835,
+ "epoch": 1.8909090909090909,
+ "grad_norm": 2.1635665893554688,
+ "learning_rate": 3.1111111111111116e-06,
+ "loss": 1.737868070602417,
+ "mean_token_accuracy": 0.6030776500701904,
+ "num_tokens": 15335424.0,
+ "step": 936
+ },
+ {
+ "entropy": 1.3964993953704834,
+ "epoch": 1.892929292929293,
+ "grad_norm": 2.262946367263794,
+ "learning_rate": 3.1090909090909095e-06,
+ "loss": 1.3927817344665527,
+ "mean_token_accuracy": 0.6546775698661804,
+ "num_tokens": 15351808.0,
+ "step": 937
+ },
+ {
+ "entropy": 1.2503907680511475,
+ "epoch": 1.894949494949495,
+ "grad_norm": 1.9837188720703125,
+ "learning_rate": 3.1070707070707074e-06,
+ "loss": 1.2529189586639404,
+ "mean_token_accuracy": 0.6965681314468384,
+ "num_tokens": 15368192.0,
+ "step": 938
+ },
+ {
+ "entropy": 0.9774144291877747,
+ "epoch": 1.896969696969697,
+ "grad_norm": 1.9201257228851318,
+ "learning_rate": 3.1050505050505052e-06,
+ "loss": 0.9864459037780762,
+ "mean_token_accuracy": 0.7534807324409485,
+ "num_tokens": 15384576.0,
+ "step": 939
+ },
+ {
+ "entropy": 1.4065781831741333,
+ "epoch": 1.898989898989899,
+ "grad_norm": 2.357865571975708,
+ "learning_rate": 3.103030303030303e-06,
+ "loss": 1.428115725517273,
+ "mean_token_accuracy": 0.6493649482727051,
+ "num_tokens": 15400960.0,
+ "step": 940
+ },
+ {
+ "entropy": 1.1632903814315796,
+ "epoch": 1.901010101010101,
+ "grad_norm": 1.8824918270111084,
+ "learning_rate": 3.1010101010101014e-06,
+ "loss": 1.1478252410888672,
+ "mean_token_accuracy": 0.7158036231994629,
+ "num_tokens": 15417344.0,
+ "step": 941
+ },
+ {
+ "entropy": 1.7470908164978027,
+ "epoch": 1.903030303030303,
+ "grad_norm": 2.2958669662475586,
+ "learning_rate": 3.0989898989898993e-06,
+ "loss": 1.7697877883911133,
+ "mean_token_accuracy": 0.5892769694328308,
+ "num_tokens": 15433728.0,
+ "step": 942
+ },
+ {
+ "entropy": 1.2897151708602905,
+ "epoch": 1.905050505050505,
+ "grad_norm": 2.155731439590454,
+ "learning_rate": 3.0969696969696972e-06,
+ "loss": 1.2971919775009155,
+ "mean_token_accuracy": 0.6731802821159363,
+ "num_tokens": 15450112.0,
+ "step": 943
+ },
+ {
+ "entropy": 1.2687026262283325,
+ "epoch": 1.907070707070707,
+ "grad_norm": 2.0896284580230713,
+ "learning_rate": 3.094949494949495e-06,
+ "loss": 1.2423906326293945,
+ "mean_token_accuracy": 0.6988885998725891,
+ "num_tokens": 15466496.0,
+ "step": 944
+ },
+ {
+ "entropy": 1.5710495710372925,
+ "epoch": 1.9090909090909092,
+ "grad_norm": 2.200758457183838,
+ "learning_rate": 3.092929292929293e-06,
+ "loss": 1.5329574346542358,
+ "mean_token_accuracy": 0.6560820937156677,
+ "num_tokens": 15482880.0,
+ "step": 945
+ },
+ {
+ "entropy": 1.1555253267288208,
+ "epoch": 1.911111111111111,
+ "grad_norm": 2.278230905532837,
+ "learning_rate": 3.090909090909091e-06,
+ "loss": 1.1469731330871582,
+ "mean_token_accuracy": 0.7220932841300964,
+ "num_tokens": 15499264.0,
+ "step": 946
+ },
+ {
+ "entropy": 1.2870018482208252,
+ "epoch": 1.9131313131313132,
+ "grad_norm": 1.8973854780197144,
+ "learning_rate": 3.088888888888889e-06,
+ "loss": 1.2787907123565674,
+ "mean_token_accuracy": 0.6885075569152832,
+ "num_tokens": 15515648.0,
+ "step": 947
+ },
+ {
+ "entropy": 1.324872612953186,
+ "epoch": 1.915151515151515,
+ "grad_norm": 2.1889214515686035,
+ "learning_rate": 3.086868686868687e-06,
+ "loss": 1.3286981582641602,
+ "mean_token_accuracy": 0.6743404865264893,
+ "num_tokens": 15532032.0,
+ "step": 948
+ },
+ {
+ "entropy": 1.5601656436920166,
+ "epoch": 1.9171717171717173,
+ "grad_norm": 2.304993152618408,
+ "learning_rate": 3.084848484848485e-06,
+ "loss": 1.5766493082046509,
+ "mean_token_accuracy": 0.6172447204589844,
+ "num_tokens": 15548416.0,
+ "step": 949
+ },
+ {
+ "entropy": 1.4633477926254272,
+ "epoch": 1.9191919191919191,
+ "grad_norm": 1.9083963632583618,
+ "learning_rate": 3.082828282828283e-06,
+ "loss": 1.4554777145385742,
+ "mean_token_accuracy": 0.6667073965072632,
+ "num_tokens": 15564800.0,
+ "step": 950
+ },
+ {
+ "epoch": 1.9191919191919191,
+ "eval_entropy": 1.4443931824737979,
+ "eval_loss": 1.5272752046585083,
+ "eval_mean_token_accuracy": 0.6459201723337173,
+ "eval_num_tokens": 15564800.0,
+ "eval_runtime": 43.7243,
+ "eval_samples_per_second": 22.642,
+ "eval_steps_per_second": 2.836,
+ "step": 950
+ },
+ {
+ "entropy": 1.3350679874420166,
+ "epoch": 1.9212121212121214,
+ "grad_norm": 2.139172315597534,
+ "learning_rate": 3.0808080808080807e-06,
+ "loss": 1.3285927772521973,
+ "mean_token_accuracy": 0.6833781003952026,
+ "num_tokens": 15581184.0,
+ "step": 951
+ },
+ {
+ "entropy": 1.2373814582824707,
+ "epoch": 1.9232323232323232,
+ "grad_norm": 2.1068115234375,
+ "learning_rate": 3.078787878787879e-06,
+ "loss": 1.2527921199798584,
+ "mean_token_accuracy": 0.6969345211982727,
+ "num_tokens": 15597568.0,
+ "step": 952
+ },
+ {
+ "entropy": 1.2182631492614746,
+ "epoch": 1.9252525252525252,
+ "grad_norm": 2.1326634883880615,
+ "learning_rate": 3.076767676767677e-06,
+ "loss": 1.2174904346466064,
+ "mean_token_accuracy": 0.6905227303504944,
+ "num_tokens": 15613952.0,
+ "step": 953
+ },
+ {
+ "entropy": 0.9768888354301453,
+ "epoch": 1.9272727272727272,
+ "grad_norm": 1.9833085536956787,
+ "learning_rate": 3.074747474747475e-06,
+ "loss": 0.9956569671630859,
+ "mean_token_accuracy": 0.7401685118675232,
+ "num_tokens": 15630336.0,
+ "step": 954
+ },
+ {
+ "entropy": 0.7818430066108704,
+ "epoch": 1.9292929292929293,
+ "grad_norm": 1.9219205379486084,
+ "learning_rate": 3.0727272727272727e-06,
+ "loss": 0.7922182083129883,
+ "mean_token_accuracy": 0.7827919125556946,
+ "num_tokens": 15646720.0,
+ "step": 955
+ },
+ {
+ "entropy": 1.6172020435333252,
+ "epoch": 1.9313131313131313,
+ "grad_norm": 1.9390695095062256,
+ "learning_rate": 3.0707070707070706e-06,
+ "loss": 1.6629729270935059,
+ "mean_token_accuracy": 0.6192598938941956,
+ "num_tokens": 15663104.0,
+ "step": 956
+ },
+ {
+ "entropy": 1.291207194328308,
+ "epoch": 1.9333333333333333,
+ "grad_norm": 1.9242857694625854,
+ "learning_rate": 3.068686868686869e-06,
+ "loss": 1.2721552848815918,
+ "mean_token_accuracy": 0.6867367029190063,
+ "num_tokens": 15679488.0,
+ "step": 957
+ },
+ {
+ "entropy": 1.3730298280715942,
+ "epoch": 1.9353535353535354,
+ "grad_norm": 1.9483041763305664,
+ "learning_rate": 3.066666666666667e-06,
+ "loss": 1.3835055828094482,
+ "mean_token_accuracy": 0.6745237112045288,
+ "num_tokens": 15695872.0,
+ "step": 958
+ },
+ {
+ "entropy": 1.1266423463821411,
+ "epoch": 1.9373737373737374,
+ "grad_norm": 2.1663992404937744,
+ "learning_rate": 3.0646464646464647e-06,
+ "loss": 1.1091173887252808,
+ "mean_token_accuracy": 0.7151318788528442,
+ "num_tokens": 15712256.0,
+ "step": 959
+ },
+ {
+ "entropy": 1.298812747001648,
+ "epoch": 1.9393939393939394,
+ "grad_norm": 2.0564448833465576,
+ "learning_rate": 3.0626262626262626e-06,
+ "loss": 1.3167932033538818,
+ "mean_token_accuracy": 0.6812408566474915,
+ "num_tokens": 15728640.0,
+ "step": 960
+ },
+ {
+ "entropy": 1.7007938623428345,
+ "epoch": 1.9414141414141413,
+ "grad_norm": 2.1043670177459717,
+ "learning_rate": 3.0606060606060605e-06,
+ "loss": 1.6956250667572021,
+ "mean_token_accuracy": 0.6194430589675903,
+ "num_tokens": 15745024.0,
+ "step": 961
+ },
+ {
+ "entropy": 1.408955693244934,
+ "epoch": 1.9434343434343435,
+ "grad_norm": 2.0432121753692627,
+ "learning_rate": 3.058585858585859e-06,
+ "loss": 1.3905062675476074,
+ "mean_token_accuracy": 0.6662799119949341,
+ "num_tokens": 15761408.0,
+ "step": 962
+ },
+ {
+ "entropy": 1.938867211341858,
+ "epoch": 1.9454545454545453,
+ "grad_norm": 2.3359744548797607,
+ "learning_rate": 3.056565656565657e-06,
+ "loss": 1.9670305252075195,
+ "mean_token_accuracy": 0.5845139026641846,
+ "num_tokens": 15777792.0,
+ "step": 963
+ },
+ {
+ "entropy": 1.5049448013305664,
+ "epoch": 1.9474747474747476,
+ "grad_norm": 2.1907997131347656,
+ "learning_rate": 3.054545454545455e-06,
+ "loss": 1.534651756286621,
+ "mean_token_accuracy": 0.652723491191864,
+ "num_tokens": 15794176.0,
+ "step": 964
+ },
+ {
+ "entropy": 1.1916700601577759,
+ "epoch": 1.9494949494949494,
+ "grad_norm": 2.098696231842041,
+ "learning_rate": 3.052525252525253e-06,
+ "loss": 1.191361904144287,
+ "mean_token_accuracy": 0.715192973613739,
+ "num_tokens": 15810560.0,
+ "step": 965
+ },
+ {
+ "entropy": 1.6707743406295776,
+ "epoch": 1.9515151515151516,
+ "grad_norm": 2.106846570968628,
+ "learning_rate": 3.0505050505050508e-06,
+ "loss": 1.6789159774780273,
+ "mean_token_accuracy": 0.6133365631103516,
+ "num_tokens": 15826944.0,
+ "step": 966
+ },
+ {
+ "entropy": 1.4679991006851196,
+ "epoch": 1.9535353535353535,
+ "grad_norm": 2.1568050384521484,
+ "learning_rate": 3.048484848484849e-06,
+ "loss": 1.4427556991577148,
+ "mean_token_accuracy": 0.6526013612747192,
+ "num_tokens": 15843328.0,
+ "step": 967
+ },
+ {
+ "entropy": 0.9881705641746521,
+ "epoch": 1.9555555555555557,
+ "grad_norm": 2.029680013656616,
+ "learning_rate": 3.046464646464647e-06,
+ "loss": 0.9961811304092407,
+ "mean_token_accuracy": 0.7408402562141418,
+ "num_tokens": 15859712.0,
+ "step": 968
+ },
+ {
+ "entropy": 1.0721628665924072,
+ "epoch": 1.9575757575757575,
+ "grad_norm": 1.9763140678405762,
+ "learning_rate": 3.044444444444445e-06,
+ "loss": 1.067029356956482,
+ "mean_token_accuracy": 0.733146071434021,
+ "num_tokens": 15876096.0,
+ "step": 969
+ },
+ {
+ "entropy": 1.7053834199905396,
+ "epoch": 1.9595959595959596,
+ "grad_norm": 2.3659849166870117,
+ "learning_rate": 3.0424242424242427e-06,
+ "loss": 1.7629203796386719,
+ "mean_token_accuracy": 0.6105275750160217,
+ "num_tokens": 15892480.0,
+ "step": 970
+ },
+ {
+ "entropy": 1.8083471059799194,
+ "epoch": 1.9616161616161616,
+ "grad_norm": 2.081869602203369,
+ "learning_rate": 3.0404040404040406e-06,
+ "loss": 1.8430367708206177,
+ "mean_token_accuracy": 0.6009404063224792,
+ "num_tokens": 15908864.0,
+ "step": 971
+ },
+ {
+ "entropy": 1.2576755285263062,
+ "epoch": 1.9636363636363636,
+ "grad_norm": 2.128230571746826,
+ "learning_rate": 3.038383838383839e-06,
+ "loss": 1.2493822574615479,
+ "mean_token_accuracy": 0.6913776397705078,
+ "num_tokens": 15925248.0,
+ "step": 972
+ },
+ {
+ "entropy": 0.8995128273963928,
+ "epoch": 1.9656565656565657,
+ "grad_norm": 2.0216031074523926,
+ "learning_rate": 3.036363636363637e-06,
+ "loss": 0.9069615602493286,
+ "mean_token_accuracy": 0.7620908617973328,
+ "num_tokens": 15941632.0,
+ "step": 973
+ },
+ {
+ "entropy": 1.615832805633545,
+ "epoch": 1.9676767676767677,
+ "grad_norm": 2.26552152633667,
+ "learning_rate": 3.0343434343434347e-06,
+ "loss": 1.6284873485565186,
+ "mean_token_accuracy": 0.6193209290504456,
+ "num_tokens": 15958016.0,
+ "step": 974
+ },
+ {
+ "entropy": 1.404674768447876,
+ "epoch": 1.9696969696969697,
+ "grad_norm": 2.1790690422058105,
+ "learning_rate": 3.0323232323232326e-06,
+ "loss": 1.4130914211273193,
+ "mean_token_accuracy": 0.6594406366348267,
+ "num_tokens": 15974400.0,
+ "step": 975
+ },
+ {
+ "entropy": 1.3440049886703491,
+ "epoch": 1.9717171717171718,
+ "grad_norm": 2.121338367462158,
+ "learning_rate": 3.0303030303030305e-06,
+ "loss": 1.3652238845825195,
+ "mean_token_accuracy": 0.6794699430465698,
+ "num_tokens": 15990784.0,
+ "step": 976
+ },
+ {
+ "entropy": 1.0894689559936523,
+ "epoch": 1.9737373737373738,
+ "grad_norm": 2.0155327320098877,
+ "learning_rate": 3.028282828282829e-06,
+ "loss": 1.0642526149749756,
+ "mean_token_accuracy": 0.7223986387252808,
+ "num_tokens": 16007168.0,
+ "step": 977
+ },
+ {
+ "entropy": 1.469588041305542,
+ "epoch": 1.9757575757575756,
+ "grad_norm": 2.2047550678253174,
+ "learning_rate": 3.0262626262626267e-06,
+ "loss": 1.498295545578003,
+ "mean_token_accuracy": 0.6441133618354797,
+ "num_tokens": 16023552.0,
+ "step": 978
+ },
+ {
+ "entropy": 1.6617093086242676,
+ "epoch": 1.9777777777777779,
+ "grad_norm": 1.9770722389221191,
+ "learning_rate": 3.0242424242424246e-06,
+ "loss": 1.6753556728363037,
+ "mean_token_accuracy": 0.6284807324409485,
+ "num_tokens": 16039936.0,
+ "step": 979
+ },
+ {
+ "entropy": 1.8029206991195679,
+ "epoch": 1.9797979797979797,
+ "grad_norm": 2.2365968227386475,
+ "learning_rate": 3.0222222222222225e-06,
+ "loss": 1.7957369089126587,
+ "mean_token_accuracy": 0.601062536239624,
+ "num_tokens": 16056320.0,
+ "step": 980
+ },
+ {
+ "entropy": 1.6331384181976318,
+ "epoch": 1.981818181818182,
+ "grad_norm": 2.0565884113311768,
+ "learning_rate": 3.0202020202020203e-06,
+ "loss": 1.673048973083496,
+ "mean_token_accuracy": 0.6203590631484985,
+ "num_tokens": 16072704.0,
+ "step": 981
+ },
+ {
+ "entropy": 1.1730273962020874,
+ "epoch": 1.9838383838383837,
+ "grad_norm": 2.0347228050231934,
+ "learning_rate": 3.0181818181818182e-06,
+ "loss": 1.1709872484207153,
+ "mean_token_accuracy": 0.7086589932441711,
+ "num_tokens": 16089088.0,
+ "step": 982
+ },
+ {
+ "entropy": 1.1109111309051514,
+ "epoch": 1.985858585858586,
+ "grad_norm": 1.9992051124572754,
+ "learning_rate": 3.0161616161616165e-06,
+ "loss": 1.097399353981018,
+ "mean_token_accuracy": 0.7245969772338867,
+ "num_tokens": 16105472.0,
+ "step": 983
+ },
+ {
+ "entropy": 1.2580686807632446,
+ "epoch": 1.9878787878787878,
+ "grad_norm": 2.1748640537261963,
+ "learning_rate": 3.0141414141414144e-06,
+ "loss": 1.2503113746643066,
+ "mean_token_accuracy": 0.6883854269981384,
+ "num_tokens": 16121856.0,
+ "step": 984
+ },
+ {
+ "entropy": 1.4884002208709717,
+ "epoch": 1.98989898989899,
+ "grad_norm": 2.3283042907714844,
+ "learning_rate": 3.0121212121212123e-06,
+ "loss": 1.4856922626495361,
+ "mean_token_accuracy": 0.6461895704269409,
+ "num_tokens": 16138240.0,
+ "step": 985
+ },
+ {
+ "entropy": 1.6224603652954102,
+ "epoch": 1.9919191919191919,
+ "grad_norm": 2.0596823692321777,
+ "learning_rate": 3.0101010101010102e-06,
+ "loss": 1.6202584505081177,
+ "mean_token_accuracy": 0.6388617753982544,
+ "num_tokens": 16154624.0,
+ "step": 986
+ },
+ {
+ "entropy": 1.559553623199463,
+ "epoch": 1.993939393939394,
+ "grad_norm": 2.100667953491211,
+ "learning_rate": 3.008080808080808e-06,
+ "loss": 1.5781179666519165,
+ "mean_token_accuracy": 0.6409379839897156,
+ "num_tokens": 16171008.0,
+ "step": 987
+ },
+ {
+ "entropy": 1.299710750579834,
+ "epoch": 1.995959595959596,
+ "grad_norm": 2.2846767902374268,
+ "learning_rate": 3.0060606060606064e-06,
+ "loss": 1.2758262157440186,
+ "mean_token_accuracy": 0.6882632970809937,
+ "num_tokens": 16187392.0,
+ "step": 988
+ },
+ {
+ "entropy": 1.3966935873031616,
+ "epoch": 1.997979797979798,
+ "grad_norm": 2.1244044303894043,
+ "learning_rate": 3.0040404040404043e-06,
+ "loss": 1.420769214630127,
+ "mean_token_accuracy": 0.6670737862586975,
+ "num_tokens": 16203776.0,
+ "step": 989
+ },
+ {
+ "entropy": 1.5851658582687378,
+ "epoch": 2.0,
+ "grad_norm": 2.277179002761841,
+ "learning_rate": 3.002020202020202e-06,
+ "loss": 1.5963382720947266,
+ "mean_token_accuracy": 0.6276868581771851,
+ "num_tokens": 16220160.0,
+ "step": 990
+ },
+ {
+ "entropy": 1.9146819114685059,
+ "epoch": 2.002020202020202,
+ "grad_norm": 2.0008981227874756,
+ "learning_rate": 3e-06,
+ "loss": 1.8820562362670898,
+ "mean_token_accuracy": 0.5895212292671204,
+ "num_tokens": 16236544.0,
+ "step": 991
+ },
+ {
+ "entropy": 1.3451478481292725,
+ "epoch": 2.004040404040404,
+ "grad_norm": 2.236388921737671,
+ "learning_rate": 2.997979797979798e-06,
+ "loss": 1.2709381580352783,
+ "mean_token_accuracy": 0.6848436594009399,
+ "num_tokens": 16252928.0,
+ "step": 992
+ },
+ {
+ "entropy": 1.4454528093338013,
+ "epoch": 2.006060606060606,
+ "grad_norm": 2.220324754714966,
+ "learning_rate": 2.9959595959595963e-06,
+ "loss": 1.387961506843567,
+ "mean_token_accuracy": 0.6642647981643677,
+ "num_tokens": 16269312.0,
+ "step": 993
+ },
+ {
+ "entropy": 1.2083219289779663,
+ "epoch": 2.008080808080808,
+ "grad_norm": 2.092430830001831,
+ "learning_rate": 2.993939393939394e-06,
+ "loss": 1.1450953483581543,
+ "mean_token_accuracy": 0.7159257531166077,
+ "num_tokens": 16285696.0,
+ "step": 994
+ },
+ {
+ "entropy": 1.3030426502227783,
+ "epoch": 2.01010101010101,
+ "grad_norm": 2.0722665786743164,
+ "learning_rate": 2.991919191919192e-06,
+ "loss": 1.2463462352752686,
+ "mean_token_accuracy": 0.6780654788017273,
+ "num_tokens": 16302080.0,
+ "step": 995
+ },
+ {
+ "entropy": 1.3418742418289185,
+ "epoch": 2.012121212121212,
+ "grad_norm": 1.9004424810409546,
+ "learning_rate": 2.98989898989899e-06,
+ "loss": 1.3043954372406006,
+ "mean_token_accuracy": 0.6850268840789795,
+ "num_tokens": 16318464.0,
+ "step": 996
+ },
+ {
+ "entropy": 1.6736825704574585,
+ "epoch": 2.014141414141414,
+ "grad_norm": 2.132983446121216,
+ "learning_rate": 2.987878787878788e-06,
+ "loss": 1.631580114364624,
+ "mean_token_accuracy": 0.619076669216156,
+ "num_tokens": 16334848.0,
+ "step": 997
+ },
+ {
+ "entropy": 1.435375690460205,
+ "epoch": 2.0161616161616163,
+ "grad_norm": 2.2508206367492676,
+ "learning_rate": 2.9858585858585857e-06,
+ "loss": 1.3867809772491455,
+ "mean_token_accuracy": 0.662432849407196,
+ "num_tokens": 16351232.0,
+ "step": 998
+ },
+ {
+ "entropy": 1.050812005996704,
+ "epoch": 2.018181818181818,
+ "grad_norm": 1.878668189048767,
+ "learning_rate": 2.983838383838384e-06,
+ "loss": 1.0428903102874756,
+ "mean_token_accuracy": 0.7329018115997314,
+ "num_tokens": 16367616.0,
+ "step": 999
+ },
+ {
+ "entropy": 1.325859546661377,
+ "epoch": 2.0202020202020203,
+ "grad_norm": 1.8450349569320679,
+ "learning_rate": 2.981818181818182e-06,
+ "loss": 1.302099347114563,
+ "mean_token_accuracy": 0.6926599740982056,
+ "num_tokens": 16384000.0,
+ "step": 1000
+ },
+ {
+ "epoch": 2.0202020202020203,
+ "eval_entropy": 1.3899660999736478,
+ "eval_loss": 1.5310550928115845,
+ "eval_mean_token_accuracy": 0.6458596015168775,
+ "eval_num_tokens": 16384000.0,
+ "eval_runtime": 43.8845,
+ "eval_samples_per_second": 22.559,
+ "eval_steps_per_second": 2.826,
+ "step": 1000
+ },
+ {
+ "entropy": 1.5226235389709473,
+ "epoch": 2.022222222222222,
+ "grad_norm": 2.0676803588867188,
+ "learning_rate": 2.97979797979798e-06,
+ "loss": 1.4971344470977783,
+ "mean_token_accuracy": 0.6517464518547058,
+ "num_tokens": 16400384.0,
+ "step": 1001
+ },
+ {
+ "entropy": 0.9506940841674805,
+ "epoch": 2.0242424242424244,
+ "grad_norm": 1.972718596458435,
+ "learning_rate": 2.9777777777777777e-06,
+ "loss": 0.9229776859283447,
+ "mean_token_accuracy": 0.7554958462715149,
+ "num_tokens": 16416768.0,
+ "step": 1002
+ },
+ {
+ "entropy": 1.8095428943634033,
+ "epoch": 2.026262626262626,
+ "grad_norm": 2.119502305984497,
+ "learning_rate": 2.9757575757575756e-06,
+ "loss": 1.8333206176757812,
+ "mean_token_accuracy": 0.5888495445251465,
+ "num_tokens": 16433152.0,
+ "step": 1003
+ },
+ {
+ "entropy": 1.110617756843567,
+ "epoch": 2.0282828282828285,
+ "grad_norm": 2.1480863094329834,
+ "learning_rate": 2.9737373737373743e-06,
+ "loss": 1.0999674797058105,
+ "mean_token_accuracy": 0.7213605046272278,
+ "num_tokens": 16449536.0,
+ "step": 1004
+ },
+ {
+ "entropy": 1.1889066696166992,
+ "epoch": 2.0303030303030303,
+ "grad_norm": 2.1086478233337402,
+ "learning_rate": 2.971717171717172e-06,
+ "loss": 1.2093305587768555,
+ "mean_token_accuracy": 0.7071323990821838,
+ "num_tokens": 16465920.0,
+ "step": 1005
+ },
+ {
+ "entropy": 1.2134279012680054,
+ "epoch": 2.0323232323232325,
+ "grad_norm": 2.020108461380005,
+ "learning_rate": 2.96969696969697e-06,
+ "loss": 1.2015900611877441,
+ "mean_token_accuracy": 0.7037127614021301,
+ "num_tokens": 16482304.0,
+ "step": 1006
+ },
+ {
+ "entropy": 1.1034245491027832,
+ "epoch": 2.0343434343434343,
+ "grad_norm": 2.1722943782806396,
+ "learning_rate": 2.967676767676768e-06,
+ "loss": 1.1243445873260498,
+ "mean_token_accuracy": 0.7051172256469727,
+ "num_tokens": 16498688.0,
+ "step": 1007
+ },
+ {
+ "entropy": 1.6888902187347412,
+ "epoch": 2.036363636363636,
+ "grad_norm": 2.1574206352233887,
+ "learning_rate": 2.9656565656565663e-06,
+ "loss": 1.7164931297302246,
+ "mean_token_accuracy": 0.6174889802932739,
+ "num_tokens": 16515072.0,
+ "step": 1008
+ },
+ {
+ "entropy": 0.8055605292320251,
+ "epoch": 2.0383838383838384,
+ "grad_norm": 1.8896156549453735,
+ "learning_rate": 2.963636363636364e-06,
+ "loss": 0.7961650490760803,
+ "mean_token_accuracy": 0.7895700931549072,
+ "num_tokens": 16531456.0,
+ "step": 1009
+ },
+ {
+ "entropy": 1.7698009014129639,
+ "epoch": 2.04040404040404,
+ "grad_norm": 2.1700897216796875,
+ "learning_rate": 2.961616161616162e-06,
+ "loss": 1.7871267795562744,
+ "mean_token_accuracy": 0.6139472126960754,
+ "num_tokens": 16547840.0,
+ "step": 1010
+ },
+ {
+ "entropy": 1.414292335510254,
+ "epoch": 2.0424242424242425,
+ "grad_norm": 2.0185277462005615,
+ "learning_rate": 2.95959595959596e-06,
+ "loss": 1.4295077323913574,
+ "mean_token_accuracy": 0.6670126914978027,
+ "num_tokens": 16564224.0,
+ "step": 1011
+ },
+ {
+ "entropy": 1.4013340473175049,
+ "epoch": 2.0444444444444443,
+ "grad_norm": 2.1133840084075928,
+ "learning_rate": 2.957575757575758e-06,
+ "loss": 1.4482200145721436,
+ "mean_token_accuracy": 0.6780654788017273,
+ "num_tokens": 16580608.0,
+ "step": 1012
+ },
+ {
+ "entropy": 1.342740774154663,
+ "epoch": 2.0464646464646465,
+ "grad_norm": 2.0520355701446533,
+ "learning_rate": 2.955555555555556e-06,
+ "loss": 1.3550419807434082,
+ "mean_token_accuracy": 0.6850879192352295,
+ "num_tokens": 16596992.0,
+ "step": 1013
+ },
+ {
+ "entropy": 1.2301024198532104,
+ "epoch": 2.0484848484848484,
+ "grad_norm": 2.1948137283325195,
+ "learning_rate": 2.953535353535354e-06,
+ "loss": 1.243154525756836,
+ "mean_token_accuracy": 0.6838055849075317,
+ "num_tokens": 16613376.0,
+ "step": 1014
+ },
+ {
+ "entropy": 1.3916606903076172,
+ "epoch": 2.0505050505050506,
+ "grad_norm": 2.241595506668091,
+ "learning_rate": 2.951515151515152e-06,
+ "loss": 1.4011318683624268,
+ "mean_token_accuracy": 0.664631187915802,
+ "num_tokens": 16629760.0,
+ "step": 1015
+ },
+ {
+ "entropy": 1.4715481996536255,
+ "epoch": 2.0525252525252524,
+ "grad_norm": 2.224256753921509,
+ "learning_rate": 2.94949494949495e-06,
+ "loss": 1.5011882781982422,
+ "mean_token_accuracy": 0.6493038535118103,
+ "num_tokens": 16646144.0,
+ "step": 1016
+ },
+ {
+ "entropy": 1.1261032819747925,
+ "epoch": 2.0545454545454547,
+ "grad_norm": 2.2784500122070312,
+ "learning_rate": 2.9474747474747477e-06,
+ "loss": 1.1250882148742676,
+ "mean_token_accuracy": 0.7178798317909241,
+ "num_tokens": 16662528.0,
+ "step": 1017
+ },
+ {
+ "entropy": 1.782008171081543,
+ "epoch": 2.0565656565656565,
+ "grad_norm": 2.1711416244506836,
+ "learning_rate": 2.9454545454545456e-06,
+ "loss": 1.8037612438201904,
+ "mean_token_accuracy": 0.6113824844360352,
+ "num_tokens": 16678912.0,
+ "step": 1018
+ },
+ {
+ "entropy": 1.521897315979004,
+ "epoch": 2.0585858585858587,
+ "grad_norm": 2.1903202533721924,
+ "learning_rate": 2.943434343434344e-06,
+ "loss": 1.5377610921859741,
+ "mean_token_accuracy": 0.642953097820282,
+ "num_tokens": 16695296.0,
+ "step": 1019
+ },
+ {
+ "entropy": 1.3112438917160034,
+ "epoch": 2.0606060606060606,
+ "grad_norm": 2.0299105644226074,
+ "learning_rate": 2.941414141414142e-06,
+ "loss": 1.3218681812286377,
+ "mean_token_accuracy": 0.6842941045761108,
+ "num_tokens": 16711680.0,
+ "step": 1020
+ },
+ {
+ "entropy": 1.3634895086288452,
+ "epoch": 2.062626262626263,
+ "grad_norm": 2.2818808555603027,
+ "learning_rate": 2.9393939393939397e-06,
+ "loss": 1.3611259460449219,
+ "mean_token_accuracy": 0.6758060455322266,
+ "num_tokens": 16728064.0,
+ "step": 1021
+ },
+ {
+ "entropy": 1.361467957496643,
+ "epoch": 2.0646464646464646,
+ "grad_norm": 2.290698289871216,
+ "learning_rate": 2.9373737373737376e-06,
+ "loss": 1.3082963228225708,
+ "mean_token_accuracy": 0.6809965968132019,
+ "num_tokens": 16744448.0,
+ "step": 1022
+ },
+ {
+ "entropy": 1.633399486541748,
+ "epoch": 2.066666666666667,
+ "grad_norm": 2.1775712966918945,
+ "learning_rate": 2.9353535353535355e-06,
+ "loss": 1.6423053741455078,
+ "mean_token_accuracy": 0.639655590057373,
+ "num_tokens": 16760832.0,
+ "step": 1023
+ },
+ {
+ "entropy": 1.4717872142791748,
+ "epoch": 2.0686868686868687,
+ "grad_norm": 2.2070701122283936,
+ "learning_rate": 2.9333333333333338e-06,
+ "loss": 1.508943796157837,
+ "mean_token_accuracy": 0.6469223499298096,
+ "num_tokens": 16777216.0,
+ "step": 1024
+ },
+ {
+ "entropy": 1.1494653224945068,
+ "epoch": 2.0707070707070705,
+ "grad_norm": 2.0239098072052,
+ "learning_rate": 2.9313131313131317e-06,
+ "loss": 1.1367695331573486,
+ "mean_token_accuracy": 0.7148265838623047,
+ "num_tokens": 16793600.0,
+ "step": 1025
+ },
+ {
+ "entropy": 1.0235252380371094,
+ "epoch": 2.0727272727272728,
+ "grad_norm": 2.071176052093506,
+ "learning_rate": 2.9292929292929295e-06,
+ "loss": 1.0369395017623901,
+ "mean_token_accuracy": 0.7396799921989441,
+ "num_tokens": 16809984.0,
+ "step": 1026
+ },
+ {
+ "entropy": 1.2325578927993774,
+ "epoch": 2.0747474747474746,
+ "grad_norm": 2.212671995162964,
+ "learning_rate": 2.9272727272727274e-06,
+ "loss": 1.2429314851760864,
+ "mean_token_accuracy": 0.688568651676178,
+ "num_tokens": 16826368.0,
+ "step": 1027
+ },
+ {
+ "entropy": 1.2879300117492676,
+ "epoch": 2.076767676767677,
+ "grad_norm": 2.0366690158843994,
+ "learning_rate": 2.9252525252525253e-06,
+ "loss": 1.2931241989135742,
+ "mean_token_accuracy": 0.6813018918037415,
+ "num_tokens": 16842752.0,
+ "step": 1028
+ },
+ {
+ "entropy": 0.9473312497138977,
+ "epoch": 2.0787878787878786,
+ "grad_norm": 2.0870468616485596,
+ "learning_rate": 2.9232323232323236e-06,
+ "loss": 0.9292516708374023,
+ "mean_token_accuracy": 0.7525036931037903,
+ "num_tokens": 16859136.0,
+ "step": 1029
+ },
+ {
+ "entropy": 1.3882765769958496,
+ "epoch": 2.080808080808081,
+ "grad_norm": 2.1285617351531982,
+ "learning_rate": 2.9212121212121215e-06,
+ "loss": 1.3798363208770752,
+ "mean_token_accuracy": 0.6768441796302795,
+ "num_tokens": 16875520.0,
+ "step": 1030
+ },
+ {
+ "entropy": 1.3588547706604004,
+ "epoch": 2.0828282828282827,
+ "grad_norm": 2.2275242805480957,
+ "learning_rate": 2.9191919191919194e-06,
+ "loss": 1.3538521528244019,
+ "mean_token_accuracy": 0.6615168452262878,
+ "num_tokens": 16891904.0,
+ "step": 1031
+ },
+ {
+ "entropy": 1.0415440797805786,
+ "epoch": 2.084848484848485,
+ "grad_norm": 2.2782297134399414,
+ "learning_rate": 2.9171717171717173e-06,
+ "loss": 1.03328537940979,
+ "mean_token_accuracy": 0.7577552795410156,
+ "num_tokens": 16908288.0,
+ "step": 1032
+ },
+ {
+ "entropy": 1.241741418838501,
+ "epoch": 2.0868686868686868,
+ "grad_norm": 2.141939163208008,
+ "learning_rate": 2.915151515151515e-06,
+ "loss": 1.2248635292053223,
+ "mean_token_accuracy": 0.6950415372848511,
+ "num_tokens": 16924672.0,
+ "step": 1033
+ },
+ {
+ "entropy": 1.1192725896835327,
+ "epoch": 2.088888888888889,
+ "grad_norm": 1.9980862140655518,
+ "learning_rate": 2.913131313131313e-06,
+ "loss": 1.1176412105560303,
+ "mean_token_accuracy": 0.7272838354110718,
+ "num_tokens": 16941056.0,
+ "step": 1034
+ },
+ {
+ "entropy": 1.1871322393417358,
+ "epoch": 2.090909090909091,
+ "grad_norm": 2.1212220191955566,
+ "learning_rate": 2.9111111111111114e-06,
+ "loss": 1.1670379638671875,
+ "mean_token_accuracy": 0.7048119306564331,
+ "num_tokens": 16957440.0,
+ "step": 1035
+ },
+ {
+ "entropy": 1.5194982290267944,
+ "epoch": 2.092929292929293,
+ "grad_norm": 2.2593677043914795,
+ "learning_rate": 2.9090909090909093e-06,
+ "loss": 1.50520920753479,
+ "mean_token_accuracy": 0.6475940346717834,
+ "num_tokens": 16973824.0,
+ "step": 1036
+ },
+ {
+ "entropy": 1.212785243988037,
+ "epoch": 2.094949494949495,
+ "grad_norm": 2.221132278442383,
+ "learning_rate": 2.907070707070707e-06,
+ "loss": 1.206315040588379,
+ "mean_token_accuracy": 0.7046898007392883,
+ "num_tokens": 16990208.0,
+ "step": 1037
+ },
+ {
+ "entropy": 1.2913068532943726,
+ "epoch": 2.096969696969697,
+ "grad_norm": 2.0525012016296387,
+ "learning_rate": 2.905050505050505e-06,
+ "loss": 1.2987749576568604,
+ "mean_token_accuracy": 0.6894235610961914,
+ "num_tokens": 17006592.0,
+ "step": 1038
+ },
+ {
+ "entropy": 1.139485239982605,
+ "epoch": 2.098989898989899,
+ "grad_norm": 2.093764543533325,
+ "learning_rate": 2.903030303030303e-06,
+ "loss": 1.1460036039352417,
+ "mean_token_accuracy": 0.7205055952072144,
+ "num_tokens": 17022976.0,
+ "step": 1039
+ },
+ {
+ "entropy": 1.3781968355178833,
+ "epoch": 2.101010101010101,
+ "grad_norm": 2.095311164855957,
+ "learning_rate": 2.9010101010101012e-06,
+ "loss": 1.3901969194412231,
+ "mean_token_accuracy": 0.679286777973175,
+ "num_tokens": 17039360.0,
+ "step": 1040
+ },
+ {
+ "entropy": 1.1370116472244263,
+ "epoch": 2.103030303030303,
+ "grad_norm": 2.139094352722168,
+ "learning_rate": 2.898989898989899e-06,
+ "loss": 1.1491400003433228,
+ "mean_token_accuracy": 0.72007817029953,
+ "num_tokens": 17055744.0,
+ "step": 1041
+ },
+ {
+ "entropy": 1.0845732688903809,
+ "epoch": 2.105050505050505,
+ "grad_norm": 2.180278778076172,
+ "learning_rate": 2.896969696969697e-06,
+ "loss": 1.0764763355255127,
+ "mean_token_accuracy": 0.7365657091140747,
+ "num_tokens": 17072128.0,
+ "step": 1042
+ },
+ {
+ "entropy": 1.4869064092636108,
+ "epoch": 2.107070707070707,
+ "grad_norm": 2.053727388381958,
+ "learning_rate": 2.894949494949495e-06,
+ "loss": 1.4862562417984009,
+ "mean_token_accuracy": 0.6643258333206177,
+ "num_tokens": 17088512.0,
+ "step": 1043
+ },
+ {
+ "entropy": 1.048888921737671,
+ "epoch": 2.109090909090909,
+ "grad_norm": 2.1302237510681152,
+ "learning_rate": 2.892929292929293e-06,
+ "loss": 1.0331683158874512,
+ "mean_token_accuracy": 0.7388250827789307,
+ "num_tokens": 17104896.0,
+ "step": 1044
+ },
+ {
+ "entropy": 1.2839856147766113,
+ "epoch": 2.111111111111111,
+ "grad_norm": 2.8510677814483643,
+ "learning_rate": 2.8909090909090907e-06,
+ "loss": 1.3016979694366455,
+ "mean_token_accuracy": 0.6904616355895996,
+ "num_tokens": 17121280.0,
+ "step": 1045
+ },
+ {
+ "entropy": 1.050045371055603,
+ "epoch": 2.113131313131313,
+ "grad_norm": 2.2457358837127686,
+ "learning_rate": 2.888888888888889e-06,
+ "loss": 1.0580981969833374,
+ "mean_token_accuracy": 0.7221543788909912,
+ "num_tokens": 17137664.0,
+ "step": 1046
+ },
+ {
+ "entropy": 1.3087437152862549,
+ "epoch": 2.1151515151515152,
+ "grad_norm": 2.0929996967315674,
+ "learning_rate": 2.8868686868686873e-06,
+ "loss": 1.2914986610412598,
+ "mean_token_accuracy": 0.6939423680305481,
+ "num_tokens": 17154048.0,
+ "step": 1047
+ },
+ {
+ "entropy": 1.7247086763381958,
+ "epoch": 2.117171717171717,
+ "grad_norm": 2.2230091094970703,
+ "learning_rate": 2.884848484848485e-06,
+ "loss": 1.7394956350326538,
+ "mean_token_accuracy": 0.6131533980369568,
+ "num_tokens": 17170432.0,
+ "step": 1048
+ },
+ {
+ "entropy": 1.5816357135772705,
+ "epoch": 2.1191919191919193,
+ "grad_norm": 2.196422576904297,
+ "learning_rate": 2.8828282828282835e-06,
+ "loss": 1.6226823329925537,
+ "mean_token_accuracy": 0.6375183463096619,
+ "num_tokens": 17186816.0,
+ "step": 1049
+ },
+ {
+ "entropy": 1.2316261529922485,
+ "epoch": 2.121212121212121,
+ "grad_norm": 2.173909902572632,
+ "learning_rate": 2.8808080808080814e-06,
+ "loss": 1.2228440046310425,
+ "mean_token_accuracy": 0.6940034031867981,
+ "num_tokens": 17203200.0,
+ "step": 1050
+ },
+ {
+ "epoch": 2.121212121212121,
+ "eval_entropy": 1.3722701356295617,
+ "eval_loss": 1.5385833978652954,
+ "eval_mean_token_accuracy": 0.6450051809510877,
+ "eval_num_tokens": 17203200.0,
+ "eval_runtime": 43.9186,
+ "eval_samples_per_second": 22.542,
+ "eval_steps_per_second": 2.823,
+ "step": 1050
+ },
+ {
+ "entropy": 1.3023537397384644,
+ "epoch": 2.1232323232323234,
+ "grad_norm": 2.1490018367767334,
+ "learning_rate": 2.8787878787878793e-06,
+ "loss": 1.284008264541626,
+ "mean_token_accuracy": 0.7057278752326965,
+ "num_tokens": 17219584.0,
+ "step": 1051
+ },
+ {
+ "entropy": 1.3198270797729492,
+ "epoch": 2.125252525252525,
+ "grad_norm": 2.582298994064331,
+ "learning_rate": 2.876767676767677e-06,
+ "loss": 1.3185001611709595,
+ "mean_token_accuracy": 0.6772105693817139,
+ "num_tokens": 17235968.0,
+ "step": 1052
+ },
+ {
+ "entropy": 0.8889825940132141,
+ "epoch": 2.1272727272727274,
+ "grad_norm": 2.0617194175720215,
+ "learning_rate": 2.874747474747475e-06,
+ "loss": 0.889773964881897,
+ "mean_token_accuracy": 0.767892062664032,
+ "num_tokens": 17252352.0,
+ "step": 1053
+ },
+ {
+ "entropy": 1.5485399961471558,
+ "epoch": 2.1292929292929292,
+ "grad_norm": 2.2752370834350586,
+ "learning_rate": 2.872727272727273e-06,
+ "loss": 1.5542051792144775,
+ "mean_token_accuracy": 0.6397166848182678,
+ "num_tokens": 17268736.0,
+ "step": 1054
+ },
+ {
+ "entropy": 1.3949565887451172,
+ "epoch": 2.1313131313131315,
+ "grad_norm": 2.0112996101379395,
+ "learning_rate": 2.8707070707070713e-06,
+ "loss": 1.4111721515655518,
+ "mean_token_accuracy": 0.6659746170043945,
+ "num_tokens": 17285120.0,
+ "step": 1055
+ },
+ {
+ "entropy": 1.0529720783233643,
+ "epoch": 2.1333333333333333,
+ "grad_norm": 2.2219948768615723,
+ "learning_rate": 2.868686868686869e-06,
+ "loss": 1.0484942197799683,
+ "mean_token_accuracy": 0.7224596738815308,
+ "num_tokens": 17301504.0,
+ "step": 1056
+ },
+ {
+ "entropy": 1.5362433195114136,
+ "epoch": 2.1353535353535356,
+ "grad_norm": 2.4097044467926025,
+ "learning_rate": 2.866666666666667e-06,
+ "loss": 1.5809426307678223,
+ "mean_token_accuracy": 0.6266487836837769,
+ "num_tokens": 17317888.0,
+ "step": 1057
+ },
+ {
+ "entropy": 1.6682685613632202,
+ "epoch": 2.1373737373737374,
+ "grad_norm": 2.121086359024048,
+ "learning_rate": 2.864646464646465e-06,
+ "loss": 1.6636815071105957,
+ "mean_token_accuracy": 0.6317782402038574,
+ "num_tokens": 17334272.0,
+ "step": 1058
+ },
+ {
+ "entropy": 1.5480221509933472,
+ "epoch": 2.1393939393939396,
+ "grad_norm": 2.2226176261901855,
+ "learning_rate": 2.862626262626263e-06,
+ "loss": 1.535983681678772,
+ "mean_token_accuracy": 0.6623717546463013,
+ "num_tokens": 17350656.0,
+ "step": 1059
+ },
+ {
+ "entropy": 1.3323101997375488,
+ "epoch": 2.1414141414141414,
+ "grad_norm": 2.2644572257995605,
+ "learning_rate": 2.860606060606061e-06,
+ "loss": 1.3316677808761597,
+ "mean_token_accuracy": 0.6741573214530945,
+ "num_tokens": 17367040.0,
+ "step": 1060
+ },
+ {
+ "entropy": 1.5107028484344482,
+ "epoch": 2.1434343434343432,
+ "grad_norm": 2.1291613578796387,
+ "learning_rate": 2.858585858585859e-06,
+ "loss": 1.4996232986450195,
+ "mean_token_accuracy": 0.6537005305290222,
+ "num_tokens": 17383424.0,
+ "step": 1061
+ },
+ {
+ "entropy": 1.5927387475967407,
+ "epoch": 2.1454545454545455,
+ "grad_norm": 2.1977617740631104,
+ "learning_rate": 2.856565656565657e-06,
+ "loss": 1.582688331604004,
+ "mean_token_accuracy": 0.6306790709495544,
+ "num_tokens": 17399808.0,
+ "step": 1062
+ },
+ {
+ "entropy": 1.0717219114303589,
+ "epoch": 2.1474747474747473,
+ "grad_norm": 2.2494494915008545,
+ "learning_rate": 2.8545454545454548e-06,
+ "loss": 1.0780384540557861,
+ "mean_token_accuracy": 0.7214826345443726,
+ "num_tokens": 17416192.0,
+ "step": 1063
+ },
+ {
+ "entropy": 1.484608769416809,
+ "epoch": 2.1494949494949496,
+ "grad_norm": 2.20664644241333,
+ "learning_rate": 2.8525252525252527e-06,
+ "loss": 1.5074207782745361,
+ "mean_token_accuracy": 0.6502808928489685,
+ "num_tokens": 17432576.0,
+ "step": 1064
+ },
+ {
+ "entropy": 1.6607396602630615,
+ "epoch": 2.1515151515151514,
+ "grad_norm": 2.123538017272949,
+ "learning_rate": 2.850505050505051e-06,
+ "loss": 1.659307837486267,
+ "mean_token_accuracy": 0.619076669216156,
+ "num_tokens": 17448960.0,
+ "step": 1065
+ },
+ {
+ "entropy": 1.1195216178894043,
+ "epoch": 2.1535353535353536,
+ "grad_norm": 2.037261486053467,
+ "learning_rate": 2.848484848484849e-06,
+ "loss": 1.127397894859314,
+ "mean_token_accuracy": 0.7209330797195435,
+ "num_tokens": 17465344.0,
+ "step": 1066
+ },
+ {
+ "entropy": 1.0512653589248657,
+ "epoch": 2.1555555555555554,
+ "grad_norm": 2.209764003753662,
+ "learning_rate": 2.8464646464646468e-06,
+ "loss": 1.073946475982666,
+ "mean_token_accuracy": 0.717391312122345,
+ "num_tokens": 17481728.0,
+ "step": 1067
+ },
+ {
+ "entropy": 1.131946325302124,
+ "epoch": 2.1575757575757577,
+ "grad_norm": 1.9188926219940186,
+ "learning_rate": 2.8444444444444446e-06,
+ "loss": 1.11629056930542,
+ "mean_token_accuracy": 0.7220322489738464,
+ "num_tokens": 17498112.0,
+ "step": 1068
+ },
+ {
+ "entropy": 1.4745967388153076,
+ "epoch": 2.1595959595959595,
+ "grad_norm": 2.285099983215332,
+ "learning_rate": 2.8424242424242425e-06,
+ "loss": 1.4795483350753784,
+ "mean_token_accuracy": 0.6466169953346252,
+ "num_tokens": 17514496.0,
+ "step": 1069
+ },
+ {
+ "entropy": 1.2614645957946777,
+ "epoch": 2.1616161616161618,
+ "grad_norm": 2.4019627571105957,
+ "learning_rate": 2.8404040404040404e-06,
+ "loss": 1.2659732103347778,
+ "mean_token_accuracy": 0.6797142028808594,
+ "num_tokens": 17530880.0,
+ "step": 1070
+ },
+ {
+ "entropy": 1.4247770309448242,
+ "epoch": 2.1636363636363636,
+ "grad_norm": 2.350632905960083,
+ "learning_rate": 2.8383838383838387e-06,
+ "loss": 1.415325403213501,
+ "mean_token_accuracy": 0.6655471324920654,
+ "num_tokens": 17547264.0,
+ "step": 1071
+ },
+ {
+ "entropy": 1.5746078491210938,
+ "epoch": 2.165656565656566,
+ "grad_norm": 2.18448543548584,
+ "learning_rate": 2.8363636363636366e-06,
+ "loss": 1.6022642850875854,
+ "mean_token_accuracy": 0.6324499249458313,
+ "num_tokens": 17563648.0,
+ "step": 1072
+ },
+ {
+ "entropy": 1.5869743824005127,
+ "epoch": 2.1676767676767676,
+ "grad_norm": 2.3553452491760254,
+ "learning_rate": 2.8343434343434345e-06,
+ "loss": 1.6163790225982666,
+ "mean_token_accuracy": 0.6199926733970642,
+ "num_tokens": 17580032.0,
+ "step": 1073
+ },
+ {
+ "entropy": 1.2412667274475098,
+ "epoch": 2.16969696969697,
+ "grad_norm": 2.209017038345337,
+ "learning_rate": 2.8323232323232324e-06,
+ "loss": 1.2491700649261475,
+ "mean_token_accuracy": 0.6911333799362183,
+ "num_tokens": 17596416.0,
+ "step": 1074
+ },
+ {
+ "entropy": 0.9127920866012573,
+ "epoch": 2.1717171717171717,
+ "grad_norm": 1.9761381149291992,
+ "learning_rate": 2.8303030303030303e-06,
+ "loss": 0.8961890339851379,
+ "mean_token_accuracy": 0.7523815631866455,
+ "num_tokens": 17612800.0,
+ "step": 1075
+ },
+ {
+ "entropy": 1.2798824310302734,
+ "epoch": 2.1737373737373735,
+ "grad_norm": 2.397031784057617,
+ "learning_rate": 2.8282828282828286e-06,
+ "loss": 1.286920189857483,
+ "mean_token_accuracy": 0.6805080771446228,
+ "num_tokens": 17629184.0,
+ "step": 1076
+ },
+ {
+ "entropy": 1.7863305807113647,
+ "epoch": 2.175757575757576,
+ "grad_norm": 2.2798852920532227,
+ "learning_rate": 2.8262626262626265e-06,
+ "loss": 1.7654080390930176,
+ "mean_token_accuracy": 0.5806057453155518,
+ "num_tokens": 17645568.0,
+ "step": 1077
+ },
+ {
+ "entropy": 1.327653408050537,
+ "epoch": 2.1777777777777776,
+ "grad_norm": 2.3473095893859863,
+ "learning_rate": 2.8242424242424244e-06,
+ "loss": 1.3175032138824463,
+ "mean_token_accuracy": 0.6750732660293579,
+ "num_tokens": 17661952.0,
+ "step": 1078
+ },
+ {
+ "entropy": 1.4917371273040771,
+ "epoch": 2.17979797979798,
+ "grad_norm": 2.181729793548584,
+ "learning_rate": 2.8222222222222223e-06,
+ "loss": 1.4738472700119019,
+ "mean_token_accuracy": 0.6619443297386169,
+ "num_tokens": 17678336.0,
+ "step": 1079
+ },
+ {
+ "entropy": 1.4644533395767212,
+ "epoch": 2.1818181818181817,
+ "grad_norm": 2.2558112144470215,
+ "learning_rate": 2.82020202020202e-06,
+ "loss": 1.4563480615615845,
+ "mean_token_accuracy": 0.6484489440917969,
+ "num_tokens": 17694720.0,
+ "step": 1080
+ },
+ {
+ "entropy": 0.7626141905784607,
+ "epoch": 2.183838383838384,
+ "grad_norm": 1.8687561750411987,
+ "learning_rate": 2.818181818181818e-06,
+ "loss": 0.7718763947486877,
+ "mean_token_accuracy": 0.7982413172721863,
+ "num_tokens": 17711104.0,
+ "step": 1081
+ },
+ {
+ "entropy": 1.0613173246383667,
+ "epoch": 2.1858585858585857,
+ "grad_norm": 2.072643995285034,
+ "learning_rate": 2.8161616161616163e-06,
+ "loss": 1.0554629564285278,
+ "mean_token_accuracy": 0.734306275844574,
+ "num_tokens": 17727488.0,
+ "step": 1082
+ },
+ {
+ "entropy": 1.0348066091537476,
+ "epoch": 2.187878787878788,
+ "grad_norm": 1.8455089330673218,
+ "learning_rate": 2.8141414141414142e-06,
+ "loss": 1.0231983661651611,
+ "mean_token_accuracy": 0.7382755279541016,
+ "num_tokens": 17743872.0,
+ "step": 1083
+ },
+ {
+ "entropy": 1.271000862121582,
+ "epoch": 2.18989898989899,
+ "grad_norm": 2.3292477130889893,
+ "learning_rate": 2.812121212121212e-06,
+ "loss": 1.2822595834732056,
+ "mean_token_accuracy": 0.6816682815551758,
+ "num_tokens": 17760256.0,
+ "step": 1084
+ },
+ {
+ "entropy": 1.62147855758667,
+ "epoch": 2.191919191919192,
+ "grad_norm": 2.3269946575164795,
+ "learning_rate": 2.81010101010101e-06,
+ "loss": 1.6017215251922607,
+ "mean_token_accuracy": 0.635808527469635,
+ "num_tokens": 17776640.0,
+ "step": 1085
+ },
+ {
+ "entropy": 1.4834544658660889,
+ "epoch": 2.193939393939394,
+ "grad_norm": 2.223590135574341,
+ "learning_rate": 2.808080808080808e-06,
+ "loss": 1.496992826461792,
+ "mean_token_accuracy": 0.6494870781898499,
+ "num_tokens": 17793024.0,
+ "step": 1086
+ },
+ {
+ "entropy": 1.4784536361694336,
+ "epoch": 2.195959595959596,
+ "grad_norm": 2.169416666030884,
+ "learning_rate": 2.806060606060606e-06,
+ "loss": 1.4766731262207031,
+ "mean_token_accuracy": 0.6543722748756409,
+ "num_tokens": 17809408.0,
+ "step": 1087
+ },
+ {
+ "entropy": 1.3650349378585815,
+ "epoch": 2.197979797979798,
+ "grad_norm": 2.1369845867156982,
+ "learning_rate": 2.804040404040404e-06,
+ "loss": 1.3475263118743896,
+ "mean_token_accuracy": 0.6712262034416199,
+ "num_tokens": 17825792.0,
+ "step": 1088
+ },
+ {
+ "entropy": 1.052316665649414,
+ "epoch": 2.2,
+ "grad_norm": 2.3176774978637695,
+ "learning_rate": 2.802020202020202e-06,
+ "loss": 1.056478500366211,
+ "mean_token_accuracy": 0.7374206185340881,
+ "num_tokens": 17842176.0,
+ "step": 1089
+ },
+ {
+ "entropy": 1.1928741931915283,
+ "epoch": 2.202020202020202,
+ "grad_norm": 2.2518808841705322,
+ "learning_rate": 2.8000000000000003e-06,
+ "loss": 1.2238969802856445,
+ "mean_token_accuracy": 0.7048119306564331,
+ "num_tokens": 17858560.0,
+ "step": 1090
+ },
+ {
+ "entropy": 1.135977864265442,
+ "epoch": 2.2040404040404042,
+ "grad_norm": 2.2688262462615967,
+ "learning_rate": 2.7979797979797986e-06,
+ "loss": 1.1330978870391846,
+ "mean_token_accuracy": 0.7053004503250122,
+ "num_tokens": 17874944.0,
+ "step": 1091
+ },
+ {
+ "entropy": 1.3542495965957642,
+ "epoch": 2.206060606060606,
+ "grad_norm": 2.1078014373779297,
+ "learning_rate": 2.7959595959595965e-06,
+ "loss": 1.3983874320983887,
+ "mean_token_accuracy": 0.6840498447418213,
+ "num_tokens": 17891328.0,
+ "step": 1092
+ },
+ {
+ "entropy": 1.1861188411712646,
+ "epoch": 2.2080808080808083,
+ "grad_norm": 2.348931312561035,
+ "learning_rate": 2.7939393939393944e-06,
+ "loss": 1.1982170343399048,
+ "mean_token_accuracy": 0.7014533281326294,
+ "num_tokens": 17907712.0,
+ "step": 1093
+ },
+ {
+ "entropy": 1.2858752012252808,
+ "epoch": 2.21010101010101,
+ "grad_norm": 2.1746175289154053,
+ "learning_rate": 2.7919191919191923e-06,
+ "loss": 1.302664041519165,
+ "mean_token_accuracy": 0.6818515062332153,
+ "num_tokens": 17924096.0,
+ "step": 1094
+ },
+ {
+ "entropy": 1.4817839860916138,
+ "epoch": 2.212121212121212,
+ "grad_norm": 1.9842815399169922,
+ "learning_rate": 2.78989898989899e-06,
+ "loss": 1.5031331777572632,
+ "mean_token_accuracy": 0.6546165347099304,
+ "num_tokens": 17940480.0,
+ "step": 1095
+ },
+ {
+ "entropy": 1.2977368831634521,
+ "epoch": 2.214141414141414,
+ "grad_norm": 2.0394232273101807,
+ "learning_rate": 2.7878787878787885e-06,
+ "loss": 1.3185319900512695,
+ "mean_token_accuracy": 0.6885075569152832,
+ "num_tokens": 17956864.0,
+ "step": 1096
+ },
+ {
+ "entropy": 1.010884404182434,
+ "epoch": 2.216161616161616,
+ "grad_norm": 2.1113393306732178,
+ "learning_rate": 2.7858585858585864e-06,
+ "loss": 1.0239849090576172,
+ "mean_token_accuracy": 0.7426111102104187,
+ "num_tokens": 17973248.0,
+ "step": 1097
+ },
+ {
+ "entropy": 1.6405178308486938,
+ "epoch": 2.2181818181818183,
+ "grad_norm": 2.220865249633789,
+ "learning_rate": 2.7838383838383842e-06,
+ "loss": 1.660733938217163,
+ "mean_token_accuracy": 0.6250610947608948,
+ "num_tokens": 17989632.0,
+ "step": 1098
+ },
+ {
+ "entropy": 1.2822861671447754,
+ "epoch": 2.22020202020202,
+ "grad_norm": 2.1409878730773926,
+ "learning_rate": 2.781818181818182e-06,
+ "loss": 1.2675114870071411,
+ "mean_token_accuracy": 0.7099413871765137,
+ "num_tokens": 18006016.0,
+ "step": 1099
+ },
+ {
+ "entropy": 1.5835071802139282,
+ "epoch": 2.2222222222222223,
+ "grad_norm": 2.32753586769104,
+ "learning_rate": 2.77979797979798e-06,
+ "loss": 1.585759162902832,
+ "mean_token_accuracy": 0.6321446299552917,
+ "num_tokens": 18022400.0,
+ "step": 1100
+ },
+ {
+ "epoch": 2.2222222222222223,
+ "eval_entropy": 1.3726552551792515,
+ "eval_loss": 1.5380089282989502,
+ "eval_mean_token_accuracy": 0.645188374384757,
+ "eval_num_tokens": 18022400.0,
+ "eval_runtime": 43.7591,
+ "eval_samples_per_second": 22.624,
+ "eval_steps_per_second": 2.834,
+ "step": 1100
+ },
+ {
+ "entropy": 1.0727043151855469,
+ "epoch": 2.224242424242424,
+ "grad_norm": 2.2024266719818115,
+ "learning_rate": 2.7777777777777783e-06,
+ "loss": 1.066201090812683,
+ "mean_token_accuracy": 0.7272838354110718,
+ "num_tokens": 18038784.0,
+ "step": 1101
+ },
+ {
+ "entropy": 1.1188048124313354,
+ "epoch": 2.2262626262626264,
+ "grad_norm": 2.032862663269043,
+ "learning_rate": 2.7757575757575762e-06,
+ "loss": 1.1292006969451904,
+ "mean_token_accuracy": 0.7198949456214905,
+ "num_tokens": 18055168.0,
+ "step": 1102
+ },
+ {
+ "entropy": 1.3658474683761597,
+ "epoch": 2.228282828282828,
+ "grad_norm": 2.081120491027832,
+ "learning_rate": 2.773737373737374e-06,
+ "loss": 1.3503609895706177,
+ "mean_token_accuracy": 0.6908280253410339,
+ "num_tokens": 18071552.0,
+ "step": 1103
+ },
+ {
+ "entropy": 1.5833252668380737,
+ "epoch": 2.2303030303030305,
+ "grad_norm": 2.174055576324463,
+ "learning_rate": 2.771717171717172e-06,
+ "loss": 1.5991246700286865,
+ "mean_token_accuracy": 0.6322056651115417,
+ "num_tokens": 18087936.0,
+ "step": 1104
+ },
+ {
+ "entropy": 1.2178122997283936,
+ "epoch": 2.2323232323232323,
+ "grad_norm": 2.328620195388794,
+ "learning_rate": 2.76969696969697e-06,
+ "loss": 1.2162861824035645,
+ "mean_token_accuracy": 0.6916218996047974,
+ "num_tokens": 18104320.0,
+ "step": 1105
+ },
+ {
+ "entropy": 1.2894606590270996,
+ "epoch": 2.2343434343434345,
+ "grad_norm": 2.236239433288574,
+ "learning_rate": 2.7676767676767678e-06,
+ "loss": 1.2695279121398926,
+ "mean_token_accuracy": 0.689728856086731,
+ "num_tokens": 18120704.0,
+ "step": 1106
+ },
+ {
+ "entropy": 1.0141552686691284,
+ "epoch": 2.2363636363636363,
+ "grad_norm": 2.000847339630127,
+ "learning_rate": 2.765656565656566e-06,
+ "loss": 1.0147396326065063,
+ "mean_token_accuracy": 0.7469467520713806,
+ "num_tokens": 18137088.0,
+ "step": 1107
+ },
+ {
+ "entropy": 1.150480031967163,
+ "epoch": 2.2383838383838386,
+ "grad_norm": 2.15156888961792,
+ "learning_rate": 2.763636363636364e-06,
+ "loss": 1.1593115329742432,
+ "mean_token_accuracy": 0.7129335403442383,
+ "num_tokens": 18153472.0,
+ "step": 1108
+ },
+ {
+ "entropy": 1.4387518167495728,
+ "epoch": 2.2404040404040404,
+ "grad_norm": 2.3262319564819336,
+ "learning_rate": 2.761616161616162e-06,
+ "loss": 1.4393596649169922,
+ "mean_token_accuracy": 0.6572422981262207,
+ "num_tokens": 18169856.0,
+ "step": 1109
+ },
+ {
+ "entropy": 1.205459475517273,
+ "epoch": 2.242424242424242,
+ "grad_norm": 2.223987579345703,
+ "learning_rate": 2.7595959595959597e-06,
+ "loss": 1.1871377229690552,
+ "mean_token_accuracy": 0.7018197178840637,
+ "num_tokens": 18186240.0,
+ "step": 1110
+ },
+ {
+ "entropy": 1.428817868232727,
+ "epoch": 2.2444444444444445,
+ "grad_norm": 2.091627836227417,
+ "learning_rate": 2.7575757575757576e-06,
+ "loss": 1.4262187480926514,
+ "mean_token_accuracy": 0.6708598136901855,
+ "num_tokens": 18202624.0,
+ "step": 1111
+ },
+ {
+ "entropy": 1.420682668685913,
+ "epoch": 2.2464646464646463,
+ "grad_norm": 2.30232310295105,
+ "learning_rate": 2.755555555555556e-06,
+ "loss": 1.439821720123291,
+ "mean_token_accuracy": 0.6508305072784424,
+ "num_tokens": 18219008.0,
+ "step": 1112
+ },
+ {
+ "entropy": 1.2365189790725708,
+ "epoch": 2.2484848484848485,
+ "grad_norm": 2.1747143268585205,
+ "learning_rate": 2.753535353535354e-06,
+ "loss": 1.2286595106124878,
+ "mean_token_accuracy": 0.7011480331420898,
+ "num_tokens": 18235392.0,
+ "step": 1113
+ },
+ {
+ "entropy": 1.0271798372268677,
+ "epoch": 2.2505050505050503,
+ "grad_norm": 2.1767842769622803,
+ "learning_rate": 2.7515151515151517e-06,
+ "loss": 1.018047571182251,
+ "mean_token_accuracy": 0.7297264337539673,
+ "num_tokens": 18251776.0,
+ "step": 1114
+ },
+ {
+ "entropy": 1.229053258895874,
+ "epoch": 2.2525252525252526,
+ "grad_norm": 2.390345811843872,
+ "learning_rate": 2.7494949494949496e-06,
+ "loss": 1.250126838684082,
+ "mean_token_accuracy": 0.6869809627532959,
+ "num_tokens": 18268160.0,
+ "step": 1115
+ },
+ {
+ "entropy": 1.4062843322753906,
+ "epoch": 2.2545454545454544,
+ "grad_norm": 2.220144271850586,
+ "learning_rate": 2.7474747474747475e-06,
+ "loss": 1.4262535572052002,
+ "mean_token_accuracy": 0.6592574715614319,
+ "num_tokens": 18284544.0,
+ "step": 1116
+ },
+ {
+ "entropy": 1.1991937160491943,
+ "epoch": 2.2565656565656567,
+ "grad_norm": 1.9910556077957153,
+ "learning_rate": 2.7454545454545454e-06,
+ "loss": 1.1869618892669678,
+ "mean_token_accuracy": 0.7049340605735779,
+ "num_tokens": 18300928.0,
+ "step": 1117
+ },
+ {
+ "entropy": 1.3147934675216675,
+ "epoch": 2.2585858585858585,
+ "grad_norm": 2.033942222595215,
+ "learning_rate": 2.7434343434343437e-06,
+ "loss": 1.3236720561981201,
+ "mean_token_accuracy": 0.6854543089866638,
+ "num_tokens": 18317312.0,
+ "step": 1118
+ },
+ {
+ "entropy": 1.2828413248062134,
+ "epoch": 2.2606060606060607,
+ "grad_norm": 2.2729172706604004,
+ "learning_rate": 2.7414141414141416e-06,
+ "loss": 1.2601265907287598,
+ "mean_token_accuracy": 0.6866145730018616,
+ "num_tokens": 18333696.0,
+ "step": 1119
+ },
+ {
+ "entropy": 1.9708930253982544,
+ "epoch": 2.2626262626262625,
+ "grad_norm": 2.4053614139556885,
+ "learning_rate": 2.7393939393939395e-06,
+ "loss": 1.9953765869140625,
+ "mean_token_accuracy": 0.5719345211982727,
+ "num_tokens": 18350080.0,
+ "step": 1120
+ },
+ {
+ "entropy": 1.071018099784851,
+ "epoch": 2.264646464646465,
+ "grad_norm": 2.0399091243743896,
+ "learning_rate": 2.7373737373737374e-06,
+ "loss": 1.0672862529754639,
+ "mean_token_accuracy": 0.72007817029953,
+ "num_tokens": 18366464.0,
+ "step": 1121
+ },
+ {
+ "entropy": 1.8741599321365356,
+ "epoch": 2.2666666666666666,
+ "grad_norm": 2.0918753147125244,
+ "learning_rate": 2.7353535353535353e-06,
+ "loss": 1.8604744672775269,
+ "mean_token_accuracy": 0.598436713218689,
+ "num_tokens": 18382848.0,
+ "step": 1122
+ },
+ {
+ "entropy": 1.0931968688964844,
+ "epoch": 2.268686868686869,
+ "grad_norm": 2.161102294921875,
+ "learning_rate": 2.7333333333333336e-06,
+ "loss": 1.1152517795562744,
+ "mean_token_accuracy": 0.7209330797195435,
+ "num_tokens": 18399232.0,
+ "step": 1123
+ },
+ {
+ "entropy": 1.3318616151809692,
+ "epoch": 2.2707070707070707,
+ "grad_norm": 2.200199842453003,
+ "learning_rate": 2.7313131313131315e-06,
+ "loss": 1.3456424474716187,
+ "mean_token_accuracy": 0.6789203882217407,
+ "num_tokens": 18415616.0,
+ "step": 1124
+ },
+ {
+ "entropy": 1.3777986764907837,
+ "epoch": 2.2727272727272725,
+ "grad_norm": 2.328887701034546,
+ "learning_rate": 2.7292929292929293e-06,
+ "loss": 1.3955169916152954,
+ "mean_token_accuracy": 0.6679897308349609,
+ "num_tokens": 18432000.0,
+ "step": 1125
+ },
+ {
+ "entropy": 1.0748711824417114,
+ "epoch": 2.2747474747474747,
+ "grad_norm": 2.151282787322998,
+ "learning_rate": 2.7272727272727272e-06,
+ "loss": 1.0380895137786865,
+ "mean_token_accuracy": 0.736932098865509,
+ "num_tokens": 18448384.0,
+ "step": 1126
+ },
+ {
+ "entropy": 1.4420548677444458,
+ "epoch": 2.276767676767677,
+ "grad_norm": 2.133190870285034,
+ "learning_rate": 2.725252525252525e-06,
+ "loss": 1.4465012550354004,
+ "mean_token_accuracy": 0.6784929037094116,
+ "num_tokens": 18464768.0,
+ "step": 1127
+ },
+ {
+ "entropy": 0.9478949904441833,
+ "epoch": 2.278787878787879,
+ "grad_norm": 2.0902628898620605,
+ "learning_rate": 2.7232323232323234e-06,
+ "loss": 0.9504396915435791,
+ "mean_token_accuracy": 0.7458475828170776,
+ "num_tokens": 18481152.0,
+ "step": 1128
+ },
+ {
+ "entropy": 1.367720127105713,
+ "epoch": 2.2808080808080806,
+ "grad_norm": 2.0413167476654053,
+ "learning_rate": 2.7212121212121213e-06,
+ "loss": 1.3884644508361816,
+ "mean_token_accuracy": 0.6762945652008057,
+ "num_tokens": 18497536.0,
+ "step": 1129
+ },
+ {
+ "entropy": 1.6931284666061401,
+ "epoch": 2.282828282828283,
+ "grad_norm": 2.188426971435547,
+ "learning_rate": 2.719191919191919e-06,
+ "loss": 1.7233154773712158,
+ "mean_token_accuracy": 0.6323277950286865,
+ "num_tokens": 18513920.0,
+ "step": 1130
+ },
+ {
+ "entropy": 1.563072681427002,
+ "epoch": 2.2848484848484847,
+ "grad_norm": 2.241102695465088,
+ "learning_rate": 2.717171717171717e-06,
+ "loss": 1.5677220821380615,
+ "mean_token_accuracy": 0.640693724155426,
+ "num_tokens": 18530304.0,
+ "step": 1131
+ },
+ {
+ "entropy": 1.3332083225250244,
+ "epoch": 2.286868686868687,
+ "grad_norm": 2.209522008895874,
+ "learning_rate": 2.715151515151516e-06,
+ "loss": 1.337693452835083,
+ "mean_token_accuracy": 0.6715925931930542,
+ "num_tokens": 18546688.0,
+ "step": 1132
+ },
+ {
+ "entropy": 1.5389035940170288,
+ "epoch": 2.2888888888888888,
+ "grad_norm": 2.212047815322876,
+ "learning_rate": 2.7131313131313137e-06,
+ "loss": 1.52158784866333,
+ "mean_token_accuracy": 0.6447850465774536,
+ "num_tokens": 18563072.0,
+ "step": 1133
+ },
+ {
+ "entropy": 1.2687056064605713,
+ "epoch": 2.290909090909091,
+ "grad_norm": 2.1673405170440674,
+ "learning_rate": 2.7111111111111116e-06,
+ "loss": 1.2689636945724487,
+ "mean_token_accuracy": 0.6911333799362183,
+ "num_tokens": 18579456.0,
+ "step": 1134
+ },
+ {
+ "entropy": 1.1464369297027588,
+ "epoch": 2.292929292929293,
+ "grad_norm": 1.9283273220062256,
+ "learning_rate": 2.7090909090909095e-06,
+ "loss": 1.1613389253616333,
+ "mean_token_accuracy": 0.7145212292671204,
+ "num_tokens": 18595840.0,
+ "step": 1135
+ },
+ {
+ "entropy": 1.327767252922058,
+ "epoch": 2.294949494949495,
+ "grad_norm": 2.2193424701690674,
+ "learning_rate": 2.7070707070707074e-06,
+ "loss": 1.3209152221679688,
+ "mean_token_accuracy": 0.6740962266921997,
+ "num_tokens": 18612224.0,
+ "step": 1136
+ },
+ {
+ "entropy": 1.5820066928863525,
+ "epoch": 2.296969696969697,
+ "grad_norm": 2.2020492553710938,
+ "learning_rate": 2.7050505050505057e-06,
+ "loss": 1.603272795677185,
+ "mean_token_accuracy": 0.6400830745697021,
+ "num_tokens": 18628608.0,
+ "step": 1137
+ },
+ {
+ "entropy": 1.279178500175476,
+ "epoch": 2.298989898989899,
+ "grad_norm": 2.1509714126586914,
+ "learning_rate": 2.7030303030303036e-06,
+ "loss": 1.2742427587509155,
+ "mean_token_accuracy": 0.689667820930481,
+ "num_tokens": 18644992.0,
+ "step": 1138
+ },
+ {
+ "entropy": 1.2199708223342896,
+ "epoch": 2.301010101010101,
+ "grad_norm": 2.0223684310913086,
+ "learning_rate": 2.7010101010101015e-06,
+ "loss": 1.2254796028137207,
+ "mean_token_accuracy": 0.6971787810325623,
+ "num_tokens": 18661376.0,
+ "step": 1139
+ },
+ {
+ "entropy": 1.1640081405639648,
+ "epoch": 2.303030303030303,
+ "grad_norm": 2.1907923221588135,
+ "learning_rate": 2.6989898989898994e-06,
+ "loss": 1.170613408088684,
+ "mean_token_accuracy": 0.6979726552963257,
+ "num_tokens": 18677760.0,
+ "step": 1140
+ },
+ {
+ "entropy": 1.3213385343551636,
+ "epoch": 2.305050505050505,
+ "grad_norm": 2.170276641845703,
+ "learning_rate": 2.6969696969696972e-06,
+ "loss": 1.3338474035263062,
+ "mean_token_accuracy": 0.672874927520752,
+ "num_tokens": 18694144.0,
+ "step": 1141
+ },
+ {
+ "entropy": 1.6036078929901123,
+ "epoch": 2.3070707070707073,
+ "grad_norm": 2.3499386310577393,
+ "learning_rate": 2.694949494949495e-06,
+ "loss": 1.5859179496765137,
+ "mean_token_accuracy": 0.6349536180496216,
+ "num_tokens": 18710528.0,
+ "step": 1142
+ },
+ {
+ "entropy": 1.2171586751937866,
+ "epoch": 2.309090909090909,
+ "grad_norm": 2.0380377769470215,
+ "learning_rate": 2.6929292929292934e-06,
+ "loss": 1.2191288471221924,
+ "mean_token_accuracy": 0.7063385248184204,
+ "num_tokens": 18726912.0,
+ "step": 1143
+ },
+ {
+ "entropy": 1.165902018547058,
+ "epoch": 2.311111111111111,
+ "grad_norm": 2.077310085296631,
+ "learning_rate": 2.6909090909090913e-06,
+ "loss": 1.1728768348693848,
+ "mean_token_accuracy": 0.7075598239898682,
+ "num_tokens": 18743296.0,
+ "step": 1144
+ },
+ {
+ "entropy": 1.5172539949417114,
+ "epoch": 2.313131313131313,
+ "grad_norm": 2.0132439136505127,
+ "learning_rate": 2.6888888888888892e-06,
+ "loss": 1.5073471069335938,
+ "mean_token_accuracy": 0.6546775698661804,
+ "num_tokens": 18759680.0,
+ "step": 1145
+ },
+ {
+ "entropy": 1.3588616847991943,
+ "epoch": 2.315151515151515,
+ "grad_norm": 2.1851465702056885,
+ "learning_rate": 2.686868686868687e-06,
+ "loss": 1.4111744165420532,
+ "mean_token_accuracy": 0.6750732660293579,
+ "num_tokens": 18776064.0,
+ "step": 1146
+ },
+ {
+ "entropy": 1.1948320865631104,
+ "epoch": 2.317171717171717,
+ "grad_norm": 2.1817259788513184,
+ "learning_rate": 2.684848484848485e-06,
+ "loss": 1.188377857208252,
+ "mean_token_accuracy": 0.7122007608413696,
+ "num_tokens": 18792448.0,
+ "step": 1147
+ },
+ {
+ "entropy": 1.7049702405929565,
+ "epoch": 2.319191919191919,
+ "grad_norm": 2.309352159500122,
+ "learning_rate": 2.6828282828282833e-06,
+ "loss": 1.6966536045074463,
+ "mean_token_accuracy": 0.6052759885787964,
+ "num_tokens": 18808832.0,
+ "step": 1148
+ },
+ {
+ "entropy": 1.0853440761566162,
+ "epoch": 2.3212121212121213,
+ "grad_norm": 2.1501059532165527,
+ "learning_rate": 2.680808080808081e-06,
+ "loss": 1.0747997760772705,
+ "mean_token_accuracy": 0.7195896506309509,
+ "num_tokens": 18825216.0,
+ "step": 1149
+ },
+ {
+ "entropy": 1.209170937538147,
+ "epoch": 2.323232323232323,
+ "grad_norm": 2.105555772781372,
+ "learning_rate": 2.678787878787879e-06,
+ "loss": 1.2055076360702515,
+ "mean_token_accuracy": 0.7062774896621704,
+ "num_tokens": 18841600.0,
+ "step": 1150
+ },
+ {
+ "epoch": 2.323232323232323,
+ "eval_entropy": 1.3799022854335847,
+ "eval_loss": 1.5365934371948242,
+ "eval_mean_token_accuracy": 0.6452573234996488,
+ "eval_num_tokens": 18841600.0,
+ "eval_runtime": 43.7362,
+ "eval_samples_per_second": 22.636,
+ "eval_steps_per_second": 2.835,
+ "step": 1150
+ },
+ {
+ "entropy": 0.9533048868179321,
+ "epoch": 2.3252525252525253,
+ "grad_norm": 1.9681413173675537,
+ "learning_rate": 2.676767676767677e-06,
+ "loss": 0.9284776449203491,
+ "mean_token_accuracy": 0.7609916925430298,
+ "num_tokens": 18857984.0,
+ "step": 1151
+ },
+ {
+ "entropy": 1.575872778892517,
+ "epoch": 2.327272727272727,
+ "grad_norm": 2.15085768699646,
+ "learning_rate": 2.674747474747475e-06,
+ "loss": 1.5716705322265625,
+ "mean_token_accuracy": 0.6415486335754395,
+ "num_tokens": 18874368.0,
+ "step": 1152
+ },
+ {
+ "entropy": 1.1680493354797363,
+ "epoch": 2.3292929292929294,
+ "grad_norm": 2.189589023590088,
+ "learning_rate": 2.6727272727272727e-06,
+ "loss": 1.1651779413223267,
+ "mean_token_accuracy": 0.7156203985214233,
+ "num_tokens": 18890752.0,
+ "step": 1153
+ },
+ {
+ "entropy": 1.1636664867401123,
+ "epoch": 2.3313131313131312,
+ "grad_norm": 1.9992915391921997,
+ "learning_rate": 2.670707070707071e-06,
+ "loss": 1.1650042533874512,
+ "mean_token_accuracy": 0.7116512060165405,
+ "num_tokens": 18907136.0,
+ "step": 1154
+ },
+ {
+ "entropy": 1.205094337463379,
+ "epoch": 2.3333333333333335,
+ "grad_norm": 2.070323944091797,
+ "learning_rate": 2.668686868686869e-06,
+ "loss": 1.2071596384048462,
+ "mean_token_accuracy": 0.7018197178840637,
+ "num_tokens": 18923520.0,
+ "step": 1155
+ },
+ {
+ "entropy": 1.2256643772125244,
+ "epoch": 2.3353535353535353,
+ "grad_norm": 2.144339084625244,
+ "learning_rate": 2.666666666666667e-06,
+ "loss": 1.2168457508087158,
+ "mean_token_accuracy": 0.7010869383811951,
+ "num_tokens": 18939904.0,
+ "step": 1156
+ },
+ {
+ "entropy": 1.631039023399353,
+ "epoch": 2.3373737373737375,
+ "grad_norm": 2.2886807918548584,
+ "learning_rate": 2.6646464646464647e-06,
+ "loss": 1.6534450054168701,
+ "mean_token_accuracy": 0.6248778700828552,
+ "num_tokens": 18956288.0,
+ "step": 1157
+ },
+ {
+ "entropy": 1.0360562801361084,
+ "epoch": 2.3393939393939394,
+ "grad_norm": 2.137491226196289,
+ "learning_rate": 2.6626262626262626e-06,
+ "loss": 1.0167723894119263,
+ "mean_token_accuracy": 0.7330239415168762,
+ "num_tokens": 18972672.0,
+ "step": 1158
+ },
+ {
+ "entropy": 1.2292720079421997,
+ "epoch": 2.341414141414141,
+ "grad_norm": 3.1518330574035645,
+ "learning_rate": 2.660606060606061e-06,
+ "loss": 1.2180495262145996,
+ "mean_token_accuracy": 0.6957743167877197,
+ "num_tokens": 18989056.0,
+ "step": 1159
+ },
+ {
+ "entropy": 1.2210739850997925,
+ "epoch": 2.3434343434343434,
+ "grad_norm": 2.1314170360565186,
+ "learning_rate": 2.658585858585859e-06,
+ "loss": 1.2399296760559082,
+ "mean_token_accuracy": 0.7047508358955383,
+ "num_tokens": 19005440.0,
+ "step": 1160
+ },
+ {
+ "entropy": 1.2762846946716309,
+ "epoch": 2.3454545454545457,
+ "grad_norm": 2.1767046451568604,
+ "learning_rate": 2.6565656565656567e-06,
+ "loss": 1.2640776634216309,
+ "mean_token_accuracy": 0.6954079270362854,
+ "num_tokens": 19021824.0,
+ "step": 1161
+ },
+ {
+ "entropy": 1.271180272102356,
+ "epoch": 2.3474747474747475,
+ "grad_norm": 2.121891975402832,
+ "learning_rate": 2.6545454545454546e-06,
+ "loss": 1.2666800022125244,
+ "mean_token_accuracy": 0.6880801320075989,
+ "num_tokens": 19038208.0,
+ "step": 1162
+ },
+ {
+ "entropy": 1.5604509115219116,
+ "epoch": 2.3494949494949493,
+ "grad_norm": 2.320765495300293,
+ "learning_rate": 2.6525252525252525e-06,
+ "loss": 1.5687410831451416,
+ "mean_token_accuracy": 0.627198338508606,
+ "num_tokens": 19054592.0,
+ "step": 1163
+ },
+ {
+ "entropy": 1.1606905460357666,
+ "epoch": 2.3515151515151516,
+ "grad_norm": 2.1167874336242676,
+ "learning_rate": 2.6505050505050508e-06,
+ "loss": 1.1879122257232666,
+ "mean_token_accuracy": 0.7130556702613831,
+ "num_tokens": 19070976.0,
+ "step": 1164
+ },
+ {
+ "entropy": 1.5421797037124634,
+ "epoch": 2.3535353535353534,
+ "grad_norm": 2.144657850265503,
+ "learning_rate": 2.6484848484848487e-06,
+ "loss": 1.5562596321105957,
+ "mean_token_accuracy": 0.6350146532058716,
+ "num_tokens": 19087360.0,
+ "step": 1165
+ },
+ {
+ "entropy": 1.2803337574005127,
+ "epoch": 2.3555555555555556,
+ "grad_norm": 2.0440914630889893,
+ "learning_rate": 2.6464646464646466e-06,
+ "loss": 1.3008983135223389,
+ "mean_token_accuracy": 0.6868588328361511,
+ "num_tokens": 19103744.0,
+ "step": 1166
+ },
+ {
+ "entropy": 1.1863176822662354,
+ "epoch": 2.3575757575757574,
+ "grad_norm": 2.231771469116211,
+ "learning_rate": 2.6444444444444444e-06,
+ "loss": 1.1863257884979248,
+ "mean_token_accuracy": 0.7069491744041443,
+ "num_tokens": 19120128.0,
+ "step": 1167
+ },
+ {
+ "entropy": 1.1186378002166748,
+ "epoch": 2.3595959595959597,
+ "grad_norm": 2.0704708099365234,
+ "learning_rate": 2.6424242424242423e-06,
+ "loss": 1.1196620464324951,
+ "mean_token_accuracy": 0.726062536239624,
+ "num_tokens": 19136512.0,
+ "step": 1168
+ },
+ {
+ "entropy": 1.2110928297042847,
+ "epoch": 2.3616161616161615,
+ "grad_norm": 2.251551866531372,
+ "learning_rate": 2.6404040404040402e-06,
+ "loss": 1.2020606994628906,
+ "mean_token_accuracy": 0.7058500051498413,
+ "num_tokens": 19152896.0,
+ "step": 1169
+ },
+ {
+ "entropy": 1.302309513092041,
+ "epoch": 2.3636363636363638,
+ "grad_norm": 2.171961545944214,
+ "learning_rate": 2.6383838383838385e-06,
+ "loss": 1.3029515743255615,
+ "mean_token_accuracy": 0.673363447189331,
+ "num_tokens": 19169280.0,
+ "step": 1170
+ },
+ {
+ "entropy": 1.6036607027053833,
+ "epoch": 2.3656565656565656,
+ "grad_norm": 2.336979627609253,
+ "learning_rate": 2.6363636363636364e-06,
+ "loss": 1.6076864004135132,
+ "mean_token_accuracy": 0.630984365940094,
+ "num_tokens": 19185664.0,
+ "step": 1171
+ },
+ {
+ "entropy": 1.431620478630066,
+ "epoch": 2.367676767676768,
+ "grad_norm": 2.115891218185425,
+ "learning_rate": 2.6343434343434343e-06,
+ "loss": 1.4436556100845337,
+ "mean_token_accuracy": 0.6623717546463013,
+ "num_tokens": 19202048.0,
+ "step": 1172
+ },
+ {
+ "entropy": 1.4537419080734253,
+ "epoch": 2.3696969696969696,
+ "grad_norm": 2.497891664505005,
+ "learning_rate": 2.632323232323232e-06,
+ "loss": 1.48027503490448,
+ "mean_token_accuracy": 0.6352589130401611,
+ "num_tokens": 19218432.0,
+ "step": 1173
+ },
+ {
+ "entropy": 1.292363166809082,
+ "epoch": 2.371717171717172,
+ "grad_norm": 2.0235633850097656,
+ "learning_rate": 2.63030303030303e-06,
+ "loss": 1.3111207485198975,
+ "mean_token_accuracy": 0.6873473525047302,
+ "num_tokens": 19234816.0,
+ "step": 1174
+ },
+ {
+ "entropy": 1.2336944341659546,
+ "epoch": 2.3737373737373737,
+ "grad_norm": 2.342643976211548,
+ "learning_rate": 2.628282828282829e-06,
+ "loss": 1.2427266836166382,
+ "mean_token_accuracy": 0.6844772696495056,
+ "num_tokens": 19251200.0,
+ "step": 1175
+ },
+ {
+ "entropy": 1.166039228439331,
+ "epoch": 2.375757575757576,
+ "grad_norm": 2.157937526702881,
+ "learning_rate": 2.6262626262626267e-06,
+ "loss": 1.167286992073059,
+ "mean_token_accuracy": 0.7115290760993958,
+ "num_tokens": 19267584.0,
+ "step": 1176
+ },
+ {
+ "entropy": 1.3824102878570557,
+ "epoch": 2.3777777777777778,
+ "grad_norm": 2.2874608039855957,
+ "learning_rate": 2.6242424242424246e-06,
+ "loss": 1.404619574546814,
+ "mean_token_accuracy": 0.6707376837730408,
+ "num_tokens": 19283968.0,
+ "step": 1177
+ },
+ {
+ "entropy": 0.8266427516937256,
+ "epoch": 2.3797979797979796,
+ "grad_norm": 2.098217010498047,
+ "learning_rate": 2.6222222222222225e-06,
+ "loss": 0.821276068687439,
+ "mean_token_accuracy": 0.7726551294326782,
+ "num_tokens": 19300352.0,
+ "step": 1178
+ },
+ {
+ "entropy": 1.4335079193115234,
+ "epoch": 2.381818181818182,
+ "grad_norm": 2.3051438331604004,
+ "learning_rate": 2.620202020202021e-06,
+ "loss": 1.437370777130127,
+ "mean_token_accuracy": 0.6520518064498901,
+ "num_tokens": 19316736.0,
+ "step": 1179
+ },
+ {
+ "entropy": 1.0816318988800049,
+ "epoch": 2.3838383838383836,
+ "grad_norm": 2.0723016262054443,
+ "learning_rate": 2.6181818181818187e-06,
+ "loss": 1.0697057247161865,
+ "mean_token_accuracy": 0.7304592132568359,
+ "num_tokens": 19333120.0,
+ "step": 1180
+ },
+ {
+ "entropy": 1.2515255212783813,
+ "epoch": 2.385858585858586,
+ "grad_norm": 2.32125186920166,
+ "learning_rate": 2.6161616161616166e-06,
+ "loss": 1.256905198097229,
+ "mean_token_accuracy": 0.697300910949707,
+ "num_tokens": 19349504.0,
+ "step": 1181
+ },
+ {
+ "entropy": 1.0271868705749512,
+ "epoch": 2.3878787878787877,
+ "grad_norm": 2.0145103931427,
+ "learning_rate": 2.6141414141414145e-06,
+ "loss": 1.0158754587173462,
+ "mean_token_accuracy": 0.7518930435180664,
+ "num_tokens": 19365888.0,
+ "step": 1182
+ },
+ {
+ "entropy": 1.2139984369277954,
+ "epoch": 2.38989898989899,
+ "grad_norm": 2.1393911838531494,
+ "learning_rate": 2.6121212121212123e-06,
+ "loss": 1.2150651216506958,
+ "mean_token_accuracy": 0.7135441899299622,
+ "num_tokens": 19382272.0,
+ "step": 1183
+ },
+ {
+ "entropy": 1.076101303100586,
+ "epoch": 2.391919191919192,
+ "grad_norm": 2.1664907932281494,
+ "learning_rate": 2.6101010101010107e-06,
+ "loss": 1.0744431018829346,
+ "mean_token_accuracy": 0.7267953157424927,
+ "num_tokens": 19398656.0,
+ "step": 1184
+ },
+ {
+ "entropy": 1.5176570415496826,
+ "epoch": 2.393939393939394,
+ "grad_norm": 2.0756311416625977,
+ "learning_rate": 2.6080808080808085e-06,
+ "loss": 1.5209624767303467,
+ "mean_token_accuracy": 0.647777259349823,
+ "num_tokens": 19415040.0,
+ "step": 1185
+ },
+ {
+ "entropy": 1.0946495532989502,
+ "epoch": 2.395959595959596,
+ "grad_norm": 2.0545496940612793,
+ "learning_rate": 2.6060606060606064e-06,
+ "loss": 1.0858100652694702,
+ "mean_token_accuracy": 0.7226428985595703,
+ "num_tokens": 19431424.0,
+ "step": 1186
+ },
+ {
+ "entropy": 0.7713431715965271,
+ "epoch": 2.397979797979798,
+ "grad_norm": 2.0606112480163574,
+ "learning_rate": 2.6040404040404043e-06,
+ "loss": 0.7782045006752014,
+ "mean_token_accuracy": 0.7804103493690491,
+ "num_tokens": 19447808.0,
+ "step": 1187
+ },
+ {
+ "entropy": 1.2803305387496948,
+ "epoch": 2.4,
+ "grad_norm": 2.2125155925750732,
+ "learning_rate": 2.602020202020202e-06,
+ "loss": 1.2971004247665405,
+ "mean_token_accuracy": 0.6787371635437012,
+ "num_tokens": 19464192.0,
+ "step": 1188
+ },
+ {
+ "entropy": 0.9156424403190613,
+ "epoch": 2.402020202020202,
+ "grad_norm": 2.032531499862671,
+ "learning_rate": 2.6e-06,
+ "loss": 0.9119267463684082,
+ "mean_token_accuracy": 0.7627015113830566,
+ "num_tokens": 19480576.0,
+ "step": 1189
+ },
+ {
+ "entropy": 1.2698813676834106,
+ "epoch": 2.404040404040404,
+ "grad_norm": 2.2204320430755615,
+ "learning_rate": 2.5979797979797984e-06,
+ "loss": 1.2934629917144775,
+ "mean_token_accuracy": 0.682584285736084,
+ "num_tokens": 19496960.0,
+ "step": 1190
+ },
+ {
+ "entropy": 1.5677871704101562,
+ "epoch": 2.4060606060606062,
+ "grad_norm": 2.229994773864746,
+ "learning_rate": 2.5959595959595963e-06,
+ "loss": 1.591191053390503,
+ "mean_token_accuracy": 0.6297020316123962,
+ "num_tokens": 19513344.0,
+ "step": 1191
+ },
+ {
+ "entropy": 1.3057011365890503,
+ "epoch": 2.408080808080808,
+ "grad_norm": 2.4158618450164795,
+ "learning_rate": 2.593939393939394e-06,
+ "loss": 1.2989580631256104,
+ "mean_token_accuracy": 0.689728856086731,
+ "num_tokens": 19529728.0,
+ "step": 1192
+ },
+ {
+ "entropy": 1.3805862665176392,
+ "epoch": 2.41010101010101,
+ "grad_norm": 2.2611825466156006,
+ "learning_rate": 2.591919191919192e-06,
+ "loss": 1.385383129119873,
+ "mean_token_accuracy": 0.6634709239006042,
+ "num_tokens": 19546112.0,
+ "step": 1193
+ },
+ {
+ "entropy": 1.8123539686203003,
+ "epoch": 2.412121212121212,
+ "grad_norm": 2.3038923740386963,
+ "learning_rate": 2.58989898989899e-06,
+ "loss": 1.80764901638031,
+ "mean_token_accuracy": 0.5925744771957397,
+ "num_tokens": 19562496.0,
+ "step": 1194
+ },
+ {
+ "entropy": 1.1476300954818726,
+ "epoch": 2.4141414141414144,
+ "grad_norm": 2.2417445182800293,
+ "learning_rate": 2.5878787878787883e-06,
+ "loss": 1.1802024841308594,
+ "mean_token_accuracy": 0.7130556702613831,
+ "num_tokens": 19578880.0,
+ "step": 1195
+ },
+ {
+ "entropy": 1.6678205728530884,
+ "epoch": 2.416161616161616,
+ "grad_norm": 2.399136781692505,
+ "learning_rate": 2.585858585858586e-06,
+ "loss": 1.6780188083648682,
+ "mean_token_accuracy": 0.6161455512046814,
+ "num_tokens": 19595264.0,
+ "step": 1196
+ },
+ {
+ "entropy": 1.053451418876648,
+ "epoch": 2.418181818181818,
+ "grad_norm": 1.9980698823928833,
+ "learning_rate": 2.583838383838384e-06,
+ "loss": 1.0241587162017822,
+ "mean_token_accuracy": 0.7372984886169434,
+ "num_tokens": 19611648.0,
+ "step": 1197
+ },
+ {
+ "entropy": 1.5389430522918701,
+ "epoch": 2.4202020202020202,
+ "grad_norm": 2.171046733856201,
+ "learning_rate": 2.581818181818182e-06,
+ "loss": 1.5499310493469238,
+ "mean_token_accuracy": 0.6422203183174133,
+ "num_tokens": 19628032.0,
+ "step": 1198
+ },
+ {
+ "entropy": 1.4151558876037598,
+ "epoch": 2.422222222222222,
+ "grad_norm": 2.185184955596924,
+ "learning_rate": 2.57979797979798e-06,
+ "loss": 1.4296989440917969,
+ "mean_token_accuracy": 0.663593053817749,
+ "num_tokens": 19644416.0,
+ "step": 1199
+ },
+ {
+ "entropy": 1.1455156803131104,
+ "epoch": 2.4242424242424243,
+ "grad_norm": 2.224477767944336,
+ "learning_rate": 2.577777777777778e-06,
+ "loss": 1.1682045459747314,
+ "mean_token_accuracy": 0.7084147334098816,
+ "num_tokens": 19660800.0,
+ "step": 1200
+ },
+ {
+ "epoch": 2.4242424242424243,
+ "eval_entropy": 1.3761802339746105,
+ "eval_loss": 1.53595769405365,
+ "eval_mean_token_accuracy": 0.6453439944213436,
+ "eval_num_tokens": 19660800.0,
+ "eval_runtime": 43.8176,
+ "eval_samples_per_second": 22.594,
+ "eval_steps_per_second": 2.83,
+ "step": 1200
+ },
+ {
+ "entropy": 1.6039199829101562,
+ "epoch": 2.426262626262626,
+ "grad_norm": 2.122072696685791,
+ "learning_rate": 2.575757575757576e-06,
+ "loss": 1.625109314918518,
+ "mean_token_accuracy": 0.6306179761886597,
+ "num_tokens": 19677184.0,
+ "step": 1201
+ },
+ {
+ "entropy": 1.2364906072616577,
+ "epoch": 2.4282828282828284,
+ "grad_norm": 2.175671339035034,
+ "learning_rate": 2.573737373737374e-06,
+ "loss": 1.2398111820220947,
+ "mean_token_accuracy": 0.6886907815933228,
+ "num_tokens": 19693568.0,
+ "step": 1202
+ },
+ {
+ "entropy": 1.2937804460525513,
+ "epoch": 2.43030303030303,
+ "grad_norm": 2.1206040382385254,
+ "learning_rate": 2.571717171717172e-06,
+ "loss": 1.3073123693466187,
+ "mean_token_accuracy": 0.6762945652008057,
+ "num_tokens": 19709952.0,
+ "step": 1203
+ },
+ {
+ "entropy": 1.0239282846450806,
+ "epoch": 2.4323232323232324,
+ "grad_norm": 1.863510012626648,
+ "learning_rate": 2.5696969696969697e-06,
+ "loss": 1.015241026878357,
+ "mean_token_accuracy": 0.7452369332313538,
+ "num_tokens": 19726336.0,
+ "step": 1204
+ },
+ {
+ "entropy": 1.2327015399932861,
+ "epoch": 2.4343434343434343,
+ "grad_norm": 1.968019962310791,
+ "learning_rate": 2.5676767676767676e-06,
+ "loss": 1.2275054454803467,
+ "mean_token_accuracy": 0.6974841356277466,
+ "num_tokens": 19742720.0,
+ "step": 1205
+ },
+ {
+ "entropy": 1.130920171737671,
+ "epoch": 2.4363636363636365,
+ "grad_norm": 2.096205949783325,
+ "learning_rate": 2.565656565656566e-06,
+ "loss": 1.1229937076568604,
+ "mean_token_accuracy": 0.7102466821670532,
+ "num_tokens": 19759104.0,
+ "step": 1206
+ },
+ {
+ "entropy": 0.9204118251800537,
+ "epoch": 2.4383838383838383,
+ "grad_norm": 1.7956496477127075,
+ "learning_rate": 2.5636363636363638e-06,
+ "loss": 0.9276518821716309,
+ "mean_token_accuracy": 0.7644113302230835,
+ "num_tokens": 19775488.0,
+ "step": 1207
+ },
+ {
+ "entropy": 0.9552480578422546,
+ "epoch": 2.4404040404040406,
+ "grad_norm": 2.0838167667388916,
+ "learning_rate": 2.5616161616161617e-06,
+ "loss": 0.9676632881164551,
+ "mean_token_accuracy": 0.7421225905418396,
+ "num_tokens": 19791872.0,
+ "step": 1208
+ },
+ {
+ "entropy": 1.0109010934829712,
+ "epoch": 2.4424242424242424,
+ "grad_norm": 2.1739392280578613,
+ "learning_rate": 2.5595959595959595e-06,
+ "loss": 1.0201005935668945,
+ "mean_token_accuracy": 0.7366267442703247,
+ "num_tokens": 19808256.0,
+ "step": 1209
+ },
+ {
+ "entropy": 1.4037907123565674,
+ "epoch": 2.4444444444444446,
+ "grad_norm": 2.151865005493164,
+ "learning_rate": 2.5575757575757574e-06,
+ "loss": 1.4110395908355713,
+ "mean_token_accuracy": 0.6682339906692505,
+ "num_tokens": 19824640.0,
+ "step": 1210
+ },
+ {
+ "entropy": 1.2449947595596313,
+ "epoch": 2.4464646464646465,
+ "grad_norm": 2.065934896469116,
+ "learning_rate": 2.5555555555555557e-06,
+ "loss": 1.2521324157714844,
+ "mean_token_accuracy": 0.6952247023582458,
+ "num_tokens": 19841024.0,
+ "step": 1211
+ },
+ {
+ "entropy": 1.2075072526931763,
+ "epoch": 2.4484848484848483,
+ "grad_norm": 2.197885274887085,
+ "learning_rate": 2.5535353535353536e-06,
+ "loss": 1.2041654586791992,
+ "mean_token_accuracy": 0.69840008020401,
+ "num_tokens": 19857408.0,
+ "step": 1212
+ },
+ {
+ "entropy": 1.0454095602035522,
+ "epoch": 2.4505050505050505,
+ "grad_norm": 2.0778188705444336,
+ "learning_rate": 2.5515151515151515e-06,
+ "loss": 1.0392906665802002,
+ "mean_token_accuracy": 0.74187833070755,
+ "num_tokens": 19873792.0,
+ "step": 1213
+ },
+ {
+ "entropy": 1.3138911724090576,
+ "epoch": 2.4525252525252528,
+ "grad_norm": 2.1871910095214844,
+ "learning_rate": 2.5494949494949494e-06,
+ "loss": 1.317048192024231,
+ "mean_token_accuracy": 0.6720811128616333,
+ "num_tokens": 19890176.0,
+ "step": 1214
+ },
+ {
+ "entropy": 1.263680338859558,
+ "epoch": 2.4545454545454546,
+ "grad_norm": 2.129873752593994,
+ "learning_rate": 2.5474747474747473e-06,
+ "loss": 1.277045726776123,
+ "mean_token_accuracy": 0.7011480331420898,
+ "num_tokens": 19906560.0,
+ "step": 1215
+ },
+ {
+ "entropy": 0.9653311967849731,
+ "epoch": 2.4565656565656564,
+ "grad_norm": 2.1161508560180664,
+ "learning_rate": 2.5454545454545456e-06,
+ "loss": 0.9699352979660034,
+ "mean_token_accuracy": 0.7437713742256165,
+ "num_tokens": 19922944.0,
+ "step": 1216
+ },
+ {
+ "entropy": 1.350392460823059,
+ "epoch": 2.4585858585858587,
+ "grad_norm": 2.0958914756774902,
+ "learning_rate": 2.5434343434343435e-06,
+ "loss": 1.3442802429199219,
+ "mean_token_accuracy": 0.6768441796302795,
+ "num_tokens": 19939328.0,
+ "step": 1217
+ },
+ {
+ "entropy": 1.2484683990478516,
+ "epoch": 2.4606060606060605,
+ "grad_norm": 2.191668748855591,
+ "learning_rate": 2.541414141414142e-06,
+ "loss": 1.2663497924804688,
+ "mean_token_accuracy": 0.6958964467048645,
+ "num_tokens": 19955712.0,
+ "step": 1218
+ },
+ {
+ "entropy": 0.9751343131065369,
+ "epoch": 2.4626262626262627,
+ "grad_norm": 2.0976815223693848,
+ "learning_rate": 2.5393939393939397e-06,
+ "loss": 0.9734562635421753,
+ "mean_token_accuracy": 0.7372373938560486,
+ "num_tokens": 19972096.0,
+ "step": 1219
+ },
+ {
+ "entropy": 1.1655267477035522,
+ "epoch": 2.4646464646464645,
+ "grad_norm": 2.3504767417907715,
+ "learning_rate": 2.537373737373738e-06,
+ "loss": 1.1775636672973633,
+ "mean_token_accuracy": 0.7023082375526428,
+ "num_tokens": 19988480.0,
+ "step": 1220
+ },
+ {
+ "entropy": 1.3543325662612915,
+ "epoch": 2.466666666666667,
+ "grad_norm": 2.038313627243042,
+ "learning_rate": 2.535353535353536e-06,
+ "loss": 1.3447773456573486,
+ "mean_token_accuracy": 0.6779433488845825,
+ "num_tokens": 20004864.0,
+ "step": 1221
+ },
+ {
+ "entropy": 1.1814380884170532,
+ "epoch": 2.4686868686868686,
+ "grad_norm": 2.186947822570801,
+ "learning_rate": 2.5333333333333338e-06,
+ "loss": 1.186347246170044,
+ "mean_token_accuracy": 0.7064606547355652,
+ "num_tokens": 20021248.0,
+ "step": 1222
+ },
+ {
+ "entropy": 0.9775183796882629,
+ "epoch": 2.470707070707071,
+ "grad_norm": 2.144923210144043,
+ "learning_rate": 2.5313131313131317e-06,
+ "loss": 1.0010652542114258,
+ "mean_token_accuracy": 0.7385808229446411,
+ "num_tokens": 20037632.0,
+ "step": 1223
+ },
+ {
+ "entropy": 1.6076546907424927,
+ "epoch": 2.4727272727272727,
+ "grad_norm": 2.1664817333221436,
+ "learning_rate": 2.5292929292929296e-06,
+ "loss": 1.618837594985962,
+ "mean_token_accuracy": 0.6297630667686462,
+ "num_tokens": 20054016.0,
+ "step": 1224
+ },
+ {
+ "entropy": 1.0457805395126343,
+ "epoch": 2.474747474747475,
+ "grad_norm": 2.154000997543335,
+ "learning_rate": 2.5272727272727274e-06,
+ "loss": 1.0406007766723633,
+ "mean_token_accuracy": 0.7322911620140076,
+ "num_tokens": 20070400.0,
+ "step": 1225
+ },
+ {
+ "entropy": 1.065863847732544,
+ "epoch": 2.4767676767676767,
+ "grad_norm": 2.062626361846924,
+ "learning_rate": 2.5252525252525258e-06,
+ "loss": 1.0432696342468262,
+ "mean_token_accuracy": 0.7371152639389038,
+ "num_tokens": 20086784.0,
+ "step": 1226
+ },
+ {
+ "entropy": 1.5166304111480713,
+ "epoch": 2.4787878787878785,
+ "grad_norm": 2.089587926864624,
+ "learning_rate": 2.5232323232323236e-06,
+ "loss": 1.5205962657928467,
+ "mean_token_accuracy": 0.653822660446167,
+ "num_tokens": 20103168.0,
+ "step": 1227
+ },
+ {
+ "entropy": 1.5969332456588745,
+ "epoch": 2.480808080808081,
+ "grad_norm": 2.297184705734253,
+ "learning_rate": 2.5212121212121215e-06,
+ "loss": 1.611903190612793,
+ "mean_token_accuracy": 0.6246336102485657,
+ "num_tokens": 20119552.0,
+ "step": 1228
+ },
+ {
+ "entropy": 1.8385323286056519,
+ "epoch": 2.482828282828283,
+ "grad_norm": 2.0568299293518066,
+ "learning_rate": 2.5191919191919194e-06,
+ "loss": 1.848689079284668,
+ "mean_token_accuracy": 0.6079018115997314,
+ "num_tokens": 20135936.0,
+ "step": 1229
+ },
+ {
+ "entropy": 1.1586394309997559,
+ "epoch": 2.484848484848485,
+ "grad_norm": 1.91037917137146,
+ "learning_rate": 2.5171717171717173e-06,
+ "loss": 1.1482139825820923,
+ "mean_token_accuracy": 0.7165364027023315,
+ "num_tokens": 20152320.0,
+ "step": 1230
+ },
+ {
+ "entropy": 1.6905426979064941,
+ "epoch": 2.4868686868686867,
+ "grad_norm": 2.2317121028900146,
+ "learning_rate": 2.5151515151515156e-06,
+ "loss": 1.7028450965881348,
+ "mean_token_accuracy": 0.6125427484512329,
+ "num_tokens": 20168704.0,
+ "step": 1231
+ },
+ {
+ "entropy": 1.1329548358917236,
+ "epoch": 2.488888888888889,
+ "grad_norm": 2.112380027770996,
+ "learning_rate": 2.5131313131313135e-06,
+ "loss": 1.120601773262024,
+ "mean_token_accuracy": 0.7181240916252136,
+ "num_tokens": 20185088.0,
+ "step": 1232
+ },
+ {
+ "entropy": 1.2013109922409058,
+ "epoch": 2.4909090909090907,
+ "grad_norm": 2.087054491043091,
+ "learning_rate": 2.5111111111111114e-06,
+ "loss": 1.2169899940490723,
+ "mean_token_accuracy": 0.7089032530784607,
+ "num_tokens": 20201472.0,
+ "step": 1233
+ },
+ {
+ "entropy": 1.187458872795105,
+ "epoch": 2.492929292929293,
+ "grad_norm": 2.431793689727783,
+ "learning_rate": 2.5090909090909093e-06,
+ "loss": 1.2183618545532227,
+ "mean_token_accuracy": 0.6925378441810608,
+ "num_tokens": 20217856.0,
+ "step": 1234
+ },
+ {
+ "entropy": 1.2123541831970215,
+ "epoch": 2.494949494949495,
+ "grad_norm": 2.1715121269226074,
+ "learning_rate": 2.507070707070707e-06,
+ "loss": 1.2254973649978638,
+ "mean_token_accuracy": 0.703285276889801,
+ "num_tokens": 20234240.0,
+ "step": 1235
+ },
+ {
+ "entropy": 1.0582001209259033,
+ "epoch": 2.496969696969697,
+ "grad_norm": 2.222578763961792,
+ "learning_rate": 2.5050505050505055e-06,
+ "loss": 1.061441421508789,
+ "mean_token_accuracy": 0.7247191071510315,
+ "num_tokens": 20250624.0,
+ "step": 1236
+ },
+ {
+ "entropy": 1.5102840662002563,
+ "epoch": 2.498989898989899,
+ "grad_norm": 2.27860951423645,
+ "learning_rate": 2.5030303030303034e-06,
+ "loss": 1.519880771636963,
+ "mean_token_accuracy": 0.6581583023071289,
+ "num_tokens": 20267008.0,
+ "step": 1237
+ },
+ {
+ "entropy": 1.222317099571228,
+ "epoch": 2.501010101010101,
+ "grad_norm": 2.182908773422241,
+ "learning_rate": 2.5010101010101013e-06,
+ "loss": 1.2382556200027466,
+ "mean_token_accuracy": 0.6908891201019287,
+ "num_tokens": 20283392.0,
+ "step": 1238
+ },
+ {
+ "entropy": 1.252193570137024,
+ "epoch": 2.503030303030303,
+ "grad_norm": 2.2065556049346924,
+ "learning_rate": 2.498989898989899e-06,
+ "loss": 1.2510521411895752,
+ "mean_token_accuracy": 0.6914386749267578,
+ "num_tokens": 20299776.0,
+ "step": 1239
+ },
+ {
+ "entropy": 1.2909793853759766,
+ "epoch": 2.505050505050505,
+ "grad_norm": 2.374338150024414,
+ "learning_rate": 2.496969696969697e-06,
+ "loss": 1.3144972324371338,
+ "mean_token_accuracy": 0.6847215294837952,
+ "num_tokens": 20316160.0,
+ "step": 1240
+ },
+ {
+ "entropy": 1.5218966007232666,
+ "epoch": 2.507070707070707,
+ "grad_norm": 2.3914759159088135,
+ "learning_rate": 2.494949494949495e-06,
+ "loss": 1.5194602012634277,
+ "mean_token_accuracy": 0.6398998498916626,
+ "num_tokens": 20332544.0,
+ "step": 1241
+ },
+ {
+ "entropy": 1.2130569219589233,
+ "epoch": 2.509090909090909,
+ "grad_norm": 2.2857706546783447,
+ "learning_rate": 2.4929292929292932e-06,
+ "loss": 1.2035828828811646,
+ "mean_token_accuracy": 0.6977283954620361,
+ "num_tokens": 20348928.0,
+ "step": 1242
+ },
+ {
+ "entropy": 1.623070478439331,
+ "epoch": 2.511111111111111,
+ "grad_norm": 2.3757028579711914,
+ "learning_rate": 2.490909090909091e-06,
+ "loss": 1.654674768447876,
+ "mean_token_accuracy": 0.6242061257362366,
+ "num_tokens": 20365312.0,
+ "step": 1243
+ },
+ {
+ "entropy": 1.6762864589691162,
+ "epoch": 2.5131313131313133,
+ "grad_norm": 2.239466667175293,
+ "learning_rate": 2.488888888888889e-06,
+ "loss": 1.6712524890899658,
+ "mean_token_accuracy": 0.6280532479286194,
+ "num_tokens": 20381696.0,
+ "step": 1244
+ },
+ {
+ "entropy": 1.6754344701766968,
+ "epoch": 2.515151515151515,
+ "grad_norm": 2.2599003314971924,
+ "learning_rate": 2.486868686868687e-06,
+ "loss": 1.6697289943695068,
+ "mean_token_accuracy": 0.6158402562141418,
+ "num_tokens": 20398080.0,
+ "step": 1245
+ },
+ {
+ "entropy": 1.6282587051391602,
+ "epoch": 2.517171717171717,
+ "grad_norm": 2.0555050373077393,
+ "learning_rate": 2.4848484848484848e-06,
+ "loss": 1.6177103519439697,
+ "mean_token_accuracy": 0.63312166929245,
+ "num_tokens": 20414464.0,
+ "step": 1246
+ },
+ {
+ "entropy": 1.3457770347595215,
+ "epoch": 2.519191919191919,
+ "grad_norm": 2.3396496772766113,
+ "learning_rate": 2.482828282828283e-06,
+ "loss": 1.3288640975952148,
+ "mean_token_accuracy": 0.6720811128616333,
+ "num_tokens": 20430848.0,
+ "step": 1247
+ },
+ {
+ "entropy": 1.5269931554794312,
+ "epoch": 2.5212121212121215,
+ "grad_norm": 2.303053617477417,
+ "learning_rate": 2.480808080808081e-06,
+ "loss": 1.4953217506408691,
+ "mean_token_accuracy": 0.639167070388794,
+ "num_tokens": 20447232.0,
+ "step": 1248
+ },
+ {
+ "entropy": 1.3399546146392822,
+ "epoch": 2.5232323232323233,
+ "grad_norm": 2.3238017559051514,
+ "learning_rate": 2.478787878787879e-06,
+ "loss": 1.337552547454834,
+ "mean_token_accuracy": 0.6790425181388855,
+ "num_tokens": 20463616.0,
+ "step": 1249
+ },
+ {
+ "entropy": 1.0954687595367432,
+ "epoch": 2.525252525252525,
+ "grad_norm": 2.2266361713409424,
+ "learning_rate": 2.476767676767677e-06,
+ "loss": 1.074552297592163,
+ "mean_token_accuracy": 0.7192232608795166,
+ "num_tokens": 20480000.0,
+ "step": 1250
+ },
+ {
+ "epoch": 2.525252525252525,
+ "eval_entropy": 1.375552624464035,
+ "eval_loss": 1.5356626510620117,
+ "eval_mean_token_accuracy": 0.6455749580937047,
+ "eval_num_tokens": 20480000.0,
+ "eval_runtime": 43.8024,
+ "eval_samples_per_second": 22.602,
+ "eval_steps_per_second": 2.831,
+ "step": 1250
+ },
+ {
+ "entropy": 1.329379916191101,
+ "epoch": 2.5272727272727273,
+ "grad_norm": 2.288421392440796,
+ "learning_rate": 2.474747474747475e-06,
+ "loss": 1.3166120052337646,
+ "mean_token_accuracy": 0.6743404865264893,
+ "num_tokens": 20496384.0,
+ "step": 1251
+ },
+ {
+ "entropy": 1.3155947923660278,
+ "epoch": 2.529292929292929,
+ "grad_norm": 2.2791004180908203,
+ "learning_rate": 2.472727272727273e-06,
+ "loss": 1.3281152248382568,
+ "mean_token_accuracy": 0.6623717546463013,
+ "num_tokens": 20512768.0,
+ "step": 1252
+ },
+ {
+ "entropy": 1.2875827550888062,
+ "epoch": 2.5313131313131314,
+ "grad_norm": 2.193981409072876,
+ "learning_rate": 2.470707070707071e-06,
+ "loss": 1.280174732208252,
+ "mean_token_accuracy": 0.6996824741363525,
+ "num_tokens": 20529152.0,
+ "step": 1253
+ },
+ {
+ "entropy": 1.1551588773727417,
+ "epoch": 2.533333333333333,
+ "grad_norm": 2.297534942626953,
+ "learning_rate": 2.468686868686869e-06,
+ "loss": 1.1548980474472046,
+ "mean_token_accuracy": 0.7089643478393555,
+ "num_tokens": 20545536.0,
+ "step": 1254
+ },
+ {
+ "entropy": 1.1836578845977783,
+ "epoch": 2.5353535353535355,
+ "grad_norm": 2.1724929809570312,
+ "learning_rate": 2.466666666666667e-06,
+ "loss": 1.1787464618682861,
+ "mean_token_accuracy": 0.705422580242157,
+ "num_tokens": 20561920.0,
+ "step": 1255
+ },
+ {
+ "entropy": 1.0938197374343872,
+ "epoch": 2.5373737373737373,
+ "grad_norm": 2.247507095336914,
+ "learning_rate": 2.464646464646465e-06,
+ "loss": 1.0977723598480225,
+ "mean_token_accuracy": 0.7205055952072144,
+ "num_tokens": 20578304.0,
+ "step": 1256
+ },
+ {
+ "entropy": 1.790277361869812,
+ "epoch": 2.5393939393939395,
+ "grad_norm": 2.314159631729126,
+ "learning_rate": 2.462626262626263e-06,
+ "loss": 1.792272925376892,
+ "mean_token_accuracy": 0.6231069564819336,
+ "num_tokens": 20594688.0,
+ "step": 1257
+ },
+ {
+ "entropy": 1.385352611541748,
+ "epoch": 2.5414141414141413,
+ "grad_norm": 2.2065250873565674,
+ "learning_rate": 2.4606060606060607e-06,
+ "loss": 1.4161381721496582,
+ "mean_token_accuracy": 0.673363447189331,
+ "num_tokens": 20611072.0,
+ "step": 1258
+ },
+ {
+ "entropy": 1.5186140537261963,
+ "epoch": 2.5434343434343436,
+ "grad_norm": 2.277163028717041,
+ "learning_rate": 2.4585858585858586e-06,
+ "loss": 1.5157082080841064,
+ "mean_token_accuracy": 0.6420371532440186,
+ "num_tokens": 20627456.0,
+ "step": 1259
+ },
+ {
+ "entropy": 1.6179429292678833,
+ "epoch": 2.5454545454545454,
+ "grad_norm": 2.1880221366882324,
+ "learning_rate": 2.456565656565657e-06,
+ "loss": 1.6365107297897339,
+ "mean_token_accuracy": 0.6164509057998657,
+ "num_tokens": 20643840.0,
+ "step": 1260
+ },
+ {
+ "entropy": 1.5839602947235107,
+ "epoch": 2.5474747474747472,
+ "grad_norm": 2.3658924102783203,
+ "learning_rate": 2.454545454545455e-06,
+ "loss": 1.593346118927002,
+ "mean_token_accuracy": 0.6339765787124634,
+ "num_tokens": 20660224.0,
+ "step": 1261
+ },
+ {
+ "entropy": 1.3825792074203491,
+ "epoch": 2.5494949494949495,
+ "grad_norm": 2.3293564319610596,
+ "learning_rate": 2.4525252525252527e-06,
+ "loss": 1.44181489944458,
+ "mean_token_accuracy": 0.6660356521606445,
+ "num_tokens": 20676608.0,
+ "step": 1262
+ },
+ {
+ "entropy": 1.0796198844909668,
+ "epoch": 2.5515151515151517,
+ "grad_norm": 2.1055870056152344,
+ "learning_rate": 2.4505050505050506e-06,
+ "loss": 1.0861892700195312,
+ "mean_token_accuracy": 0.7263067960739136,
+ "num_tokens": 20692992.0,
+ "step": 1263
+ },
+ {
+ "entropy": 1.4059629440307617,
+ "epoch": 2.5535353535353535,
+ "grad_norm": 2.137063980102539,
+ "learning_rate": 2.4484848484848485e-06,
+ "loss": 1.4127235412597656,
+ "mean_token_accuracy": 0.6654250025749207,
+ "num_tokens": 20709376.0,
+ "step": 1264
+ },
+ {
+ "entropy": 1.2902467250823975,
+ "epoch": 2.5555555555555554,
+ "grad_norm": 2.175401210784912,
+ "learning_rate": 2.4464646464646468e-06,
+ "loss": 1.2854642868041992,
+ "mean_token_accuracy": 0.6966292262077332,
+ "num_tokens": 20725760.0,
+ "step": 1265
+ },
+ {
+ "entropy": 1.7958943843841553,
+ "epoch": 2.5575757575757576,
+ "grad_norm": 2.3500523567199707,
+ "learning_rate": 2.4444444444444447e-06,
+ "loss": 1.811906337738037,
+ "mean_token_accuracy": 0.5961162447929382,
+ "num_tokens": 20742144.0,
+ "step": 1266
+ },
+ {
+ "entropy": 1.7406131029129028,
+ "epoch": 2.55959595959596,
+ "grad_norm": 2.3911163806915283,
+ "learning_rate": 2.4424242424242426e-06,
+ "loss": 1.7608356475830078,
+ "mean_token_accuracy": 0.6002076268196106,
+ "num_tokens": 20758528.0,
+ "step": 1267
+ },
+ {
+ "entropy": 1.440137267112732,
+ "epoch": 2.5616161616161617,
+ "grad_norm": 2.057145833969116,
+ "learning_rate": 2.4404040404040404e-06,
+ "loss": 1.4450485706329346,
+ "mean_token_accuracy": 0.6570591330528259,
+ "num_tokens": 20774912.0,
+ "step": 1268
+ },
+ {
+ "entropy": 1.510685682296753,
+ "epoch": 2.5636363636363635,
+ "grad_norm": 2.379920244216919,
+ "learning_rate": 2.4383838383838383e-06,
+ "loss": 1.5113954544067383,
+ "mean_token_accuracy": 0.6450293064117432,
+ "num_tokens": 20791296.0,
+ "step": 1269
+ },
+ {
+ "entropy": 1.3595963716506958,
+ "epoch": 2.5656565656565657,
+ "grad_norm": 2.226229190826416,
+ "learning_rate": 2.4363636363636366e-06,
+ "loss": 1.3598928451538086,
+ "mean_token_accuracy": 0.6723864078521729,
+ "num_tokens": 20807680.0,
+ "step": 1270
+ },
+ {
+ "entropy": 1.3053373098373413,
+ "epoch": 2.5676767676767676,
+ "grad_norm": 2.181692361831665,
+ "learning_rate": 2.4343434343434345e-06,
+ "loss": 1.283501148223877,
+ "mean_token_accuracy": 0.672874927520752,
+ "num_tokens": 20824064.0,
+ "step": 1271
+ },
+ {
+ "entropy": 0.9631388187408447,
+ "epoch": 2.56969696969697,
+ "grad_norm": 2.083442449569702,
+ "learning_rate": 2.432323232323233e-06,
+ "loss": 0.9463216066360474,
+ "mean_token_accuracy": 0.7476184368133545,
+ "num_tokens": 20840448.0,
+ "step": 1272
+ },
+ {
+ "entropy": 1.0792899131774902,
+ "epoch": 2.5717171717171716,
+ "grad_norm": 2.204257011413574,
+ "learning_rate": 2.4303030303030307e-06,
+ "loss": 1.0787214040756226,
+ "mean_token_accuracy": 0.7241694927215576,
+ "num_tokens": 20856832.0,
+ "step": 1273
+ },
+ {
+ "entropy": 1.0843226909637451,
+ "epoch": 2.573737373737374,
+ "grad_norm": 2.0145976543426514,
+ "learning_rate": 2.4282828282828286e-06,
+ "loss": 1.0722554922103882,
+ "mean_token_accuracy": 0.7263678312301636,
+ "num_tokens": 20873216.0,
+ "step": 1274
+ },
+ {
+ "entropy": 1.5040221214294434,
+ "epoch": 2.5757575757575757,
+ "grad_norm": 2.3076000213623047,
+ "learning_rate": 2.4262626262626265e-06,
+ "loss": 1.5152506828308105,
+ "mean_token_accuracy": 0.6420371532440186,
+ "num_tokens": 20889600.0,
+ "step": 1275
+ },
+ {
+ "entropy": 1.5882561206817627,
+ "epoch": 2.5777777777777775,
+ "grad_norm": 2.093400716781616,
+ "learning_rate": 2.4242424242424244e-06,
+ "loss": 1.5827500820159912,
+ "mean_token_accuracy": 0.6293356418609619,
+ "num_tokens": 20905984.0,
+ "step": 1276
+ },
+ {
+ "entropy": 1.1761770248413086,
+ "epoch": 2.5797979797979798,
+ "grad_norm": 1.9723676443099976,
+ "learning_rate": 2.4222222222222223e-06,
+ "loss": 1.1729084253311157,
+ "mean_token_accuracy": 0.7160478830337524,
+ "num_tokens": 20922368.0,
+ "step": 1277
+ },
+ {
+ "entropy": 1.5020796060562134,
+ "epoch": 2.581818181818182,
+ "grad_norm": 2.198739767074585,
+ "learning_rate": 2.4202020202020206e-06,
+ "loss": 1.5012025833129883,
+ "mean_token_accuracy": 0.6408768892288208,
+ "num_tokens": 20938752.0,
+ "step": 1278
+ },
+ {
+ "entropy": 1.0234061479568481,
+ "epoch": 2.583838383838384,
+ "grad_norm": 2.0784571170806885,
+ "learning_rate": 2.4181818181818185e-06,
+ "loss": 1.0343588590621948,
+ "mean_token_accuracy": 0.7372984886169434,
+ "num_tokens": 20955136.0,
+ "step": 1279
+ },
+ {
+ "entropy": 1.6372766494750977,
+ "epoch": 2.5858585858585856,
+ "grad_norm": 2.339104652404785,
+ "learning_rate": 2.4161616161616164e-06,
+ "loss": 1.6200227737426758,
+ "mean_token_accuracy": 0.6249389052391052,
+ "num_tokens": 20971520.0,
+ "step": 1280
+ },
+ {
+ "entropy": 1.3991668224334717,
+ "epoch": 2.587878787878788,
+ "grad_norm": 2.33329439163208,
+ "learning_rate": 2.4141414141414143e-06,
+ "loss": 1.434525489807129,
+ "mean_token_accuracy": 0.6656692624092102,
+ "num_tokens": 20987904.0,
+ "step": 1281
+ },
+ {
+ "entropy": 0.9680843949317932,
+ "epoch": 2.58989898989899,
+ "grad_norm": 2.0454914569854736,
+ "learning_rate": 2.412121212121212e-06,
+ "loss": 0.9875590801239014,
+ "mean_token_accuracy": 0.7512823939323425,
+ "num_tokens": 21004288.0,
+ "step": 1282
+ },
+ {
+ "entropy": 1.1564005613327026,
+ "epoch": 2.591919191919192,
+ "grad_norm": 2.0994997024536133,
+ "learning_rate": 2.4101010101010105e-06,
+ "loss": 1.1473734378814697,
+ "mean_token_accuracy": 0.709208607673645,
+ "num_tokens": 21020672.0,
+ "step": 1283
+ },
+ {
+ "entropy": 1.435438871383667,
+ "epoch": 2.5939393939393938,
+ "grad_norm": 2.369999408721924,
+ "learning_rate": 2.4080808080808083e-06,
+ "loss": 1.4502406120300293,
+ "mean_token_accuracy": 0.6505251526832581,
+ "num_tokens": 21037056.0,
+ "step": 1284
+ },
+ {
+ "entropy": 1.358853816986084,
+ "epoch": 2.595959595959596,
+ "grad_norm": 4.034506320953369,
+ "learning_rate": 2.4060606060606062e-06,
+ "loss": 1.3811532258987427,
+ "mean_token_accuracy": 0.6720200181007385,
+ "num_tokens": 21053440.0,
+ "step": 1285
+ },
+ {
+ "entropy": 1.2741261720657349,
+ "epoch": 2.597979797979798,
+ "grad_norm": 2.231471061706543,
+ "learning_rate": 2.404040404040404e-06,
+ "loss": 1.2811863422393799,
+ "mean_token_accuracy": 0.6806302070617676,
+ "num_tokens": 21069824.0,
+ "step": 1286
+ },
+ {
+ "entropy": 1.3789101839065552,
+ "epoch": 2.6,
+ "grad_norm": 2.287457227706909,
+ "learning_rate": 2.402020202020202e-06,
+ "loss": 1.389050006866455,
+ "mean_token_accuracy": 0.6687836050987244,
+ "num_tokens": 21086208.0,
+ "step": 1287
+ },
+ {
+ "entropy": 1.0021989345550537,
+ "epoch": 2.602020202020202,
+ "grad_norm": 1.9996925592422485,
+ "learning_rate": 2.4000000000000003e-06,
+ "loss": 1.028086543083191,
+ "mean_token_accuracy": 0.7479237914085388,
+ "num_tokens": 21102592.0,
+ "step": 1288
+ },
+ {
+ "entropy": 1.4363199472427368,
+ "epoch": 2.604040404040404,
+ "grad_norm": 2.203239917755127,
+ "learning_rate": 2.397979797979798e-06,
+ "loss": 1.4645861387252808,
+ "mean_token_accuracy": 0.6590132117271423,
+ "num_tokens": 21118976.0,
+ "step": 1289
+ },
+ {
+ "entropy": 1.307484745979309,
+ "epoch": 2.606060606060606,
+ "grad_norm": 1.9925199747085571,
+ "learning_rate": 2.395959595959596e-06,
+ "loss": 1.3217129707336426,
+ "mean_token_accuracy": 0.6787371635437012,
+ "num_tokens": 21135360.0,
+ "step": 1290
+ },
+ {
+ "entropy": 1.2430635690689087,
+ "epoch": 2.608080808080808,
+ "grad_norm": 2.1385529041290283,
+ "learning_rate": 2.393939393939394e-06,
+ "loss": 1.238807201385498,
+ "mean_token_accuracy": 0.6949804425239563,
+ "num_tokens": 21151744.0,
+ "step": 1291
+ },
+ {
+ "entropy": 1.0351316928863525,
+ "epoch": 2.61010101010101,
+ "grad_norm": 2.2897744178771973,
+ "learning_rate": 2.3919191919191923e-06,
+ "loss": 1.032408595085144,
+ "mean_token_accuracy": 0.7267953157424927,
+ "num_tokens": 21168128.0,
+ "step": 1292
+ },
+ {
+ "entropy": 1.2167868614196777,
+ "epoch": 2.6121212121212123,
+ "grad_norm": 2.021771192550659,
+ "learning_rate": 2.38989898989899e-06,
+ "loss": 1.223731279373169,
+ "mean_token_accuracy": 0.7067049145698547,
+ "num_tokens": 21184512.0,
+ "step": 1293
+ },
+ {
+ "entropy": 1.0386477708816528,
+ "epoch": 2.614141414141414,
+ "grad_norm": 2.048285722732544,
+ "learning_rate": 2.387878787878788e-06,
+ "loss": 1.0437978506088257,
+ "mean_token_accuracy": 0.7332682013511658,
+ "num_tokens": 21200896.0,
+ "step": 1294
+ },
+ {
+ "entropy": 1.5832154750823975,
+ "epoch": 2.616161616161616,
+ "grad_norm": 2.214010715484619,
+ "learning_rate": 2.385858585858586e-06,
+ "loss": 1.5869958400726318,
+ "mean_token_accuracy": 0.6240839958190918,
+ "num_tokens": 21217280.0,
+ "step": 1295
+ },
+ {
+ "entropy": 1.4032398462295532,
+ "epoch": 2.618181818181818,
+ "grad_norm": 2.069399833679199,
+ "learning_rate": 2.3838383838383843e-06,
+ "loss": 1.40470290184021,
+ "mean_token_accuracy": 0.6671348214149475,
+ "num_tokens": 21233664.0,
+ "step": 1296
+ },
+ {
+ "entropy": 1.1057566404342651,
+ "epoch": 2.6202020202020204,
+ "grad_norm": 2.090847969055176,
+ "learning_rate": 2.381818181818182e-06,
+ "loss": 1.099064826965332,
+ "mean_token_accuracy": 0.7186736464500427,
+ "num_tokens": 21250048.0,
+ "step": 1297
+ },
+ {
+ "entropy": 1.1217924356460571,
+ "epoch": 2.6222222222222222,
+ "grad_norm": 2.2802088260650635,
+ "learning_rate": 2.37979797979798e-06,
+ "loss": 1.1425195932388306,
+ "mean_token_accuracy": 0.717452347278595,
+ "num_tokens": 21266432.0,
+ "step": 1298
+ },
+ {
+ "entropy": 1.3103245496749878,
+ "epoch": 2.624242424242424,
+ "grad_norm": 2.2961838245391846,
+ "learning_rate": 2.377777777777778e-06,
+ "loss": 1.325084924697876,
+ "mean_token_accuracy": 0.6704323291778564,
+ "num_tokens": 21282816.0,
+ "step": 1299
+ },
+ {
+ "entropy": 1.397556185722351,
+ "epoch": 2.6262626262626263,
+ "grad_norm": 2.14385986328125,
+ "learning_rate": 2.375757575757576e-06,
+ "loss": 1.3891851902008057,
+ "mean_token_accuracy": 0.6722642779350281,
+ "num_tokens": 21299200.0,
+ "step": 1300
+ },
+ {
+ "epoch": 2.6262626262626263,
+ "eval_entropy": 1.377457697064646,
+ "eval_loss": 1.5343767404556274,
+ "eval_mean_token_accuracy": 0.6456591679203895,
+ "eval_num_tokens": 21299200.0,
+ "eval_runtime": 43.8661,
+ "eval_samples_per_second": 22.569,
+ "eval_steps_per_second": 2.827,
+ "step": 1300
+ },
+ {
+ "entropy": 1.4542138576507568,
+ "epoch": 2.6282828282828286,
+ "grad_norm": 2.2326884269714355,
+ "learning_rate": 2.373737373737374e-06,
+ "loss": 1.459869384765625,
+ "mean_token_accuracy": 0.6545554399490356,
+ "num_tokens": 21315584.0,
+ "step": 1301
+ },
+ {
+ "entropy": 1.144904613494873,
+ "epoch": 2.6303030303030304,
+ "grad_norm": 2.0662593841552734,
+ "learning_rate": 2.371717171717172e-06,
+ "loss": 1.151012897491455,
+ "mean_token_accuracy": 0.7232535481452942,
+ "num_tokens": 21331968.0,
+ "step": 1302
+ },
+ {
+ "entropy": 1.2142722606658936,
+ "epoch": 2.632323232323232,
+ "grad_norm": 2.206763982772827,
+ "learning_rate": 2.36969696969697e-06,
+ "loss": 1.2040069103240967,
+ "mean_token_accuracy": 0.7002320289611816,
+ "num_tokens": 21348352.0,
+ "step": 1303
+ },
+ {
+ "entropy": 1.7695715427398682,
+ "epoch": 2.6343434343434344,
+ "grad_norm": 2.1346793174743652,
+ "learning_rate": 2.367676767676768e-06,
+ "loss": 1.7356187105178833,
+ "mean_token_accuracy": 0.6051538586616516,
+ "num_tokens": 21364736.0,
+ "step": 1304
+ },
+ {
+ "entropy": 1.351786494255066,
+ "epoch": 2.6363636363636362,
+ "grad_norm": 2.291290283203125,
+ "learning_rate": 2.3656565656565657e-06,
+ "loss": 1.3471777439117432,
+ "mean_token_accuracy": 0.6797142028808594,
+ "num_tokens": 21381120.0,
+ "step": 1305
+ },
+ {
+ "entropy": 1.202124834060669,
+ "epoch": 2.6383838383838385,
+ "grad_norm": 2.1541965007781982,
+ "learning_rate": 2.363636363636364e-06,
+ "loss": 1.2085479497909546,
+ "mean_token_accuracy": 0.7031631469726562,
+ "num_tokens": 21397504.0,
+ "step": 1306
+ },
+ {
+ "entropy": 0.9269154667854309,
+ "epoch": 2.6404040404040403,
+ "grad_norm": 2.1681666374206543,
+ "learning_rate": 2.361616161616162e-06,
+ "loss": 0.9319549798965454,
+ "mean_token_accuracy": 0.7553126811981201,
+ "num_tokens": 21413888.0,
+ "step": 1307
+ },
+ {
+ "entropy": 1.0244951248168945,
+ "epoch": 2.6424242424242426,
+ "grad_norm": 1.9392226934432983,
+ "learning_rate": 2.3595959595959598e-06,
+ "loss": 0.9993175268173218,
+ "mean_token_accuracy": 0.7588544487953186,
+ "num_tokens": 21430272.0,
+ "step": 1308
+ },
+ {
+ "entropy": 1.256042718887329,
+ "epoch": 2.6444444444444444,
+ "grad_norm": 2.346327304840088,
+ "learning_rate": 2.3575757575757577e-06,
+ "loss": 1.2512779235839844,
+ "mean_token_accuracy": 0.6858206987380981,
+ "num_tokens": 21446656.0,
+ "step": 1309
+ },
+ {
+ "entropy": 1.0318644046783447,
+ "epoch": 2.6464646464646466,
+ "grad_norm": 1.988356113433838,
+ "learning_rate": 2.3555555555555555e-06,
+ "loss": 1.0428593158721924,
+ "mean_token_accuracy": 0.7383365631103516,
+ "num_tokens": 21463040.0,
+ "step": 1310
+ },
+ {
+ "entropy": 1.3145314455032349,
+ "epoch": 2.6484848484848484,
+ "grad_norm": 2.1777143478393555,
+ "learning_rate": 2.3535353535353534e-06,
+ "loss": 1.3207441568374634,
+ "mean_token_accuracy": 0.6881411671638489,
+ "num_tokens": 21479424.0,
+ "step": 1311
+ },
+ {
+ "entropy": 1.2858450412750244,
+ "epoch": 2.6505050505050507,
+ "grad_norm": 2.2654671669006348,
+ "learning_rate": 2.3515151515151517e-06,
+ "loss": 1.2940181493759155,
+ "mean_token_accuracy": 0.6831338405609131,
+ "num_tokens": 21495808.0,
+ "step": 1312
+ },
+ {
+ "entropy": 1.4315612316131592,
+ "epoch": 2.6525252525252525,
+ "grad_norm": 2.140456438064575,
+ "learning_rate": 2.3494949494949496e-06,
+ "loss": 1.4298607110977173,
+ "mean_token_accuracy": 0.6585246920585632,
+ "num_tokens": 21512192.0,
+ "step": 1313
+ },
+ {
+ "entropy": 1.2628042697906494,
+ "epoch": 2.6545454545454543,
+ "grad_norm": 2.387079954147339,
+ "learning_rate": 2.347474747474748e-06,
+ "loss": 1.2526830434799194,
+ "mean_token_accuracy": 0.6882632970809937,
+ "num_tokens": 21528576.0,
+ "step": 1314
+ },
+ {
+ "entropy": 1.3501923084259033,
+ "epoch": 2.6565656565656566,
+ "grad_norm": 2.17873215675354,
+ "learning_rate": 2.345454545454546e-06,
+ "loss": 1.3459683656692505,
+ "mean_token_accuracy": 0.6783707737922668,
+ "num_tokens": 21544960.0,
+ "step": 1315
+ },
+ {
+ "entropy": 1.4369540214538574,
+ "epoch": 2.658585858585859,
+ "grad_norm": 2.335825204849243,
+ "learning_rate": 2.3434343434343437e-06,
+ "loss": 1.4296600818634033,
+ "mean_token_accuracy": 0.6534562706947327,
+ "num_tokens": 21561344.0,
+ "step": 1316
+ },
+ {
+ "entropy": 1.1612765789031982,
+ "epoch": 2.6606060606060606,
+ "grad_norm": 2.194164276123047,
+ "learning_rate": 2.3414141414141416e-06,
+ "loss": 1.1527385711669922,
+ "mean_token_accuracy": 0.7140327095985413,
+ "num_tokens": 21577728.0,
+ "step": 1317
+ },
+ {
+ "entropy": 1.0844523906707764,
+ "epoch": 2.6626262626262625,
+ "grad_norm": 2.1018459796905518,
+ "learning_rate": 2.3393939393939395e-06,
+ "loss": 1.0732126235961914,
+ "mean_token_accuracy": 0.7245969772338867,
+ "num_tokens": 21594112.0,
+ "step": 1318
+ },
+ {
+ "entropy": 1.1760655641555786,
+ "epoch": 2.6646464646464647,
+ "grad_norm": 2.1674582958221436,
+ "learning_rate": 2.337373737373738e-06,
+ "loss": 1.1928012371063232,
+ "mean_token_accuracy": 0.693453848361969,
+ "num_tokens": 21610496.0,
+ "step": 1319
+ },
+ {
+ "entropy": 1.3627854585647583,
+ "epoch": 2.6666666666666665,
+ "grad_norm": 2.2469325065612793,
+ "learning_rate": 2.3353535353535357e-06,
+ "loss": 1.375096321105957,
+ "mean_token_accuracy": 0.6670737862586975,
+ "num_tokens": 21626880.0,
+ "step": 1320
+ },
+ {
+ "entropy": 1.0562385320663452,
+ "epoch": 2.6686868686868688,
+ "grad_norm": 2.3479180335998535,
+ "learning_rate": 2.3333333333333336e-06,
+ "loss": 1.0513578653335571,
+ "mean_token_accuracy": 0.7401685118675232,
+ "num_tokens": 21643264.0,
+ "step": 1321
+ },
+ {
+ "entropy": 1.1622484922409058,
+ "epoch": 2.6707070707070706,
+ "grad_norm": 2.218021869659424,
+ "learning_rate": 2.3313131313131315e-06,
+ "loss": 1.172579288482666,
+ "mean_token_accuracy": 0.7042623162269592,
+ "num_tokens": 21659648.0,
+ "step": 1322
+ },
+ {
+ "entropy": 1.2139835357666016,
+ "epoch": 2.672727272727273,
+ "grad_norm": 2.2875492572784424,
+ "learning_rate": 2.3292929292929294e-06,
+ "loss": 1.2015337944030762,
+ "mean_token_accuracy": 0.7027357220649719,
+ "num_tokens": 21676032.0,
+ "step": 1323
+ },
+ {
+ "entropy": 1.7094130516052246,
+ "epoch": 2.6747474747474747,
+ "grad_norm": 2.2344024181365967,
+ "learning_rate": 2.3272727272727277e-06,
+ "loss": 1.7565287351608276,
+ "mean_token_accuracy": 0.6025891304016113,
+ "num_tokens": 21692416.0,
+ "step": 1324
+ },
+ {
+ "entropy": 1.1033966541290283,
+ "epoch": 2.676767676767677,
+ "grad_norm": 2.2697556018829346,
+ "learning_rate": 2.3252525252525256e-06,
+ "loss": 1.1173789501190186,
+ "mean_token_accuracy": 0.7180629968643188,
+ "num_tokens": 21708800.0,
+ "step": 1325
+ },
+ {
+ "entropy": 0.9456706643104553,
+ "epoch": 2.6787878787878787,
+ "grad_norm": 2.1194941997528076,
+ "learning_rate": 2.3232323232323234e-06,
+ "loss": 0.9613161087036133,
+ "mean_token_accuracy": 0.7573277950286865,
+ "num_tokens": 21725184.0,
+ "step": 1326
+ },
+ {
+ "entropy": 1.098917841911316,
+ "epoch": 2.680808080808081,
+ "grad_norm": 2.2719058990478516,
+ "learning_rate": 2.3212121212121213e-06,
+ "loss": 1.116957664489746,
+ "mean_token_accuracy": 0.709269642829895,
+ "num_tokens": 21741568.0,
+ "step": 1327
+ },
+ {
+ "entropy": 1.5637823343276978,
+ "epoch": 2.682828282828283,
+ "grad_norm": 2.257596731185913,
+ "learning_rate": 2.3191919191919192e-06,
+ "loss": 1.5726745128631592,
+ "mean_token_accuracy": 0.6359306573867798,
+ "num_tokens": 21757952.0,
+ "step": 1328
+ },
+ {
+ "entropy": 0.9321528673171997,
+ "epoch": 2.6848484848484846,
+ "grad_norm": 2.0641798973083496,
+ "learning_rate": 2.317171717171717e-06,
+ "loss": 0.9297301769256592,
+ "mean_token_accuracy": 0.7602589130401611,
+ "num_tokens": 21774336.0,
+ "step": 1329
+ },
+ {
+ "entropy": 1.2314496040344238,
+ "epoch": 2.686868686868687,
+ "grad_norm": 2.143329620361328,
+ "learning_rate": 2.3151515151515154e-06,
+ "loss": 1.2292556762695312,
+ "mean_token_accuracy": 0.6940034031867981,
+ "num_tokens": 21790720.0,
+ "step": 1330
+ },
+ {
+ "entropy": 1.195270299911499,
+ "epoch": 2.688888888888889,
+ "grad_norm": 2.291335105895996,
+ "learning_rate": 2.3131313131313133e-06,
+ "loss": 1.1904417276382446,
+ "mean_token_accuracy": 0.7023082375526428,
+ "num_tokens": 21807104.0,
+ "step": 1331
+ },
+ {
+ "entropy": 0.8783636093139648,
+ "epoch": 2.690909090909091,
+ "grad_norm": 2.069147825241089,
+ "learning_rate": 2.311111111111111e-06,
+ "loss": 0.8936513066291809,
+ "mean_token_accuracy": 0.7703346610069275,
+ "num_tokens": 21823488.0,
+ "step": 1332
+ },
+ {
+ "entropy": 1.3100625276565552,
+ "epoch": 2.6929292929292927,
+ "grad_norm": 2.396043300628662,
+ "learning_rate": 2.309090909090909e-06,
+ "loss": 1.3054237365722656,
+ "mean_token_accuracy": 0.6758671402931213,
+ "num_tokens": 21839872.0,
+ "step": 1333
+ },
+ {
+ "entropy": 0.9413285255432129,
+ "epoch": 2.694949494949495,
+ "grad_norm": 2.0738699436187744,
+ "learning_rate": 2.307070707070707e-06,
+ "loss": 0.9171029329299927,
+ "mean_token_accuracy": 0.7571446299552917,
+ "num_tokens": 21856256.0,
+ "step": 1334
+ },
+ {
+ "entropy": 1.268250584602356,
+ "epoch": 2.6969696969696972,
+ "grad_norm": 2.0988986492156982,
+ "learning_rate": 2.3050505050505053e-06,
+ "loss": 1.257792353630066,
+ "mean_token_accuracy": 0.6866145730018616,
+ "num_tokens": 21872640.0,
+ "step": 1335
+ },
+ {
+ "entropy": 1.3402502536773682,
+ "epoch": 2.698989898989899,
+ "grad_norm": 2.08109712600708,
+ "learning_rate": 2.303030303030303e-06,
+ "loss": 1.3316476345062256,
+ "mean_token_accuracy": 0.6869198679924011,
+ "num_tokens": 21889024.0,
+ "step": 1336
+ },
+ {
+ "entropy": 1.5612709522247314,
+ "epoch": 2.701010101010101,
+ "grad_norm": 2.2410786151885986,
+ "learning_rate": 2.3010101010101015e-06,
+ "loss": 1.5604108572006226,
+ "mean_token_accuracy": 0.6339765787124634,
+ "num_tokens": 21905408.0,
+ "step": 1337
+ },
+ {
+ "entropy": 1.0962555408477783,
+ "epoch": 2.703030303030303,
+ "grad_norm": 2.289069890975952,
+ "learning_rate": 2.2989898989898994e-06,
+ "loss": 1.1016948223114014,
+ "mean_token_accuracy": 0.714093804359436,
+ "num_tokens": 21921792.0,
+ "step": 1338
+ },
+ {
+ "entropy": 1.600198745727539,
+ "epoch": 2.705050505050505,
+ "grad_norm": 2.152244806289673,
+ "learning_rate": 2.2969696969696973e-06,
+ "loss": 1.608877420425415,
+ "mean_token_accuracy": 0.6394724249839783,
+ "num_tokens": 21938176.0,
+ "step": 1339
+ },
+ {
+ "entropy": 1.4823815822601318,
+ "epoch": 2.707070707070707,
+ "grad_norm": 2.142822742462158,
+ "learning_rate": 2.294949494949495e-06,
+ "loss": 1.4775569438934326,
+ "mean_token_accuracy": 0.6656082272529602,
+ "num_tokens": 21954560.0,
+ "step": 1340
+ },
+ {
+ "entropy": 1.1100481748580933,
+ "epoch": 2.709090909090909,
+ "grad_norm": 2.271402597427368,
+ "learning_rate": 2.292929292929293e-06,
+ "loss": 1.1097328662872314,
+ "mean_token_accuracy": 0.7238031029701233,
+ "num_tokens": 21970944.0,
+ "step": 1341
+ },
+ {
+ "entropy": 1.3482670783996582,
+ "epoch": 2.7111111111111112,
+ "grad_norm": 2.136296510696411,
+ "learning_rate": 2.2909090909090913e-06,
+ "loss": 1.3715920448303223,
+ "mean_token_accuracy": 0.670615553855896,
+ "num_tokens": 21987328.0,
+ "step": 1342
+ },
+ {
+ "entropy": 1.3454313278198242,
+ "epoch": 2.713131313131313,
+ "grad_norm": 1.8919764757156372,
+ "learning_rate": 2.2888888888888892e-06,
+ "loss": 1.3545129299163818,
+ "mean_token_accuracy": 0.6950415372848511,
+ "num_tokens": 22003712.0,
+ "step": 1343
+ },
+ {
+ "entropy": 1.314072847366333,
+ "epoch": 2.7151515151515153,
+ "grad_norm": 2.0495684146881104,
+ "learning_rate": 2.286868686868687e-06,
+ "loss": 1.3117344379425049,
+ "mean_token_accuracy": 0.6800195574760437,
+ "num_tokens": 22020096.0,
+ "step": 1344
+ },
+ {
+ "entropy": 0.7977169156074524,
+ "epoch": 2.717171717171717,
+ "grad_norm": 2.050006866455078,
+ "learning_rate": 2.284848484848485e-06,
+ "loss": 0.7982609868049622,
+ "mean_token_accuracy": 0.7774181962013245,
+ "num_tokens": 22036480.0,
+ "step": 1345
+ },
+ {
+ "entropy": 1.6121824979782104,
+ "epoch": 2.7191919191919194,
+ "grad_norm": 2.1058602333068848,
+ "learning_rate": 2.282828282828283e-06,
+ "loss": 1.6340572834014893,
+ "mean_token_accuracy": 0.6278700828552246,
+ "num_tokens": 22052864.0,
+ "step": 1346
+ },
+ {
+ "entropy": 1.2698943614959717,
+ "epoch": 2.721212121212121,
+ "grad_norm": 1.9856007099151611,
+ "learning_rate": 2.2808080808080808e-06,
+ "loss": 1.2857555150985718,
+ "mean_token_accuracy": 0.7088422179222107,
+ "num_tokens": 22069248.0,
+ "step": 1347
+ },
+ {
+ "entropy": 1.6614608764648438,
+ "epoch": 2.723232323232323,
+ "grad_norm": 2.1818666458129883,
+ "learning_rate": 2.278787878787879e-06,
+ "loss": 1.6615056991577148,
+ "mean_token_accuracy": 0.6217635273933411,
+ "num_tokens": 22085632.0,
+ "step": 1348
+ },
+ {
+ "entropy": 1.542864203453064,
+ "epoch": 2.7252525252525253,
+ "grad_norm": 2.1022093296051025,
+ "learning_rate": 2.276767676767677e-06,
+ "loss": 1.5696978569030762,
+ "mean_token_accuracy": 0.6462506055831909,
+ "num_tokens": 22102016.0,
+ "step": 1349
+ },
+ {
+ "entropy": 1.0613912343978882,
+ "epoch": 2.7272727272727275,
+ "grad_norm": 2.213463068008423,
+ "learning_rate": 2.274747474747475e-06,
+ "loss": 1.0789921283721924,
+ "mean_token_accuracy": 0.7288104295730591,
+ "num_tokens": 22118400.0,
+ "step": 1350
+ },
+ {
+ "epoch": 2.7272727272727275,
+ "eval_entropy": 1.362602738603469,
+ "eval_loss": 1.5345665216445923,
+ "eval_mean_token_accuracy": 0.6458812678052533,
+ "eval_num_tokens": 22118400.0,
+ "eval_runtime": 43.734,
+ "eval_samples_per_second": 22.637,
+ "eval_steps_per_second": 2.835,
+ "step": 1350
+ },
+ {
+ "entropy": 1.175439715385437,
+ "epoch": 2.7292929292929293,
+ "grad_norm": 2.3636603355407715,
+ "learning_rate": 2.2727272727272728e-06,
+ "loss": 1.2032190561294556,
+ "mean_token_accuracy": 0.689667820930481,
+ "num_tokens": 22134784.0,
+ "step": 1351
+ },
+ {
+ "entropy": 1.056649923324585,
+ "epoch": 2.731313131313131,
+ "grad_norm": 2.111971139907837,
+ "learning_rate": 2.2707070707070706e-06,
+ "loss": 1.0646127462387085,
+ "mean_token_accuracy": 0.7132999300956726,
+ "num_tokens": 22151168.0,
+ "step": 1352
+ },
+ {
+ "entropy": 1.401559591293335,
+ "epoch": 2.7333333333333334,
+ "grad_norm": 2.269573211669922,
+ "learning_rate": 2.268686868686869e-06,
+ "loss": 1.405207633972168,
+ "mean_token_accuracy": 0.6612115502357483,
+ "num_tokens": 22167552.0,
+ "step": 1353
+ },
+ {
+ "entropy": 0.9708455204963684,
+ "epoch": 2.735353535353535,
+ "grad_norm": 2.3020989894866943,
+ "learning_rate": 2.266666666666667e-06,
+ "loss": 0.9864540100097656,
+ "mean_token_accuracy": 0.7361993193626404,
+ "num_tokens": 22183936.0,
+ "step": 1354
+ },
+ {
+ "entropy": 1.2840858697891235,
+ "epoch": 2.7373737373737375,
+ "grad_norm": 2.4489457607269287,
+ "learning_rate": 2.2646464646464647e-06,
+ "loss": 1.2992041110992432,
+ "mean_token_accuracy": 0.6783097386360168,
+ "num_tokens": 22200320.0,
+ "step": 1355
+ },
+ {
+ "entropy": 1.6704803705215454,
+ "epoch": 2.7393939393939393,
+ "grad_norm": 2.213287591934204,
+ "learning_rate": 2.262626262626263e-06,
+ "loss": 1.7022172212600708,
+ "mean_token_accuracy": 0.6433194875717163,
+ "num_tokens": 22216704.0,
+ "step": 1356
+ },
+ {
+ "entropy": 1.2570815086364746,
+ "epoch": 2.7414141414141415,
+ "grad_norm": 2.250062942504883,
+ "learning_rate": 2.260606060606061e-06,
+ "loss": 1.2799533605575562,
+ "mean_token_accuracy": 0.6823400259017944,
+ "num_tokens": 22233088.0,
+ "step": 1357
+ },
+ {
+ "entropy": 1.0966694355010986,
+ "epoch": 2.7434343434343433,
+ "grad_norm": 2.24507212638855,
+ "learning_rate": 2.258585858585859e-06,
+ "loss": 1.0954806804656982,
+ "mean_token_accuracy": 0.7264289259910583,
+ "num_tokens": 22249472.0,
+ "step": 1358
+ },
+ {
+ "entropy": 1.6675825119018555,
+ "epoch": 2.7454545454545456,
+ "grad_norm": 2.1572930812835693,
+ "learning_rate": 2.2565656565656567e-06,
+ "loss": 1.6607719659805298,
+ "mean_token_accuracy": 0.6113214492797852,
+ "num_tokens": 22265856.0,
+ "step": 1359
+ },
+ {
+ "entropy": 1.247696042060852,
+ "epoch": 2.7474747474747474,
+ "grad_norm": 2.105797290802002,
+ "learning_rate": 2.254545454545455e-06,
+ "loss": 1.2314106225967407,
+ "mean_token_accuracy": 0.6982169151306152,
+ "num_tokens": 22282240.0,
+ "step": 1360
+ },
+ {
+ "entropy": 1.6081417798995972,
+ "epoch": 2.7494949494949497,
+ "grad_norm": 2.1624395847320557,
+ "learning_rate": 2.252525252525253e-06,
+ "loss": 1.6011556386947632,
+ "mean_token_accuracy": 0.6282364726066589,
+ "num_tokens": 22298624.0,
+ "step": 1361
+ },
+ {
+ "entropy": 1.1124155521392822,
+ "epoch": 2.7515151515151515,
+ "grad_norm": 2.141758918762207,
+ "learning_rate": 2.250505050505051e-06,
+ "loss": 1.1052089929580688,
+ "mean_token_accuracy": 0.7202003002166748,
+ "num_tokens": 22315008.0,
+ "step": 1362
+ },
+ {
+ "entropy": 1.093716025352478,
+ "epoch": 2.7535353535353533,
+ "grad_norm": 2.2610108852386475,
+ "learning_rate": 2.2484848484848487e-06,
+ "loss": 1.095299482345581,
+ "mean_token_accuracy": 0.714154839515686,
+ "num_tokens": 22331392.0,
+ "step": 1363
+ },
+ {
+ "entropy": 1.353977918624878,
+ "epoch": 2.7555555555555555,
+ "grad_norm": 2.171783447265625,
+ "learning_rate": 2.2464646464646466e-06,
+ "loss": 1.3497436046600342,
+ "mean_token_accuracy": 0.6914386749267578,
+ "num_tokens": 22347776.0,
+ "step": 1364
+ },
+ {
+ "entropy": 1.4595965147018433,
+ "epoch": 2.757575757575758,
+ "grad_norm": 2.2337677478790283,
+ "learning_rate": 2.2444444444444445e-06,
+ "loss": 1.4730737209320068,
+ "mean_token_accuracy": 0.6578529477119446,
+ "num_tokens": 22364160.0,
+ "step": 1365
+ },
+ {
+ "entropy": 1.5668566226959229,
+ "epoch": 2.7595959595959596,
+ "grad_norm": 2.2195982933044434,
+ "learning_rate": 2.2424242424242428e-06,
+ "loss": 1.5661592483520508,
+ "mean_token_accuracy": 0.6397166848182678,
+ "num_tokens": 22380544.0,
+ "step": 1366
+ },
+ {
+ "entropy": 1.4419786930084229,
+ "epoch": 2.7616161616161614,
+ "grad_norm": 2.202221393585205,
+ "learning_rate": 2.2404040404040407e-06,
+ "loss": 1.4466135501861572,
+ "mean_token_accuracy": 0.6698827743530273,
+ "num_tokens": 22396928.0,
+ "step": 1367
+ },
+ {
+ "entropy": 1.43394136428833,
+ "epoch": 2.7636363636363637,
+ "grad_norm": 2.1969001293182373,
+ "learning_rate": 2.2383838383838385e-06,
+ "loss": 1.433483362197876,
+ "mean_token_accuracy": 0.6655471324920654,
+ "num_tokens": 22413312.0,
+ "step": 1368
+ },
+ {
+ "entropy": 1.048527717590332,
+ "epoch": 2.765656565656566,
+ "grad_norm": 2.142869472503662,
+ "learning_rate": 2.2363636363636364e-06,
+ "loss": 1.0373283624649048,
+ "mean_token_accuracy": 0.7396189570426941,
+ "num_tokens": 22429696.0,
+ "step": 1369
+ },
+ {
+ "entropy": 1.4282819032669067,
+ "epoch": 2.7676767676767677,
+ "grad_norm": 2.1550097465515137,
+ "learning_rate": 2.2343434343434343e-06,
+ "loss": 1.419884443283081,
+ "mean_token_accuracy": 0.6712262034416199,
+ "num_tokens": 22446080.0,
+ "step": 1370
+ },
+ {
+ "entropy": 1.5361018180847168,
+ "epoch": 2.7696969696969695,
+ "grad_norm": 2.1175451278686523,
+ "learning_rate": 2.2323232323232326e-06,
+ "loss": 1.5433859825134277,
+ "mean_token_accuracy": 0.636907696723938,
+ "num_tokens": 22462464.0,
+ "step": 1371
+ },
+ {
+ "entropy": 1.326242208480835,
+ "epoch": 2.771717171717172,
+ "grad_norm": 2.084632635116577,
+ "learning_rate": 2.2303030303030305e-06,
+ "loss": 1.3364207744598389,
+ "mean_token_accuracy": 0.6809965968132019,
+ "num_tokens": 22478848.0,
+ "step": 1372
+ },
+ {
+ "entropy": 1.255878210067749,
+ "epoch": 2.7737373737373736,
+ "grad_norm": 2.142777919769287,
+ "learning_rate": 2.2282828282828284e-06,
+ "loss": 1.264795184135437,
+ "mean_token_accuracy": 0.6875916123390198,
+ "num_tokens": 22495232.0,
+ "step": 1373
+ },
+ {
+ "entropy": 1.2916159629821777,
+ "epoch": 2.775757575757576,
+ "grad_norm": 2.115644693374634,
+ "learning_rate": 2.2262626262626263e-06,
+ "loss": 1.2945363521575928,
+ "mean_token_accuracy": 0.6895456910133362,
+ "num_tokens": 22511616.0,
+ "step": 1374
+ },
+ {
+ "entropy": 1.4490385055541992,
+ "epoch": 2.7777777777777777,
+ "grad_norm": 2.2078094482421875,
+ "learning_rate": 2.224242424242424e-06,
+ "loss": 1.4401804208755493,
+ "mean_token_accuracy": 0.6637151837348938,
+ "num_tokens": 22528000.0,
+ "step": 1375
+ },
+ {
+ "entropy": 1.2710940837860107,
+ "epoch": 2.77979797979798,
+ "grad_norm": 2.1877286434173584,
+ "learning_rate": 2.222222222222222e-06,
+ "loss": 1.2672982215881348,
+ "mean_token_accuracy": 0.6958353519439697,
+ "num_tokens": 22544384.0,
+ "step": 1376
+ },
+ {
+ "entropy": 1.3981144428253174,
+ "epoch": 2.7818181818181817,
+ "grad_norm": 2.2816240787506104,
+ "learning_rate": 2.2202020202020204e-06,
+ "loss": 1.4073718786239624,
+ "mean_token_accuracy": 0.6583414673805237,
+ "num_tokens": 22560768.0,
+ "step": 1377
+ },
+ {
+ "entropy": 1.541896104812622,
+ "epoch": 2.783838383838384,
+ "grad_norm": 2.3664891719818115,
+ "learning_rate": 2.2181818181818187e-06,
+ "loss": 1.5782896280288696,
+ "mean_token_accuracy": 0.6264045238494873,
+ "num_tokens": 22577152.0,
+ "step": 1378
+ },
+ {
+ "entropy": 1.1470069885253906,
+ "epoch": 2.785858585858586,
+ "grad_norm": 2.334867000579834,
+ "learning_rate": 2.2161616161616166e-06,
+ "loss": 1.1734097003936768,
+ "mean_token_accuracy": 0.6981558203697205,
+ "num_tokens": 22593536.0,
+ "step": 1379
+ },
+ {
+ "entropy": 1.3938566446304321,
+ "epoch": 2.787878787878788,
+ "grad_norm": 2.493093967437744,
+ "learning_rate": 2.2141414141414145e-06,
+ "loss": 1.4161372184753418,
+ "mean_token_accuracy": 0.6610283255577087,
+ "num_tokens": 22609920.0,
+ "step": 1380
+ },
+ {
+ "entropy": 1.2948062419891357,
+ "epoch": 2.78989898989899,
+ "grad_norm": 2.227708578109741,
+ "learning_rate": 2.2121212121212124e-06,
+ "loss": 1.3157434463500977,
+ "mean_token_accuracy": 0.6835002303123474,
+ "num_tokens": 22626304.0,
+ "step": 1381
+ },
+ {
+ "entropy": 0.9506068229675293,
+ "epoch": 2.7919191919191917,
+ "grad_norm": 2.1482889652252197,
+ "learning_rate": 2.2101010101010102e-06,
+ "loss": 0.950823187828064,
+ "mean_token_accuracy": 0.7500610947608948,
+ "num_tokens": 22642688.0,
+ "step": 1382
+ },
+ {
+ "entropy": 1.4429880380630493,
+ "epoch": 2.793939393939394,
+ "grad_norm": 2.8189656734466553,
+ "learning_rate": 2.208080808080808e-06,
+ "loss": 1.4801222085952759,
+ "mean_token_accuracy": 0.6469223499298096,
+ "num_tokens": 22659072.0,
+ "step": 1383
+ },
+ {
+ "entropy": 1.3517380952835083,
+ "epoch": 2.795959595959596,
+ "grad_norm": 2.047236442565918,
+ "learning_rate": 2.2060606060606064e-06,
+ "loss": 1.3715345859527588,
+ "mean_token_accuracy": 0.6921714544296265,
+ "num_tokens": 22675456.0,
+ "step": 1384
+ },
+ {
+ "entropy": 1.3647958040237427,
+ "epoch": 2.797979797979798,
+ "grad_norm": 2.296548366546631,
+ "learning_rate": 2.2040404040404043e-06,
+ "loss": 1.3669397830963135,
+ "mean_token_accuracy": 0.6769663095474243,
+ "num_tokens": 22691840.0,
+ "step": 1385
+ },
+ {
+ "entropy": 1.2830337285995483,
+ "epoch": 2.8,
+ "grad_norm": 2.222038745880127,
+ "learning_rate": 2.2020202020202022e-06,
+ "loss": 1.2938398122787476,
+ "mean_token_accuracy": 0.6786150336265564,
+ "num_tokens": 22708224.0,
+ "step": 1386
+ },
+ {
+ "entropy": 1.1581721305847168,
+ "epoch": 2.802020202020202,
+ "grad_norm": 1.9979426860809326,
+ "learning_rate": 2.2e-06,
+ "loss": 1.1363141536712646,
+ "mean_token_accuracy": 0.7106130719184875,
+ "num_tokens": 22724608.0,
+ "step": 1387
+ },
+ {
+ "entropy": 1.4855602979660034,
+ "epoch": 2.804040404040404,
+ "grad_norm": 2.2763257026672363,
+ "learning_rate": 2.197979797979798e-06,
+ "loss": 1.5020172595977783,
+ "mean_token_accuracy": 0.6478993892669678,
+ "num_tokens": 22740992.0,
+ "step": 1388
+ },
+ {
+ "entropy": 1.1942899227142334,
+ "epoch": 2.806060606060606,
+ "grad_norm": 2.1685962677001953,
+ "learning_rate": 2.1959595959595963e-06,
+ "loss": 1.188063621520996,
+ "mean_token_accuracy": 0.705483615398407,
+ "num_tokens": 22757376.0,
+ "step": 1389
+ },
+ {
+ "entropy": 1.1369596719741821,
+ "epoch": 2.808080808080808,
+ "grad_norm": 2.1820614337921143,
+ "learning_rate": 2.193939393939394e-06,
+ "loss": 1.1348206996917725,
+ "mean_token_accuracy": 0.7253297567367554,
+ "num_tokens": 22773760.0,
+ "step": 1390
+ },
+ {
+ "entropy": 1.2539693117141724,
+ "epoch": 2.81010101010101,
+ "grad_norm": 2.0384390354156494,
+ "learning_rate": 2.191919191919192e-06,
+ "loss": 1.250205636024475,
+ "mean_token_accuracy": 0.6993771195411682,
+ "num_tokens": 22790144.0,
+ "step": 1391
+ },
+ {
+ "entropy": 1.7445363998413086,
+ "epoch": 2.812121212121212,
+ "grad_norm": 2.55062198638916,
+ "learning_rate": 2.18989898989899e-06,
+ "loss": 1.7788689136505127,
+ "mean_token_accuracy": 0.607107937335968,
+ "num_tokens": 22806528.0,
+ "step": 1392
+ },
+ {
+ "entropy": 1.2282429933547974,
+ "epoch": 2.8141414141414143,
+ "grad_norm": 2.1791000366210938,
+ "learning_rate": 2.187878787878788e-06,
+ "loss": 1.2460708618164062,
+ "mean_token_accuracy": 0.697239875793457,
+ "num_tokens": 22822912.0,
+ "step": 1393
+ },
+ {
+ "entropy": 1.2007265090942383,
+ "epoch": 2.816161616161616,
+ "grad_norm": 2.4067599773406982,
+ "learning_rate": 2.1858585858585858e-06,
+ "loss": 1.2013460397720337,
+ "mean_token_accuracy": 0.6963238716125488,
+ "num_tokens": 22839296.0,
+ "step": 1394
+ },
+ {
+ "entropy": 1.7315549850463867,
+ "epoch": 2.8181818181818183,
+ "grad_norm": 2.15682053565979,
+ "learning_rate": 2.183838383838384e-06,
+ "loss": 1.6833488941192627,
+ "mean_token_accuracy": 0.6170004606246948,
+ "num_tokens": 22855680.0,
+ "step": 1395
+ },
+ {
+ "entropy": 1.325459599494934,
+ "epoch": 2.82020202020202,
+ "grad_norm": 2.0464699268341064,
+ "learning_rate": 2.181818181818182e-06,
+ "loss": 1.3265891075134277,
+ "mean_token_accuracy": 0.6802638173103333,
+ "num_tokens": 22872064.0,
+ "step": 1396
+ },
+ {
+ "entropy": 1.0391966104507446,
+ "epoch": 2.822222222222222,
+ "grad_norm": 2.3595499992370605,
+ "learning_rate": 2.17979797979798e-06,
+ "loss": 1.0515706539154053,
+ "mean_token_accuracy": 0.7299706935882568,
+ "num_tokens": 22888448.0,
+ "step": 1397
+ },
+ {
+ "entropy": 1.5484130382537842,
+ "epoch": 2.824242424242424,
+ "grad_norm": 2.24381422996521,
+ "learning_rate": 2.1777777777777777e-06,
+ "loss": 1.5656720399856567,
+ "mean_token_accuracy": 0.6390449404716492,
+ "num_tokens": 22904832.0,
+ "step": 1398
+ },
+ {
+ "entropy": 1.2196807861328125,
+ "epoch": 2.8262626262626265,
+ "grad_norm": 2.3043432235717773,
+ "learning_rate": 2.175757575757576e-06,
+ "loss": 1.2172417640686035,
+ "mean_token_accuracy": 0.6985833048820496,
+ "num_tokens": 22921216.0,
+ "step": 1399
+ },
+ {
+ "entropy": 1.171323537826538,
+ "epoch": 2.8282828282828283,
+ "grad_norm": 2.34149169921875,
+ "learning_rate": 2.173737373737374e-06,
+ "loss": 1.1752846240997314,
+ "mean_token_accuracy": 0.6979726552963257,
+ "num_tokens": 22937600.0,
+ "step": 1400
+ },
+ {
+ "epoch": 2.8282828282828283,
+ "eval_entropy": 1.3776377598124165,
+ "eval_loss": 1.533128261566162,
+ "eval_mean_token_accuracy": 0.645915245336871,
+ "eval_num_tokens": 22937600.0,
+ "eval_runtime": 43.7952,
+ "eval_samples_per_second": 22.605,
+ "eval_steps_per_second": 2.831,
+ "step": 1400
+ },
+ {
+ "entropy": 1.2173237800598145,
+ "epoch": 2.83030303030303,
+ "grad_norm": 2.245490789413452,
+ "learning_rate": 2.171717171717172e-06,
+ "loss": 1.1928956508636475,
+ "mean_token_accuracy": 0.6988885998725891,
+ "num_tokens": 22953984.0,
+ "step": 1401
+ },
+ {
+ "entropy": 1.4213385581970215,
+ "epoch": 2.8323232323232324,
+ "grad_norm": 2.192051649093628,
+ "learning_rate": 2.16969696969697e-06,
+ "loss": 1.4340262413024902,
+ "mean_token_accuracy": 0.6622496247291565,
+ "num_tokens": 22970368.0,
+ "step": 1402
+ },
+ {
+ "entropy": 1.922193169593811,
+ "epoch": 2.8343434343434346,
+ "grad_norm": 2.0441250801086426,
+ "learning_rate": 2.167676767676768e-06,
+ "loss": 1.9164612293243408,
+ "mean_token_accuracy": 0.5942232608795166,
+ "num_tokens": 22986752.0,
+ "step": 1403
+ },
+ {
+ "entropy": 1.3124938011169434,
+ "epoch": 2.8363636363636364,
+ "grad_norm": 2.245614528656006,
+ "learning_rate": 2.165656565656566e-06,
+ "loss": 1.3055529594421387,
+ "mean_token_accuracy": 0.680385947227478,
+ "num_tokens": 23003136.0,
+ "step": 1404
+ },
+ {
+ "entropy": 0.9893879294395447,
+ "epoch": 2.8383838383838382,
+ "grad_norm": 2.1041340827941895,
+ "learning_rate": 2.163636363636364e-06,
+ "loss": 0.9646933078765869,
+ "mean_token_accuracy": 0.7441377639770508,
+ "num_tokens": 23019520.0,
+ "step": 1405
+ },
+ {
+ "entropy": 1.4011882543563843,
+ "epoch": 2.8404040404040405,
+ "grad_norm": 2.5371105670928955,
+ "learning_rate": 2.1616161616161617e-06,
+ "loss": 1.381878137588501,
+ "mean_token_accuracy": 0.6522960662841797,
+ "num_tokens": 23035904.0,
+ "step": 1406
+ },
+ {
+ "entropy": 1.4563852548599243,
+ "epoch": 2.8424242424242423,
+ "grad_norm": 2.1308720111846924,
+ "learning_rate": 2.15959595959596e-06,
+ "loss": 1.4763004779815674,
+ "mean_token_accuracy": 0.6792256832122803,
+ "num_tokens": 23052288.0,
+ "step": 1407
+ },
+ {
+ "entropy": 0.8676514625549316,
+ "epoch": 2.8444444444444446,
+ "grad_norm": 2.016085147857666,
+ "learning_rate": 2.157575757575758e-06,
+ "loss": 0.8672611713409424,
+ "mean_token_accuracy": 0.7660601139068604,
+ "num_tokens": 23068672.0,
+ "step": 1408
+ },
+ {
+ "entropy": 1.2011700868606567,
+ "epoch": 2.8464646464646464,
+ "grad_norm": 2.1641454696655273,
+ "learning_rate": 2.1555555555555558e-06,
+ "loss": 1.2113897800445557,
+ "mean_token_accuracy": 0.7180019617080688,
+ "num_tokens": 23085056.0,
+ "step": 1409
+ },
+ {
+ "entropy": 1.7813633680343628,
+ "epoch": 2.8484848484848486,
+ "grad_norm": 2.1768839359283447,
+ "learning_rate": 2.1535353535353537e-06,
+ "loss": 1.7897224426269531,
+ "mean_token_accuracy": 0.6129701733589172,
+ "num_tokens": 23101440.0,
+ "step": 1410
+ },
+ {
+ "entropy": 1.0219851732254028,
+ "epoch": 2.8505050505050504,
+ "grad_norm": 2.299229145050049,
+ "learning_rate": 2.1515151515151515e-06,
+ "loss": 1.0073617696762085,
+ "mean_token_accuracy": 0.7240473628044128,
+ "num_tokens": 23117824.0,
+ "step": 1411
+ },
+ {
+ "entropy": 1.3342534303665161,
+ "epoch": 2.8525252525252527,
+ "grad_norm": 2.165161371231079,
+ "learning_rate": 2.1494949494949494e-06,
+ "loss": 1.3219788074493408,
+ "mean_token_accuracy": 0.6764777898788452,
+ "num_tokens": 23134208.0,
+ "step": 1412
+ },
+ {
+ "entropy": 1.1618846654891968,
+ "epoch": 2.8545454545454545,
+ "grad_norm": 2.1276957988739014,
+ "learning_rate": 2.1474747474747477e-06,
+ "loss": 1.1667187213897705,
+ "mean_token_accuracy": 0.7214215993881226,
+ "num_tokens": 23150592.0,
+ "step": 1413
+ },
+ {
+ "entropy": 1.3673025369644165,
+ "epoch": 2.8565656565656568,
+ "grad_norm": 2.249702215194702,
+ "learning_rate": 2.1454545454545456e-06,
+ "loss": 1.380744218826294,
+ "mean_token_accuracy": 0.6656082272529602,
+ "num_tokens": 23166976.0,
+ "step": 1414
+ },
+ {
+ "entropy": 1.4118674993515015,
+ "epoch": 2.8585858585858586,
+ "grad_norm": 2.1488327980041504,
+ "learning_rate": 2.1434343434343435e-06,
+ "loss": 1.4007214307785034,
+ "mean_token_accuracy": 0.6663410067558289,
+ "num_tokens": 23183360.0,
+ "step": 1415
+ },
+ {
+ "entropy": 1.4398424625396729,
+ "epoch": 2.8606060606060604,
+ "grad_norm": 2.2609732151031494,
+ "learning_rate": 2.1414141414141414e-06,
+ "loss": 1.4657073020935059,
+ "mean_token_accuracy": 0.6483268141746521,
+ "num_tokens": 23199744.0,
+ "step": 1416
+ },
+ {
+ "entropy": 1.2922199964523315,
+ "epoch": 2.8626262626262626,
+ "grad_norm": 2.1911425590515137,
+ "learning_rate": 2.1393939393939393e-06,
+ "loss": 1.3118690252304077,
+ "mean_token_accuracy": 0.6951025724411011,
+ "num_tokens": 23216128.0,
+ "step": 1417
+ },
+ {
+ "entropy": 1.6888933181762695,
+ "epoch": 2.864646464646465,
+ "grad_norm": 2.2805919647216797,
+ "learning_rate": 2.1373737373737376e-06,
+ "loss": 1.6726739406585693,
+ "mean_token_accuracy": 0.6136419177055359,
+ "num_tokens": 23232512.0,
+ "step": 1418
+ },
+ {
+ "entropy": 1.289890170097351,
+ "epoch": 2.8666666666666667,
+ "grad_norm": 2.092263698577881,
+ "learning_rate": 2.1353535353535355e-06,
+ "loss": 1.2876317501068115,
+ "mean_token_accuracy": 0.6835613250732422,
+ "num_tokens": 23248896.0,
+ "step": 1419
+ },
+ {
+ "entropy": 1.361393690109253,
+ "epoch": 2.8686868686868685,
+ "grad_norm": 2.1863481998443604,
+ "learning_rate": 2.133333333333334e-06,
+ "loss": 1.3719209432601929,
+ "mean_token_accuracy": 0.663532018661499,
+ "num_tokens": 23265280.0,
+ "step": 1420
+ },
+ {
+ "entropy": 1.1187894344329834,
+ "epoch": 2.8707070707070708,
+ "grad_norm": 2.256434679031372,
+ "learning_rate": 2.1313131313131317e-06,
+ "loss": 1.1177198886871338,
+ "mean_token_accuracy": 0.7082926034927368,
+ "num_tokens": 23281664.0,
+ "step": 1421
+ },
+ {
+ "entropy": 0.8779795169830322,
+ "epoch": 2.8727272727272726,
+ "grad_norm": 2.095226526260376,
+ "learning_rate": 2.1292929292929296e-06,
+ "loss": 0.8675153851509094,
+ "mean_token_accuracy": 0.7647777199745178,
+ "num_tokens": 23298048.0,
+ "step": 1422
+ },
+ {
+ "entropy": 1.458146095275879,
+ "epoch": 2.874747474747475,
+ "grad_norm": 2.225179433822632,
+ "learning_rate": 2.1272727272727275e-06,
+ "loss": 1.4488263130187988,
+ "mean_token_accuracy": 0.6640205383300781,
+ "num_tokens": 23314432.0,
+ "step": 1423
+ },
+ {
+ "entropy": 1.187269926071167,
+ "epoch": 2.8767676767676766,
+ "grad_norm": 2.023710250854492,
+ "learning_rate": 2.1252525252525254e-06,
+ "loss": 1.182060718536377,
+ "mean_token_accuracy": 0.7017586827278137,
+ "num_tokens": 23330816.0,
+ "step": 1424
+ },
+ {
+ "entropy": 1.1446877717971802,
+ "epoch": 2.878787878787879,
+ "grad_norm": 2.301314115524292,
+ "learning_rate": 2.1232323232323237e-06,
+ "loss": 1.1507803201675415,
+ "mean_token_accuracy": 0.7048119306564331,
+ "num_tokens": 23347200.0,
+ "step": 1425
+ },
+ {
+ "entropy": 1.3453346490859985,
+ "epoch": 2.8808080808080807,
+ "grad_norm": 2.0338571071624756,
+ "learning_rate": 2.1212121212121216e-06,
+ "loss": 1.345799207687378,
+ "mean_token_accuracy": 0.6966292262077332,
+ "num_tokens": 23363584.0,
+ "step": 1426
+ },
+ {
+ "entropy": 1.2638238668441772,
+ "epoch": 2.882828282828283,
+ "grad_norm": 2.1575632095336914,
+ "learning_rate": 2.1191919191919194e-06,
+ "loss": 1.2882521152496338,
+ "mean_token_accuracy": 0.6901563405990601,
+ "num_tokens": 23379968.0,
+ "step": 1427
+ },
+ {
+ "entropy": 1.4490493535995483,
+ "epoch": 2.8848484848484848,
+ "grad_norm": 2.1735994815826416,
+ "learning_rate": 2.1171717171717173e-06,
+ "loss": 1.455702781677246,
+ "mean_token_accuracy": 0.6538837552070618,
+ "num_tokens": 23396352.0,
+ "step": 1428
+ },
+ {
+ "entropy": 1.3873618841171265,
+ "epoch": 2.886868686868687,
+ "grad_norm": 2.3870506286621094,
+ "learning_rate": 2.1151515151515152e-06,
+ "loss": 1.3657323122024536,
+ "mean_token_accuracy": 0.6692110300064087,
+ "num_tokens": 23412736.0,
+ "step": 1429
+ },
+ {
+ "entropy": 1.298308253288269,
+ "epoch": 2.888888888888889,
+ "grad_norm": 2.079481363296509,
+ "learning_rate": 2.113131313131313e-06,
+ "loss": 1.30775785446167,
+ "mean_token_accuracy": 0.6834391951560974,
+ "num_tokens": 23429120.0,
+ "step": 1430
+ },
+ {
+ "entropy": 1.1544564962387085,
+ "epoch": 2.8909090909090907,
+ "grad_norm": 2.2400357723236084,
+ "learning_rate": 2.1111111111111114e-06,
+ "loss": 1.17466139793396,
+ "mean_token_accuracy": 0.7093307375907898,
+ "num_tokens": 23445504.0,
+ "step": 1431
+ },
+ {
+ "entropy": 1.4711883068084717,
+ "epoch": 2.892929292929293,
+ "grad_norm": 2.1779074668884277,
+ "learning_rate": 2.1090909090909093e-06,
+ "loss": 1.4626137018203735,
+ "mean_token_accuracy": 0.6602955460548401,
+ "num_tokens": 23461888.0,
+ "step": 1432
+ },
+ {
+ "entropy": 1.4876407384872437,
+ "epoch": 2.894949494949495,
+ "grad_norm": 2.278554677963257,
+ "learning_rate": 2.107070707070707e-06,
+ "loss": 1.5031921863555908,
+ "mean_token_accuracy": 0.6570591330528259,
+ "num_tokens": 23478272.0,
+ "step": 1433
+ },
+ {
+ "entropy": 1.328355073928833,
+ "epoch": 2.896969696969697,
+ "grad_norm": 2.034842014312744,
+ "learning_rate": 2.105050505050505e-06,
+ "loss": 1.3163859844207764,
+ "mean_token_accuracy": 0.6878358721733093,
+ "num_tokens": 23494656.0,
+ "step": 1434
+ },
+ {
+ "entropy": 1.487100601196289,
+ "epoch": 2.898989898989899,
+ "grad_norm": 2.1980724334716797,
+ "learning_rate": 2.103030303030303e-06,
+ "loss": 1.4886071681976318,
+ "mean_token_accuracy": 0.6521739363670349,
+ "num_tokens": 23511040.0,
+ "step": 1435
+ },
+ {
+ "entropy": 1.4826854467391968,
+ "epoch": 2.901010101010101,
+ "grad_norm": 2.5214874744415283,
+ "learning_rate": 2.1010101010101013e-06,
+ "loss": 1.4610626697540283,
+ "mean_token_accuracy": 0.6503419876098633,
+ "num_tokens": 23527424.0,
+ "step": 1436
+ },
+ {
+ "entropy": 1.00849187374115,
+ "epoch": 2.9030303030303033,
+ "grad_norm": 2.1452033519744873,
+ "learning_rate": 2.098989898989899e-06,
+ "loss": 0.9949107766151428,
+ "mean_token_accuracy": 0.7457864880561829,
+ "num_tokens": 23543808.0,
+ "step": 1437
+ },
+ {
+ "entropy": 1.0621757507324219,
+ "epoch": 2.905050505050505,
+ "grad_norm": 2.1871836185455322,
+ "learning_rate": 2.096969696969697e-06,
+ "loss": 1.0466980934143066,
+ "mean_token_accuracy": 0.7275280952453613,
+ "num_tokens": 23560192.0,
+ "step": 1438
+ },
+ {
+ "entropy": 1.3413732051849365,
+ "epoch": 2.907070707070707,
+ "grad_norm": 2.2954704761505127,
+ "learning_rate": 2.094949494949495e-06,
+ "loss": 1.3602681159973145,
+ "mean_token_accuracy": 0.6686003804206848,
+ "num_tokens": 23576576.0,
+ "step": 1439
+ },
+ {
+ "entropy": 0.8138323426246643,
+ "epoch": 2.909090909090909,
+ "grad_norm": 1.9476791620254517,
+ "learning_rate": 2.092929292929293e-06,
+ "loss": 0.8208089470863342,
+ "mean_token_accuracy": 0.7796775698661804,
+ "num_tokens": 23592960.0,
+ "step": 1440
+ },
+ {
+ "entropy": 0.9035854339599609,
+ "epoch": 2.911111111111111,
+ "grad_norm": 1.9753291606903076,
+ "learning_rate": 2.090909090909091e-06,
+ "loss": 0.8889603614807129,
+ "mean_token_accuracy": 0.7610527873039246,
+ "num_tokens": 23609344.0,
+ "step": 1441
+ },
+ {
+ "entropy": 0.9301351308822632,
+ "epoch": 2.9131313131313132,
+ "grad_norm": 2.244597911834717,
+ "learning_rate": 2.088888888888889e-06,
+ "loss": 0.9369313716888428,
+ "mean_token_accuracy": 0.7603810429573059,
+ "num_tokens": 23625728.0,
+ "step": 1442
+ },
+ {
+ "entropy": 0.8740416765213013,
+ "epoch": 2.915151515151515,
+ "grad_norm": 2.0207667350769043,
+ "learning_rate": 2.0868686868686873e-06,
+ "loss": 0.8751406073570251,
+ "mean_token_accuracy": 0.7634342908859253,
+ "num_tokens": 23642112.0,
+ "step": 1443
+ },
+ {
+ "entropy": 1.044225811958313,
+ "epoch": 2.9171717171717173,
+ "grad_norm": 2.8100409507751465,
+ "learning_rate": 2.0848484848484852e-06,
+ "loss": 1.0568115711212158,
+ "mean_token_accuracy": 0.7291157841682434,
+ "num_tokens": 23658496.0,
+ "step": 1444
+ },
+ {
+ "entropy": 1.1093257665634155,
+ "epoch": 2.919191919191919,
+ "grad_norm": 2.2957236766815186,
+ "learning_rate": 2.082828282828283e-06,
+ "loss": 1.1235377788543701,
+ "mean_token_accuracy": 0.7108573317527771,
+ "num_tokens": 23674880.0,
+ "step": 1445
+ },
+ {
+ "entropy": 1.2211518287658691,
+ "epoch": 2.9212121212121214,
+ "grad_norm": 2.022287130355835,
+ "learning_rate": 2.080808080808081e-06,
+ "loss": 1.241473913192749,
+ "mean_token_accuracy": 0.7020029425621033,
+ "num_tokens": 23691264.0,
+ "step": 1446
+ },
+ {
+ "entropy": 1.3636398315429688,
+ "epoch": 2.923232323232323,
+ "grad_norm": 2.1484291553497314,
+ "learning_rate": 2.078787878787879e-06,
+ "loss": 1.3876409530639648,
+ "mean_token_accuracy": 0.6714093685150146,
+ "num_tokens": 23707648.0,
+ "step": 1447
+ },
+ {
+ "entropy": 1.1615006923675537,
+ "epoch": 2.9252525252525254,
+ "grad_norm": 2.775181293487549,
+ "learning_rate": 2.0767676767676768e-06,
+ "loss": 1.1792278289794922,
+ "mean_token_accuracy": 0.7070102691650391,
+ "num_tokens": 23724032.0,
+ "step": 1448
+ },
+ {
+ "entropy": 1.6308530569076538,
+ "epoch": 2.9272727272727272,
+ "grad_norm": 2.3813729286193848,
+ "learning_rate": 2.074747474747475e-06,
+ "loss": 1.6642422676086426,
+ "mean_token_accuracy": 0.6171225905418396,
+ "num_tokens": 23740416.0,
+ "step": 1449
+ },
+ {
+ "entropy": 1.2764222621917725,
+ "epoch": 2.929292929292929,
+ "grad_norm": 2.234309196472168,
+ "learning_rate": 2.072727272727273e-06,
+ "loss": 1.2756493091583252,
+ "mean_token_accuracy": 0.6824010610580444,
+ "num_tokens": 23756800.0,
+ "step": 1450
+ },
+ {
+ "epoch": 2.929292929292929,
+ "eval_entropy": 1.366065975639128,
+ "eval_loss": 1.5328131914138794,
+ "eval_mean_token_accuracy": 0.6461491642459747,
+ "eval_num_tokens": 23756800.0,
+ "eval_runtime": 43.8548,
+ "eval_samples_per_second": 22.574,
+ "eval_steps_per_second": 2.828,
+ "step": 1450
+ },
+ {
+ "entropy": 1.178619623184204,
+ "epoch": 2.9313131313131313,
+ "grad_norm": 2.0946547985076904,
+ "learning_rate": 2.070707070707071e-06,
+ "loss": 1.1751608848571777,
+ "mean_token_accuracy": 0.704384446144104,
+ "num_tokens": 23773184.0,
+ "step": 1451
+ },
+ {
+ "entropy": 1.319455862045288,
+ "epoch": 2.9333333333333336,
+ "grad_norm": 2.1925816535949707,
+ "learning_rate": 2.0686868686868688e-06,
+ "loss": 1.3229246139526367,
+ "mean_token_accuracy": 0.6867977380752563,
+ "num_tokens": 23789568.0,
+ "step": 1452
+ },
+ {
+ "entropy": 1.787271499633789,
+ "epoch": 2.9353535353535354,
+ "grad_norm": 2.374494791030884,
+ "learning_rate": 2.0666666666666666e-06,
+ "loss": 1.7788429260253906,
+ "mean_token_accuracy": 0.589154839515686,
+ "num_tokens": 23805952.0,
+ "step": 1453
+ },
+ {
+ "entropy": 1.25193452835083,
+ "epoch": 2.937373737373737,
+ "grad_norm": 2.3499720096588135,
+ "learning_rate": 2.064646464646465e-06,
+ "loss": 1.2651758193969727,
+ "mean_token_accuracy": 0.6863092184066772,
+ "num_tokens": 23822336.0,
+ "step": 1454
+ },
+ {
+ "entropy": 1.355589747428894,
+ "epoch": 2.9393939393939394,
+ "grad_norm": 2.228123426437378,
+ "learning_rate": 2.062626262626263e-06,
+ "loss": 1.3623197078704834,
+ "mean_token_accuracy": 0.6623717546463013,
+ "num_tokens": 23838720.0,
+ "step": 1455
+ },
+ {
+ "entropy": 1.2293426990509033,
+ "epoch": 2.9414141414141413,
+ "grad_norm": 2.4237186908721924,
+ "learning_rate": 2.0606060606060607e-06,
+ "loss": 1.2226693630218506,
+ "mean_token_accuracy": 0.6903395056724548,
+ "num_tokens": 23855104.0,
+ "step": 1456
+ },
+ {
+ "entropy": 1.2312723398208618,
+ "epoch": 2.9434343434343435,
+ "grad_norm": 2.2350525856018066,
+ "learning_rate": 2.0585858585858586e-06,
+ "loss": 1.235978364944458,
+ "mean_token_accuracy": 0.696201741695404,
+ "num_tokens": 23871488.0,
+ "step": 1457
+ },
+ {
+ "entropy": 1.2731300592422485,
+ "epoch": 2.9454545454545453,
+ "grad_norm": 2.488929033279419,
+ "learning_rate": 2.0565656565656565e-06,
+ "loss": 1.2763257026672363,
+ "mean_token_accuracy": 0.6765388250350952,
+ "num_tokens": 23887872.0,
+ "step": 1458
+ },
+ {
+ "entropy": 1.1283669471740723,
+ "epoch": 2.9474747474747476,
+ "grad_norm": 2.116241693496704,
+ "learning_rate": 2.054545454545455e-06,
+ "loss": 1.1279038190841675,
+ "mean_token_accuracy": 0.7123228907585144,
+ "num_tokens": 23904256.0,
+ "step": 1459
+ },
+ {
+ "entropy": 1.4969894886016846,
+ "epoch": 2.9494949494949494,
+ "grad_norm": 2.17519474029541,
+ "learning_rate": 2.0525252525252527e-06,
+ "loss": 1.5177810192108154,
+ "mean_token_accuracy": 0.6544333100318909,
+ "num_tokens": 23920640.0,
+ "step": 1460
+ },
+ {
+ "entropy": 1.3133295774459839,
+ "epoch": 2.9515151515151516,
+ "grad_norm": 2.149486780166626,
+ "learning_rate": 2.0505050505050506e-06,
+ "loss": 1.2985190153121948,
+ "mean_token_accuracy": 0.6745237112045288,
+ "num_tokens": 23937024.0,
+ "step": 1461
+ },
+ {
+ "entropy": 1.2777348756790161,
+ "epoch": 2.9535353535353535,
+ "grad_norm": 2.124922752380371,
+ "learning_rate": 2.0484848484848485e-06,
+ "loss": 1.2731128931045532,
+ "mean_token_accuracy": 0.692415714263916,
+ "num_tokens": 23953408.0,
+ "step": 1462
+ },
+ {
+ "entropy": 1.0544995069503784,
+ "epoch": 2.9555555555555557,
+ "grad_norm": 2.177363872528076,
+ "learning_rate": 2.046464646464647e-06,
+ "loss": 1.0588994026184082,
+ "mean_token_accuracy": 0.7378480434417725,
+ "num_tokens": 23969792.0,
+ "step": 1463
+ },
+ {
+ "entropy": 1.441129446029663,
+ "epoch": 2.9575757575757575,
+ "grad_norm": 2.5355494022369385,
+ "learning_rate": 2.0444444444444447e-06,
+ "loss": 1.4684169292449951,
+ "mean_token_accuracy": 0.6495481133460999,
+ "num_tokens": 23986176.0,
+ "step": 1464
+ },
+ {
+ "entropy": 1.3359375,
+ "epoch": 2.9595959595959593,
+ "grad_norm": 2.309727191925049,
+ "learning_rate": 2.0424242424242426e-06,
+ "loss": 1.339503288269043,
+ "mean_token_accuracy": 0.6707376837730408,
+ "num_tokens": 24002560.0,
+ "step": 1465
+ },
+ {
+ "entropy": 1.5922675132751465,
+ "epoch": 2.9616161616161616,
+ "grad_norm": 2.191904067993164,
+ "learning_rate": 2.0404040404040405e-06,
+ "loss": 1.613649845123291,
+ "mean_token_accuracy": 0.635869562625885,
+ "num_tokens": 24018944.0,
+ "step": 1466
+ },
+ {
+ "entropy": 1.152869701385498,
+ "epoch": 2.963636363636364,
+ "grad_norm": 2.1656837463378906,
+ "learning_rate": 2.0383838383838388e-06,
+ "loss": 1.1791510581970215,
+ "mean_token_accuracy": 0.7013311982154846,
+ "num_tokens": 24035328.0,
+ "step": 1467
+ },
+ {
+ "entropy": 1.491494059562683,
+ "epoch": 2.9656565656565657,
+ "grad_norm": 2.397916555404663,
+ "learning_rate": 2.0363636363636367e-06,
+ "loss": 1.4803133010864258,
+ "mean_token_accuracy": 0.6395334601402283,
+ "num_tokens": 24051712.0,
+ "step": 1468
+ },
+ {
+ "entropy": 1.165432095527649,
+ "epoch": 2.9676767676767675,
+ "grad_norm": 2.0728659629821777,
+ "learning_rate": 2.0343434343434345e-06,
+ "loss": 1.1676859855651855,
+ "mean_token_accuracy": 0.7170249223709106,
+ "num_tokens": 24068096.0,
+ "step": 1469
+ },
+ {
+ "entropy": 0.7055315971374512,
+ "epoch": 2.9696969696969697,
+ "grad_norm": 1.949097752571106,
+ "learning_rate": 2.0323232323232324e-06,
+ "loss": 0.7094426155090332,
+ "mean_token_accuracy": 0.8047752976417542,
+ "num_tokens": 24084480.0,
+ "step": 1470
+ },
+ {
+ "entropy": 1.5079377889633179,
+ "epoch": 2.971717171717172,
+ "grad_norm": 2.282525062561035,
+ "learning_rate": 2.0303030303030303e-06,
+ "loss": 1.5309293270111084,
+ "mean_token_accuracy": 0.65486079454422,
+ "num_tokens": 24100864.0,
+ "step": 1471
+ },
+ {
+ "entropy": 1.6969871520996094,
+ "epoch": 2.973737373737374,
+ "grad_norm": 2.2211408615112305,
+ "learning_rate": 2.0282828282828286e-06,
+ "loss": 1.6919132471084595,
+ "mean_token_accuracy": 0.6259770393371582,
+ "num_tokens": 24117248.0,
+ "step": 1472
+ },
+ {
+ "entropy": 1.3981496095657349,
+ "epoch": 2.9757575757575756,
+ "grad_norm": 2.27559232711792,
+ "learning_rate": 2.0262626262626265e-06,
+ "loss": 1.4188807010650635,
+ "mean_token_accuracy": 0.6502198576927185,
+ "num_tokens": 24133632.0,
+ "step": 1473
+ },
+ {
+ "entropy": 1.2966978549957275,
+ "epoch": 2.977777777777778,
+ "grad_norm": 2.2315874099731445,
+ "learning_rate": 2.0242424242424244e-06,
+ "loss": 1.3034720420837402,
+ "mean_token_accuracy": 0.6849047541618347,
+ "num_tokens": 24150016.0,
+ "step": 1474
+ },
+ {
+ "entropy": 1.4312105178833008,
+ "epoch": 2.9797979797979797,
+ "grad_norm": 2.3630731105804443,
+ "learning_rate": 2.0222222222222223e-06,
+ "loss": 1.412591814994812,
+ "mean_token_accuracy": 0.6673180460929871,
+ "num_tokens": 24166400.0,
+ "step": 1475
+ },
+ {
+ "entropy": 1.246411681175232,
+ "epoch": 2.981818181818182,
+ "grad_norm": 2.101142644882202,
+ "learning_rate": 2.02020202020202e-06,
+ "loss": 1.2674810886383057,
+ "mean_token_accuracy": 0.691255509853363,
+ "num_tokens": 24182784.0,
+ "step": 1476
+ },
+ {
+ "entropy": 1.2367678880691528,
+ "epoch": 2.9838383838383837,
+ "grad_norm": 2.3414289951324463,
+ "learning_rate": 2.0181818181818185e-06,
+ "loss": 1.254493236541748,
+ "mean_token_accuracy": 0.6811797618865967,
+ "num_tokens": 24199168.0,
+ "step": 1477
+ },
+ {
+ "entropy": 1.3566304445266724,
+ "epoch": 2.985858585858586,
+ "grad_norm": 2.3050434589385986,
+ "learning_rate": 2.0161616161616164e-06,
+ "loss": 1.3336361646652222,
+ "mean_token_accuracy": 0.6761724352836609,
+ "num_tokens": 24215552.0,
+ "step": 1478
+ },
+ {
+ "entropy": 1.3928223848342896,
+ "epoch": 2.987878787878788,
+ "grad_norm": 2.2100086212158203,
+ "learning_rate": 2.0141414141414143e-06,
+ "loss": 1.4050222635269165,
+ "mean_token_accuracy": 0.6573033928871155,
+ "num_tokens": 24231936.0,
+ "step": 1479
+ },
+ {
+ "entropy": 1.067913293838501,
+ "epoch": 2.98989898989899,
+ "grad_norm": 2.0196874141693115,
+ "learning_rate": 2.012121212121212e-06,
+ "loss": 1.0836842060089111,
+ "mean_token_accuracy": 0.7427943348884583,
+ "num_tokens": 24248320.0,
+ "step": 1480
+ },
+ {
+ "entropy": 1.077475905418396,
+ "epoch": 2.991919191919192,
+ "grad_norm": 2.117917776107788,
+ "learning_rate": 2.01010101010101e-06,
+ "loss": 1.0825674533843994,
+ "mean_token_accuracy": 0.7288104295730591,
+ "num_tokens": 24264704.0,
+ "step": 1481
+ },
+ {
+ "entropy": 1.092706561088562,
+ "epoch": 2.993939393939394,
+ "grad_norm": 2.143916130065918,
+ "learning_rate": 2.008080808080808e-06,
+ "loss": 1.1073365211486816,
+ "mean_token_accuracy": 0.7271006107330322,
+ "num_tokens": 24281088.0,
+ "step": 1482
+ },
+ {
+ "entropy": 1.3734595775604248,
+ "epoch": 2.995959595959596,
+ "grad_norm": 2.3651299476623535,
+ "learning_rate": 2.0060606060606062e-06,
+ "loss": 1.3319274187088013,
+ "mean_token_accuracy": 0.6729360222816467,
+ "num_tokens": 24297472.0,
+ "step": 1483
+ },
+ {
+ "entropy": 1.5602202415466309,
+ "epoch": 2.9979797979797977,
+ "grad_norm": 2.4872968196868896,
+ "learning_rate": 2.004040404040404e-06,
+ "loss": 1.5969634056091309,
+ "mean_token_accuracy": 0.6352589130401611,
+ "num_tokens": 24313856.0,
+ "step": 1484
+ },
+ {
+ "entropy": 1.4174549579620361,
+ "epoch": 3.0,
+ "grad_norm": 2.1410610675811768,
+ "learning_rate": 2.0020202020202024e-06,
+ "loss": 1.4155133962631226,
+ "mean_token_accuracy": 0.6683561205863953,
+ "num_tokens": 24330240.0,
+ "step": 1485
+ },
+ {
+ "entropy": 1.5710192918777466,
+ "epoch": 3.002020202020202,
+ "grad_norm": 2.1068179607391357,
+ "learning_rate": 2.0000000000000003e-06,
+ "loss": 1.5049738883972168,
+ "mean_token_accuracy": 0.6563873887062073,
+ "num_tokens": 24346624.0,
+ "step": 1486
+ },
+ {
+ "entropy": 1.2311826944351196,
+ "epoch": 3.004040404040404,
+ "grad_norm": 2.238032579421997,
+ "learning_rate": 1.9979797979797982e-06,
+ "loss": 1.1740195751190186,
+ "mean_token_accuracy": 0.7018197178840637,
+ "num_tokens": 24363008.0,
+ "step": 1487
+ },
+ {
+ "entropy": 1.0681109428405762,
+ "epoch": 3.006060606060606,
+ "grad_norm": 2.0808138847351074,
+ "learning_rate": 1.995959595959596e-06,
+ "loss": 1.0269291400909424,
+ "mean_token_accuracy": 0.739130437374115,
+ "num_tokens": 24379392.0,
+ "step": 1488
+ },
+ {
+ "entropy": 1.2553848028182983,
+ "epoch": 3.008080808080808,
+ "grad_norm": 2.155320644378662,
+ "learning_rate": 1.993939393939394e-06,
+ "loss": 1.209822654724121,
+ "mean_token_accuracy": 0.7072545289993286,
+ "num_tokens": 24395776.0,
+ "step": 1489
+ },
+ {
+ "entropy": 1.5376615524291992,
+ "epoch": 3.01010101010101,
+ "grad_norm": 2.376460075378418,
+ "learning_rate": 1.9919191919191923e-06,
+ "loss": 1.485018014907837,
+ "mean_token_accuracy": 0.6547996997833252,
+ "num_tokens": 24412160.0,
+ "step": 1490
+ },
+ {
+ "entropy": 1.0649843215942383,
+ "epoch": 3.012121212121212,
+ "grad_norm": 2.1056692600250244,
+ "learning_rate": 1.98989898989899e-06,
+ "loss": 1.037630558013916,
+ "mean_token_accuracy": 0.7341841459274292,
+ "num_tokens": 24428544.0,
+ "step": 1491
+ },
+ {
+ "entropy": 1.3711966276168823,
+ "epoch": 3.014141414141414,
+ "grad_norm": 2.238981008529663,
+ "learning_rate": 1.987878787878788e-06,
+ "loss": 1.3336507081985474,
+ "mean_token_accuracy": 0.6914386749267578,
+ "num_tokens": 24444928.0,
+ "step": 1492
+ },
+ {
+ "entropy": 1.3955618143081665,
+ "epoch": 3.0161616161616163,
+ "grad_norm": 2.183464765548706,
+ "learning_rate": 1.985858585858586e-06,
+ "loss": 1.3517842292785645,
+ "mean_token_accuracy": 0.6735466718673706,
+ "num_tokens": 24461312.0,
+ "step": 1493
+ },
+ {
+ "entropy": 1.0063103437423706,
+ "epoch": 3.018181818181818,
+ "grad_norm": 2.1911609172821045,
+ "learning_rate": 1.983838383838384e-06,
+ "loss": 0.9973118305206299,
+ "mean_token_accuracy": 0.7280166149139404,
+ "num_tokens": 24477696.0,
+ "step": 1494
+ },
+ {
+ "entropy": 1.3080904483795166,
+ "epoch": 3.0202020202020203,
+ "grad_norm": 2.1129884719848633,
+ "learning_rate": 1.981818181818182e-06,
+ "loss": 1.2758756875991821,
+ "mean_token_accuracy": 0.7027357220649719,
+ "num_tokens": 24494080.0,
+ "step": 1495
+ },
+ {
+ "entropy": 1.5414087772369385,
+ "epoch": 3.022222222222222,
+ "grad_norm": 2.1628341674804688,
+ "learning_rate": 1.97979797979798e-06,
+ "loss": 1.527054786682129,
+ "mean_token_accuracy": 0.6569370031356812,
+ "num_tokens": 24510464.0,
+ "step": 1496
+ },
+ {
+ "entropy": 1.44732666015625,
+ "epoch": 3.0242424242424244,
+ "grad_norm": 2.2525594234466553,
+ "learning_rate": 1.977777777777778e-06,
+ "loss": 1.3968576192855835,
+ "mean_token_accuracy": 0.6604787707328796,
+ "num_tokens": 24526848.0,
+ "step": 1497
+ },
+ {
+ "entropy": 1.522241234779358,
+ "epoch": 3.026262626262626,
+ "grad_norm": 2.1826529502868652,
+ "learning_rate": 1.975757575757576e-06,
+ "loss": 1.529256820678711,
+ "mean_token_accuracy": 0.6493649482727051,
+ "num_tokens": 24543232.0,
+ "step": 1498
+ },
+ {
+ "entropy": 0.8186452984809875,
+ "epoch": 3.0282828282828285,
+ "grad_norm": 2.1004087924957275,
+ "learning_rate": 1.9737373737373737e-06,
+ "loss": 0.8136417865753174,
+ "mean_token_accuracy": 0.7819370031356812,
+ "num_tokens": 24559616.0,
+ "step": 1499
+ },
+ {
+ "entropy": 1.2122654914855957,
+ "epoch": 3.0303030303030303,
+ "grad_norm": 2.4567720890045166,
+ "learning_rate": 1.9717171717171716e-06,
+ "loss": 1.236933946609497,
+ "mean_token_accuracy": 0.6828895807266235,
+ "num_tokens": 24576000.0,
+ "step": 1500
+ },
+ {
+ "epoch": 3.0303030303030303,
+ "eval_entropy": 1.3221501962792488,
+ "eval_loss": 1.5418624877929688,
+ "eval_mean_token_accuracy": 0.6455981039231823,
+ "eval_num_tokens": 24576000.0,
+ "eval_runtime": 43.8085,
+ "eval_samples_per_second": 22.598,
+ "eval_steps_per_second": 2.831,
+ "step": 1500
+ },
+ {
+ "entropy": 0.9493642449378967,
+ "epoch": 3.0323232323232325,
+ "grad_norm": 2.278731346130371,
+ "learning_rate": 1.96969696969697e-06,
+ "loss": 0.9456083178520203,
+ "mean_token_accuracy": 0.7484733462333679,
+ "num_tokens": 24592384.0,
+ "step": 1501
+ },
+ {
+ "entropy": 1.4509791135787964,
+ "epoch": 3.0343434343434343,
+ "grad_norm": 2.112172842025757,
+ "learning_rate": 1.967676767676768e-06,
+ "loss": 1.451171875,
+ "mean_token_accuracy": 0.6633487939834595,
+ "num_tokens": 24608768.0,
+ "step": 1502
+ },
+ {
+ "entropy": 1.6493220329284668,
+ "epoch": 3.036363636363636,
+ "grad_norm": 2.0969111919403076,
+ "learning_rate": 1.9656565656565657e-06,
+ "loss": 1.6576147079467773,
+ "mean_token_accuracy": 0.628724992275238,
+ "num_tokens": 24625152.0,
+ "step": 1503
+ },
+ {
+ "entropy": 1.5533742904663086,
+ "epoch": 3.0383838383838384,
+ "grad_norm": 2.1630685329437256,
+ "learning_rate": 1.9636363636363636e-06,
+ "loss": 1.556575894355774,
+ "mean_token_accuracy": 0.6745237112045288,
+ "num_tokens": 24641536.0,
+ "step": 1504
+ },
+ {
+ "entropy": 1.382328987121582,
+ "epoch": 3.04040404040404,
+ "grad_norm": 2.1493992805480957,
+ "learning_rate": 1.9616161616161615e-06,
+ "loss": 1.3725603818893433,
+ "mean_token_accuracy": 0.6811797618865967,
+ "num_tokens": 24657920.0,
+ "step": 1505
+ },
+ {
+ "entropy": 1.276682734489441,
+ "epoch": 3.0424242424242425,
+ "grad_norm": 2.342043161392212,
+ "learning_rate": 1.9595959595959598e-06,
+ "loss": 1.2861356735229492,
+ "mean_token_accuracy": 0.6843551397323608,
+ "num_tokens": 24674304.0,
+ "step": 1506
+ },
+ {
+ "entropy": 1.3813475370407104,
+ "epoch": 3.0444444444444443,
+ "grad_norm": 2.246891736984253,
+ "learning_rate": 1.9575757575757577e-06,
+ "loss": 1.3686306476593018,
+ "mean_token_accuracy": 0.6649364829063416,
+ "num_tokens": 24690688.0,
+ "step": 1507
+ },
+ {
+ "entropy": 1.2101274728775024,
+ "epoch": 3.0464646464646465,
+ "grad_norm": 2.2917656898498535,
+ "learning_rate": 1.955555555555556e-06,
+ "loss": 1.2131311893463135,
+ "mean_token_accuracy": 0.6958353519439697,
+ "num_tokens": 24707072.0,
+ "step": 1508
+ },
+ {
+ "entropy": 0.7796710133552551,
+ "epoch": 3.0484848484848484,
+ "grad_norm": 2.1023900508880615,
+ "learning_rate": 1.953535353535354e-06,
+ "loss": 0.7828338146209717,
+ "mean_token_accuracy": 0.7879824042320251,
+ "num_tokens": 24723456.0,
+ "step": 1509
+ },
+ {
+ "entropy": 0.8980224132537842,
+ "epoch": 3.0505050505050506,
+ "grad_norm": 2.259763240814209,
+ "learning_rate": 1.9515151515151518e-06,
+ "loss": 0.8986867070198059,
+ "mean_token_accuracy": 0.751038134098053,
+ "num_tokens": 24739840.0,
+ "step": 1510
+ },
+ {
+ "entropy": 1.1927038431167603,
+ "epoch": 3.0525252525252524,
+ "grad_norm": 2.2032012939453125,
+ "learning_rate": 1.9494949494949496e-06,
+ "loss": 1.1801973581314087,
+ "mean_token_accuracy": 0.7048729658126831,
+ "num_tokens": 24756224.0,
+ "step": 1511
+ },
+ {
+ "entropy": 0.8241261839866638,
+ "epoch": 3.0545454545454547,
+ "grad_norm": 1.9027519226074219,
+ "learning_rate": 1.9474747474747475e-06,
+ "loss": 0.8152725100517273,
+ "mean_token_accuracy": 0.7793111801147461,
+ "num_tokens": 24772608.0,
+ "step": 1512
+ },
+ {
+ "entropy": 1.1086668968200684,
+ "epoch": 3.0565656565656565,
+ "grad_norm": 2.4891581535339355,
+ "learning_rate": 1.945454545454546e-06,
+ "loss": 1.1167728900909424,
+ "mean_token_accuracy": 0.7162310481071472,
+ "num_tokens": 24788992.0,
+ "step": 1513
+ },
+ {
+ "entropy": 1.0804646015167236,
+ "epoch": 3.0585858585858587,
+ "grad_norm": 2.3115813732147217,
+ "learning_rate": 1.9434343434343437e-06,
+ "loss": 1.1059165000915527,
+ "mean_token_accuracy": 0.7208109498023987,
+ "num_tokens": 24805376.0,
+ "step": 1514
+ },
+ {
+ "entropy": 1.2628663778305054,
+ "epoch": 3.0606060606060606,
+ "grad_norm": 2.077954053878784,
+ "learning_rate": 1.9414141414141416e-06,
+ "loss": 1.2748162746429443,
+ "mean_token_accuracy": 0.697300910949707,
+ "num_tokens": 24821760.0,
+ "step": 1515
+ },
+ {
+ "entropy": 1.1143958568572998,
+ "epoch": 3.062626262626263,
+ "grad_norm": 2.3562631607055664,
+ "learning_rate": 1.9393939393939395e-06,
+ "loss": 1.127720832824707,
+ "mean_token_accuracy": 0.7080483436584473,
+ "num_tokens": 24838144.0,
+ "step": 1516
+ },
+ {
+ "entropy": 1.3625105619430542,
+ "epoch": 3.0646464646464646,
+ "grad_norm": 2.592428684234619,
+ "learning_rate": 1.9373737373737374e-06,
+ "loss": 1.3909248113632202,
+ "mean_token_accuracy": 0.6637151837348938,
+ "num_tokens": 24854528.0,
+ "step": 1517
+ },
+ {
+ "entropy": 1.032320261001587,
+ "epoch": 3.066666666666667,
+ "grad_norm": 2.400594711303711,
+ "learning_rate": 1.9353535353535353e-06,
+ "loss": 1.0697423219680786,
+ "mean_token_accuracy": 0.7302759885787964,
+ "num_tokens": 24870912.0,
+ "step": 1518
+ },
+ {
+ "entropy": 1.019578456878662,
+ "epoch": 3.0686868686868687,
+ "grad_norm": 2.137122869491577,
+ "learning_rate": 1.9333333333333336e-06,
+ "loss": 0.9904043674468994,
+ "mean_token_accuracy": 0.7461528778076172,
+ "num_tokens": 24887296.0,
+ "step": 1519
+ },
+ {
+ "entropy": 1.4399018287658691,
+ "epoch": 3.0707070707070705,
+ "grad_norm": 2.343580484390259,
+ "learning_rate": 1.9313131313131315e-06,
+ "loss": 1.429957628250122,
+ "mean_token_accuracy": 0.6615168452262878,
+ "num_tokens": 24903680.0,
+ "step": 1520
+ },
+ {
+ "entropy": 0.946161150932312,
+ "epoch": 3.0727272727272728,
+ "grad_norm": 1.9037182331085205,
+ "learning_rate": 1.9292929292929294e-06,
+ "loss": 0.943518877029419,
+ "mean_token_accuracy": 0.7626404762268066,
+ "num_tokens": 24920064.0,
+ "step": 1521
+ },
+ {
+ "entropy": 1.2314492464065552,
+ "epoch": 3.0747474747474746,
+ "grad_norm": 2.0896897315979004,
+ "learning_rate": 1.9272727272727273e-06,
+ "loss": 1.2434325218200684,
+ "mean_token_accuracy": 0.7122007608413696,
+ "num_tokens": 24936448.0,
+ "step": 1522
+ },
+ {
+ "entropy": 1.4557143449783325,
+ "epoch": 3.076767676767677,
+ "grad_norm": 2.162768602371216,
+ "learning_rate": 1.925252525252525e-06,
+ "loss": 1.4258036613464355,
+ "mean_token_accuracy": 0.6709819436073303,
+ "num_tokens": 24952832.0,
+ "step": 1523
+ },
+ {
+ "entropy": 1.1379952430725098,
+ "epoch": 3.0787878787878786,
+ "grad_norm": 2.322523832321167,
+ "learning_rate": 1.9232323232323235e-06,
+ "loss": 1.1321945190429688,
+ "mean_token_accuracy": 0.7206888198852539,
+ "num_tokens": 24969216.0,
+ "step": 1524
+ },
+ {
+ "entropy": 1.4143598079681396,
+ "epoch": 3.080808080808081,
+ "grad_norm": 2.2130038738250732,
+ "learning_rate": 1.9212121212121213e-06,
+ "loss": 1.4286584854125977,
+ "mean_token_accuracy": 0.6595627665519714,
+ "num_tokens": 24985600.0,
+ "step": 1525
+ },
+ {
+ "entropy": 0.9206982254981995,
+ "epoch": 3.0828282828282827,
+ "grad_norm": 2.150385618209839,
+ "learning_rate": 1.9191919191919192e-06,
+ "loss": 0.9122598171234131,
+ "mean_token_accuracy": 0.7564728856086731,
+ "num_tokens": 25001984.0,
+ "step": 1526
+ },
+ {
+ "entropy": 1.3168023824691772,
+ "epoch": 3.084848484848485,
+ "grad_norm": 2.2106544971466064,
+ "learning_rate": 1.9171717171717175e-06,
+ "loss": 1.3181350231170654,
+ "mean_token_accuracy": 0.6904616355895996,
+ "num_tokens": 25018368.0,
+ "step": 1527
+ },
+ {
+ "entropy": 1.2383761405944824,
+ "epoch": 3.0868686868686868,
+ "grad_norm": 2.1377766132354736,
+ "learning_rate": 1.9151515151515154e-06,
+ "loss": 1.2224464416503906,
+ "mean_token_accuracy": 0.7129335403442383,
+ "num_tokens": 25034752.0,
+ "step": 1528
+ },
+ {
+ "entropy": 1.6698395013809204,
+ "epoch": 3.088888888888889,
+ "grad_norm": 2.354219913482666,
+ "learning_rate": 1.9131313131313133e-06,
+ "loss": 1.67041015625,
+ "mean_token_accuracy": 0.6486932039260864,
+ "num_tokens": 25051136.0,
+ "step": 1529
+ },
+ {
+ "entropy": 1.1346160173416138,
+ "epoch": 3.090909090909091,
+ "grad_norm": 2.0619447231292725,
+ "learning_rate": 1.9111111111111112e-06,
+ "loss": 1.1436500549316406,
+ "mean_token_accuracy": 0.721177339553833,
+ "num_tokens": 25067520.0,
+ "step": 1530
+ },
+ {
+ "entropy": 1.4769412279129028,
+ "epoch": 3.092929292929293,
+ "grad_norm": 2.272189140319824,
+ "learning_rate": 1.9090909090909095e-06,
+ "loss": 1.4578258991241455,
+ "mean_token_accuracy": 0.6602955460548401,
+ "num_tokens": 25083904.0,
+ "step": 1531
+ },
+ {
+ "entropy": 1.3661279678344727,
+ "epoch": 3.094949494949495,
+ "grad_norm": 2.381265878677368,
+ "learning_rate": 1.9070707070707072e-06,
+ "loss": 1.3745051622390747,
+ "mean_token_accuracy": 0.6734245419502258,
+ "num_tokens": 25100288.0,
+ "step": 1532
+ },
+ {
+ "entropy": 1.0367696285247803,
+ "epoch": 3.096969696969697,
+ "grad_norm": 2.337775707244873,
+ "learning_rate": 1.9050505050505053e-06,
+ "loss": 1.0290615558624268,
+ "mean_token_accuracy": 0.7355275750160217,
+ "num_tokens": 25116672.0,
+ "step": 1533
+ },
+ {
+ "entropy": 0.9273203015327454,
+ "epoch": 3.098989898989899,
+ "grad_norm": 2.0960872173309326,
+ "learning_rate": 1.9030303030303032e-06,
+ "loss": 0.9162085056304932,
+ "mean_token_accuracy": 0.7639839053153992,
+ "num_tokens": 25133056.0,
+ "step": 1534
+ },
+ {
+ "entropy": 1.018664002418518,
+ "epoch": 3.101010101010101,
+ "grad_norm": 2.243028163909912,
+ "learning_rate": 1.9010101010101013e-06,
+ "loss": 1.019313097000122,
+ "mean_token_accuracy": 0.7399242520332336,
+ "num_tokens": 25149440.0,
+ "step": 1535
+ },
+ {
+ "entropy": 1.3178181648254395,
+ "epoch": 3.103030303030303,
+ "grad_norm": 2.305586338043213,
+ "learning_rate": 1.8989898989898992e-06,
+ "loss": 1.3278884887695312,
+ "mean_token_accuracy": 0.6838055849075317,
+ "num_tokens": 25165824.0,
+ "step": 1536
+ },
+ {
+ "entropy": 1.0808820724487305,
+ "epoch": 3.105050505050505,
+ "grad_norm": 2.2185559272766113,
+ "learning_rate": 1.896969696969697e-06,
+ "loss": 1.0852123498916626,
+ "mean_token_accuracy": 0.7187347412109375,
+ "num_tokens": 25182208.0,
+ "step": 1537
+ },
+ {
+ "entropy": 1.1828639507293701,
+ "epoch": 3.107070707070707,
+ "grad_norm": 2.5469112396240234,
+ "learning_rate": 1.8949494949494952e-06,
+ "loss": 1.1897902488708496,
+ "mean_token_accuracy": 0.697239875793457,
+ "num_tokens": 25198592.0,
+ "step": 1538
+ },
+ {
+ "entropy": 1.3447293043136597,
+ "epoch": 3.109090909090909,
+ "grad_norm": 2.2092692852020264,
+ "learning_rate": 1.892929292929293e-06,
+ "loss": 1.3312615156173706,
+ "mean_token_accuracy": 0.6797142028808594,
+ "num_tokens": 25214976.0,
+ "step": 1539
+ },
+ {
+ "entropy": 1.392472743988037,
+ "epoch": 3.111111111111111,
+ "grad_norm": 2.3445911407470703,
+ "learning_rate": 1.890909090909091e-06,
+ "loss": 1.3892791271209717,
+ "mean_token_accuracy": 0.6742794513702393,
+ "num_tokens": 25231360.0,
+ "step": 1540
+ },
+ {
+ "entropy": 1.422629475593567,
+ "epoch": 3.113131313131313,
+ "grad_norm": 2.4753165245056152,
+ "learning_rate": 1.888888888888889e-06,
+ "loss": 1.432596206665039,
+ "mean_token_accuracy": 0.6670126914978027,
+ "num_tokens": 25247744.0,
+ "step": 1541
+ },
+ {
+ "entropy": 1.5371577739715576,
+ "epoch": 3.1151515151515152,
+ "grad_norm": 2.3083088397979736,
+ "learning_rate": 1.886868686868687e-06,
+ "loss": 1.5318076610565186,
+ "mean_token_accuracy": 0.6301905512809753,
+ "num_tokens": 25264128.0,
+ "step": 1542
+ },
+ {
+ "entropy": 1.1179860830307007,
+ "epoch": 3.117171717171717,
+ "grad_norm": 2.1037089824676514,
+ "learning_rate": 1.884848484848485e-06,
+ "loss": 1.123746395111084,
+ "mean_token_accuracy": 0.7327185869216919,
+ "num_tokens": 25280512.0,
+ "step": 1543
+ },
+ {
+ "entropy": 1.046632170677185,
+ "epoch": 3.1191919191919193,
+ "grad_norm": 2.267493963241577,
+ "learning_rate": 1.882828282828283e-06,
+ "loss": 1.027912974357605,
+ "mean_token_accuracy": 0.7275280952453613,
+ "num_tokens": 25296896.0,
+ "step": 1544
+ },
+ {
+ "entropy": 1.2848094701766968,
+ "epoch": 3.121212121212121,
+ "grad_norm": 2.1797454357147217,
+ "learning_rate": 1.8808080808080808e-06,
+ "loss": 1.3054044246673584,
+ "mean_token_accuracy": 0.7058500051498413,
+ "num_tokens": 25313280.0,
+ "step": 1545
+ },
+ {
+ "entropy": 1.132463812828064,
+ "epoch": 3.1232323232323234,
+ "grad_norm": 2.2171123027801514,
+ "learning_rate": 1.878787878787879e-06,
+ "loss": 1.1395090818405151,
+ "mean_token_accuracy": 0.7204445600509644,
+ "num_tokens": 25329664.0,
+ "step": 1546
+ },
+ {
+ "entropy": 1.084518551826477,
+ "epoch": 3.125252525252525,
+ "grad_norm": 2.2163641452789307,
+ "learning_rate": 1.8767676767676768e-06,
+ "loss": 1.093268871307373,
+ "mean_token_accuracy": 0.7297264337539673,
+ "num_tokens": 25346048.0,
+ "step": 1547
+ },
+ {
+ "entropy": 1.1031688451766968,
+ "epoch": 3.1272727272727274,
+ "grad_norm": 2.3715343475341797,
+ "learning_rate": 1.874747474747475e-06,
+ "loss": 1.1064910888671875,
+ "mean_token_accuracy": 0.7130556702613831,
+ "num_tokens": 25362432.0,
+ "step": 1548
+ },
+ {
+ "entropy": 1.515089988708496,
+ "epoch": 3.1292929292929292,
+ "grad_norm": 2.0995519161224365,
+ "learning_rate": 1.872727272727273e-06,
+ "loss": 1.5236327648162842,
+ "mean_token_accuracy": 0.6491206884384155,
+ "num_tokens": 25378816.0,
+ "step": 1549
+ },
+ {
+ "entropy": 1.26030695438385,
+ "epoch": 3.1313131313131315,
+ "grad_norm": 2.2167913913726807,
+ "learning_rate": 1.8707070707070709e-06,
+ "loss": 1.2750146389007568,
+ "mean_token_accuracy": 0.6991939544677734,
+ "num_tokens": 25395200.0,
+ "step": 1550
+ },
+ {
+ "epoch": 3.1313131313131315,
+ "eval_entropy": 1.3187109231948853,
+ "eval_loss": 1.5490814447402954,
+ "eval_mean_token_accuracy": 0.6446880385760339,
+ "eval_num_tokens": 25395200.0,
+ "eval_runtime": 43.7612,
+ "eval_samples_per_second": 22.623,
+ "eval_steps_per_second": 2.834,
+ "step": 1550
+ },
+ {
+ "entropy": 1.044468641281128,
+ "epoch": 3.1333333333333333,
+ "grad_norm": 2.189656972885132,
+ "learning_rate": 1.868686868686869e-06,
+ "loss": 1.035596251487732,
+ "mean_token_accuracy": 0.7405959963798523,
+ "num_tokens": 25411584.0,
+ "step": 1551
+ },
+ {
+ "entropy": 1.286029577255249,
+ "epoch": 3.1353535353535356,
+ "grad_norm": 2.3911969661712646,
+ "learning_rate": 1.8666666666666669e-06,
+ "loss": 1.287235975265503,
+ "mean_token_accuracy": 0.6918661594390869,
+ "num_tokens": 25427968.0,
+ "step": 1552
+ },
+ {
+ "entropy": 1.4401317834854126,
+ "epoch": 3.1373737373737374,
+ "grad_norm": 2.452929973602295,
+ "learning_rate": 1.864646464646465e-06,
+ "loss": 1.4605298042297363,
+ "mean_token_accuracy": 0.6466169953346252,
+ "num_tokens": 25444352.0,
+ "step": 1553
+ },
+ {
+ "entropy": 1.316934585571289,
+ "epoch": 3.1393939393939396,
+ "grad_norm": 2.4121034145355225,
+ "learning_rate": 1.8626262626262629e-06,
+ "loss": 1.3266665935516357,
+ "mean_token_accuracy": 0.6759892702102661,
+ "num_tokens": 25460736.0,
+ "step": 1554
+ },
+ {
+ "entropy": 1.2644363641738892,
+ "epoch": 3.1414141414141414,
+ "grad_norm": 2.2631289958953857,
+ "learning_rate": 1.8606060606060607e-06,
+ "loss": 1.2905628681182861,
+ "mean_token_accuracy": 0.6833170652389526,
+ "num_tokens": 25477120.0,
+ "step": 1555
+ },
+ {
+ "entropy": 1.3172475099563599,
+ "epoch": 3.1434343434343432,
+ "grad_norm": 2.357954502105713,
+ "learning_rate": 1.8585858585858588e-06,
+ "loss": 1.316823124885559,
+ "mean_token_accuracy": 0.6763556599617004,
+ "num_tokens": 25493504.0,
+ "step": 1556
+ },
+ {
+ "entropy": 1.392463207244873,
+ "epoch": 3.1454545454545455,
+ "grad_norm": 2.491037130355835,
+ "learning_rate": 1.8565656565656567e-06,
+ "loss": 1.3718677759170532,
+ "mean_token_accuracy": 0.6508915424346924,
+ "num_tokens": 25509888.0,
+ "step": 1557
+ },
+ {
+ "entropy": 1.3607147932052612,
+ "epoch": 3.1474747474747473,
+ "grad_norm": 2.306100845336914,
+ "learning_rate": 1.8545454545454546e-06,
+ "loss": 1.3523163795471191,
+ "mean_token_accuracy": 0.6557157039642334,
+ "num_tokens": 25526272.0,
+ "step": 1558
+ },
+ {
+ "entropy": 0.9888757467269897,
+ "epoch": 3.1494949494949496,
+ "grad_norm": 2.125560998916626,
+ "learning_rate": 1.8525252525252527e-06,
+ "loss": 0.9713444709777832,
+ "mean_token_accuracy": 0.7584269642829895,
+ "num_tokens": 25542656.0,
+ "step": 1559
+ },
+ {
+ "entropy": 1.286476492881775,
+ "epoch": 3.1515151515151514,
+ "grad_norm": 2.149113416671753,
+ "learning_rate": 1.8505050505050506e-06,
+ "loss": 1.2688651084899902,
+ "mean_token_accuracy": 0.6882022619247437,
+ "num_tokens": 25559040.0,
+ "step": 1560
+ },
+ {
+ "entropy": 0.9255673289299011,
+ "epoch": 3.1535353535353536,
+ "grad_norm": 2.1615419387817383,
+ "learning_rate": 1.8484848484848487e-06,
+ "loss": 0.9265275597572327,
+ "mean_token_accuracy": 0.7542135119438171,
+ "num_tokens": 25575424.0,
+ "step": 1561
+ },
+ {
+ "entropy": 1.4960277080535889,
+ "epoch": 3.1555555555555554,
+ "grad_norm": 2.183372974395752,
+ "learning_rate": 1.8464646464646466e-06,
+ "loss": 1.4737393856048584,
+ "mean_token_accuracy": 0.6516854166984558,
+ "num_tokens": 25591808.0,
+ "step": 1562
+ },
+ {
+ "entropy": 1.2294487953186035,
+ "epoch": 3.1575757575757577,
+ "grad_norm": 2.389885425567627,
+ "learning_rate": 1.8444444444444445e-06,
+ "loss": 1.210254192352295,
+ "mean_token_accuracy": 0.6988885998725891,
+ "num_tokens": 25608192.0,
+ "step": 1563
+ },
+ {
+ "entropy": 1.0153220891952515,
+ "epoch": 3.1595959595959595,
+ "grad_norm": 2.131989002227783,
+ "learning_rate": 1.8424242424242426e-06,
+ "loss": 1.0005543231964111,
+ "mean_token_accuracy": 0.7426722049713135,
+ "num_tokens": 25624576.0,
+ "step": 1564
+ },
+ {
+ "entropy": 1.2591265439987183,
+ "epoch": 3.1616161616161618,
+ "grad_norm": 2.150913715362549,
+ "learning_rate": 1.8404040404040405e-06,
+ "loss": 1.2284668684005737,
+ "mean_token_accuracy": 0.6993771195411682,
+ "num_tokens": 25640960.0,
+ "step": 1565
+ },
+ {
+ "entropy": 1.009209394454956,
+ "epoch": 3.1636363636363636,
+ "grad_norm": 2.090205192565918,
+ "learning_rate": 1.8383838383838384e-06,
+ "loss": 0.9917510747909546,
+ "mean_token_accuracy": 0.7412066459655762,
+ "num_tokens": 25657344.0,
+ "step": 1566
+ },
+ {
+ "entropy": 1.716966986656189,
+ "epoch": 3.165656565656566,
+ "grad_norm": 2.2880618572235107,
+ "learning_rate": 1.8363636363636365e-06,
+ "loss": 1.7331494092941284,
+ "mean_token_accuracy": 0.6106497049331665,
+ "num_tokens": 25673728.0,
+ "step": 1567
+ },
+ {
+ "entropy": 1.5511447191238403,
+ "epoch": 3.1676767676767676,
+ "grad_norm": 2.371710777282715,
+ "learning_rate": 1.8343434343434343e-06,
+ "loss": 1.5570940971374512,
+ "mean_token_accuracy": 0.6477161645889282,
+ "num_tokens": 25690112.0,
+ "step": 1568
+ },
+ {
+ "entropy": 1.325405240058899,
+ "epoch": 3.16969696969697,
+ "grad_norm": 2.3747830390930176,
+ "learning_rate": 1.8323232323232324e-06,
+ "loss": 1.3260624408721924,
+ "mean_token_accuracy": 0.6791035532951355,
+ "num_tokens": 25706496.0,
+ "step": 1569
+ },
+ {
+ "entropy": 1.2529335021972656,
+ "epoch": 3.1717171717171717,
+ "grad_norm": 2.162775754928589,
+ "learning_rate": 1.8303030303030305e-06,
+ "loss": 1.257356882095337,
+ "mean_token_accuracy": 0.7066438794136047,
+ "num_tokens": 25722880.0,
+ "step": 1570
+ },
+ {
+ "entropy": 1.116769790649414,
+ "epoch": 3.1737373737373735,
+ "grad_norm": 2.217358350753784,
+ "learning_rate": 1.8282828282828286e-06,
+ "loss": 1.1458317041397095,
+ "mean_token_accuracy": 0.7087811231613159,
+ "num_tokens": 25739264.0,
+ "step": 1571
+ },
+ {
+ "entropy": 1.3465793132781982,
+ "epoch": 3.175757575757576,
+ "grad_norm": 2.225062370300293,
+ "learning_rate": 1.8262626262626265e-06,
+ "loss": 1.380183219909668,
+ "mean_token_accuracy": 0.6872862577438354,
+ "num_tokens": 25755648.0,
+ "step": 1572
+ },
+ {
+ "entropy": 1.237407922744751,
+ "epoch": 3.1777777777777776,
+ "grad_norm": 2.31829833984375,
+ "learning_rate": 1.8242424242424244e-06,
+ "loss": 1.247010588645935,
+ "mean_token_accuracy": 0.7034074068069458,
+ "num_tokens": 25772032.0,
+ "step": 1573
+ },
+ {
+ "entropy": 1.3967171907424927,
+ "epoch": 3.17979797979798,
+ "grad_norm": 2.4075193405151367,
+ "learning_rate": 1.8222222222222225e-06,
+ "loss": 1.4189220666885376,
+ "mean_token_accuracy": 0.6609672904014587,
+ "num_tokens": 25788416.0,
+ "step": 1574
+ },
+ {
+ "entropy": 1.4262871742248535,
+ "epoch": 3.1818181818181817,
+ "grad_norm": 2.507885217666626,
+ "learning_rate": 1.8202020202020204e-06,
+ "loss": 1.4657065868377686,
+ "mean_token_accuracy": 0.6502198576927185,
+ "num_tokens": 25804800.0,
+ "step": 1575
+ },
+ {
+ "entropy": 1.1763924360275269,
+ "epoch": 3.183838383838384,
+ "grad_norm": 2.179891586303711,
+ "learning_rate": 1.8181818181818183e-06,
+ "loss": 1.1837804317474365,
+ "mean_token_accuracy": 0.7209941148757935,
+ "num_tokens": 25821184.0,
+ "step": 1576
+ },
+ {
+ "entropy": 1.119240403175354,
+ "epoch": 3.1858585858585857,
+ "grad_norm": 2.300708532333374,
+ "learning_rate": 1.8161616161616164e-06,
+ "loss": 1.1251145601272583,
+ "mean_token_accuracy": 0.7134220600128174,
+ "num_tokens": 25837568.0,
+ "step": 1577
+ },
+ {
+ "entropy": 1.3045564889907837,
+ "epoch": 3.187878787878788,
+ "grad_norm": 2.2904021739959717,
+ "learning_rate": 1.8141414141414143e-06,
+ "loss": 1.3114243745803833,
+ "mean_token_accuracy": 0.691316545009613,
+ "num_tokens": 25853952.0,
+ "step": 1578
+ },
+ {
+ "entropy": 1.0507739782333374,
+ "epoch": 3.18989898989899,
+ "grad_norm": 2.0942912101745605,
+ "learning_rate": 1.8121212121212124e-06,
+ "loss": 1.043527364730835,
+ "mean_token_accuracy": 0.7244748473167419,
+ "num_tokens": 25870336.0,
+ "step": 1579
+ },
+ {
+ "entropy": 1.2050808668136597,
+ "epoch": 3.191919191919192,
+ "grad_norm": 2.131643295288086,
+ "learning_rate": 1.8101010101010103e-06,
+ "loss": 1.1917791366577148,
+ "mean_token_accuracy": 0.7046898007392883,
+ "num_tokens": 25886720.0,
+ "step": 1580
+ },
+ {
+ "entropy": 1.3607940673828125,
+ "epoch": 3.193939393939394,
+ "grad_norm": 2.2744040489196777,
+ "learning_rate": 1.8080808080808082e-06,
+ "loss": 1.347360610961914,
+ "mean_token_accuracy": 0.6780043840408325,
+ "num_tokens": 25903104.0,
+ "step": 1581
+ },
+ {
+ "entropy": 1.4996237754821777,
+ "epoch": 3.195959595959596,
+ "grad_norm": 2.3050780296325684,
+ "learning_rate": 1.8060606060606063e-06,
+ "loss": 1.5023889541625977,
+ "mean_token_accuracy": 0.6423424482345581,
+ "num_tokens": 25919488.0,
+ "step": 1582
+ },
+ {
+ "entropy": 1.1950902938842773,
+ "epoch": 3.197979797979798,
+ "grad_norm": 2.408799648284912,
+ "learning_rate": 1.8040404040404041e-06,
+ "loss": 1.1868454217910767,
+ "mean_token_accuracy": 0.6998046040534973,
+ "num_tokens": 25935872.0,
+ "step": 1583
+ },
+ {
+ "entropy": 1.5395914316177368,
+ "epoch": 3.2,
+ "grad_norm": 2.4104502201080322,
+ "learning_rate": 1.802020202020202e-06,
+ "loss": 1.5414860248565674,
+ "mean_token_accuracy": 0.6382511258125305,
+ "num_tokens": 25952256.0,
+ "step": 1584
+ },
+ {
+ "entropy": 1.0701062679290771,
+ "epoch": 3.202020202020202,
+ "grad_norm": 2.2250685691833496,
+ "learning_rate": 1.8000000000000001e-06,
+ "loss": 1.0787923336029053,
+ "mean_token_accuracy": 0.7307645082473755,
+ "num_tokens": 25968640.0,
+ "step": 1585
+ },
+ {
+ "entropy": 1.0968927145004272,
+ "epoch": 3.2040404040404042,
+ "grad_norm": 2.3169026374816895,
+ "learning_rate": 1.797979797979798e-06,
+ "loss": 1.1023577451705933,
+ "mean_token_accuracy": 0.7217879891395569,
+ "num_tokens": 25985024.0,
+ "step": 1586
+ },
+ {
+ "entropy": 1.0547802448272705,
+ "epoch": 3.206060606060606,
+ "grad_norm": 2.1187734603881836,
+ "learning_rate": 1.7959595959595961e-06,
+ "loss": 1.0536352396011353,
+ "mean_token_accuracy": 0.733146071434021,
+ "num_tokens": 26001408.0,
+ "step": 1587
+ },
+ {
+ "entropy": 1.821456789970398,
+ "epoch": 3.2080808080808083,
+ "grad_norm": 2.2364695072174072,
+ "learning_rate": 1.793939393939394e-06,
+ "loss": 1.8386787176132202,
+ "mean_token_accuracy": 0.5963605046272278,
+ "num_tokens": 26017792.0,
+ "step": 1588
+ },
+ {
+ "entropy": 1.5173720121383667,
+ "epoch": 3.21010101010101,
+ "grad_norm": 2.2785775661468506,
+ "learning_rate": 1.7919191919191919e-06,
+ "loss": 1.5099977254867554,
+ "mean_token_accuracy": 0.6417317986488342,
+ "num_tokens": 26034176.0,
+ "step": 1589
+ },
+ {
+ "entropy": 1.5751056671142578,
+ "epoch": 3.212121212121212,
+ "grad_norm": 2.2048606872558594,
+ "learning_rate": 1.78989898989899e-06,
+ "loss": 1.5845966339111328,
+ "mean_token_accuracy": 0.633671224117279,
+ "num_tokens": 26050560.0,
+ "step": 1590
+ },
+ {
+ "entropy": 1.3307859897613525,
+ "epoch": 3.214141414141414,
+ "grad_norm": 2.310291051864624,
+ "learning_rate": 1.787878787878788e-06,
+ "loss": 1.3058912754058838,
+ "mean_token_accuracy": 0.6883854269981384,
+ "num_tokens": 26066944.0,
+ "step": 1591
+ },
+ {
+ "entropy": 0.9954612255096436,
+ "epoch": 3.216161616161616,
+ "grad_norm": 2.1447651386260986,
+ "learning_rate": 1.7858585858585862e-06,
+ "loss": 0.975112795829773,
+ "mean_token_accuracy": 0.7406570315361023,
+ "num_tokens": 26083328.0,
+ "step": 1592
+ },
+ {
+ "entropy": 1.1513571739196777,
+ "epoch": 3.2181818181818183,
+ "grad_norm": 2.3595945835113525,
+ "learning_rate": 1.783838383838384e-06,
+ "loss": 1.1416239738464355,
+ "mean_token_accuracy": 0.7137274146080017,
+ "num_tokens": 26099712.0,
+ "step": 1593
+ },
+ {
+ "entropy": 0.9992931485176086,
+ "epoch": 3.22020202020202,
+ "grad_norm": 2.1605300903320312,
+ "learning_rate": 1.781818181818182e-06,
+ "loss": 1.0023880004882812,
+ "mean_token_accuracy": 0.738092303276062,
+ "num_tokens": 26116096.0,
+ "step": 1594
+ },
+ {
+ "entropy": 1.6680811643600464,
+ "epoch": 3.2222222222222223,
+ "grad_norm": 2.3036210536956787,
+ "learning_rate": 1.77979797979798e-06,
+ "loss": 1.7154381275177002,
+ "mean_token_accuracy": 0.6182217597961426,
+ "num_tokens": 26132480.0,
+ "step": 1595
+ },
+ {
+ "entropy": 1.113036036491394,
+ "epoch": 3.224242424242424,
+ "grad_norm": 2.3098108768463135,
+ "learning_rate": 1.777777777777778e-06,
+ "loss": 1.1006368398666382,
+ "mean_token_accuracy": 0.7120175957679749,
+ "num_tokens": 26148864.0,
+ "step": 1596
+ },
+ {
+ "entropy": 1.0055886507034302,
+ "epoch": 3.2262626262626264,
+ "grad_norm": 2.3227128982543945,
+ "learning_rate": 1.775757575757576e-06,
+ "loss": 1.0274525880813599,
+ "mean_token_accuracy": 0.7253297567367554,
+ "num_tokens": 26165248.0,
+ "step": 1597
+ },
+ {
+ "entropy": 1.139039397239685,
+ "epoch": 3.228282828282828,
+ "grad_norm": 2.415011405944824,
+ "learning_rate": 1.773737373737374e-06,
+ "loss": 1.1494395732879639,
+ "mean_token_accuracy": 0.7061553597450256,
+ "num_tokens": 26181632.0,
+ "step": 1598
+ },
+ {
+ "entropy": 1.1573477983474731,
+ "epoch": 3.2303030303030305,
+ "grad_norm": 2.2666749954223633,
+ "learning_rate": 1.7717171717171718e-06,
+ "loss": 1.1615169048309326,
+ "mean_token_accuracy": 0.7168416976928711,
+ "num_tokens": 26198016.0,
+ "step": 1599
+ },
+ {
+ "entropy": 1.1301828622817993,
+ "epoch": 3.2323232323232323,
+ "grad_norm": 2.345222234725952,
+ "learning_rate": 1.76969696969697e-06,
+ "loss": 1.1062344312667847,
+ "mean_token_accuracy": 0.7164753079414368,
+ "num_tokens": 26214400.0,
+ "step": 1600
+ },
+ {
+ "epoch": 3.2323232323232323,
+ "eval_entropy": 1.3245953331070561,
+ "eval_loss": 1.5489962100982666,
+ "eval_mean_token_accuracy": 0.644533898080549,
+ "eval_num_tokens": 26214400.0,
+ "eval_runtime": 43.7619,
+ "eval_samples_per_second": 22.622,
+ "eval_steps_per_second": 2.834,
+ "step": 1600
+ },
+ {
+ "entropy": 1.4168665409088135,
+ "epoch": 3.2343434343434345,
+ "grad_norm": 2.3160741329193115,
+ "learning_rate": 1.7676767676767678e-06,
+ "loss": 1.403707504272461,
+ "mean_token_accuracy": 0.6696385145187378,
+ "num_tokens": 26230784.0,
+ "step": 1601
+ },
+ {
+ "entropy": 1.4689940214157104,
+ "epoch": 3.2363636363636363,
+ "grad_norm": 2.1691131591796875,
+ "learning_rate": 1.7656565656565657e-06,
+ "loss": 1.463477611541748,
+ "mean_token_accuracy": 0.6482046842575073,
+ "num_tokens": 26247168.0,
+ "step": 1602
+ },
+ {
+ "entropy": 1.2312567234039307,
+ "epoch": 3.2383838383838386,
+ "grad_norm": 2.4240810871124268,
+ "learning_rate": 1.7636363636363638e-06,
+ "loss": 1.2253142595291138,
+ "mean_token_accuracy": 0.6933317184448242,
+ "num_tokens": 26263552.0,
+ "step": 1603
+ },
+ {
+ "entropy": 1.5346348285675049,
+ "epoch": 3.2404040404040404,
+ "grad_norm": 2.417025566101074,
+ "learning_rate": 1.7616161616161617e-06,
+ "loss": 1.5418915748596191,
+ "mean_token_accuracy": 0.6389228105545044,
+ "num_tokens": 26279936.0,
+ "step": 1604
+ },
+ {
+ "entropy": 1.4432504177093506,
+ "epoch": 3.242424242424242,
+ "grad_norm": 2.2379117012023926,
+ "learning_rate": 1.7595959595959598e-06,
+ "loss": 1.4543676376342773,
+ "mean_token_accuracy": 0.6502198576927185,
+ "num_tokens": 26296320.0,
+ "step": 1605
+ },
+ {
+ "entropy": 1.54794442653656,
+ "epoch": 3.2444444444444445,
+ "grad_norm": 2.293386936187744,
+ "learning_rate": 1.7575757575757577e-06,
+ "loss": 1.525251865386963,
+ "mean_token_accuracy": 0.652723491191864,
+ "num_tokens": 26312704.0,
+ "step": 1606
+ },
+ {
+ "entropy": 1.3418866395950317,
+ "epoch": 3.2464646464646463,
+ "grad_norm": 2.143143892288208,
+ "learning_rate": 1.7555555555555556e-06,
+ "loss": 1.3268392086029053,
+ "mean_token_accuracy": 0.6758060455322266,
+ "num_tokens": 26329088.0,
+ "step": 1607
+ },
+ {
+ "entropy": 1.3032151460647583,
+ "epoch": 3.2484848484848485,
+ "grad_norm": 2.0813350677490234,
+ "learning_rate": 1.7535353535353537e-06,
+ "loss": 1.3105076551437378,
+ "mean_token_accuracy": 0.6873473525047302,
+ "num_tokens": 26345472.0,
+ "step": 1608
+ },
+ {
+ "entropy": 1.258314847946167,
+ "epoch": 3.2505050505050503,
+ "grad_norm": 2.346703052520752,
+ "learning_rate": 1.7515151515151516e-06,
+ "loss": 1.2453927993774414,
+ "mean_token_accuracy": 0.6931484937667847,
+ "num_tokens": 26361856.0,
+ "step": 1609
+ },
+ {
+ "entropy": 1.1887613534927368,
+ "epoch": 3.2525252525252526,
+ "grad_norm": 2.2838189601898193,
+ "learning_rate": 1.7494949494949494e-06,
+ "loss": 1.2008028030395508,
+ "mean_token_accuracy": 0.7044455409049988,
+ "num_tokens": 26378240.0,
+ "step": 1610
+ },
+ {
+ "entropy": 1.2221968173980713,
+ "epoch": 3.2545454545454544,
+ "grad_norm": 2.201186418533325,
+ "learning_rate": 1.7474747474747475e-06,
+ "loss": 1.2126820087432861,
+ "mean_token_accuracy": 0.6948583126068115,
+ "num_tokens": 26394624.0,
+ "step": 1611
+ },
+ {
+ "entropy": 1.3306292295455933,
+ "epoch": 3.2565656565656567,
+ "grad_norm": 2.1501541137695312,
+ "learning_rate": 1.7454545454545456e-06,
+ "loss": 1.3301104307174683,
+ "mean_token_accuracy": 0.6844772696495056,
+ "num_tokens": 26411008.0,
+ "step": 1612
+ },
+ {
+ "entropy": 1.0881599187850952,
+ "epoch": 3.2585858585858585,
+ "grad_norm": 2.092081069946289,
+ "learning_rate": 1.7434343434343437e-06,
+ "loss": 1.0868645906448364,
+ "mean_token_accuracy": 0.7172691822052002,
+ "num_tokens": 26427392.0,
+ "step": 1613
+ },
+ {
+ "entropy": 1.1937274932861328,
+ "epoch": 3.2606060606060607,
+ "grad_norm": 2.563627243041992,
+ "learning_rate": 1.7414141414141416e-06,
+ "loss": 1.1793221235275269,
+ "mean_token_accuracy": 0.7015144228935242,
+ "num_tokens": 26443776.0,
+ "step": 1614
+ },
+ {
+ "entropy": 1.1501706838607788,
+ "epoch": 3.2626262626262625,
+ "grad_norm": 2.2964398860931396,
+ "learning_rate": 1.7393939393939397e-06,
+ "loss": 1.1438164710998535,
+ "mean_token_accuracy": 0.7105520367622375,
+ "num_tokens": 26460160.0,
+ "step": 1615
+ },
+ {
+ "entropy": 1.276473045349121,
+ "epoch": 3.264646464646465,
+ "grad_norm": 2.4301693439483643,
+ "learning_rate": 1.7373737373737376e-06,
+ "loss": 1.2897876501083374,
+ "mean_token_accuracy": 0.6933927536010742,
+ "num_tokens": 26476544.0,
+ "step": 1616
+ },
+ {
+ "entropy": 1.1926265954971313,
+ "epoch": 3.2666666666666666,
+ "grad_norm": 2.1503496170043945,
+ "learning_rate": 1.7353535353535355e-06,
+ "loss": 1.1882628202438354,
+ "mean_token_accuracy": 0.7013922929763794,
+ "num_tokens": 26492928.0,
+ "step": 1617
+ },
+ {
+ "entropy": 1.2843140363693237,
+ "epoch": 3.268686868686869,
+ "grad_norm": 2.312652587890625,
+ "learning_rate": 1.7333333333333336e-06,
+ "loss": 1.3045732975006104,
+ "mean_token_accuracy": 0.6849047541618347,
+ "num_tokens": 26509312.0,
+ "step": 1618
+ },
+ {
+ "entropy": 1.0693352222442627,
+ "epoch": 3.2707070707070707,
+ "grad_norm": 2.2860965728759766,
+ "learning_rate": 1.7313131313131315e-06,
+ "loss": 1.044942855834961,
+ "mean_token_accuracy": 0.72832190990448,
+ "num_tokens": 26525696.0,
+ "step": 1619
+ },
+ {
+ "entropy": 1.5462334156036377,
+ "epoch": 3.2727272727272725,
+ "grad_norm": 2.395378589630127,
+ "learning_rate": 1.7292929292929294e-06,
+ "loss": 1.5529801845550537,
+ "mean_token_accuracy": 0.6301294565200806,
+ "num_tokens": 26542080.0,
+ "step": 1620
+ },
+ {
+ "entropy": 1.0582746267318726,
+ "epoch": 3.2747474747474747,
+ "grad_norm": 2.198370933532715,
+ "learning_rate": 1.7272727272727275e-06,
+ "loss": 1.0674792528152466,
+ "mean_token_accuracy": 0.7289936542510986,
+ "num_tokens": 26558464.0,
+ "step": 1621
+ },
+ {
+ "entropy": 1.133894920349121,
+ "epoch": 3.276767676767677,
+ "grad_norm": 2.2087979316711426,
+ "learning_rate": 1.7252525252525254e-06,
+ "loss": 1.1322801113128662,
+ "mean_token_accuracy": 0.7148876190185547,
+ "num_tokens": 26574848.0,
+ "step": 1622
+ },
+ {
+ "entropy": 1.0465091466903687,
+ "epoch": 3.278787878787879,
+ "grad_norm": 2.4500746726989746,
+ "learning_rate": 1.7232323232323235e-06,
+ "loss": 1.042628526687622,
+ "mean_token_accuracy": 0.72832190990448,
+ "num_tokens": 26591232.0,
+ "step": 1623
+ },
+ {
+ "entropy": 1.3930310010910034,
+ "epoch": 3.2808080808080806,
+ "grad_norm": 2.2968733310699463,
+ "learning_rate": 1.7212121212121214e-06,
+ "loss": 1.3903234004974365,
+ "mean_token_accuracy": 0.6658524870872498,
+ "num_tokens": 26607616.0,
+ "step": 1624
+ },
+ {
+ "entropy": 1.4873614311218262,
+ "epoch": 3.282828282828283,
+ "grad_norm": 2.404604434967041,
+ "learning_rate": 1.7191919191919192e-06,
+ "loss": 1.4828845262527466,
+ "mean_token_accuracy": 0.6601123809814453,
+ "num_tokens": 26624000.0,
+ "step": 1625
+ },
+ {
+ "entropy": 1.4326503276824951,
+ "epoch": 3.2848484848484847,
+ "grad_norm": 2.3051655292510986,
+ "learning_rate": 1.7171717171717173e-06,
+ "loss": 1.4187182188034058,
+ "mean_token_accuracy": 0.6783097386360168,
+ "num_tokens": 26640384.0,
+ "step": 1626
+ },
+ {
+ "entropy": 1.067203402519226,
+ "epoch": 3.286868686868687,
+ "grad_norm": 2.1361143589019775,
+ "learning_rate": 1.7151515151515152e-06,
+ "loss": 1.0571379661560059,
+ "mean_token_accuracy": 0.7386419177055359,
+ "num_tokens": 26656768.0,
+ "step": 1627
+ },
+ {
+ "entropy": 0.9872021675109863,
+ "epoch": 3.2888888888888888,
+ "grad_norm": 2.084920883178711,
+ "learning_rate": 1.7131313131313131e-06,
+ "loss": 0.9779852628707886,
+ "mean_token_accuracy": 0.7591597437858582,
+ "num_tokens": 26673152.0,
+ "step": 1628
+ },
+ {
+ "entropy": 1.261454701423645,
+ "epoch": 3.290909090909091,
+ "grad_norm": 2.2008469104766846,
+ "learning_rate": 1.7111111111111112e-06,
+ "loss": 1.2701423168182373,
+ "mean_token_accuracy": 0.6892403364181519,
+ "num_tokens": 26689536.0,
+ "step": 1629
+ },
+ {
+ "entropy": 1.6490918397903442,
+ "epoch": 3.292929292929293,
+ "grad_norm": 2.2967329025268555,
+ "learning_rate": 1.7090909090909091e-06,
+ "loss": 1.6521049737930298,
+ "mean_token_accuracy": 0.6223741769790649,
+ "num_tokens": 26705920.0,
+ "step": 1630
+ },
+ {
+ "entropy": 1.241626262664795,
+ "epoch": 3.294949494949495,
+ "grad_norm": 2.3993594646453857,
+ "learning_rate": 1.707070707070707e-06,
+ "loss": 1.2557122707366943,
+ "mean_token_accuracy": 0.6982169151306152,
+ "num_tokens": 26722304.0,
+ "step": 1631
+ },
+ {
+ "entropy": 1.2038613557815552,
+ "epoch": 3.296969696969697,
+ "grad_norm": 2.6470985412597656,
+ "learning_rate": 1.705050505050505e-06,
+ "loss": 1.220262885093689,
+ "mean_token_accuracy": 0.702186107635498,
+ "num_tokens": 26738688.0,
+ "step": 1632
+ },
+ {
+ "entropy": 1.0352909564971924,
+ "epoch": 3.298989898989899,
+ "grad_norm": 2.30973219871521,
+ "learning_rate": 1.703030303030303e-06,
+ "loss": 1.0416737794876099,
+ "mean_token_accuracy": 0.7299095988273621,
+ "num_tokens": 26755072.0,
+ "step": 1633
+ },
+ {
+ "entropy": 1.0178903341293335,
+ "epoch": 3.301010101010101,
+ "grad_norm": 2.3068888187408447,
+ "learning_rate": 1.7010101010101013e-06,
+ "loss": 1.021026849746704,
+ "mean_token_accuracy": 0.7370542287826538,
+ "num_tokens": 26771456.0,
+ "step": 1634
+ },
+ {
+ "entropy": 1.1840031147003174,
+ "epoch": 3.303030303030303,
+ "grad_norm": 2.1070966720581055,
+ "learning_rate": 1.6989898989898992e-06,
+ "loss": 1.191172480583191,
+ "mean_token_accuracy": 0.7166585326194763,
+ "num_tokens": 26787840.0,
+ "step": 1635
+ },
+ {
+ "entropy": 1.3296533823013306,
+ "epoch": 3.305050505050505,
+ "grad_norm": 2.4398109912872314,
+ "learning_rate": 1.6969696969696973e-06,
+ "loss": 1.338945984840393,
+ "mean_token_accuracy": 0.67666095495224,
+ "num_tokens": 26804224.0,
+ "step": 1636
+ },
+ {
+ "entropy": 1.1331745386123657,
+ "epoch": 3.3070707070707073,
+ "grad_norm": 2.103158473968506,
+ "learning_rate": 1.6949494949494952e-06,
+ "loss": 1.139038324356079,
+ "mean_token_accuracy": 0.7154372334480286,
+ "num_tokens": 26820608.0,
+ "step": 1637
+ },
+ {
+ "entropy": 1.3300442695617676,
+ "epoch": 3.309090909090909,
+ "grad_norm": 2.4576940536499023,
+ "learning_rate": 1.692929292929293e-06,
+ "loss": 1.3464535474777222,
+ "mean_token_accuracy": 0.6715925931930542,
+ "num_tokens": 26836992.0,
+ "step": 1638
+ },
+ {
+ "entropy": 1.5538347959518433,
+ "epoch": 3.311111111111111,
+ "grad_norm": 2.4972009658813477,
+ "learning_rate": 1.6909090909090912e-06,
+ "loss": 1.544262409210205,
+ "mean_token_accuracy": 0.6377015113830566,
+ "num_tokens": 26853376.0,
+ "step": 1639
+ },
+ {
+ "entropy": 1.2610400915145874,
+ "epoch": 3.313131313131313,
+ "grad_norm": 2.354304790496826,
+ "learning_rate": 1.688888888888889e-06,
+ "loss": 1.2615927457809448,
+ "mean_token_accuracy": 0.6805691123008728,
+ "num_tokens": 26869760.0,
+ "step": 1640
+ },
+ {
+ "entropy": 0.9232268333435059,
+ "epoch": 3.315151515151515,
+ "grad_norm": 2.2459871768951416,
+ "learning_rate": 1.6868686868686871e-06,
+ "loss": 0.9295459985733032,
+ "mean_token_accuracy": 0.7508549094200134,
+ "num_tokens": 26886144.0,
+ "step": 1641
+ },
+ {
+ "entropy": 1.3030803203582764,
+ "epoch": 3.317171717171717,
+ "grad_norm": 2.086177110671997,
+ "learning_rate": 1.684848484848485e-06,
+ "loss": 1.3260836601257324,
+ "mean_token_accuracy": 0.6850268840789795,
+ "num_tokens": 26902528.0,
+ "step": 1642
+ },
+ {
+ "entropy": 1.0115952491760254,
+ "epoch": 3.319191919191919,
+ "grad_norm": 2.3342647552490234,
+ "learning_rate": 1.682828282828283e-06,
+ "loss": 1.0125452280044556,
+ "mean_token_accuracy": 0.7252076268196106,
+ "num_tokens": 26918912.0,
+ "step": 1643
+ },
+ {
+ "entropy": 0.9997871518135071,
+ "epoch": 3.3212121212121213,
+ "grad_norm": 2.0844249725341797,
+ "learning_rate": 1.680808080808081e-06,
+ "loss": 1.0233726501464844,
+ "mean_token_accuracy": 0.7490839958190918,
+ "num_tokens": 26935296.0,
+ "step": 1644
+ },
+ {
+ "entropy": 1.3131269216537476,
+ "epoch": 3.323232323232323,
+ "grad_norm": 2.279741048812866,
+ "learning_rate": 1.678787878787879e-06,
+ "loss": 1.3082493543624878,
+ "mean_token_accuracy": 0.6856985688209534,
+ "num_tokens": 26951680.0,
+ "step": 1645
+ },
+ {
+ "entropy": 1.5594273805618286,
+ "epoch": 3.3252525252525253,
+ "grad_norm": 2.423882246017456,
+ "learning_rate": 1.6767676767676768e-06,
+ "loss": 1.5775840282440186,
+ "mean_token_accuracy": 0.662493884563446,
+ "num_tokens": 26968064.0,
+ "step": 1646
+ },
+ {
+ "entropy": 1.2298411130905151,
+ "epoch": 3.327272727272727,
+ "grad_norm": 2.237107515335083,
+ "learning_rate": 1.674747474747475e-06,
+ "loss": 1.1941357851028442,
+ "mean_token_accuracy": 0.7106130719184875,
+ "num_tokens": 26984448.0,
+ "step": 1647
+ },
+ {
+ "entropy": 0.986072301864624,
+ "epoch": 3.3292929292929294,
+ "grad_norm": 2.2843995094299316,
+ "learning_rate": 1.6727272727272728e-06,
+ "loss": 0.9964872598648071,
+ "mean_token_accuracy": 0.7433438897132874,
+ "num_tokens": 27000832.0,
+ "step": 1648
+ },
+ {
+ "entropy": 1.2612992525100708,
+ "epoch": 3.3313131313131312,
+ "grad_norm": 2.2401392459869385,
+ "learning_rate": 1.6707070707070707e-06,
+ "loss": 1.2401535511016846,
+ "mean_token_accuracy": 0.6914997696876526,
+ "num_tokens": 27017216.0,
+ "step": 1649
+ },
+ {
+ "entropy": 0.8876240849494934,
+ "epoch": 3.3333333333333335,
+ "grad_norm": 2.055438995361328,
+ "learning_rate": 1.6686868686868688e-06,
+ "loss": 0.8796688914299011,
+ "mean_token_accuracy": 0.7739985585212708,
+ "num_tokens": 27033600.0,
+ "step": 1650
+ },
+ {
+ "epoch": 3.3333333333333335,
+ "eval_entropy": 1.3213856729768938,
+ "eval_loss": 1.5484445095062256,
+ "eval_mean_token_accuracy": 0.6446653873689713,
+ "eval_num_tokens": 27033600.0,
+ "eval_runtime": 43.7594,
+ "eval_samples_per_second": 22.624,
+ "eval_steps_per_second": 2.834,
+ "step": 1650
+ },
+ {
+ "entropy": 1.8024967908859253,
+ "epoch": 3.3353535353535353,
+ "grad_norm": 2.316970109939575,
+ "learning_rate": 1.6666666666666667e-06,
+ "loss": 1.8035298585891724,
+ "mean_token_accuracy": 0.6057034730911255,
+ "num_tokens": 27049984.0,
+ "step": 1651
+ },
+ {
+ "entropy": 0.8543419241905212,
+ "epoch": 3.3373737373737375,
+ "grad_norm": 2.2566277980804443,
+ "learning_rate": 1.6646464646464648e-06,
+ "loss": 0.8558261394500732,
+ "mean_token_accuracy": 0.7656326293945312,
+ "num_tokens": 27066368.0,
+ "step": 1652
+ },
+ {
+ "entropy": 1.3101993799209595,
+ "epoch": 3.3393939393939394,
+ "grad_norm": 2.490753650665283,
+ "learning_rate": 1.6626262626262626e-06,
+ "loss": 1.326491355895996,
+ "mean_token_accuracy": 0.6656082272529602,
+ "num_tokens": 27082752.0,
+ "step": 1653
+ },
+ {
+ "entropy": 1.226317048072815,
+ "epoch": 3.341414141414141,
+ "grad_norm": 2.438649892807007,
+ "learning_rate": 1.6606060606060605e-06,
+ "loss": 1.2096238136291504,
+ "mean_token_accuracy": 0.6954079270362854,
+ "num_tokens": 27099136.0,
+ "step": 1654
+ },
+ {
+ "entropy": 1.1575508117675781,
+ "epoch": 3.3434343434343434,
+ "grad_norm": 2.130772113800049,
+ "learning_rate": 1.6585858585858588e-06,
+ "loss": 1.1718693971633911,
+ "mean_token_accuracy": 0.7256350517272949,
+ "num_tokens": 27115520.0,
+ "step": 1655
+ },
+ {
+ "entropy": 1.4135949611663818,
+ "epoch": 3.3454545454545457,
+ "grad_norm": 2.3037710189819336,
+ "learning_rate": 1.6565656565656567e-06,
+ "loss": 1.408097505569458,
+ "mean_token_accuracy": 0.6669516563415527,
+ "num_tokens": 27131904.0,
+ "step": 1656
+ },
+ {
+ "entropy": 0.977189838886261,
+ "epoch": 3.3474747474747475,
+ "grad_norm": 2.3096542358398438,
+ "learning_rate": 1.6545454545454548e-06,
+ "loss": 0.9848534464836121,
+ "mean_token_accuracy": 0.7306423783302307,
+ "num_tokens": 27148288.0,
+ "step": 1657
+ },
+ {
+ "entropy": 1.6641992330551147,
+ "epoch": 3.3494949494949493,
+ "grad_norm": 2.2562549114227295,
+ "learning_rate": 1.6525252525252527e-06,
+ "loss": 1.6742463111877441,
+ "mean_token_accuracy": 0.6157181262969971,
+ "num_tokens": 27164672.0,
+ "step": 1658
+ },
+ {
+ "entropy": 1.2698633670806885,
+ "epoch": 3.3515151515151516,
+ "grad_norm": 2.426100492477417,
+ "learning_rate": 1.6505050505050508e-06,
+ "loss": 1.2856690883636475,
+ "mean_token_accuracy": 0.6808133721351624,
+ "num_tokens": 27181056.0,
+ "step": 1659
+ },
+ {
+ "entropy": 1.4621158838272095,
+ "epoch": 3.3535353535353534,
+ "grad_norm": 2.3147833347320557,
+ "learning_rate": 1.6484848484848487e-06,
+ "loss": 1.468501091003418,
+ "mean_token_accuracy": 0.6610283255577087,
+ "num_tokens": 27197440.0,
+ "step": 1660
+ },
+ {
+ "entropy": 1.249279499053955,
+ "epoch": 3.3555555555555556,
+ "grad_norm": 2.237351894378662,
+ "learning_rate": 1.6464646464646466e-06,
+ "loss": 1.2397043704986572,
+ "mean_token_accuracy": 0.6951636672019958,
+ "num_tokens": 27213824.0,
+ "step": 1661
+ },
+ {
+ "entropy": 0.9608144164085388,
+ "epoch": 3.3575757575757574,
+ "grad_norm": 2.2147974967956543,
+ "learning_rate": 1.6444444444444447e-06,
+ "loss": 0.9720965623855591,
+ "mean_token_accuracy": 0.7529922127723694,
+ "num_tokens": 27230208.0,
+ "step": 1662
+ },
+ {
+ "entropy": 1.5832558870315552,
+ "epoch": 3.3595959595959597,
+ "grad_norm": 2.325833559036255,
+ "learning_rate": 1.6424242424242426e-06,
+ "loss": 1.5746217966079712,
+ "mean_token_accuracy": 0.6457010507583618,
+ "num_tokens": 27246592.0,
+ "step": 1663
+ },
+ {
+ "entropy": 1.35304856300354,
+ "epoch": 3.3616161616161615,
+ "grad_norm": 2.509624481201172,
+ "learning_rate": 1.6404040404040405e-06,
+ "loss": 1.3489930629730225,
+ "mean_token_accuracy": 0.6699438095092773,
+ "num_tokens": 27262976.0,
+ "step": 1664
+ },
+ {
+ "entropy": 1.3161576986312866,
+ "epoch": 3.3636363636363638,
+ "grad_norm": 2.393994092941284,
+ "learning_rate": 1.6383838383838386e-06,
+ "loss": 1.303264856338501,
+ "mean_token_accuracy": 0.6855764389038086,
+ "num_tokens": 27279360.0,
+ "step": 1665
+ },
+ {
+ "entropy": 1.1878900527954102,
+ "epoch": 3.3656565656565656,
+ "grad_norm": 2.180389165878296,
+ "learning_rate": 1.6363636363636365e-06,
+ "loss": 1.182593584060669,
+ "mean_token_accuracy": 0.7091475129127502,
+ "num_tokens": 27295744.0,
+ "step": 1666
+ },
+ {
+ "entropy": 1.3966656923294067,
+ "epoch": 3.367676767676768,
+ "grad_norm": 2.097033739089966,
+ "learning_rate": 1.6343434343434344e-06,
+ "loss": 1.3977530002593994,
+ "mean_token_accuracy": 0.6893014311790466,
+ "num_tokens": 27312128.0,
+ "step": 1667
+ },
+ {
+ "entropy": 1.298252820968628,
+ "epoch": 3.3696969696969696,
+ "grad_norm": 2.301867723464966,
+ "learning_rate": 1.6323232323232325e-06,
+ "loss": 1.2884492874145508,
+ "mean_token_accuracy": 0.6819125413894653,
+ "num_tokens": 27328512.0,
+ "step": 1668
+ },
+ {
+ "entropy": 1.0709348917007446,
+ "epoch": 3.371717171717172,
+ "grad_norm": 2.323345422744751,
+ "learning_rate": 1.6303030303030303e-06,
+ "loss": 1.0587565898895264,
+ "mean_token_accuracy": 0.7345505356788635,
+ "num_tokens": 27344896.0,
+ "step": 1669
+ },
+ {
+ "entropy": 1.4100518226623535,
+ "epoch": 3.3737373737373737,
+ "grad_norm": 2.3652331829071045,
+ "learning_rate": 1.6282828282828284e-06,
+ "loss": 1.4043725728988647,
+ "mean_token_accuracy": 0.670615553855896,
+ "num_tokens": 27361280.0,
+ "step": 1670
+ },
+ {
+ "entropy": 1.3682993650436401,
+ "epoch": 3.375757575757576,
+ "grad_norm": 2.4122917652130127,
+ "learning_rate": 1.6262626262626263e-06,
+ "loss": 1.3637261390686035,
+ "mean_token_accuracy": 0.665730357170105,
+ "num_tokens": 27377664.0,
+ "step": 1671
+ },
+ {
+ "entropy": 1.1528232097625732,
+ "epoch": 3.3777777777777778,
+ "grad_norm": 2.1029880046844482,
+ "learning_rate": 1.6242424242424242e-06,
+ "loss": 1.1693168878555298,
+ "mean_token_accuracy": 0.7155593633651733,
+ "num_tokens": 27394048.0,
+ "step": 1672
+ },
+ {
+ "entropy": 1.0737035274505615,
+ "epoch": 3.3797979797979796,
+ "grad_norm": 2.0809051990509033,
+ "learning_rate": 1.6222222222222223e-06,
+ "loss": 1.0787606239318848,
+ "mean_token_accuracy": 0.7236199378967285,
+ "num_tokens": 27410432.0,
+ "step": 1673
+ },
+ {
+ "entropy": 0.992008626461029,
+ "epoch": 3.381818181818182,
+ "grad_norm": 2.216033935546875,
+ "learning_rate": 1.6202020202020202e-06,
+ "loss": 0.9949617385864258,
+ "mean_token_accuracy": 0.7416340708732605,
+ "num_tokens": 27426816.0,
+ "step": 1674
+ },
+ {
+ "entropy": 1.3799123764038086,
+ "epoch": 3.3838383838383836,
+ "grad_norm": 2.218331813812256,
+ "learning_rate": 1.618181818181818e-06,
+ "loss": 1.400660514831543,
+ "mean_token_accuracy": 0.6707376837730408,
+ "num_tokens": 27443200.0,
+ "step": 1675
+ },
+ {
+ "entropy": 1.125901222229004,
+ "epoch": 3.385858585858586,
+ "grad_norm": 2.1194028854370117,
+ "learning_rate": 1.6161616161616164e-06,
+ "loss": 1.128243088722229,
+ "mean_token_accuracy": 0.7427332401275635,
+ "num_tokens": 27459584.0,
+ "step": 1676
+ },
+ {
+ "entropy": 1.347123384475708,
+ "epoch": 3.3878787878787877,
+ "grad_norm": 2.3231191635131836,
+ "learning_rate": 1.6141414141414145e-06,
+ "loss": 1.3301351070404053,
+ "mean_token_accuracy": 0.6792256832122803,
+ "num_tokens": 27475968.0,
+ "step": 1677
+ },
+ {
+ "entropy": 1.3429654836654663,
+ "epoch": 3.38989898989899,
+ "grad_norm": 2.3116424083709717,
+ "learning_rate": 1.6121212121212124e-06,
+ "loss": 1.3509732484817505,
+ "mean_token_accuracy": 0.6730581521987915,
+ "num_tokens": 27492352.0,
+ "step": 1678
+ },
+ {
+ "entropy": 1.0976272821426392,
+ "epoch": 3.391919191919192,
+ "grad_norm": 2.2770488262176514,
+ "learning_rate": 1.6101010101010103e-06,
+ "loss": 1.0915334224700928,
+ "mean_token_accuracy": 0.72826087474823,
+ "num_tokens": 27508736.0,
+ "step": 1679
+ },
+ {
+ "entropy": 1.6415719985961914,
+ "epoch": 3.393939393939394,
+ "grad_norm": 2.224179267883301,
+ "learning_rate": 1.6080808080808084e-06,
+ "loss": 1.6517691612243652,
+ "mean_token_accuracy": 0.6277479529380798,
+ "num_tokens": 27525120.0,
+ "step": 1680
+ },
+ {
+ "entropy": 1.2124820947647095,
+ "epoch": 3.395959595959596,
+ "grad_norm": 2.1096651554107666,
+ "learning_rate": 1.6060606060606063e-06,
+ "loss": 1.230252981185913,
+ "mean_token_accuracy": 0.7074987888336182,
+ "num_tokens": 27541504.0,
+ "step": 1681
+ },
+ {
+ "entropy": 1.4807898998260498,
+ "epoch": 3.397979797979798,
+ "grad_norm": 2.1753411293029785,
+ "learning_rate": 1.6040404040404042e-06,
+ "loss": 1.4894174337387085,
+ "mean_token_accuracy": 0.6585246920585632,
+ "num_tokens": 27557888.0,
+ "step": 1682
+ },
+ {
+ "entropy": 1.31308913230896,
+ "epoch": 3.4,
+ "grad_norm": 2.251997947692871,
+ "learning_rate": 1.6020202020202023e-06,
+ "loss": 1.3479615449905396,
+ "mean_token_accuracy": 0.6800805926322937,
+ "num_tokens": 27574272.0,
+ "step": 1683
+ },
+ {
+ "entropy": 1.1820056438446045,
+ "epoch": 3.402020202020202,
+ "grad_norm": 2.074507474899292,
+ "learning_rate": 1.6000000000000001e-06,
+ "loss": 1.1995149850845337,
+ "mean_token_accuracy": 0.7117733359336853,
+ "num_tokens": 27590656.0,
+ "step": 1684
+ },
+ {
+ "entropy": 1.2405987977981567,
+ "epoch": 3.404040404040404,
+ "grad_norm": 1.9572449922561646,
+ "learning_rate": 1.597979797979798e-06,
+ "loss": 1.228973627090454,
+ "mean_token_accuracy": 0.7143990993499756,
+ "num_tokens": 27607040.0,
+ "step": 1685
+ },
+ {
+ "entropy": 1.3054970502853394,
+ "epoch": 3.4060606060606062,
+ "grad_norm": 2.174598217010498,
+ "learning_rate": 1.5959595959595961e-06,
+ "loss": 1.2918202877044678,
+ "mean_token_accuracy": 0.7015754580497742,
+ "num_tokens": 27623424.0,
+ "step": 1686
+ },
+ {
+ "entropy": 1.2568256855010986,
+ "epoch": 3.408080808080808,
+ "grad_norm": 2.1647098064422607,
+ "learning_rate": 1.593939393939394e-06,
+ "loss": 1.2464665174484253,
+ "mean_token_accuracy": 0.7046898007392883,
+ "num_tokens": 27639808.0,
+ "step": 1687
+ },
+ {
+ "entropy": 0.9985859990119934,
+ "epoch": 3.41010101010101,
+ "grad_norm": 2.230548143386841,
+ "learning_rate": 1.5919191919191921e-06,
+ "loss": 0.9941741824150085,
+ "mean_token_accuracy": 0.7424889802932739,
+ "num_tokens": 27656192.0,
+ "step": 1688
+ },
+ {
+ "entropy": 1.2002249956130981,
+ "epoch": 3.412121212121212,
+ "grad_norm": 2.211632490158081,
+ "learning_rate": 1.58989898989899e-06,
+ "loss": 1.1901870965957642,
+ "mean_token_accuracy": 0.7168416976928711,
+ "num_tokens": 27672576.0,
+ "step": 1689
+ },
+ {
+ "entropy": 1.5074162483215332,
+ "epoch": 3.4141414141414144,
+ "grad_norm": 2.3280093669891357,
+ "learning_rate": 1.5878787878787879e-06,
+ "loss": 1.5252172946929932,
+ "mean_token_accuracy": 0.6465559601783752,
+ "num_tokens": 27688960.0,
+ "step": 1690
+ },
+ {
+ "entropy": 1.1828655004501343,
+ "epoch": 3.416161616161616,
+ "grad_norm": 2.3439552783966064,
+ "learning_rate": 1.585858585858586e-06,
+ "loss": 1.196863055229187,
+ "mean_token_accuracy": 0.7009037733078003,
+ "num_tokens": 27705344.0,
+ "step": 1691
+ },
+ {
+ "entropy": 1.176308035850525,
+ "epoch": 3.418181818181818,
+ "grad_norm": 2.460371255874634,
+ "learning_rate": 1.5838383838383839e-06,
+ "loss": 1.1818437576293945,
+ "mean_token_accuracy": 0.6998656392097473,
+ "num_tokens": 27721728.0,
+ "step": 1692
+ },
+ {
+ "entropy": 1.1989773511886597,
+ "epoch": 3.4202020202020202,
+ "grad_norm": 2.406575918197632,
+ "learning_rate": 1.5818181818181818e-06,
+ "loss": 1.2158204317092896,
+ "mean_token_accuracy": 0.6949194073677063,
+ "num_tokens": 27738112.0,
+ "step": 1693
+ },
+ {
+ "entropy": 1.537143588066101,
+ "epoch": 3.422222222222222,
+ "grad_norm": 2.298275947570801,
+ "learning_rate": 1.5797979797979799e-06,
+ "loss": 1.5335557460784912,
+ "mean_token_accuracy": 0.6571812629699707,
+ "num_tokens": 27754496.0,
+ "step": 1694
+ },
+ {
+ "entropy": 1.3271749019622803,
+ "epoch": 3.4242424242424243,
+ "grad_norm": 2.309699058532715,
+ "learning_rate": 1.5777777777777778e-06,
+ "loss": 1.3499958515167236,
+ "mean_token_accuracy": 0.6943087577819824,
+ "num_tokens": 27770880.0,
+ "step": 1695
+ },
+ {
+ "entropy": 1.4381791353225708,
+ "epoch": 3.426262626262626,
+ "grad_norm": 2.2183644771575928,
+ "learning_rate": 1.5757575757575759e-06,
+ "loss": 1.4350523948669434,
+ "mean_token_accuracy": 0.6568148732185364,
+ "num_tokens": 27787264.0,
+ "step": 1696
+ },
+ {
+ "entropy": 0.9867562055587769,
+ "epoch": 3.4282828282828284,
+ "grad_norm": 2.13663387298584,
+ "learning_rate": 1.5737373737373737e-06,
+ "loss": 0.9871019124984741,
+ "mean_token_accuracy": 0.7538471221923828,
+ "num_tokens": 27803648.0,
+ "step": 1697
+ },
+ {
+ "entropy": 1.0085591077804565,
+ "epoch": 3.43030303030303,
+ "grad_norm": 2.3447134494781494,
+ "learning_rate": 1.571717171717172e-06,
+ "loss": 1.0313187837600708,
+ "mean_token_accuracy": 0.7325354218482971,
+ "num_tokens": 27820032.0,
+ "step": 1698
+ },
+ {
+ "entropy": 0.9242914915084839,
+ "epoch": 3.4323232323232324,
+ "grad_norm": 2.16290545463562,
+ "learning_rate": 1.56969696969697e-06,
+ "loss": 0.9127025604248047,
+ "mean_token_accuracy": 0.7653273344039917,
+ "num_tokens": 27836416.0,
+ "step": 1699
+ },
+ {
+ "entropy": 1.4173065423965454,
+ "epoch": 3.4343434343434343,
+ "grad_norm": 2.192754030227661,
+ "learning_rate": 1.5676767676767678e-06,
+ "loss": 1.4122503995895386,
+ "mean_token_accuracy": 0.6770884394645691,
+ "num_tokens": 27852800.0,
+ "step": 1700
+ },
+ {
+ "epoch": 3.4343434343434343,
+ "eval_entropy": 1.3203289614569755,
+ "eval_loss": 1.5483064651489258,
+ "eval_mean_token_accuracy": 0.6447958845284677,
+ "eval_num_tokens": 27852800.0,
+ "eval_runtime": 43.7896,
+ "eval_samples_per_second": 22.608,
+ "eval_steps_per_second": 2.832,
+ "step": 1700
+ },
+ {
+ "entropy": 1.4677248001098633,
+ "epoch": 3.4363636363636365,
+ "grad_norm": 2.07183575630188,
+ "learning_rate": 1.565656565656566e-06,
+ "loss": 1.4772744178771973,
+ "mean_token_accuracy": 0.6927821040153503,
+ "num_tokens": 27869184.0,
+ "step": 1701
+ },
+ {
+ "entropy": 1.845194935798645,
+ "epoch": 3.4383838383838383,
+ "grad_norm": 1.9974539279937744,
+ "learning_rate": 1.5636363636363638e-06,
+ "loss": 1.8598425388336182,
+ "mean_token_accuracy": 0.5933683514595032,
+ "num_tokens": 27885568.0,
+ "step": 1702
+ },
+ {
+ "entropy": 1.1044740676879883,
+ "epoch": 3.4404040404040406,
+ "grad_norm": 2.199014186859131,
+ "learning_rate": 1.5616161616161617e-06,
+ "loss": 1.119746446609497,
+ "mean_token_accuracy": 0.7175744771957397,
+ "num_tokens": 27901952.0,
+ "step": 1703
+ },
+ {
+ "entropy": 0.920816957950592,
+ "epoch": 3.4424242424242424,
+ "grad_norm": 2.3172857761383057,
+ "learning_rate": 1.5595959595959598e-06,
+ "loss": 0.908664882183075,
+ "mean_token_accuracy": 0.7598314881324768,
+ "num_tokens": 27918336.0,
+ "step": 1704
+ },
+ {
+ "entropy": 1.4610254764556885,
+ "epoch": 3.4444444444444446,
+ "grad_norm": 2.5196661949157715,
+ "learning_rate": 1.5575757575757577e-06,
+ "loss": 1.4767801761627197,
+ "mean_token_accuracy": 0.652662456035614,
+ "num_tokens": 27934720.0,
+ "step": 1705
+ },
+ {
+ "entropy": 1.0131516456604004,
+ "epoch": 3.4464646464646465,
+ "grad_norm": 2.2522189617156982,
+ "learning_rate": 1.5555555555555558e-06,
+ "loss": 1.0143696069717407,
+ "mean_token_accuracy": 0.7348558902740479,
+ "num_tokens": 27951104.0,
+ "step": 1706
+ },
+ {
+ "entropy": 1.2054252624511719,
+ "epoch": 3.4484848484848483,
+ "grad_norm": 2.1769580841064453,
+ "learning_rate": 1.5535353535353537e-06,
+ "loss": 1.2020167112350464,
+ "mean_token_accuracy": 0.7076209187507629,
+ "num_tokens": 27967488.0,
+ "step": 1707
+ },
+ {
+ "entropy": 1.1004403829574585,
+ "epoch": 3.4505050505050505,
+ "grad_norm": 2.3803768157958984,
+ "learning_rate": 1.5515151515151516e-06,
+ "loss": 1.1220088005065918,
+ "mean_token_accuracy": 0.712506115436554,
+ "num_tokens": 27983872.0,
+ "step": 1708
+ },
+ {
+ "entropy": 1.423991322517395,
+ "epoch": 3.4525252525252528,
+ "grad_norm": 2.2648093700408936,
+ "learning_rate": 1.5494949494949497e-06,
+ "loss": 1.432944893836975,
+ "mean_token_accuracy": 0.6545554399490356,
+ "num_tokens": 28000256.0,
+ "step": 1709
+ },
+ {
+ "entropy": 1.5308680534362793,
+ "epoch": 3.4545454545454546,
+ "grad_norm": 2.1772451400756836,
+ "learning_rate": 1.5474747474747476e-06,
+ "loss": 1.5395504236221313,
+ "mean_token_accuracy": 0.6433805823326111,
+ "num_tokens": 28016640.0,
+ "step": 1710
+ },
+ {
+ "entropy": 1.3868193626403809,
+ "epoch": 3.4565656565656564,
+ "grad_norm": 2.1782312393188477,
+ "learning_rate": 1.5454545454545454e-06,
+ "loss": 1.3730117082595825,
+ "mean_token_accuracy": 0.6857596635818481,
+ "num_tokens": 28033024.0,
+ "step": 1711
+ },
+ {
+ "entropy": 0.6769964694976807,
+ "epoch": 3.4585858585858587,
+ "grad_norm": 1.941037654876709,
+ "learning_rate": 1.5434343434343435e-06,
+ "loss": 0.6863309144973755,
+ "mean_token_accuracy": 0.8126526474952698,
+ "num_tokens": 28049408.0,
+ "step": 1712
+ },
+ {
+ "entropy": 1.1113249063491821,
+ "epoch": 3.4606060606060605,
+ "grad_norm": 2.0039165019989014,
+ "learning_rate": 1.5414141414141414e-06,
+ "loss": 1.1056404113769531,
+ "mean_token_accuracy": 0.7314362525939941,
+ "num_tokens": 28065792.0,
+ "step": 1713
+ },
+ {
+ "entropy": 0.8827037811279297,
+ "epoch": 3.4626262626262627,
+ "grad_norm": 2.2235255241394043,
+ "learning_rate": 1.5393939393939395e-06,
+ "loss": 0.8903871178627014,
+ "mean_token_accuracy": 0.7547020316123962,
+ "num_tokens": 28082176.0,
+ "step": 1714
+ },
+ {
+ "entropy": 1.2060964107513428,
+ "epoch": 3.4646464646464645,
+ "grad_norm": 2.2997398376464844,
+ "learning_rate": 1.5373737373737374e-06,
+ "loss": 1.2247741222381592,
+ "mean_token_accuracy": 0.6979726552963257,
+ "num_tokens": 28098560.0,
+ "step": 1715
+ },
+ {
+ "entropy": 1.2543354034423828,
+ "epoch": 3.466666666666667,
+ "grad_norm": 2.2603039741516113,
+ "learning_rate": 1.5353535353535353e-06,
+ "loss": 1.2638390064239502,
+ "mean_token_accuracy": 0.6919271945953369,
+ "num_tokens": 28114944.0,
+ "step": 1716
+ },
+ {
+ "entropy": 1.4569451808929443,
+ "epoch": 3.4686868686868686,
+ "grad_norm": 2.3623344898223877,
+ "learning_rate": 1.5333333333333334e-06,
+ "loss": 1.4669182300567627,
+ "mean_token_accuracy": 0.6596238613128662,
+ "num_tokens": 28131328.0,
+ "step": 1717
+ },
+ {
+ "entropy": 1.0979474782943726,
+ "epoch": 3.470707070707071,
+ "grad_norm": 2.282487154006958,
+ "learning_rate": 1.5313131313131313e-06,
+ "loss": 1.123124361038208,
+ "mean_token_accuracy": 0.7218490242958069,
+ "num_tokens": 28147712.0,
+ "step": 1718
+ },
+ {
+ "entropy": 1.3850172758102417,
+ "epoch": 3.4727272727272727,
+ "grad_norm": 2.406338930130005,
+ "learning_rate": 1.5292929292929296e-06,
+ "loss": 1.361925482749939,
+ "mean_token_accuracy": 0.672874927520752,
+ "num_tokens": 28164096.0,
+ "step": 1719
+ },
+ {
+ "entropy": 1.3522284030914307,
+ "epoch": 3.474747474747475,
+ "grad_norm": 2.171316385269165,
+ "learning_rate": 1.5272727272727275e-06,
+ "loss": 1.3782949447631836,
+ "mean_token_accuracy": 0.6841108798980713,
+ "num_tokens": 28180480.0,
+ "step": 1720
+ },
+ {
+ "entropy": 1.4793171882629395,
+ "epoch": 3.4767676767676767,
+ "grad_norm": 2.29586124420166,
+ "learning_rate": 1.5252525252525254e-06,
+ "loss": 1.4899311065673828,
+ "mean_token_accuracy": 0.6422203183174133,
+ "num_tokens": 28196864.0,
+ "step": 1721
+ },
+ {
+ "entropy": 1.0572689771652222,
+ "epoch": 3.4787878787878785,
+ "grad_norm": 2.0798633098602295,
+ "learning_rate": 1.5232323232323235e-06,
+ "loss": 1.0417135953903198,
+ "mean_token_accuracy": 0.7372373938560486,
+ "num_tokens": 28213248.0,
+ "step": 1722
+ },
+ {
+ "entropy": 1.0050398111343384,
+ "epoch": 3.480808080808081,
+ "grad_norm": 2.205551862716675,
+ "learning_rate": 1.5212121212121214e-06,
+ "loss": 1.0072510242462158,
+ "mean_token_accuracy": 0.7414509057998657,
+ "num_tokens": 28229632.0,
+ "step": 1723
+ },
+ {
+ "entropy": 0.9438449144363403,
+ "epoch": 3.482828282828283,
+ "grad_norm": 2.177335262298584,
+ "learning_rate": 1.5191919191919195e-06,
+ "loss": 0.9125795364379883,
+ "mean_token_accuracy": 0.7605642676353455,
+ "num_tokens": 28246016.0,
+ "step": 1724
+ },
+ {
+ "entropy": 0.8600634336471558,
+ "epoch": 3.484848484848485,
+ "grad_norm": 2.286794662475586,
+ "learning_rate": 1.5171717171717174e-06,
+ "loss": 0.8473736047744751,
+ "mean_token_accuracy": 0.7839521169662476,
+ "num_tokens": 28262400.0,
+ "step": 1725
+ },
+ {
+ "entropy": 1.0827349424362183,
+ "epoch": 3.4868686868686867,
+ "grad_norm": 2.176316738128662,
+ "learning_rate": 1.5151515151515152e-06,
+ "loss": 1.0760138034820557,
+ "mean_token_accuracy": 0.7195896506309509,
+ "num_tokens": 28278784.0,
+ "step": 1726
+ },
+ {
+ "entropy": 1.3604737520217896,
+ "epoch": 3.488888888888889,
+ "grad_norm": 2.3393819332122803,
+ "learning_rate": 1.5131313131313133e-06,
+ "loss": 1.3466029167175293,
+ "mean_token_accuracy": 0.6731802821159363,
+ "num_tokens": 28295168.0,
+ "step": 1727
+ },
+ {
+ "entropy": 1.4611977338790894,
+ "epoch": 3.4909090909090907,
+ "grad_norm": 2.3396732807159424,
+ "learning_rate": 1.5111111111111112e-06,
+ "loss": 1.474691390991211,
+ "mean_token_accuracy": 0.6497313380241394,
+ "num_tokens": 28311552.0,
+ "step": 1728
+ },
+ {
+ "entropy": 1.1734099388122559,
+ "epoch": 3.492929292929293,
+ "grad_norm": 2.154437780380249,
+ "learning_rate": 1.5090909090909091e-06,
+ "loss": 1.1631931066513062,
+ "mean_token_accuracy": 0.7048119306564331,
+ "num_tokens": 28327936.0,
+ "step": 1729
+ },
+ {
+ "entropy": 1.0015299320220947,
+ "epoch": 3.494949494949495,
+ "grad_norm": 2.273815393447876,
+ "learning_rate": 1.5070707070707072e-06,
+ "loss": 0.9818742871284485,
+ "mean_token_accuracy": 0.7361382246017456,
+ "num_tokens": 28344320.0,
+ "step": 1730
+ },
+ {
+ "entropy": 1.0342366695404053,
+ "epoch": 3.496969696969697,
+ "grad_norm": 2.1011879444122314,
+ "learning_rate": 1.5050505050505051e-06,
+ "loss": 1.02262282371521,
+ "mean_token_accuracy": 0.7448094487190247,
+ "num_tokens": 28360704.0,
+ "step": 1731
+ },
+ {
+ "entropy": 1.6318060159683228,
+ "epoch": 3.498989898989899,
+ "grad_norm": 2.338040590286255,
+ "learning_rate": 1.5030303030303032e-06,
+ "loss": 1.6215972900390625,
+ "mean_token_accuracy": 0.6225574016571045,
+ "num_tokens": 28377088.0,
+ "step": 1732
+ },
+ {
+ "entropy": 1.2391737699508667,
+ "epoch": 3.501010101010101,
+ "grad_norm": 2.2958035469055176,
+ "learning_rate": 1.501010101010101e-06,
+ "loss": 1.2481398582458496,
+ "mean_token_accuracy": 0.6922325491905212,
+ "num_tokens": 28393472.0,
+ "step": 1733
+ },
+ {
+ "entropy": 1.09321129322052,
+ "epoch": 3.503030303030303,
+ "grad_norm": 2.117506980895996,
+ "learning_rate": 1.498989898989899e-06,
+ "loss": 1.100616455078125,
+ "mean_token_accuracy": 0.7240473628044128,
+ "num_tokens": 28409856.0,
+ "step": 1734
+ },
+ {
+ "entropy": 1.2746680974960327,
+ "epoch": 3.505050505050505,
+ "grad_norm": 2.2428109645843506,
+ "learning_rate": 1.496969696969697e-06,
+ "loss": 1.278875470161438,
+ "mean_token_accuracy": 0.688629686832428,
+ "num_tokens": 28426240.0,
+ "step": 1735
+ },
+ {
+ "entropy": 0.9300931096076965,
+ "epoch": 3.507070707070707,
+ "grad_norm": 2.0250492095947266,
+ "learning_rate": 1.494949494949495e-06,
+ "loss": 0.9304848909378052,
+ "mean_token_accuracy": 0.7613580822944641,
+ "num_tokens": 28442624.0,
+ "step": 1736
+ },
+ {
+ "entropy": 1.5743480920791626,
+ "epoch": 3.509090909090909,
+ "grad_norm": 2.251974582672119,
+ "learning_rate": 1.4929292929292929e-06,
+ "loss": 1.570695400238037,
+ "mean_token_accuracy": 0.6520518064498901,
+ "num_tokens": 28459008.0,
+ "step": 1737
+ },
+ {
+ "entropy": 1.0316240787506104,
+ "epoch": 3.511111111111111,
+ "grad_norm": 2.187208652496338,
+ "learning_rate": 1.490909090909091e-06,
+ "loss": 1.022985816001892,
+ "mean_token_accuracy": 0.7444430589675903,
+ "num_tokens": 28475392.0,
+ "step": 1738
+ },
+ {
+ "entropy": 1.0719387531280518,
+ "epoch": 3.5131313131313133,
+ "grad_norm": 2.1497936248779297,
+ "learning_rate": 1.4888888888888888e-06,
+ "loss": 1.087444543838501,
+ "mean_token_accuracy": 0.7261846661567688,
+ "num_tokens": 28491776.0,
+ "step": 1739
+ },
+ {
+ "entropy": 1.679359793663025,
+ "epoch": 3.515151515151515,
+ "grad_norm": 2.2430360317230225,
+ "learning_rate": 1.4868686868686872e-06,
+ "loss": 1.6981315612792969,
+ "mean_token_accuracy": 0.6258549094200134,
+ "num_tokens": 28508160.0,
+ "step": 1740
+ },
+ {
+ "entropy": 1.0614526271820068,
+ "epoch": 3.517171717171717,
+ "grad_norm": 2.220466375350952,
+ "learning_rate": 1.484848484848485e-06,
+ "loss": 1.0519301891326904,
+ "mean_token_accuracy": 0.7313751578330994,
+ "num_tokens": 28524544.0,
+ "step": 1741
+ },
+ {
+ "entropy": 1.267195224761963,
+ "epoch": 3.519191919191919,
+ "grad_norm": 1.9448764324188232,
+ "learning_rate": 1.4828282828282831e-06,
+ "loss": 1.2594044208526611,
+ "mean_token_accuracy": 0.7093917727470398,
+ "num_tokens": 28540928.0,
+ "step": 1742
+ },
+ {
+ "entropy": 1.0900983810424805,
+ "epoch": 3.5212121212121215,
+ "grad_norm": 2.292336940765381,
+ "learning_rate": 1.480808080808081e-06,
+ "loss": 1.0952692031860352,
+ "mean_token_accuracy": 0.7146433591842651,
+ "num_tokens": 28557312.0,
+ "step": 1743
+ },
+ {
+ "entropy": 1.4188793897628784,
+ "epoch": 3.5232323232323233,
+ "grad_norm": 2.355440139770508,
+ "learning_rate": 1.478787878787879e-06,
+ "loss": 1.4508819580078125,
+ "mean_token_accuracy": 0.6642037034034729,
+ "num_tokens": 28573696.0,
+ "step": 1744
+ },
+ {
+ "entropy": 1.52566397190094,
+ "epoch": 3.525252525252525,
+ "grad_norm": 2.2767279148101807,
+ "learning_rate": 1.476767676767677e-06,
+ "loss": 1.5568758249282837,
+ "mean_token_accuracy": 0.6472276449203491,
+ "num_tokens": 28590080.0,
+ "step": 1745
+ },
+ {
+ "entropy": 1.4695135354995728,
+ "epoch": 3.5272727272727273,
+ "grad_norm": 2.3080222606658936,
+ "learning_rate": 1.474747474747475e-06,
+ "loss": 1.4671947956085205,
+ "mean_token_accuracy": 0.6530898809432983,
+ "num_tokens": 28606464.0,
+ "step": 1746
+ },
+ {
+ "entropy": 1.438908338546753,
+ "epoch": 3.529292929292929,
+ "grad_norm": 2.485170364379883,
+ "learning_rate": 1.4727272727272728e-06,
+ "loss": 1.4531956911087036,
+ "mean_token_accuracy": 0.6664020419120789,
+ "num_tokens": 28622848.0,
+ "step": 1747
+ },
+ {
+ "entropy": 1.5307536125183105,
+ "epoch": 3.5313131313131314,
+ "grad_norm": 2.3280844688415527,
+ "learning_rate": 1.470707070707071e-06,
+ "loss": 1.5298030376434326,
+ "mean_token_accuracy": 0.642892062664032,
+ "num_tokens": 28639232.0,
+ "step": 1748
+ },
+ {
+ "entropy": 1.429180383682251,
+ "epoch": 3.533333333333333,
+ "grad_norm": 2.36387038230896,
+ "learning_rate": 1.4686868686868688e-06,
+ "loss": 1.4191480875015259,
+ "mean_token_accuracy": 0.6469833850860596,
+ "num_tokens": 28655616.0,
+ "step": 1749
+ },
+ {
+ "entropy": 0.9336722493171692,
+ "epoch": 3.5353535353535355,
+ "grad_norm": 2.1873295307159424,
+ "learning_rate": 1.4666666666666669e-06,
+ "loss": 0.9257663488388062,
+ "mean_token_accuracy": 0.7629457712173462,
+ "num_tokens": 28672000.0,
+ "step": 1750
+ },
+ {
+ "epoch": 3.5353535353535355,
+ "eval_entropy": 1.3185442157330052,
+ "eval_loss": 1.548416018486023,
+ "eval_mean_token_accuracy": 0.6448259276728476,
+ "eval_num_tokens": 28672000.0,
+ "eval_runtime": 43.8376,
+ "eval_samples_per_second": 22.583,
+ "eval_steps_per_second": 2.829,
+ "step": 1750
+ },
+ {
+ "entropy": 1.2647572755813599,
+ "epoch": 3.5373737373737373,
+ "grad_norm": 2.1273581981658936,
+ "learning_rate": 1.4646464646464648e-06,
+ "loss": 1.2557384967803955,
+ "mean_token_accuracy": 0.6935759782791138,
+ "num_tokens": 28688384.0,
+ "step": 1751
+ },
+ {
+ "entropy": 1.4252692461013794,
+ "epoch": 3.5393939393939395,
+ "grad_norm": 2.052034616470337,
+ "learning_rate": 1.4626262626262627e-06,
+ "loss": 1.43434739112854,
+ "mean_token_accuracy": 0.6618832349777222,
+ "num_tokens": 28704768.0,
+ "step": 1752
+ },
+ {
+ "entropy": 1.6918041706085205,
+ "epoch": 3.5414141414141413,
+ "grad_norm": 2.20965313911438,
+ "learning_rate": 1.4606060606060608e-06,
+ "loss": 1.707175850868225,
+ "mean_token_accuracy": 0.6127870082855225,
+ "num_tokens": 28721152.0,
+ "step": 1753
+ },
+ {
+ "entropy": 1.5122125148773193,
+ "epoch": 3.5434343434343436,
+ "grad_norm": 2.2061564922332764,
+ "learning_rate": 1.4585858585858586e-06,
+ "loss": 1.5119690895080566,
+ "mean_token_accuracy": 0.6488153338432312,
+ "num_tokens": 28737536.0,
+ "step": 1754
+ },
+ {
+ "entropy": 0.9536482095718384,
+ "epoch": 3.5454545454545454,
+ "grad_norm": 2.310809373855591,
+ "learning_rate": 1.4565656565656565e-06,
+ "loss": 0.9411349296569824,
+ "mean_token_accuracy": 0.7589765787124634,
+ "num_tokens": 28753920.0,
+ "step": 1755
+ },
+ {
+ "entropy": 1.5147753953933716,
+ "epoch": 3.5474747474747472,
+ "grad_norm": 2.520862340927124,
+ "learning_rate": 1.4545454545454546e-06,
+ "loss": 1.5047588348388672,
+ "mean_token_accuracy": 0.639167070388794,
+ "num_tokens": 28770304.0,
+ "step": 1756
+ },
+ {
+ "entropy": 1.3881752490997314,
+ "epoch": 3.5494949494949495,
+ "grad_norm": 2.3421478271484375,
+ "learning_rate": 1.4525252525252525e-06,
+ "loss": 1.3775359392166138,
+ "mean_token_accuracy": 0.6719589829444885,
+ "num_tokens": 28786688.0,
+ "step": 1757
+ },
+ {
+ "entropy": 1.0432544946670532,
+ "epoch": 3.5515151515151517,
+ "grad_norm": 2.235539674758911,
+ "learning_rate": 1.4505050505050506e-06,
+ "loss": 1.055377721786499,
+ "mean_token_accuracy": 0.7360160946846008,
+ "num_tokens": 28803072.0,
+ "step": 1758
+ },
+ {
+ "entropy": 1.2739083766937256,
+ "epoch": 3.5535353535353535,
+ "grad_norm": 2.295605421066284,
+ "learning_rate": 1.4484848484848485e-06,
+ "loss": 1.289074420928955,
+ "mean_token_accuracy": 0.6969345211982727,
+ "num_tokens": 28819456.0,
+ "step": 1759
+ },
+ {
+ "entropy": 1.192219853401184,
+ "epoch": 3.5555555555555554,
+ "grad_norm": 2.1039023399353027,
+ "learning_rate": 1.4464646464646464e-06,
+ "loss": 1.2043861150741577,
+ "mean_token_accuracy": 0.7080483436584473,
+ "num_tokens": 28835840.0,
+ "step": 1760
+ },
+ {
+ "entropy": 1.002448320388794,
+ "epoch": 3.5575757575757576,
+ "grad_norm": 2.476088523864746,
+ "learning_rate": 1.4444444444444445e-06,
+ "loss": 1.0271326303482056,
+ "mean_token_accuracy": 0.7325354218482971,
+ "num_tokens": 28852224.0,
+ "step": 1761
+ },
+ {
+ "entropy": 1.4694463014602661,
+ "epoch": 3.55959595959596,
+ "grad_norm": 2.298769235610962,
+ "learning_rate": 1.4424242424242426e-06,
+ "loss": 1.4576467275619507,
+ "mean_token_accuracy": 0.6591353416442871,
+ "num_tokens": 28868608.0,
+ "step": 1762
+ },
+ {
+ "entropy": 1.4025219678878784,
+ "epoch": 3.5616161616161617,
+ "grad_norm": 2.148528575897217,
+ "learning_rate": 1.4404040404040407e-06,
+ "loss": 1.3884464502334595,
+ "mean_token_accuracy": 0.6655471324920654,
+ "num_tokens": 28884992.0,
+ "step": 1763
+ },
+ {
+ "entropy": 1.2829358577728271,
+ "epoch": 3.5636363636363635,
+ "grad_norm": 2.1344001293182373,
+ "learning_rate": 1.4383838383838386e-06,
+ "loss": 1.3100465536117554,
+ "mean_token_accuracy": 0.6872862577438354,
+ "num_tokens": 28901376.0,
+ "step": 1764
+ },
+ {
+ "entropy": 1.2500309944152832,
+ "epoch": 3.5656565656565657,
+ "grad_norm": 2.1933422088623047,
+ "learning_rate": 1.4363636363636365e-06,
+ "loss": 1.2566546201705933,
+ "mean_token_accuracy": 0.7057889699935913,
+ "num_tokens": 28917760.0,
+ "step": 1765
+ },
+ {
+ "entropy": 1.4471250772476196,
+ "epoch": 3.5676767676767676,
+ "grad_norm": 2.2238707542419434,
+ "learning_rate": 1.4343434343434346e-06,
+ "loss": 1.4434895515441895,
+ "mean_token_accuracy": 0.6715925931930542,
+ "num_tokens": 28934144.0,
+ "step": 1766
+ },
+ {
+ "entropy": 1.2484838962554932,
+ "epoch": 3.56969696969697,
+ "grad_norm": 2.272259473800659,
+ "learning_rate": 1.4323232323232325e-06,
+ "loss": 1.2546260356903076,
+ "mean_token_accuracy": 0.6996824741363525,
+ "num_tokens": 28950528.0,
+ "step": 1767
+ },
+ {
+ "entropy": 1.326412558555603,
+ "epoch": 3.5717171717171716,
+ "grad_norm": 2.157593250274658,
+ "learning_rate": 1.4303030303030306e-06,
+ "loss": 1.3175407648086548,
+ "mean_token_accuracy": 0.7037127614021301,
+ "num_tokens": 28966912.0,
+ "step": 1768
+ },
+ {
+ "entropy": 1.314650297164917,
+ "epoch": 3.573737373737374,
+ "grad_norm": 2.146393299102783,
+ "learning_rate": 1.4282828282828284e-06,
+ "loss": 1.3232550621032715,
+ "mean_token_accuracy": 0.6864924430847168,
+ "num_tokens": 28983296.0,
+ "step": 1769
+ },
+ {
+ "entropy": 1.2547006607055664,
+ "epoch": 3.5757575757575757,
+ "grad_norm": 2.2809762954711914,
+ "learning_rate": 1.4262626262626263e-06,
+ "loss": 1.2578558921813965,
+ "mean_token_accuracy": 0.689667820930481,
+ "num_tokens": 28999680.0,
+ "step": 1770
+ },
+ {
+ "entropy": 1.1902955770492554,
+ "epoch": 3.5777777777777775,
+ "grad_norm": 2.5193467140197754,
+ "learning_rate": 1.4242424242424244e-06,
+ "loss": 1.1852655410766602,
+ "mean_token_accuracy": 0.699499249458313,
+ "num_tokens": 29016064.0,
+ "step": 1771
+ },
+ {
+ "entropy": 1.075490951538086,
+ "epoch": 3.5797979797979798,
+ "grad_norm": 2.055223226547241,
+ "learning_rate": 1.4222222222222223e-06,
+ "loss": 1.09010648727417,
+ "mean_token_accuracy": 0.7302759885787964,
+ "num_tokens": 29032448.0,
+ "step": 1772
+ },
+ {
+ "entropy": 1.1549168825149536,
+ "epoch": 3.581818181818182,
+ "grad_norm": 2.1802778244018555,
+ "learning_rate": 1.4202020202020202e-06,
+ "loss": 1.1465229988098145,
+ "mean_token_accuracy": 0.7187957763671875,
+ "num_tokens": 29048832.0,
+ "step": 1773
+ },
+ {
+ "entropy": 1.0966418981552124,
+ "epoch": 3.583838383838384,
+ "grad_norm": 2.354775905609131,
+ "learning_rate": 1.4181818181818183e-06,
+ "loss": 1.0915825366973877,
+ "mean_token_accuracy": 0.7263067960739136,
+ "num_tokens": 29065216.0,
+ "step": 1774
+ },
+ {
+ "entropy": 1.3264915943145752,
+ "epoch": 3.5858585858585856,
+ "grad_norm": 2.291811943054199,
+ "learning_rate": 1.4161616161616162e-06,
+ "loss": 1.3168433904647827,
+ "mean_token_accuracy": 0.6811797618865967,
+ "num_tokens": 29081600.0,
+ "step": 1775
+ },
+ {
+ "entropy": 1.4051839113235474,
+ "epoch": 3.587878787878788,
+ "grad_norm": 2.4314849376678467,
+ "learning_rate": 1.4141414141414143e-06,
+ "loss": 1.4150385856628418,
+ "mean_token_accuracy": 0.6616389751434326,
+ "num_tokens": 29097984.0,
+ "step": 1776
+ },
+ {
+ "entropy": 1.045767903327942,
+ "epoch": 3.58989898989899,
+ "grad_norm": 2.361785650253296,
+ "learning_rate": 1.4121212121212122e-06,
+ "loss": 1.0414667129516602,
+ "mean_token_accuracy": 0.726062536239624,
+ "num_tokens": 29114368.0,
+ "step": 1777
+ },
+ {
+ "entropy": 1.0930231809616089,
+ "epoch": 3.591919191919192,
+ "grad_norm": 2.2943761348724365,
+ "learning_rate": 1.41010101010101e-06,
+ "loss": 1.116988182067871,
+ "mean_token_accuracy": 0.7142159342765808,
+ "num_tokens": 29130752.0,
+ "step": 1778
+ },
+ {
+ "entropy": 2.053565502166748,
+ "epoch": 3.5939393939393938,
+ "grad_norm": 2.274879217147827,
+ "learning_rate": 1.4080808080808082e-06,
+ "loss": 2.012974262237549,
+ "mean_token_accuracy": 0.5475085377693176,
+ "num_tokens": 29147136.0,
+ "step": 1779
+ },
+ {
+ "entropy": 1.666791319847107,
+ "epoch": 3.595959595959596,
+ "grad_norm": 2.3387227058410645,
+ "learning_rate": 1.406060606060606e-06,
+ "loss": 1.6848305463790894,
+ "mean_token_accuracy": 0.6218246221542358,
+ "num_tokens": 29163520.0,
+ "step": 1780
+ },
+ {
+ "entropy": 1.2312790155410767,
+ "epoch": 3.597979797979798,
+ "grad_norm": 2.1445071697235107,
+ "learning_rate": 1.404040404040404e-06,
+ "loss": 1.2197356224060059,
+ "mean_token_accuracy": 0.7146433591842651,
+ "num_tokens": 29179904.0,
+ "step": 1781
+ },
+ {
+ "entropy": 1.2540696859359741,
+ "epoch": 3.6,
+ "grad_norm": 2.3021488189697266,
+ "learning_rate": 1.402020202020202e-06,
+ "loss": 1.2441315650939941,
+ "mean_token_accuracy": 0.697239875793457,
+ "num_tokens": 29196288.0,
+ "step": 1782
+ },
+ {
+ "entropy": 1.3095312118530273,
+ "epoch": 3.602020202020202,
+ "grad_norm": 2.3379111289978027,
+ "learning_rate": 1.4000000000000001e-06,
+ "loss": 1.299248218536377,
+ "mean_token_accuracy": 0.6789814233779907,
+ "num_tokens": 29212672.0,
+ "step": 1783
+ },
+ {
+ "entropy": 1.0785331726074219,
+ "epoch": 3.604040404040404,
+ "grad_norm": 2.1612391471862793,
+ "learning_rate": 1.3979797979797982e-06,
+ "loss": 1.0741904973983765,
+ "mean_token_accuracy": 0.7294821739196777,
+ "num_tokens": 29229056.0,
+ "step": 1784
+ },
+ {
+ "entropy": 1.5178064107894897,
+ "epoch": 3.606060606060606,
+ "grad_norm": 2.7717669010162354,
+ "learning_rate": 1.3959595959595961e-06,
+ "loss": 1.4960123300552368,
+ "mean_token_accuracy": 0.6555935740470886,
+ "num_tokens": 29245440.0,
+ "step": 1785
+ },
+ {
+ "entropy": 0.9794894456863403,
+ "epoch": 3.608080808080808,
+ "grad_norm": 2.128119707107544,
+ "learning_rate": 1.3939393939393942e-06,
+ "loss": 0.964859664440155,
+ "mean_token_accuracy": 0.7516487836837769,
+ "num_tokens": 29261824.0,
+ "step": 1786
+ },
+ {
+ "entropy": 1.1770693063735962,
+ "epoch": 3.61010101010101,
+ "grad_norm": 2.2583603858947754,
+ "learning_rate": 1.3919191919191921e-06,
+ "loss": 1.163985252380371,
+ "mean_token_accuracy": 0.7167806625366211,
+ "num_tokens": 29278208.0,
+ "step": 1787
+ },
+ {
+ "entropy": 1.2855029106140137,
+ "epoch": 3.6121212121212123,
+ "grad_norm": 2.1924304962158203,
+ "learning_rate": 1.38989898989899e-06,
+ "loss": 1.3204423189163208,
+ "mean_token_accuracy": 0.6933927536010742,
+ "num_tokens": 29294592.0,
+ "step": 1788
+ },
+ {
+ "entropy": 1.1483845710754395,
+ "epoch": 3.614141414141414,
+ "grad_norm": 2.414332151412964,
+ "learning_rate": 1.3878787878787881e-06,
+ "loss": 1.154718041419983,
+ "mean_token_accuracy": 0.6999267339706421,
+ "num_tokens": 29310976.0,
+ "step": 1789
+ },
+ {
+ "entropy": 1.3682032823562622,
+ "epoch": 3.616161616161616,
+ "grad_norm": 2.2198476791381836,
+ "learning_rate": 1.385858585858586e-06,
+ "loss": 1.3729476928710938,
+ "mean_token_accuracy": 0.6736077070236206,
+ "num_tokens": 29327360.0,
+ "step": 1790
+ },
+ {
+ "entropy": 1.0998579263687134,
+ "epoch": 3.618181818181818,
+ "grad_norm": 2.328645706176758,
+ "learning_rate": 1.3838383838383839e-06,
+ "loss": 1.0919283628463745,
+ "mean_token_accuracy": 0.7213605046272278,
+ "num_tokens": 29343744.0,
+ "step": 1791
+ },
+ {
+ "entropy": 1.4821339845657349,
+ "epoch": 3.6202020202020204,
+ "grad_norm": 2.346853256225586,
+ "learning_rate": 1.381818181818182e-06,
+ "loss": 1.4857574701309204,
+ "mean_token_accuracy": 0.6529677510261536,
+ "num_tokens": 29360128.0,
+ "step": 1792
+ },
+ {
+ "entropy": 1.2080271244049072,
+ "epoch": 3.6222222222222222,
+ "grad_norm": 2.4182422161102295,
+ "learning_rate": 1.3797979797979799e-06,
+ "loss": 1.2055883407592773,
+ "mean_token_accuracy": 0.7100635170936584,
+ "num_tokens": 29376512.0,
+ "step": 1793
+ },
+ {
+ "entropy": 1.2058414220809937,
+ "epoch": 3.624242424242424,
+ "grad_norm": 2.216625690460205,
+ "learning_rate": 1.377777777777778e-06,
+ "loss": 1.2196989059448242,
+ "mean_token_accuracy": 0.7110405564308167,
+ "num_tokens": 29392896.0,
+ "step": 1794
+ },
+ {
+ "entropy": 1.2888880968093872,
+ "epoch": 3.6262626262626263,
+ "grad_norm": 2.339319944381714,
+ "learning_rate": 1.3757575757575759e-06,
+ "loss": 1.277637004852295,
+ "mean_token_accuracy": 0.6922935843467712,
+ "num_tokens": 29409280.0,
+ "step": 1795
+ },
+ {
+ "entropy": 1.3091189861297607,
+ "epoch": 3.6282828282828286,
+ "grad_norm": 2.2296347618103027,
+ "learning_rate": 1.3737373737373738e-06,
+ "loss": 1.3214037418365479,
+ "mean_token_accuracy": 0.6828285455703735,
+ "num_tokens": 29425664.0,
+ "step": 1796
+ },
+ {
+ "entropy": 1.3359358310699463,
+ "epoch": 3.6303030303030304,
+ "grad_norm": 2.3027234077453613,
+ "learning_rate": 1.3717171717171718e-06,
+ "loss": 1.3537523746490479,
+ "mean_token_accuracy": 0.6944919228553772,
+ "num_tokens": 29442048.0,
+ "step": 1797
+ },
+ {
+ "entropy": 1.4622148275375366,
+ "epoch": 3.632323232323232,
+ "grad_norm": 2.2217557430267334,
+ "learning_rate": 1.3696969696969697e-06,
+ "loss": 1.4778163433074951,
+ "mean_token_accuracy": 0.6425867080688477,
+ "num_tokens": 29458432.0,
+ "step": 1798
+ },
+ {
+ "entropy": 1.1139435768127441,
+ "epoch": 3.6343434343434344,
+ "grad_norm": 2.1497981548309326,
+ "learning_rate": 1.3676767676767676e-06,
+ "loss": 1.1079224348068237,
+ "mean_token_accuracy": 0.7202613353729248,
+ "num_tokens": 29474816.0,
+ "step": 1799
+ },
+ {
+ "entropy": 1.4551507234573364,
+ "epoch": 3.6363636363636362,
+ "grad_norm": 2.138988971710205,
+ "learning_rate": 1.3656565656565657e-06,
+ "loss": 1.469781517982483,
+ "mean_token_accuracy": 0.6572422981262207,
+ "num_tokens": 29491200.0,
+ "step": 1800
+ },
+ {
+ "epoch": 3.6363636363636362,
+ "eval_entropy": 1.3177791665638647,
+ "eval_loss": 1.5487139225006104,
+ "eval_mean_token_accuracy": 0.6448490720602774,
+ "eval_num_tokens": 29491200.0,
+ "eval_runtime": 43.82,
+ "eval_samples_per_second": 22.592,
+ "eval_steps_per_second": 2.83,
+ "step": 1800
+ },
+ {
+ "entropy": 1.3909705877304077,
+ "epoch": 3.6383838383838385,
+ "grad_norm": 2.2257578372955322,
+ "learning_rate": 1.3636363636363636e-06,
+ "loss": 1.4063682556152344,
+ "mean_token_accuracy": 0.6611504554748535,
+ "num_tokens": 29507584.0,
+ "step": 1801
+ },
+ {
+ "entropy": 1.2972772121429443,
+ "epoch": 3.6404040404040403,
+ "grad_norm": 2.3228883743286133,
+ "learning_rate": 1.3616161616161617e-06,
+ "loss": 1.2978641986846924,
+ "mean_token_accuracy": 0.6862481832504272,
+ "num_tokens": 29523968.0,
+ "step": 1802
+ },
+ {
+ "entropy": 0.8828009366989136,
+ "epoch": 3.6424242424242426,
+ "grad_norm": 2.294520616531372,
+ "learning_rate": 1.3595959595959596e-06,
+ "loss": 0.8818602561950684,
+ "mean_token_accuracy": 0.7627626061439514,
+ "num_tokens": 29540352.0,
+ "step": 1803
+ },
+ {
+ "entropy": 1.3855260610580444,
+ "epoch": 3.6444444444444444,
+ "grad_norm": 2.160240411758423,
+ "learning_rate": 1.357575757575758e-06,
+ "loss": 1.3808461427688599,
+ "mean_token_accuracy": 0.6831949353218079,
+ "num_tokens": 29556736.0,
+ "step": 1804
+ },
+ {
+ "entropy": 1.1665101051330566,
+ "epoch": 3.6464646464646466,
+ "grad_norm": 2.3000075817108154,
+ "learning_rate": 1.3555555555555558e-06,
+ "loss": 1.1676980257034302,
+ "mean_token_accuracy": 0.7067049145698547,
+ "num_tokens": 29573120.0,
+ "step": 1805
+ },
+ {
+ "entropy": 1.05939519405365,
+ "epoch": 3.6484848484848484,
+ "grad_norm": 2.204202651977539,
+ "learning_rate": 1.3535353535353537e-06,
+ "loss": 1.0654125213623047,
+ "mean_token_accuracy": 0.7197728157043457,
+ "num_tokens": 29589504.0,
+ "step": 1806
+ },
+ {
+ "entropy": 1.355467677116394,
+ "epoch": 3.6505050505050507,
+ "grad_norm": 2.264705181121826,
+ "learning_rate": 1.3515151515151518e-06,
+ "loss": 1.3571414947509766,
+ "mean_token_accuracy": 0.6802638173103333,
+ "num_tokens": 29605888.0,
+ "step": 1807
+ },
+ {
+ "entropy": 1.1876951456069946,
+ "epoch": 3.6525252525252525,
+ "grad_norm": 2.2944509983062744,
+ "learning_rate": 1.3494949494949497e-06,
+ "loss": 1.1937611103057861,
+ "mean_token_accuracy": 0.7105520367622375,
+ "num_tokens": 29622272.0,
+ "step": 1808
+ },
+ {
+ "entropy": 1.410723090171814,
+ "epoch": 3.6545454545454543,
+ "grad_norm": 2.329960584640503,
+ "learning_rate": 1.3474747474747476e-06,
+ "loss": 1.4132624864578247,
+ "mean_token_accuracy": 0.656448483467102,
+ "num_tokens": 29638656.0,
+ "step": 1809
+ },
+ {
+ "entropy": 1.018915057182312,
+ "epoch": 3.6565656565656566,
+ "grad_norm": 2.1073148250579834,
+ "learning_rate": 1.3454545454545457e-06,
+ "loss": 1.017066478729248,
+ "mean_token_accuracy": 0.7398021221160889,
+ "num_tokens": 29655040.0,
+ "step": 1810
+ },
+ {
+ "entropy": 1.385291337966919,
+ "epoch": 3.658585858585859,
+ "grad_norm": 2.4099128246307373,
+ "learning_rate": 1.3434343434343436e-06,
+ "loss": 1.3809027671813965,
+ "mean_token_accuracy": 0.6530288457870483,
+ "num_tokens": 29671424.0,
+ "step": 1811
+ },
+ {
+ "entropy": 1.0589433908462524,
+ "epoch": 3.6606060606060606,
+ "grad_norm": 2.3987228870391846,
+ "learning_rate": 1.3414141414141417e-06,
+ "loss": 1.051498293876648,
+ "mean_token_accuracy": 0.7296043038368225,
+ "num_tokens": 29687808.0,
+ "step": 1812
+ },
+ {
+ "entropy": 1.0399458408355713,
+ "epoch": 3.6626262626262625,
+ "grad_norm": 1.9951106309890747,
+ "learning_rate": 1.3393939393939395e-06,
+ "loss": 1.0482317209243774,
+ "mean_token_accuracy": 0.745175838470459,
+ "num_tokens": 29704192.0,
+ "step": 1813
+ },
+ {
+ "entropy": 1.3024410009384155,
+ "epoch": 3.6646464646464647,
+ "grad_norm": 2.1363778114318848,
+ "learning_rate": 1.3373737373737374e-06,
+ "loss": 1.2995185852050781,
+ "mean_token_accuracy": 0.6889960765838623,
+ "num_tokens": 29720576.0,
+ "step": 1814
+ },
+ {
+ "entropy": 1.604906678199768,
+ "epoch": 3.6666666666666665,
+ "grad_norm": 2.2837791442871094,
+ "learning_rate": 1.3353535353535355e-06,
+ "loss": 1.6025669574737549,
+ "mean_token_accuracy": 0.6281143426895142,
+ "num_tokens": 29736960.0,
+ "step": 1815
+ },
+ {
+ "entropy": 1.0479960441589355,
+ "epoch": 3.6686868686868688,
+ "grad_norm": 2.200199842453003,
+ "learning_rate": 1.3333333333333334e-06,
+ "loss": 1.0443310737609863,
+ "mean_token_accuracy": 0.7302149534225464,
+ "num_tokens": 29753344.0,
+ "step": 1816
+ },
+ {
+ "entropy": 1.1220474243164062,
+ "epoch": 3.6707070707070706,
+ "grad_norm": 2.278193712234497,
+ "learning_rate": 1.3313131313131313e-06,
+ "loss": 1.1253700256347656,
+ "mean_token_accuracy": 0.717391312122345,
+ "num_tokens": 29769728.0,
+ "step": 1817
+ },
+ {
+ "entropy": 0.7910162210464478,
+ "epoch": 3.672727272727273,
+ "grad_norm": 2.171074151992798,
+ "learning_rate": 1.3292929292929294e-06,
+ "loss": 0.7818440198898315,
+ "mean_token_accuracy": 0.7832193374633789,
+ "num_tokens": 29786112.0,
+ "step": 1818
+ },
+ {
+ "entropy": 1.2416874170303345,
+ "epoch": 3.6747474747474747,
+ "grad_norm": 2.41068959236145,
+ "learning_rate": 1.3272727272727273e-06,
+ "loss": 1.2441900968551636,
+ "mean_token_accuracy": 0.6918050646781921,
+ "num_tokens": 29802496.0,
+ "step": 1819
+ },
+ {
+ "entropy": 1.1947519779205322,
+ "epoch": 3.676767676767677,
+ "grad_norm": 2.3515799045562744,
+ "learning_rate": 1.3252525252525254e-06,
+ "loss": 1.1756401062011719,
+ "mean_token_accuracy": 0.7001098990440369,
+ "num_tokens": 29818880.0,
+ "step": 1820
+ },
+ {
+ "entropy": 1.0655851364135742,
+ "epoch": 3.6787878787878787,
+ "grad_norm": 2.3352251052856445,
+ "learning_rate": 1.3232323232323233e-06,
+ "loss": 1.0540491342544556,
+ "mean_token_accuracy": 0.7285051345825195,
+ "num_tokens": 29835264.0,
+ "step": 1821
+ },
+ {
+ "entropy": 1.1881550550460815,
+ "epoch": 3.680808080808081,
+ "grad_norm": 2.483917474746704,
+ "learning_rate": 1.3212121212121212e-06,
+ "loss": 1.199643611907959,
+ "mean_token_accuracy": 0.6998046040534973,
+ "num_tokens": 29851648.0,
+ "step": 1822
+ },
+ {
+ "entropy": 1.2031608819961548,
+ "epoch": 3.682828282828283,
+ "grad_norm": 2.1765546798706055,
+ "learning_rate": 1.3191919191919193e-06,
+ "loss": 1.2054451704025269,
+ "mean_token_accuracy": 0.7191011309623718,
+ "num_tokens": 29868032.0,
+ "step": 1823
+ },
+ {
+ "entropy": 0.8889259099960327,
+ "epoch": 3.6848484848484846,
+ "grad_norm": 2.3284575939178467,
+ "learning_rate": 1.3171717171717172e-06,
+ "loss": 0.8793114423751831,
+ "mean_token_accuracy": 0.7661211490631104,
+ "num_tokens": 29884416.0,
+ "step": 1824
+ },
+ {
+ "entropy": 1.1559280157089233,
+ "epoch": 3.686868686868687,
+ "grad_norm": 2.2942442893981934,
+ "learning_rate": 1.315151515151515e-06,
+ "loss": 1.1748852729797363,
+ "mean_token_accuracy": 0.7139105796813965,
+ "num_tokens": 29900800.0,
+ "step": 1825
+ },
+ {
+ "entropy": 1.1315678358078003,
+ "epoch": 3.688888888888889,
+ "grad_norm": 2.0435519218444824,
+ "learning_rate": 1.3131313131313134e-06,
+ "loss": 1.123443603515625,
+ "mean_token_accuracy": 0.7311308979988098,
+ "num_tokens": 29917184.0,
+ "step": 1826
+ },
+ {
+ "entropy": 1.063825249671936,
+ "epoch": 3.690909090909091,
+ "grad_norm": 2.3425393104553223,
+ "learning_rate": 1.3111111111111112e-06,
+ "loss": 1.0752053260803223,
+ "mean_token_accuracy": 0.7281997799873352,
+ "num_tokens": 29933568.0,
+ "step": 1827
+ },
+ {
+ "entropy": 0.9652242660522461,
+ "epoch": 3.6929292929292927,
+ "grad_norm": 2.2079033851623535,
+ "learning_rate": 1.3090909090909093e-06,
+ "loss": 0.969928503036499,
+ "mean_token_accuracy": 0.7507938742637634,
+ "num_tokens": 29949952.0,
+ "step": 1828
+ },
+ {
+ "entropy": 1.4131009578704834,
+ "epoch": 3.694949494949495,
+ "grad_norm": 2.368713617324829,
+ "learning_rate": 1.3070707070707072e-06,
+ "loss": 1.4113342761993408,
+ "mean_token_accuracy": 0.6670737862586975,
+ "num_tokens": 29966336.0,
+ "step": 1829
+ },
+ {
+ "entropy": 1.4447262287139893,
+ "epoch": 3.6969696969696972,
+ "grad_norm": 2.142131805419922,
+ "learning_rate": 1.3050505050505053e-06,
+ "loss": 1.4544841051101685,
+ "mean_token_accuracy": 0.669516384601593,
+ "num_tokens": 29982720.0,
+ "step": 1830
+ },
+ {
+ "entropy": 1.2524710893630981,
+ "epoch": 3.698989898989899,
+ "grad_norm": 2.096489429473877,
+ "learning_rate": 1.3030303030303032e-06,
+ "loss": 1.2251232862472534,
+ "mean_token_accuracy": 0.7231924533843994,
+ "num_tokens": 29999104.0,
+ "step": 1831
+ },
+ {
+ "entropy": 1.2173584699630737,
+ "epoch": 3.701010101010101,
+ "grad_norm": 2.265334367752075,
+ "learning_rate": 1.301010101010101e-06,
+ "loss": 1.2278194427490234,
+ "mean_token_accuracy": 0.7136052846908569,
+ "num_tokens": 30015488.0,
+ "step": 1832
+ },
+ {
+ "entropy": 1.1699647903442383,
+ "epoch": 3.703030303030303,
+ "grad_norm": 2.274156093597412,
+ "learning_rate": 1.2989898989898992e-06,
+ "loss": 1.1550461053848267,
+ "mean_token_accuracy": 0.7139716744422913,
+ "num_tokens": 30031872.0,
+ "step": 1833
+ },
+ {
+ "entropy": 1.113942265510559,
+ "epoch": 3.705050505050505,
+ "grad_norm": 2.386925220489502,
+ "learning_rate": 1.296969696969697e-06,
+ "loss": 1.1161434650421143,
+ "mean_token_accuracy": 0.7212994694709778,
+ "num_tokens": 30048256.0,
+ "step": 1834
+ },
+ {
+ "entropy": 1.4958065748214722,
+ "epoch": 3.707070707070707,
+ "grad_norm": 2.2243921756744385,
+ "learning_rate": 1.294949494949495e-06,
+ "loss": 1.4940879344940186,
+ "mean_token_accuracy": 0.6489985585212708,
+ "num_tokens": 30064640.0,
+ "step": 1835
+ },
+ {
+ "entropy": 1.047438144683838,
+ "epoch": 3.709090909090909,
+ "grad_norm": 2.237632989883423,
+ "learning_rate": 1.292929292929293e-06,
+ "loss": 1.0492076873779297,
+ "mean_token_accuracy": 0.7387640476226807,
+ "num_tokens": 30081024.0,
+ "step": 1836
+ },
+ {
+ "entropy": 1.4816311597824097,
+ "epoch": 3.7111111111111112,
+ "grad_norm": 2.2184505462646484,
+ "learning_rate": 1.290909090909091e-06,
+ "loss": 1.483510971069336,
+ "mean_token_accuracy": 0.660906195640564,
+ "num_tokens": 30097408.0,
+ "step": 1837
+ },
+ {
+ "entropy": 1.1734580993652344,
+ "epoch": 3.713131313131313,
+ "grad_norm": 2.284940242767334,
+ "learning_rate": 1.288888888888889e-06,
+ "loss": 1.1763725280761719,
+ "mean_token_accuracy": 0.708109438419342,
+ "num_tokens": 30113792.0,
+ "step": 1838
+ },
+ {
+ "entropy": 1.1073024272918701,
+ "epoch": 3.7151515151515153,
+ "grad_norm": 2.381448268890381,
+ "learning_rate": 1.286868686868687e-06,
+ "loss": 1.1168363094329834,
+ "mean_token_accuracy": 0.722337543964386,
+ "num_tokens": 30130176.0,
+ "step": 1839
+ },
+ {
+ "entropy": 1.2699110507965088,
+ "epoch": 3.717171717171717,
+ "grad_norm": 2.23447585105896,
+ "learning_rate": 1.2848484848484848e-06,
+ "loss": 1.2582776546478271,
+ "mean_token_accuracy": 0.6878358721733093,
+ "num_tokens": 30146560.0,
+ "step": 1840
+ },
+ {
+ "entropy": 1.421061635017395,
+ "epoch": 3.7191919191919194,
+ "grad_norm": 2.1020593643188477,
+ "learning_rate": 1.282828282828283e-06,
+ "loss": 1.415670394897461,
+ "mean_token_accuracy": 0.6722032427787781,
+ "num_tokens": 30162944.0,
+ "step": 1841
+ },
+ {
+ "entropy": 0.8924360871315002,
+ "epoch": 3.721212121212121,
+ "grad_norm": 2.1613609790802,
+ "learning_rate": 1.2808080808080808e-06,
+ "loss": 0.8864036202430725,
+ "mean_token_accuracy": 0.7616023421287537,
+ "num_tokens": 30179328.0,
+ "step": 1842
+ },
+ {
+ "entropy": 1.2242110967636108,
+ "epoch": 3.723232323232323,
+ "grad_norm": 2.472944498062134,
+ "learning_rate": 1.2787878787878787e-06,
+ "loss": 1.227367877960205,
+ "mean_token_accuracy": 0.6939423680305481,
+ "num_tokens": 30195712.0,
+ "step": 1843
+ },
+ {
+ "entropy": 1.2816494703292847,
+ "epoch": 3.7252525252525253,
+ "grad_norm": 2.123680353164673,
+ "learning_rate": 1.2767676767676768e-06,
+ "loss": 1.2535889148712158,
+ "mean_token_accuracy": 0.6894845962524414,
+ "num_tokens": 30212096.0,
+ "step": 1844
+ },
+ {
+ "entropy": 1.0055866241455078,
+ "epoch": 3.7272727272727275,
+ "grad_norm": 2.2397797107696533,
+ "learning_rate": 1.2747474747474747e-06,
+ "loss": 1.0046356916427612,
+ "mean_token_accuracy": 0.7375427484512329,
+ "num_tokens": 30228480.0,
+ "step": 1845
+ },
+ {
+ "entropy": 0.9162007570266724,
+ "epoch": 3.7292929292929293,
+ "grad_norm": 2.2464940547943115,
+ "learning_rate": 1.2727272727272728e-06,
+ "loss": 0.9192468523979187,
+ "mean_token_accuracy": 0.7609306573867798,
+ "num_tokens": 30244864.0,
+ "step": 1846
+ },
+ {
+ "entropy": 1.3691668510437012,
+ "epoch": 3.731313131313131,
+ "grad_norm": 2.766839027404785,
+ "learning_rate": 1.270707070707071e-06,
+ "loss": 1.3715851306915283,
+ "mean_token_accuracy": 0.6667684316635132,
+ "num_tokens": 30261248.0,
+ "step": 1847
+ },
+ {
+ "entropy": 1.140328049659729,
+ "epoch": 3.7333333333333334,
+ "grad_norm": 2.1093955039978027,
+ "learning_rate": 1.268686868686869e-06,
+ "loss": 1.1491321325302124,
+ "mean_token_accuracy": 0.7158646583557129,
+ "num_tokens": 30277632.0,
+ "step": 1848
+ },
+ {
+ "entropy": 1.3393927812576294,
+ "epoch": 3.735353535353535,
+ "grad_norm": 2.216357469558716,
+ "learning_rate": 1.2666666666666669e-06,
+ "loss": 1.3406658172607422,
+ "mean_token_accuracy": 0.6829506754875183,
+ "num_tokens": 30294016.0,
+ "step": 1849
+ },
+ {
+ "entropy": 1.0507596731185913,
+ "epoch": 3.7373737373737375,
+ "grad_norm": 2.287649154663086,
+ "learning_rate": 1.2646464646464648e-06,
+ "loss": 1.0469536781311035,
+ "mean_token_accuracy": 0.7306423783302307,
+ "num_tokens": 30310400.0,
+ "step": 1850
+ },
+ {
+ "epoch": 3.7373737373737375,
+ "eval_entropy": 1.3188492747083786,
+ "eval_loss": 1.5484505891799927,
+ "eval_mean_token_accuracy": 0.6448682820604693,
+ "eval_num_tokens": 30310400.0,
+ "eval_runtime": 43.7649,
+ "eval_samples_per_second": 22.621,
+ "eval_steps_per_second": 2.833,
+ "step": 1850
+ },
+ {
+ "entropy": 1.0057004690170288,
+ "epoch": 3.7393939393939393,
+ "grad_norm": 2.1135971546173096,
+ "learning_rate": 1.2626262626262629e-06,
+ "loss": 1.0220353603363037,
+ "mean_token_accuracy": 0.7348558902740479,
+ "num_tokens": 30326784.0,
+ "step": 1851
+ },
+ {
+ "entropy": 1.185084581375122,
+ "epoch": 3.7414141414141415,
+ "grad_norm": 2.388561725616455,
+ "learning_rate": 1.2606060606060608e-06,
+ "loss": 1.205190658569336,
+ "mean_token_accuracy": 0.7070102691650391,
+ "num_tokens": 30343168.0,
+ "step": 1852
+ },
+ {
+ "entropy": 1.2007622718811035,
+ "epoch": 3.7434343434343433,
+ "grad_norm": 2.3002219200134277,
+ "learning_rate": 1.2585858585858587e-06,
+ "loss": 1.196336269378662,
+ "mean_token_accuracy": 0.7073155641555786,
+ "num_tokens": 30359552.0,
+ "step": 1853
+ },
+ {
+ "entropy": 0.8792212605476379,
+ "epoch": 3.7454545454545456,
+ "grad_norm": 2.1559953689575195,
+ "learning_rate": 1.2565656565656568e-06,
+ "loss": 0.8786029815673828,
+ "mean_token_accuracy": 0.7723497748374939,
+ "num_tokens": 30375936.0,
+ "step": 1854
+ },
+ {
+ "entropy": 1.0509400367736816,
+ "epoch": 3.7474747474747474,
+ "grad_norm": 2.3014485836029053,
+ "learning_rate": 1.2545454545454546e-06,
+ "loss": 1.0421940088272095,
+ "mean_token_accuracy": 0.741389811038971,
+ "num_tokens": 30392320.0,
+ "step": 1855
+ },
+ {
+ "entropy": 1.4965882301330566,
+ "epoch": 3.7494949494949497,
+ "grad_norm": 2.255364179611206,
+ "learning_rate": 1.2525252525252527e-06,
+ "loss": 1.5137383937835693,
+ "mean_token_accuracy": 0.6535173654556274,
+ "num_tokens": 30408704.0,
+ "step": 1856
+ },
+ {
+ "entropy": 1.5112299919128418,
+ "epoch": 3.7515151515151515,
+ "grad_norm": 2.356682062149048,
+ "learning_rate": 1.2505050505050506e-06,
+ "loss": 1.4965360164642334,
+ "mean_token_accuracy": 0.639167070388794,
+ "num_tokens": 30425088.0,
+ "step": 1857
+ },
+ {
+ "entropy": 0.7364286780357361,
+ "epoch": 3.7535353535353533,
+ "grad_norm": 2.0973432064056396,
+ "learning_rate": 1.2484848484848485e-06,
+ "loss": 0.7423363327980042,
+ "mean_token_accuracy": 0.7923790812492371,
+ "num_tokens": 30441472.0,
+ "step": 1858
+ },
+ {
+ "entropy": 1.094409704208374,
+ "epoch": 3.7555555555555555,
+ "grad_norm": 2.174180746078491,
+ "learning_rate": 1.2464646464646466e-06,
+ "loss": 1.0954921245574951,
+ "mean_token_accuracy": 0.7165974378585815,
+ "num_tokens": 30457856.0,
+ "step": 1859
+ },
+ {
+ "entropy": 1.1821874380111694,
+ "epoch": 3.757575757575758,
+ "grad_norm": 2.253767728805542,
+ "learning_rate": 1.2444444444444445e-06,
+ "loss": 1.1888916492462158,
+ "mean_token_accuracy": 0.7046898007392883,
+ "num_tokens": 30474240.0,
+ "step": 1860
+ },
+ {
+ "entropy": 1.1281737089157104,
+ "epoch": 3.7595959595959596,
+ "grad_norm": 2.378880023956299,
+ "learning_rate": 1.2424242424242424e-06,
+ "loss": 1.1276686191558838,
+ "mean_token_accuracy": 0.712506115436554,
+ "num_tokens": 30490624.0,
+ "step": 1861
+ },
+ {
+ "entropy": 1.1477409601211548,
+ "epoch": 3.7616161616161614,
+ "grad_norm": 2.2524328231811523,
+ "learning_rate": 1.2404040404040405e-06,
+ "loss": 1.169572114944458,
+ "mean_token_accuracy": 0.7145823240280151,
+ "num_tokens": 30507008.0,
+ "step": 1862
+ },
+ {
+ "entropy": 1.1877309083938599,
+ "epoch": 3.7636363636363637,
+ "grad_norm": 2.5215635299682617,
+ "learning_rate": 1.2383838383838386e-06,
+ "loss": 1.1812067031860352,
+ "mean_token_accuracy": 0.7001709938049316,
+ "num_tokens": 30523392.0,
+ "step": 1863
+ },
+ {
+ "entropy": 1.2082164287567139,
+ "epoch": 3.765656565656566,
+ "grad_norm": 2.248847484588623,
+ "learning_rate": 1.2363636363636365e-06,
+ "loss": 1.2150661945343018,
+ "mean_token_accuracy": 0.7056668400764465,
+ "num_tokens": 30539776.0,
+ "step": 1864
+ },
+ {
+ "entropy": 1.0533868074417114,
+ "epoch": 3.7676767676767677,
+ "grad_norm": 2.4208297729492188,
+ "learning_rate": 1.2343434343434346e-06,
+ "loss": 1.0599069595336914,
+ "mean_token_accuracy": 0.7183683514595032,
+ "num_tokens": 30556160.0,
+ "step": 1865
+ },
+ {
+ "entropy": 1.3612878322601318,
+ "epoch": 3.7696969696969695,
+ "grad_norm": 2.2831575870513916,
+ "learning_rate": 1.2323232323232325e-06,
+ "loss": 1.359129548072815,
+ "mean_token_accuracy": 0.681485116481781,
+ "num_tokens": 30572544.0,
+ "step": 1866
+ },
+ {
+ "entropy": 1.3154997825622559,
+ "epoch": 3.771717171717172,
+ "grad_norm": 2.2212936878204346,
+ "learning_rate": 1.2303030303030304e-06,
+ "loss": 1.3217754364013672,
+ "mean_token_accuracy": 0.6781265139579773,
+ "num_tokens": 30588928.0,
+ "step": 1867
+ },
+ {
+ "entropy": 1.3559744358062744,
+ "epoch": 3.7737373737373736,
+ "grad_norm": 2.317963123321533,
+ "learning_rate": 1.2282828282828285e-06,
+ "loss": 1.3600891828536987,
+ "mean_token_accuracy": 0.6749511361122131,
+ "num_tokens": 30605312.0,
+ "step": 1868
+ },
+ {
+ "entropy": 1.0591025352478027,
+ "epoch": 3.775757575757576,
+ "grad_norm": 2.2434401512145996,
+ "learning_rate": 1.2262626262626263e-06,
+ "loss": 1.069009780883789,
+ "mean_token_accuracy": 0.7206888198852539,
+ "num_tokens": 30621696.0,
+ "step": 1869
+ },
+ {
+ "entropy": 1.4170074462890625,
+ "epoch": 3.7777777777777777,
+ "grad_norm": 2.395284652709961,
+ "learning_rate": 1.2242424242424242e-06,
+ "loss": 1.4480173587799072,
+ "mean_token_accuracy": 0.6687225103378296,
+ "num_tokens": 30638080.0,
+ "step": 1870
+ },
+ {
+ "entropy": 1.2455501556396484,
+ "epoch": 3.77979797979798,
+ "grad_norm": 2.308814525604248,
+ "learning_rate": 1.2222222222222223e-06,
+ "loss": 1.238303542137146,
+ "mean_token_accuracy": 0.693514883518219,
+ "num_tokens": 30654464.0,
+ "step": 1871
+ },
+ {
+ "entropy": 1.167949914932251,
+ "epoch": 3.7818181818181817,
+ "grad_norm": 2.292120933532715,
+ "learning_rate": 1.2202020202020202e-06,
+ "loss": 1.16925847530365,
+ "mean_token_accuracy": 0.7136052846908569,
+ "num_tokens": 30670848.0,
+ "step": 1872
+ },
+ {
+ "entropy": 1.1319386959075928,
+ "epoch": 3.783838383838384,
+ "grad_norm": 2.2560858726501465,
+ "learning_rate": 1.2181818181818183e-06,
+ "loss": 1.1299757957458496,
+ "mean_token_accuracy": 0.7148265838623047,
+ "num_tokens": 30687232.0,
+ "step": 1873
+ },
+ {
+ "entropy": 1.1161472797393799,
+ "epoch": 3.785858585858586,
+ "grad_norm": 2.375944137573242,
+ "learning_rate": 1.2161616161616164e-06,
+ "loss": 1.1136269569396973,
+ "mean_token_accuracy": 0.7096971273422241,
+ "num_tokens": 30703616.0,
+ "step": 1874
+ },
+ {
+ "entropy": 1.223272442817688,
+ "epoch": 3.787878787878788,
+ "grad_norm": 2.5566787719726562,
+ "learning_rate": 1.2141414141414143e-06,
+ "loss": 1.2140392065048218,
+ "mean_token_accuracy": 0.6939423680305481,
+ "num_tokens": 30720000.0,
+ "step": 1875
+ },
+ {
+ "entropy": 1.5127382278442383,
+ "epoch": 3.78989898989899,
+ "grad_norm": 2.2809665203094482,
+ "learning_rate": 1.2121212121212122e-06,
+ "loss": 1.508399248123169,
+ "mean_token_accuracy": 0.6579140424728394,
+ "num_tokens": 30736384.0,
+ "step": 1876
+ },
+ {
+ "entropy": 1.344763994216919,
+ "epoch": 3.7919191919191917,
+ "grad_norm": 2.3228111267089844,
+ "learning_rate": 1.2101010101010103e-06,
+ "loss": 1.3470673561096191,
+ "mean_token_accuracy": 0.6806302070617676,
+ "num_tokens": 30752768.0,
+ "step": 1877
+ },
+ {
+ "entropy": 1.2828526496887207,
+ "epoch": 3.793939393939394,
+ "grad_norm": 2.51246976852417,
+ "learning_rate": 1.2080808080808082e-06,
+ "loss": 1.2934420108795166,
+ "mean_token_accuracy": 0.6673790812492371,
+ "num_tokens": 30769152.0,
+ "step": 1878
+ },
+ {
+ "entropy": 0.9060937762260437,
+ "epoch": 3.795959595959596,
+ "grad_norm": 2.1415040493011475,
+ "learning_rate": 1.206060606060606e-06,
+ "loss": 0.8956457376480103,
+ "mean_token_accuracy": 0.7670371532440186,
+ "num_tokens": 30785536.0,
+ "step": 1879
+ },
+ {
+ "entropy": 1.1543211936950684,
+ "epoch": 3.797979797979798,
+ "grad_norm": 2.426781415939331,
+ "learning_rate": 1.2040404040404042e-06,
+ "loss": 1.1588597297668457,
+ "mean_token_accuracy": 0.7090253829956055,
+ "num_tokens": 30801920.0,
+ "step": 1880
+ },
+ {
+ "entropy": 1.3253799676895142,
+ "epoch": 3.8,
+ "grad_norm": 2.4828150272369385,
+ "learning_rate": 1.202020202020202e-06,
+ "loss": 1.3082318305969238,
+ "mean_token_accuracy": 0.6839887499809265,
+ "num_tokens": 30818304.0,
+ "step": 1881
+ },
+ {
+ "entropy": 1.1644330024719238,
+ "epoch": 3.802020202020202,
+ "grad_norm": 2.3429148197174072,
+ "learning_rate": 1.2000000000000002e-06,
+ "loss": 1.1645771265029907,
+ "mean_token_accuracy": 0.7025524973869324,
+ "num_tokens": 30834688.0,
+ "step": 1882
+ },
+ {
+ "entropy": 1.1643813848495483,
+ "epoch": 3.804040404040404,
+ "grad_norm": 2.2955002784729004,
+ "learning_rate": 1.197979797979798e-06,
+ "loss": 1.1721268892288208,
+ "mean_token_accuracy": 0.7041401863098145,
+ "num_tokens": 30851072.0,
+ "step": 1883
+ },
+ {
+ "entropy": 1.0957778692245483,
+ "epoch": 3.806060606060606,
+ "grad_norm": 2.244861125946045,
+ "learning_rate": 1.1959595959595961e-06,
+ "loss": 1.1006194353103638,
+ "mean_token_accuracy": 0.7057889699935913,
+ "num_tokens": 30867456.0,
+ "step": 1884
+ },
+ {
+ "entropy": 1.420525074005127,
+ "epoch": 3.808080808080808,
+ "grad_norm": 2.376477003097534,
+ "learning_rate": 1.193939393939394e-06,
+ "loss": 1.4241583347320557,
+ "mean_token_accuracy": 0.662493884563446,
+ "num_tokens": 30883840.0,
+ "step": 1885
+ },
+ {
+ "entropy": 1.4050343036651611,
+ "epoch": 3.81010101010101,
+ "grad_norm": 2.310051202774048,
+ "learning_rate": 1.1919191919191921e-06,
+ "loss": 1.4288474321365356,
+ "mean_token_accuracy": 0.6623107194900513,
+ "num_tokens": 30900224.0,
+ "step": 1886
+ },
+ {
+ "entropy": 1.1658165454864502,
+ "epoch": 3.812121212121212,
+ "grad_norm": 2.1743791103363037,
+ "learning_rate": 1.18989898989899e-06,
+ "loss": 1.1644586324691772,
+ "mean_token_accuracy": 0.7122007608413696,
+ "num_tokens": 30916608.0,
+ "step": 1887
+ },
+ {
+ "entropy": 1.1667358875274658,
+ "epoch": 3.8141414141414143,
+ "grad_norm": 2.092580556869507,
+ "learning_rate": 1.187878787878788e-06,
+ "loss": 1.1762518882751465,
+ "mean_token_accuracy": 0.718551516532898,
+ "num_tokens": 30932992.0,
+ "step": 1888
+ },
+ {
+ "entropy": 1.2293628454208374,
+ "epoch": 3.816161616161616,
+ "grad_norm": 2.2432591915130615,
+ "learning_rate": 1.185858585858586e-06,
+ "loss": 1.249151587486267,
+ "mean_token_accuracy": 0.7038959264755249,
+ "num_tokens": 30949376.0,
+ "step": 1889
+ },
+ {
+ "entropy": 1.104671597480774,
+ "epoch": 3.8181818181818183,
+ "grad_norm": 2.7913901805877686,
+ "learning_rate": 1.183838383838384e-06,
+ "loss": 1.107041597366333,
+ "mean_token_accuracy": 0.7236809730529785,
+ "num_tokens": 30965760.0,
+ "step": 1890
+ },
+ {
+ "entropy": 0.9601510763168335,
+ "epoch": 3.82020202020202,
+ "grad_norm": 2.0736911296844482,
+ "learning_rate": 1.181818181818182e-06,
+ "loss": 0.9680359363555908,
+ "mean_token_accuracy": 0.7562286257743835,
+ "num_tokens": 30982144.0,
+ "step": 1891
+ },
+ {
+ "entropy": 1.2994087934494019,
+ "epoch": 3.822222222222222,
+ "grad_norm": 2.2076780796051025,
+ "learning_rate": 1.1797979797979799e-06,
+ "loss": 1.3263574838638306,
+ "mean_token_accuracy": 0.6968123912811279,
+ "num_tokens": 30998528.0,
+ "step": 1892
+ },
+ {
+ "entropy": 1.3631376028060913,
+ "epoch": 3.824242424242424,
+ "grad_norm": 2.340932846069336,
+ "learning_rate": 1.1777777777777778e-06,
+ "loss": 1.3866732120513916,
+ "mean_token_accuracy": 0.6775158643722534,
+ "num_tokens": 31014912.0,
+ "step": 1893
+ },
+ {
+ "entropy": 0.8883916735649109,
+ "epoch": 3.8262626262626265,
+ "grad_norm": 2.2482478618621826,
+ "learning_rate": 1.1757575757575759e-06,
+ "loss": 0.8972976207733154,
+ "mean_token_accuracy": 0.7602589130401611,
+ "num_tokens": 31031296.0,
+ "step": 1894
+ },
+ {
+ "entropy": 1.2881001234054565,
+ "epoch": 3.8282828282828283,
+ "grad_norm": 2.2452962398529053,
+ "learning_rate": 1.173737373737374e-06,
+ "loss": 1.298166036605835,
+ "mean_token_accuracy": 0.6886907815933228,
+ "num_tokens": 31047680.0,
+ "step": 1895
+ },
+ {
+ "entropy": 1.2942661046981812,
+ "epoch": 3.83030303030303,
+ "grad_norm": 2.5083208084106445,
+ "learning_rate": 1.1717171717171719e-06,
+ "loss": 1.3013086318969727,
+ "mean_token_accuracy": 0.6803248524665833,
+ "num_tokens": 31064064.0,
+ "step": 1896
+ },
+ {
+ "entropy": 1.1769400835037231,
+ "epoch": 3.8323232323232324,
+ "grad_norm": 2.2185699939727783,
+ "learning_rate": 1.1696969696969697e-06,
+ "loss": 1.1732335090637207,
+ "mean_token_accuracy": 0.7107352018356323,
+ "num_tokens": 31080448.0,
+ "step": 1897
+ },
+ {
+ "entropy": 1.3242368698120117,
+ "epoch": 3.8343434343434346,
+ "grad_norm": 2.153888702392578,
+ "learning_rate": 1.1676767676767678e-06,
+ "loss": 1.3406283855438232,
+ "mean_token_accuracy": 0.6897899508476257,
+ "num_tokens": 31096832.0,
+ "step": 1898
+ },
+ {
+ "entropy": 0.9292052984237671,
+ "epoch": 3.8363636363636364,
+ "grad_norm": 2.1435623168945312,
+ "learning_rate": 1.1656565656565657e-06,
+ "loss": 0.9380660057067871,
+ "mean_token_accuracy": 0.7523204684257507,
+ "num_tokens": 31113216.0,
+ "step": 1899
+ },
+ {
+ "entropy": 1.2560123205184937,
+ "epoch": 3.8383838383838382,
+ "grad_norm": 2.292471408843994,
+ "learning_rate": 1.1636363636363638e-06,
+ "loss": 1.2524765729904175,
+ "mean_token_accuracy": 0.6988885998725891,
+ "num_tokens": 31129600.0,
+ "step": 1900
+ },
+ {
+ "epoch": 3.8383838383838382,
+ "eval_entropy": 1.323313660679325,
+ "eval_loss": 1.5475536584854126,
+ "eval_mean_token_accuracy": 0.6447510724106142,
+ "eval_num_tokens": 31129600.0,
+ "eval_runtime": 43.7651,
+ "eval_samples_per_second": 22.621,
+ "eval_steps_per_second": 2.833,
+ "step": 1900
+ },
+ {
+ "entropy": 1.0595868825912476,
+ "epoch": 3.8404040404040405,
+ "grad_norm": 2.2940235137939453,
+ "learning_rate": 1.1616161616161617e-06,
+ "loss": 1.0685131549835205,
+ "mean_token_accuracy": 0.7346116304397583,
+ "num_tokens": 31145984.0,
+ "step": 1901
+ },
+ {
+ "entropy": 1.1705970764160156,
+ "epoch": 3.8424242424242423,
+ "grad_norm": 2.200145959854126,
+ "learning_rate": 1.1595959595959596e-06,
+ "loss": 1.1744719743728638,
+ "mean_token_accuracy": 0.7096360325813293,
+ "num_tokens": 31162368.0,
+ "step": 1902
+ },
+ {
+ "entropy": 1.1404083967208862,
+ "epoch": 3.8444444444444446,
+ "grad_norm": 2.3402581214904785,
+ "learning_rate": 1.1575757575757577e-06,
+ "loss": 1.154109239578247,
+ "mean_token_accuracy": 0.7090253829956055,
+ "num_tokens": 31178752.0,
+ "step": 1903
+ },
+ {
+ "entropy": 1.0651684999465942,
+ "epoch": 3.8464646464646464,
+ "grad_norm": 2.1707308292388916,
+ "learning_rate": 1.1555555555555556e-06,
+ "loss": 1.0661014318466187,
+ "mean_token_accuracy": 0.7322300672531128,
+ "num_tokens": 31195136.0,
+ "step": 1904
+ },
+ {
+ "entropy": 1.4612106084823608,
+ "epoch": 3.8484848484848486,
+ "grad_norm": 2.3446035385131836,
+ "learning_rate": 1.1535353535353535e-06,
+ "loss": 1.4408762454986572,
+ "mean_token_accuracy": 0.6595017313957214,
+ "num_tokens": 31211520.0,
+ "step": 1905
+ },
+ {
+ "entropy": 1.2062592506408691,
+ "epoch": 3.8505050505050504,
+ "grad_norm": 2.244398593902588,
+ "learning_rate": 1.1515151515151516e-06,
+ "loss": 1.2011905908584595,
+ "mean_token_accuracy": 0.7075598239898682,
+ "num_tokens": 31227904.0,
+ "step": 1906
+ },
+ {
+ "entropy": 1.0715900659561157,
+ "epoch": 3.8525252525252527,
+ "grad_norm": 2.2543070316314697,
+ "learning_rate": 1.1494949494949497e-06,
+ "loss": 1.0634649991989136,
+ "mean_token_accuracy": 0.7305202484130859,
+ "num_tokens": 31244288.0,
+ "step": 1907
+ },
+ {
+ "entropy": 1.6116880178451538,
+ "epoch": 3.8545454545454545,
+ "grad_norm": 2.3540494441986084,
+ "learning_rate": 1.1474747474747476e-06,
+ "loss": 1.6299524307250977,
+ "mean_token_accuracy": 0.6351978778839111,
+ "num_tokens": 31260672.0,
+ "step": 1908
+ },
+ {
+ "entropy": 1.470803141593933,
+ "epoch": 3.8565656565656568,
+ "grad_norm": 2.2863407135009766,
+ "learning_rate": 1.1454545454545457e-06,
+ "loss": 1.4692970514297485,
+ "mean_token_accuracy": 0.6576697826385498,
+ "num_tokens": 31277056.0,
+ "step": 1909
+ },
+ {
+ "entropy": 1.0824670791625977,
+ "epoch": 3.8585858585858586,
+ "grad_norm": 2.3184683322906494,
+ "learning_rate": 1.1434343434343436e-06,
+ "loss": 1.0677670240402222,
+ "mean_token_accuracy": 0.7219101190567017,
+ "num_tokens": 31293440.0,
+ "step": 1910
+ },
+ {
+ "entropy": 1.027206540107727,
+ "epoch": 3.8606060606060604,
+ "grad_norm": 2.4096450805664062,
+ "learning_rate": 1.1414141414141414e-06,
+ "loss": 1.0224218368530273,
+ "mean_token_accuracy": 0.7347947955131531,
+ "num_tokens": 31309824.0,
+ "step": 1911
+ },
+ {
+ "entropy": 1.1143090724945068,
+ "epoch": 3.8626262626262626,
+ "grad_norm": 2.0057880878448486,
+ "learning_rate": 1.1393939393939395e-06,
+ "loss": 1.1075199842453003,
+ "mean_token_accuracy": 0.7245969772338867,
+ "num_tokens": 31326208.0,
+ "step": 1912
+ },
+ {
+ "entropy": 1.299835205078125,
+ "epoch": 3.864646464646465,
+ "grad_norm": 2.554501533508301,
+ "learning_rate": 1.1373737373737374e-06,
+ "loss": 1.2906074523925781,
+ "mean_token_accuracy": 0.681485116481781,
+ "num_tokens": 31342592.0,
+ "step": 1913
+ },
+ {
+ "entropy": 1.289318561553955,
+ "epoch": 3.8666666666666667,
+ "grad_norm": 2.4257376194000244,
+ "learning_rate": 1.1353535353535353e-06,
+ "loss": 1.281335711479187,
+ "mean_token_accuracy": 0.678798258304596,
+ "num_tokens": 31358976.0,
+ "step": 1914
+ },
+ {
+ "entropy": 1.0814216136932373,
+ "epoch": 3.8686868686868685,
+ "grad_norm": 2.5448758602142334,
+ "learning_rate": 1.1333333333333334e-06,
+ "loss": 1.0837101936340332,
+ "mean_token_accuracy": 0.7264899611473083,
+ "num_tokens": 31375360.0,
+ "step": 1915
+ },
+ {
+ "entropy": 0.8491103649139404,
+ "epoch": 3.8707070707070708,
+ "grad_norm": 2.0053293704986572,
+ "learning_rate": 1.1313131313131315e-06,
+ "loss": 0.8459927439689636,
+ "mean_token_accuracy": 0.775036633014679,
+ "num_tokens": 31391744.0,
+ "step": 1916
+ },
+ {
+ "entropy": 1.2331805229187012,
+ "epoch": 3.8727272727272726,
+ "grad_norm": 2.1101794242858887,
+ "learning_rate": 1.1292929292929294e-06,
+ "loss": 1.241546869277954,
+ "mean_token_accuracy": 0.7051783204078674,
+ "num_tokens": 31408128.0,
+ "step": 1917
+ },
+ {
+ "entropy": 0.8649861812591553,
+ "epoch": 3.874747474747475,
+ "grad_norm": 2.098615884780884,
+ "learning_rate": 1.1272727272727275e-06,
+ "loss": 0.8645999431610107,
+ "mean_token_accuracy": 0.7702125310897827,
+ "num_tokens": 31424512.0,
+ "step": 1918
+ },
+ {
+ "entropy": 1.1396632194519043,
+ "epoch": 3.8767676767676766,
+ "grad_norm": 2.2155094146728516,
+ "learning_rate": 1.1252525252525254e-06,
+ "loss": 1.1333739757537842,
+ "mean_token_accuracy": 0.7166585326194763,
+ "num_tokens": 31440896.0,
+ "step": 1919
+ },
+ {
+ "entropy": 1.375836968421936,
+ "epoch": 3.878787878787879,
+ "grad_norm": 2.3003385066986084,
+ "learning_rate": 1.1232323232323233e-06,
+ "loss": 1.379508376121521,
+ "mean_token_accuracy": 0.6699438095092773,
+ "num_tokens": 31457280.0,
+ "step": 1920
+ },
+ {
+ "entropy": 1.2705941200256348,
+ "epoch": 3.8808080808080807,
+ "grad_norm": 2.156363010406494,
+ "learning_rate": 1.1212121212121214e-06,
+ "loss": 1.266032099723816,
+ "mean_token_accuracy": 0.6869198679924011,
+ "num_tokens": 31473664.0,
+ "step": 1921
+ },
+ {
+ "entropy": 1.6494578123092651,
+ "epoch": 3.882828282828283,
+ "grad_norm": 2.420069932937622,
+ "learning_rate": 1.1191919191919193e-06,
+ "loss": 1.6762641668319702,
+ "mean_token_accuracy": 0.6144357323646545,
+ "num_tokens": 31490048.0,
+ "step": 1922
+ },
+ {
+ "entropy": 0.9549956917762756,
+ "epoch": 3.8848484848484848,
+ "grad_norm": 2.199296236038208,
+ "learning_rate": 1.1171717171717172e-06,
+ "loss": 0.9642987251281738,
+ "mean_token_accuracy": 0.7506717443466187,
+ "num_tokens": 31506432.0,
+ "step": 1923
+ },
+ {
+ "entropy": 1.5379301309585571,
+ "epoch": 3.886868686868687,
+ "grad_norm": 2.2627663612365723,
+ "learning_rate": 1.1151515151515153e-06,
+ "loss": 1.5375440120697021,
+ "mean_token_accuracy": 0.6497313380241394,
+ "num_tokens": 31522816.0,
+ "step": 1924
+ },
+ {
+ "entropy": 1.009248971939087,
+ "epoch": 3.888888888888889,
+ "grad_norm": 2.142307758331299,
+ "learning_rate": 1.1131313131313131e-06,
+ "loss": 1.014758586883545,
+ "mean_token_accuracy": 0.7470077872276306,
+ "num_tokens": 31539200.0,
+ "step": 1925
+ },
+ {
+ "entropy": 1.1543548107147217,
+ "epoch": 3.8909090909090907,
+ "grad_norm": 2.2896368503570557,
+ "learning_rate": 1.111111111111111e-06,
+ "loss": 1.169635534286499,
+ "mean_token_accuracy": 0.7079873085021973,
+ "num_tokens": 31555584.0,
+ "step": 1926
+ },
+ {
+ "entropy": 1.1072580814361572,
+ "epoch": 3.892929292929293,
+ "grad_norm": 2.4316368103027344,
+ "learning_rate": 1.1090909090909093e-06,
+ "loss": 1.0753037929534912,
+ "mean_token_accuracy": 0.7245358824729919,
+ "num_tokens": 31571968.0,
+ "step": 1927
+ },
+ {
+ "entropy": 1.0660669803619385,
+ "epoch": 3.894949494949495,
+ "grad_norm": 2.2569425106048584,
+ "learning_rate": 1.1070707070707072e-06,
+ "loss": 1.0584138631820679,
+ "mean_token_accuracy": 0.7323521971702576,
+ "num_tokens": 31588352.0,
+ "step": 1928
+ },
+ {
+ "entropy": 1.4635326862335205,
+ "epoch": 3.896969696969697,
+ "grad_norm": 2.4725160598754883,
+ "learning_rate": 1.1050505050505051e-06,
+ "loss": 1.4772144556045532,
+ "mean_token_accuracy": 0.6534562706947327,
+ "num_tokens": 31604736.0,
+ "step": 1929
+ },
+ {
+ "entropy": 1.2802455425262451,
+ "epoch": 3.898989898989899,
+ "grad_norm": 2.188870906829834,
+ "learning_rate": 1.1030303030303032e-06,
+ "loss": 1.2900553941726685,
+ "mean_token_accuracy": 0.6868588328361511,
+ "num_tokens": 31621120.0,
+ "step": 1930
+ },
+ {
+ "entropy": 1.0175695419311523,
+ "epoch": 3.901010101010101,
+ "grad_norm": 2.177603244781494,
+ "learning_rate": 1.1010101010101011e-06,
+ "loss": 1.0199849605560303,
+ "mean_token_accuracy": 0.7407791614532471,
+ "num_tokens": 31637504.0,
+ "step": 1931
+ },
+ {
+ "entropy": 1.0328937768936157,
+ "epoch": 3.9030303030303033,
+ "grad_norm": 2.3460423946380615,
+ "learning_rate": 1.098989898989899e-06,
+ "loss": 1.0519663095474243,
+ "mean_token_accuracy": 0.7301538586616516,
+ "num_tokens": 31653888.0,
+ "step": 1932
+ },
+ {
+ "entropy": 1.3149690628051758,
+ "epoch": 3.905050505050505,
+ "grad_norm": 2.4549903869628906,
+ "learning_rate": 1.096969696969697e-06,
+ "loss": 1.311675786972046,
+ "mean_token_accuracy": 0.6631045341491699,
+ "num_tokens": 31670272.0,
+ "step": 1933
+ },
+ {
+ "entropy": 1.0760291814804077,
+ "epoch": 3.907070707070707,
+ "grad_norm": 2.1368870735168457,
+ "learning_rate": 1.094949494949495e-06,
+ "loss": 1.0474696159362793,
+ "mean_token_accuracy": 0.740290641784668,
+ "num_tokens": 31686656.0,
+ "step": 1934
+ },
+ {
+ "entropy": 1.4127235412597656,
+ "epoch": 3.909090909090909,
+ "grad_norm": 2.329832077026367,
+ "learning_rate": 1.0929292929292929e-06,
+ "loss": 1.4108169078826904,
+ "mean_token_accuracy": 0.6663410067558289,
+ "num_tokens": 31703040.0,
+ "step": 1935
+ },
+ {
+ "entropy": 1.5287094116210938,
+ "epoch": 3.911111111111111,
+ "grad_norm": 2.2737598419189453,
+ "learning_rate": 1.090909090909091e-06,
+ "loss": 1.5544226169586182,
+ "mean_token_accuracy": 0.6364802122116089,
+ "num_tokens": 31719424.0,
+ "step": 1936
+ },
+ {
+ "entropy": 1.1318615674972534,
+ "epoch": 3.9131313131313132,
+ "grad_norm": 2.3448705673217773,
+ "learning_rate": 1.0888888888888889e-06,
+ "loss": 1.1228995323181152,
+ "mean_token_accuracy": 0.7198339104652405,
+ "num_tokens": 31735808.0,
+ "step": 1937
+ },
+ {
+ "entropy": 1.0484049320220947,
+ "epoch": 3.915151515151515,
+ "grad_norm": 2.5756354331970215,
+ "learning_rate": 1.086868686868687e-06,
+ "loss": 1.0507314205169678,
+ "mean_token_accuracy": 0.7214215993881226,
+ "num_tokens": 31752192.0,
+ "step": 1938
+ },
+ {
+ "entropy": 0.9962558150291443,
+ "epoch": 3.9171717171717173,
+ "grad_norm": 2.379638910293579,
+ "learning_rate": 1.084848484848485e-06,
+ "loss": 0.9964162707328796,
+ "mean_token_accuracy": 0.7351001501083374,
+ "num_tokens": 31768576.0,
+ "step": 1939
+ },
+ {
+ "entropy": 1.0813647508621216,
+ "epoch": 3.919191919191919,
+ "grad_norm": 2.202664852142334,
+ "learning_rate": 1.082828282828283e-06,
+ "loss": 1.0924289226531982,
+ "mean_token_accuracy": 0.7243527173995972,
+ "num_tokens": 31784960.0,
+ "step": 1940
+ },
+ {
+ "entropy": 1.3451746702194214,
+ "epoch": 3.9212121212121214,
+ "grad_norm": 2.2476084232330322,
+ "learning_rate": 1.0808080808080808e-06,
+ "loss": 1.3437637090682983,
+ "mean_token_accuracy": 0.681485116481781,
+ "num_tokens": 31801344.0,
+ "step": 1941
+ },
+ {
+ "entropy": 1.1332415342330933,
+ "epoch": 3.923232323232323,
+ "grad_norm": 2.414275884628296,
+ "learning_rate": 1.078787878787879e-06,
+ "loss": 1.1400260925292969,
+ "mean_token_accuracy": 0.7132388949394226,
+ "num_tokens": 31817728.0,
+ "step": 1942
+ },
+ {
+ "entropy": 1.087522029876709,
+ "epoch": 3.9252525252525254,
+ "grad_norm": 2.1204850673675537,
+ "learning_rate": 1.0767676767676768e-06,
+ "loss": 1.0787461996078491,
+ "mean_token_accuracy": 0.7150708436965942,
+ "num_tokens": 31834112.0,
+ "step": 1943
+ },
+ {
+ "entropy": 1.1856560707092285,
+ "epoch": 3.9272727272727272,
+ "grad_norm": 2.270273447036743,
+ "learning_rate": 1.0747474747474747e-06,
+ "loss": 1.1918301582336426,
+ "mean_token_accuracy": 0.7099413871765137,
+ "num_tokens": 31850496.0,
+ "step": 1944
+ },
+ {
+ "entropy": 1.1141650676727295,
+ "epoch": 3.929292929292929,
+ "grad_norm": 2.357293128967285,
+ "learning_rate": 1.0727272727272728e-06,
+ "loss": 1.0945638418197632,
+ "mean_token_accuracy": 0.7214826345443726,
+ "num_tokens": 31866880.0,
+ "step": 1945
+ },
+ {
+ "entropy": 1.4306713342666626,
+ "epoch": 3.9313131313131313,
+ "grad_norm": 2.1920864582061768,
+ "learning_rate": 1.0707070707070707e-06,
+ "loss": 1.4291378259658813,
+ "mean_token_accuracy": 0.6696995496749878,
+ "num_tokens": 31883264.0,
+ "step": 1946
+ },
+ {
+ "entropy": 1.0448427200317383,
+ "epoch": 3.9333333333333336,
+ "grad_norm": 2.2893853187561035,
+ "learning_rate": 1.0686868686868688e-06,
+ "loss": 1.0350053310394287,
+ "mean_token_accuracy": 0.7333292365074158,
+ "num_tokens": 31899648.0,
+ "step": 1947
+ },
+ {
+ "entropy": 0.9403035044670105,
+ "epoch": 3.9353535353535354,
+ "grad_norm": 2.036454200744629,
+ "learning_rate": 1.066666666666667e-06,
+ "loss": 0.9342324137687683,
+ "mean_token_accuracy": 0.7620298266410828,
+ "num_tokens": 31916032.0,
+ "step": 1948
+ },
+ {
+ "entropy": 1.1731034517288208,
+ "epoch": 3.937373737373737,
+ "grad_norm": 2.388814687728882,
+ "learning_rate": 1.0646464646464648e-06,
+ "loss": 1.1642717123031616,
+ "mean_token_accuracy": 0.705422580242157,
+ "num_tokens": 31932416.0,
+ "step": 1949
+ },
+ {
+ "entropy": 1.4009122848510742,
+ "epoch": 3.9393939393939394,
+ "grad_norm": 2.3011586666107178,
+ "learning_rate": 1.0626262626262627e-06,
+ "loss": 1.3879787921905518,
+ "mean_token_accuracy": 0.6805080771446228,
+ "num_tokens": 31948800.0,
+ "step": 1950
+ },
+ {
+ "epoch": 3.9393939393939394,
+ "eval_entropy": 1.319143979299453,
+ "eval_loss": 1.5476959943771362,
+ "eval_mean_token_accuracy": 0.6448943807232764,
+ "eval_num_tokens": 31948800.0,
+ "eval_runtime": 43.7596,
+ "eval_samples_per_second": 22.624,
+ "eval_steps_per_second": 2.834,
+ "step": 1950
+ },
+ {
+ "entropy": 1.2538156509399414,
+ "epoch": 3.9414141414141413,
+ "grad_norm": 2.5176167488098145,
+ "learning_rate": 1.0606060606060608e-06,
+ "loss": 1.2646058797836304,
+ "mean_token_accuracy": 0.6989496946334839,
+ "num_tokens": 31965184.0,
+ "step": 1951
+ },
+ {
+ "entropy": 1.4409568309783936,
+ "epoch": 3.9434343434343435,
+ "grad_norm": 2.3068838119506836,
+ "learning_rate": 1.0585858585858587e-06,
+ "loss": 1.4423179626464844,
+ "mean_token_accuracy": 0.6614558100700378,
+ "num_tokens": 31981568.0,
+ "step": 1952
+ },
+ {
+ "entropy": 1.4900083541870117,
+ "epoch": 3.9454545454545453,
+ "grad_norm": 2.3615455627441406,
+ "learning_rate": 1.0565656565656566e-06,
+ "loss": 1.4933650493621826,
+ "mean_token_accuracy": 0.6474108695983887,
+ "num_tokens": 31997952.0,
+ "step": 1953
+ },
+ {
+ "entropy": 1.2034417390823364,
+ "epoch": 3.9474747474747476,
+ "grad_norm": 2.4732308387756348,
+ "learning_rate": 1.0545454545454547e-06,
+ "loss": 1.209761619567871,
+ "mean_token_accuracy": 0.6927210688591003,
+ "num_tokens": 32014336.0,
+ "step": 1954
+ },
+ {
+ "entropy": 1.1165226697921753,
+ "epoch": 3.9494949494949494,
+ "grad_norm": 2.4827890396118164,
+ "learning_rate": 1.0525252525252525e-06,
+ "loss": 1.1289414167404175,
+ "mean_token_accuracy": 0.7070713043212891,
+ "num_tokens": 32030720.0,
+ "step": 1955
+ },
+ {
+ "entropy": 1.2843526601791382,
+ "epoch": 3.9515151515151516,
+ "grad_norm": 2.32797908782959,
+ "learning_rate": 1.0505050505050506e-06,
+ "loss": 1.2694849967956543,
+ "mean_token_accuracy": 0.6875916123390198,
+ "num_tokens": 32047104.0,
+ "step": 1956
+ },
+ {
+ "entropy": 1.3727971315383911,
+ "epoch": 3.9535353535353535,
+ "grad_norm": 2.320406198501587,
+ "learning_rate": 1.0484848484848485e-06,
+ "loss": 1.3948733806610107,
+ "mean_token_accuracy": 0.6679286956787109,
+ "num_tokens": 32063488.0,
+ "step": 1957
+ },
+ {
+ "entropy": 1.4869349002838135,
+ "epoch": 3.9555555555555557,
+ "grad_norm": 2.2239322662353516,
+ "learning_rate": 1.0464646464646464e-06,
+ "loss": 1.5088552236557007,
+ "mean_token_accuracy": 0.6551660895347595,
+ "num_tokens": 32079872.0,
+ "step": 1958
+ },
+ {
+ "entropy": 1.2804311513900757,
+ "epoch": 3.9575757575757575,
+ "grad_norm": 2.355632781982422,
+ "learning_rate": 1.0444444444444445e-06,
+ "loss": 1.2881189584732056,
+ "mean_token_accuracy": 0.6883854269981384,
+ "num_tokens": 32096256.0,
+ "step": 1959
+ },
+ {
+ "entropy": 1.469780683517456,
+ "epoch": 3.9595959595959593,
+ "grad_norm": 2.475818395614624,
+ "learning_rate": 1.0424242424242426e-06,
+ "loss": 1.4946942329406738,
+ "mean_token_accuracy": 0.6443576216697693,
+ "num_tokens": 32112640.0,
+ "step": 1960
+ },
+ {
+ "entropy": 1.0433298349380493,
+ "epoch": 3.9616161616161616,
+ "grad_norm": 2.335348606109619,
+ "learning_rate": 1.0404040404040405e-06,
+ "loss": 1.051847219467163,
+ "mean_token_accuracy": 0.7260014414787292,
+ "num_tokens": 32129024.0,
+ "step": 1961
+ },
+ {
+ "entropy": 1.060207486152649,
+ "epoch": 3.963636363636364,
+ "grad_norm": 2.465827465057373,
+ "learning_rate": 1.0383838383838384e-06,
+ "loss": 1.0738749504089355,
+ "mean_token_accuracy": 0.7250854969024658,
+ "num_tokens": 32145408.0,
+ "step": 1962
+ },
+ {
+ "entropy": 1.3710187673568726,
+ "epoch": 3.9656565656565657,
+ "grad_norm": 2.1859865188598633,
+ "learning_rate": 1.0363636363636365e-06,
+ "loss": 1.3790408372879028,
+ "mean_token_accuracy": 0.6800195574760437,
+ "num_tokens": 32161792.0,
+ "step": 1963
+ },
+ {
+ "entropy": 1.4380125999450684,
+ "epoch": 3.9676767676767675,
+ "grad_norm": 2.3186328411102295,
+ "learning_rate": 1.0343434343434344e-06,
+ "loss": 1.4353742599487305,
+ "mean_token_accuracy": 0.6627992391586304,
+ "num_tokens": 32178176.0,
+ "step": 1964
+ },
+ {
+ "entropy": 1.346064567565918,
+ "epoch": 3.9696969696969697,
+ "grad_norm": 2.455488920211792,
+ "learning_rate": 1.0323232323232325e-06,
+ "loss": 1.3504667282104492,
+ "mean_token_accuracy": 0.6808744668960571,
+ "num_tokens": 32194560.0,
+ "step": 1965
+ },
+ {
+ "entropy": 1.111870527267456,
+ "epoch": 3.971717171717172,
+ "grad_norm": 2.216954231262207,
+ "learning_rate": 1.0303030303030304e-06,
+ "loss": 1.1173250675201416,
+ "mean_token_accuracy": 0.7161700129508972,
+ "num_tokens": 32210944.0,
+ "step": 1966
+ },
+ {
+ "entropy": 1.570885181427002,
+ "epoch": 3.973737373737374,
+ "grad_norm": 2.5180883407592773,
+ "learning_rate": 1.0282828282828283e-06,
+ "loss": 1.550491213798523,
+ "mean_token_accuracy": 0.6203590631484985,
+ "num_tokens": 32227328.0,
+ "step": 1967
+ },
+ {
+ "entropy": 0.887984037399292,
+ "epoch": 3.9757575757575756,
+ "grad_norm": 2.193727970123291,
+ "learning_rate": 1.0262626262626264e-06,
+ "loss": 0.8939201831817627,
+ "mean_token_accuracy": 0.7674645781517029,
+ "num_tokens": 32243712.0,
+ "step": 1968
+ },
+ {
+ "entropy": 1.3493150472640991,
+ "epoch": 3.977777777777778,
+ "grad_norm": 2.320697784423828,
+ "learning_rate": 1.0242424242424242e-06,
+ "loss": 1.3643516302108765,
+ "mean_token_accuracy": 0.67659991979599,
+ "num_tokens": 32260096.0,
+ "step": 1969
+ },
+ {
+ "entropy": 1.2187929153442383,
+ "epoch": 3.9797979797979797,
+ "grad_norm": 2.2966790199279785,
+ "learning_rate": 1.0222222222222223e-06,
+ "loss": 1.2266913652420044,
+ "mean_token_accuracy": 0.701636552810669,
+ "num_tokens": 32276480.0,
+ "step": 1970
+ },
+ {
+ "entropy": 1.3222001791000366,
+ "epoch": 3.981818181818182,
+ "grad_norm": 2.484215497970581,
+ "learning_rate": 1.0202020202020202e-06,
+ "loss": 1.3209818601608276,
+ "mean_token_accuracy": 0.6782486438751221,
+ "num_tokens": 32292864.0,
+ "step": 1971
+ },
+ {
+ "entropy": 1.2982250452041626,
+ "epoch": 3.9838383838383837,
+ "grad_norm": 2.320852518081665,
+ "learning_rate": 1.0181818181818183e-06,
+ "loss": 1.3085607290267944,
+ "mean_token_accuracy": 0.6881411671638489,
+ "num_tokens": 32309248.0,
+ "step": 1972
+ },
+ {
+ "entropy": 1.3396943807601929,
+ "epoch": 3.985858585858586,
+ "grad_norm": 2.224574565887451,
+ "learning_rate": 1.0161616161616162e-06,
+ "loss": 1.3511459827423096,
+ "mean_token_accuracy": 0.6810576319694519,
+ "num_tokens": 32325632.0,
+ "step": 1973
+ },
+ {
+ "entropy": 1.2554755210876465,
+ "epoch": 3.987878787878788,
+ "grad_norm": 2.3757498264312744,
+ "learning_rate": 1.0141414141414143e-06,
+ "loss": 1.247807502746582,
+ "mean_token_accuracy": 0.687530517578125,
+ "num_tokens": 32342016.0,
+ "step": 1974
+ },
+ {
+ "entropy": 0.9622905850410461,
+ "epoch": 3.98989898989899,
+ "grad_norm": 2.2012438774108887,
+ "learning_rate": 1.0121212121212122e-06,
+ "loss": 0.9560307860374451,
+ "mean_token_accuracy": 0.7637396454811096,
+ "num_tokens": 32358400.0,
+ "step": 1975
+ },
+ {
+ "entropy": 1.1193549633026123,
+ "epoch": 3.991919191919192,
+ "grad_norm": 2.3705174922943115,
+ "learning_rate": 1.01010101010101e-06,
+ "loss": 1.11350679397583,
+ "mean_token_accuracy": 0.7154372334480286,
+ "num_tokens": 32374784.0,
+ "step": 1976
+ },
+ {
+ "entropy": 0.9361278414726257,
+ "epoch": 3.993939393939394,
+ "grad_norm": 2.174281358718872,
+ "learning_rate": 1.0080808080808082e-06,
+ "loss": 0.9388691186904907,
+ "mean_token_accuracy": 0.7485344409942627,
+ "num_tokens": 32391168.0,
+ "step": 1977
+ },
+ {
+ "entropy": 1.4282293319702148,
+ "epoch": 3.995959595959596,
+ "grad_norm": 2.1972262859344482,
+ "learning_rate": 1.006060606060606e-06,
+ "loss": 1.4244005680084229,
+ "mean_token_accuracy": 0.685332179069519,
+ "num_tokens": 32407552.0,
+ "step": 1978
+ },
+ {
+ "entropy": 1.221082091331482,
+ "epoch": 3.9979797979797977,
+ "grad_norm": 2.3935673236846924,
+ "learning_rate": 1.004040404040404e-06,
+ "loss": 1.1999341249465942,
+ "mean_token_accuracy": 0.6952857971191406,
+ "num_tokens": 32423936.0,
+ "step": 1979
+ },
+ {
+ "entropy": 1.1347371339797974,
+ "epoch": 4.0,
+ "grad_norm": 2.228233575820923,
+ "learning_rate": 1.002020202020202e-06,
+ "loss": 1.1378669738769531,
+ "mean_token_accuracy": 0.700537383556366,
+ "num_tokens": 32440320.0,
+ "step": 1980
+ },
+ {
+ "entropy": 1.1821788549423218,
+ "epoch": 4.002020202020202,
+ "grad_norm": 2.356572389602661,
+ "learning_rate": 1.0000000000000002e-06,
+ "loss": 1.1377968788146973,
+ "mean_token_accuracy": 0.708170473575592,
+ "num_tokens": 32456704.0,
+ "step": 1981
+ },
+ {
+ "entropy": 1.444516658782959,
+ "epoch": 4.004040404040404,
+ "grad_norm": 2.34165358543396,
+ "learning_rate": 9.97979797979798e-07,
+ "loss": 1.4065756797790527,
+ "mean_token_accuracy": 0.6689057350158691,
+ "num_tokens": 32473088.0,
+ "step": 1982
+ },
+ {
+ "entropy": 1.5600178241729736,
+ "epoch": 4.006060606060606,
+ "grad_norm": 2.147475004196167,
+ "learning_rate": 9.959595959595962e-07,
+ "loss": 1.5237784385681152,
+ "mean_token_accuracy": 0.6602345108985901,
+ "num_tokens": 32489472.0,
+ "step": 1983
+ },
+ {
+ "entropy": 1.3860907554626465,
+ "epoch": 4.008080808080808,
+ "grad_norm": 2.5050148963928223,
+ "learning_rate": 9.93939393939394e-07,
+ "loss": 1.3332314491271973,
+ "mean_token_accuracy": 0.6679897308349609,
+ "num_tokens": 32505856.0,
+ "step": 1984
+ },
+ {
+ "entropy": 1.2262948751449585,
+ "epoch": 4.01010101010101,
+ "grad_norm": 2.3707785606384277,
+ "learning_rate": 9.91919191919192e-07,
+ "loss": 1.1803091764450073,
+ "mean_token_accuracy": 0.6957743167877197,
+ "num_tokens": 32522240.0,
+ "step": 1985
+ },
+ {
+ "entropy": 1.2782310247421265,
+ "epoch": 4.012121212121212,
+ "grad_norm": 2.1337549686431885,
+ "learning_rate": 9.8989898989899e-07,
+ "loss": 1.2508152723312378,
+ "mean_token_accuracy": 0.6898509860038757,
+ "num_tokens": 32538624.0,
+ "step": 1986
+ },
+ {
+ "entropy": 1.2618422508239746,
+ "epoch": 4.014141414141414,
+ "grad_norm": 2.1014909744262695,
+ "learning_rate": 9.87878787878788e-07,
+ "loss": 1.2586122751235962,
+ "mean_token_accuracy": 0.7060942649841309,
+ "num_tokens": 32555008.0,
+ "step": 1987
+ },
+ {
+ "entropy": 1.4927153587341309,
+ "epoch": 4.016161616161616,
+ "grad_norm": 2.3997669219970703,
+ "learning_rate": 9.858585858585858e-07,
+ "loss": 1.458584189414978,
+ "mean_token_accuracy": 0.6642037034034729,
+ "num_tokens": 32571392.0,
+ "step": 1988
+ },
+ {
+ "entropy": 0.9929779767990112,
+ "epoch": 4.0181818181818185,
+ "grad_norm": 2.1199800968170166,
+ "learning_rate": 9.83838383838384e-07,
+ "loss": 0.9945173263549805,
+ "mean_token_accuracy": 0.7606863975524902,
+ "num_tokens": 32587776.0,
+ "step": 1989
+ },
+ {
+ "entropy": 1.2815021276474,
+ "epoch": 4.02020202020202,
+ "grad_norm": 2.2822794914245605,
+ "learning_rate": 9.818181818181818e-07,
+ "loss": 1.263106107711792,
+ "mean_token_accuracy": 0.7038959264755249,
+ "num_tokens": 32604160.0,
+ "step": 1990
+ },
+ {
+ "entropy": 1.2504572868347168,
+ "epoch": 4.022222222222222,
+ "grad_norm": 2.0446224212646484,
+ "learning_rate": 9.797979797979799e-07,
+ "loss": 1.2274131774902344,
+ "mean_token_accuracy": 0.7008426785469055,
+ "num_tokens": 32620544.0,
+ "step": 1991
+ },
+ {
+ "entropy": 1.1070255041122437,
+ "epoch": 4.024242424242424,
+ "grad_norm": 2.0727381706237793,
+ "learning_rate": 9.77777777777778e-07,
+ "loss": 1.082715392112732,
+ "mean_token_accuracy": 0.7302759885787964,
+ "num_tokens": 32636928.0,
+ "step": 1992
+ },
+ {
+ "entropy": 1.0761677026748657,
+ "epoch": 4.026262626262627,
+ "grad_norm": 2.0985822677612305,
+ "learning_rate": 9.757575757575759e-07,
+ "loss": 1.0548713207244873,
+ "mean_token_accuracy": 0.7333292365074158,
+ "num_tokens": 32653312.0,
+ "step": 1993
+ },
+ {
+ "entropy": 1.5109375715255737,
+ "epoch": 4.028282828282828,
+ "grad_norm": 2.426588535308838,
+ "learning_rate": 9.737373737373738e-07,
+ "loss": 1.4922651052474976,
+ "mean_token_accuracy": 0.6560820937156677,
+ "num_tokens": 32669696.0,
+ "step": 1994
+ },
+ {
+ "entropy": 1.2393261194229126,
+ "epoch": 4.03030303030303,
+ "grad_norm": 2.1131463050842285,
+ "learning_rate": 9.717171717171719e-07,
+ "loss": 1.2221450805664062,
+ "mean_token_accuracy": 0.7113458514213562,
+ "num_tokens": 32686080.0,
+ "step": 1995
+ },
+ {
+ "entropy": 1.2733023166656494,
+ "epoch": 4.0323232323232325,
+ "grad_norm": 2.42077898979187,
+ "learning_rate": 9.696969696969698e-07,
+ "loss": 1.266562581062317,
+ "mean_token_accuracy": 0.6916218996047974,
+ "num_tokens": 32702464.0,
+ "step": 1996
+ },
+ {
+ "entropy": 1.1015594005584717,
+ "epoch": 4.034343434343434,
+ "grad_norm": 2.3131461143493652,
+ "learning_rate": 9.676767676767676e-07,
+ "loss": 1.0699245929718018,
+ "mean_token_accuracy": 0.7295432090759277,
+ "num_tokens": 32718848.0,
+ "step": 1997
+ },
+ {
+ "entropy": 1.1488311290740967,
+ "epoch": 4.036363636363636,
+ "grad_norm": 2.460294723510742,
+ "learning_rate": 9.656565656565657e-07,
+ "loss": 1.1216062307357788,
+ "mean_token_accuracy": 0.7164753079414368,
+ "num_tokens": 32735232.0,
+ "step": 1998
+ },
+ {
+ "entropy": 1.1301594972610474,
+ "epoch": 4.038383838383838,
+ "grad_norm": 2.248311996459961,
+ "learning_rate": 9.636363636363636e-07,
+ "loss": 1.1271123886108398,
+ "mean_token_accuracy": 0.7147044539451599,
+ "num_tokens": 32751616.0,
+ "step": 1999
+ },
+ {
+ "entropy": 1.2287960052490234,
+ "epoch": 4.040404040404041,
+ "grad_norm": 2.191375732421875,
+ "learning_rate": 9.616161616161617e-07,
+ "loss": 1.2168409824371338,
+ "mean_token_accuracy": 0.7156814932823181,
+ "num_tokens": 32768000.0,
+ "step": 2000
+ },
+ {
+ "epoch": 4.040404040404041,
+ "eval_entropy": 1.294078712021151,
+ "eval_loss": 1.5553377866744995,
+ "eval_mean_token_accuracy": 0.6444841599272143,
+ "eval_num_tokens": 32768000.0,
+ "eval_runtime": 43.7303,
+ "eval_samples_per_second": 22.639,
+ "eval_steps_per_second": 2.836,
+ "step": 2000
+ }
+ ],
+ "logging_steps": 1,
+ "max_steps": 2475,
+ "num_input_tokens_seen": 0,
+ "num_train_epochs": 5,
+ "save_steps": 500,
+ "stateful_callbacks": {
+ "TrainerControl": {
+ "args": {
+ "should_epoch_stop": false,
+ "should_evaluate": false,
+ "should_log": false,
+ "should_save": true,
+ "should_training_stop": false
+ },
+ "attributes": {}
+ }
+ },
+ "total_flos": 2.77101290717184e+17,
+ "train_batch_size": 4,
+ "trial_name": null,
+ "trial_params": null
+}
diff --git a/qwen3-1.7b-teutonic-5e6/checkpoint-2000/training_args.bin b/qwen3-1.7b-teutonic-5e6/checkpoint-2000/training_args.bin
new file mode 100644
index 0000000000000000000000000000000000000000..37649d4bae96a1df88666daf20b4e5b6e092d976
--- /dev/null
+++ b/qwen3-1.7b-teutonic-5e6/checkpoint-2000/training_args.bin
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:58bbcabfe99a1cf24f2e76aab66b507c2b720d9271dc9f17c8208d741a9c3a65
+size 7121
diff --git a/qwen3-1.7b-teutonic-5e6/checkpoint-2000/zero_to_fp32.py b/qwen3-1.7b-teutonic-5e6/checkpoint-2000/zero_to_fp32.py
new file mode 100644
index 0000000000000000000000000000000000000000..3970548c400fd4361bd923b763db90e963ddaf8c
--- /dev/null
+++ b/qwen3-1.7b-teutonic-5e6/checkpoint-2000/zero_to_fp32.py
@@ -0,0 +1,854 @@
+#!/usr/bin/env python
+
+# Copyright (c) Microsoft Corporation.
+# SPDX-License-Identifier: Apache-2.0
+
+# DeepSpeed Team
+
+# This script extracts fp32 consolidated weights from a zero 1, 2 and 3 DeepSpeed checkpoints. It gets
+# copied into the top level checkpoint dir, so the user can easily do the conversion at any point in
+# the future. Once extracted, the weights don't require DeepSpeed and can be used in any
+# application.
+#
+# example:
+# python zero_to_fp32.py . output_dir/
+# or
+# python zero_to_fp32.py . output_dir/ --safe_serialization
+
+import argparse
+import torch
+import glob
+import math
+import os
+import re
+import gc
+import json
+import numpy as np
+from tqdm import tqdm
+from collections import OrderedDict
+from dataclasses import dataclass
+
+# while this script doesn't use deepspeed to recover data, since the checkpoints are pickled with
+# DeepSpeed data structures it has to be available in the current python environment.
+from deepspeed.utils import logger
+from deepspeed.checkpoint.constants import (DS_VERSION, OPTIMIZER_STATE_DICT, SINGLE_PARTITION_OF_FP32_GROUPS,
+ FP32_FLAT_GROUPS, ZERO_STAGE, PARTITION_COUNT, PARAM_SHAPES, BUFFER_NAMES,
+ FROZEN_PARAM_SHAPES, FROZEN_PARAM_FRAGMENTS, AUTOEP_LAYERS_KEY,
+ AUTOEP_LAYERS_KEY_LEGACY, AUTOEP_ZERO3_EXPERT_STATE_FORMAT_KEY,
+ AUTOEP_ZERO3_PARTITIONED_EXPERT_STATE_FORMAT, PARAM_ALIGNMENT_PADDINGS)
+
+
+@dataclass
+class zero_model_state:
+ buffers: dict()
+ param_shapes: dict()
+ shared_params: list
+ ds_version: int
+ frozen_param_shapes: dict()
+ frozen_param_fragments: dict()
+
+
+debug = 0
+
+# load to cpu
+device = torch.device('cpu')
+
+OUTPUT_DTYPE_NAMES = {
+ 'float32': torch.float32,
+ 'fp32': torch.float32,
+ 'float16': torch.float16,
+ 'fp16': torch.float16,
+ 'bfloat16': torch.bfloat16,
+ 'bf16': torch.bfloat16,
+}
+
+
+def _resolve_output_dtype(dtype):
+ requested_dtype = dtype
+ if isinstance(dtype, str):
+ dtype_name = dtype[6:] if dtype.startswith('torch.') else dtype
+ dtype = OUTPUT_DTYPE_NAMES.get(dtype_name.lower())
+ if dtype not in set(OUTPUT_DTYPE_NAMES.values()):
+ supported = ', '.join(sorted(OUTPUT_DTYPE_NAMES))
+ raise ValueError(f"Unsupported output dtype {requested_dtype!r}. Choose one of: {supported}")
+ return dtype
+
+
+def atoi(text):
+ return int(text) if text.isdigit() else text
+
+
+def natural_keys(text):
+ '''
+ alist.sort(key=natural_keys) sorts in human order
+ http://nedbatchelder.com/blog/200712/human_sorting.html
+ (See Toothy's implementation in the comments)
+ '''
+ return [atoi(c) for c in re.split(r'(\d+)', text)]
+
+
+def get_model_state_file(checkpoint_dir, zero_stage):
+ if not os.path.isdir(checkpoint_dir):
+ raise FileNotFoundError(f"Directory '{checkpoint_dir}' doesn't exist")
+
+ # there should be only one file
+ if zero_stage <= 2:
+ file = os.path.join(checkpoint_dir, "mp_rank_00_model_states.pt")
+ elif zero_stage == 3:
+ file = os.path.join(checkpoint_dir, "zero_pp_rank_0_mp_rank_00_model_states.pt")
+
+ if not os.path.exists(file):
+ raise FileNotFoundError(f"can't find model states file at '{file}'")
+
+ return file
+
+
+def get_checkpoint_files(checkpoint_dir, glob_pattern):
+ # XXX: need to test that this simple glob rule works for multi-node setup too
+ ckpt_files = sorted(glob.glob(os.path.join(checkpoint_dir, glob_pattern)), key=natural_keys)
+
+ if len(ckpt_files) == 0:
+ raise FileNotFoundError(f"can't find {glob_pattern} files in directory '{checkpoint_dir}'")
+
+ return ckpt_files
+
+
+def get_optim_files(checkpoint_dir):
+ return get_checkpoint_files(checkpoint_dir, "*_optim_states.pt")
+
+
+def get_model_state_files(checkpoint_dir):
+ return get_checkpoint_files(checkpoint_dir, "*_model_states.pt")
+
+
+def _has_autoep_zero3_partitioned_metadata(state_dict):
+ autoep_layers = state_dict.get(AUTOEP_LAYERS_KEY)
+ if autoep_layers is None:
+ autoep_layers = state_dict.get(AUTOEP_LAYERS_KEY_LEGACY)
+ if not isinstance(autoep_layers, list):
+ return False
+ return any(
+ isinstance(entry, dict)
+ and entry.get(AUTOEP_ZERO3_EXPERT_STATE_FORMAT_KEY) == AUTOEP_ZERO3_PARTITIONED_EXPERT_STATE_FORMAT
+ for entry in autoep_layers)
+
+
+def _raise_if_autoep_zero3_partitioned_state(state_dict):
+ if _has_autoep_zero3_partitioned_metadata(state_dict):
+ raise NotImplementedError("zero_to_fp32 does not support AutoEP ZeRO-3 partition-native checkpoints. "
+ "AutoEP expert parameters are partitioned over expert replica groups, so "
+ "global data-parallel consolidation would produce incomplete expert tensors. "
+ "Use ds_to_universal.py for expert-aware conversion.")
+
+
+def _raise_if_autoep_zero3_partitioned_checkpoint(model_files):
+ for file in model_files:
+ state_dict = torch.load(file, map_location=device, weights_only=False)
+ _raise_if_autoep_zero3_partitioned_state(state_dict)
+
+
+def parse_model_states(files):
+ zero_model_states = []
+ for file in files:
+ state_dict = torch.load(file, map_location=device, weights_only=False)
+ _raise_if_autoep_zero3_partitioned_state(state_dict)
+
+ if BUFFER_NAMES not in state_dict:
+ raise ValueError(f"{file} is not a model state checkpoint")
+ buffer_names = state_dict[BUFFER_NAMES]
+ if debug:
+ print("Found buffers:", buffer_names)
+
+ # recover just the buffers while restoring them to fp32 if they were saved in fp16
+ buffers = {k: v.float() for k, v in state_dict["module"].items() if k in buffer_names}
+ param_shapes = state_dict[PARAM_SHAPES]
+
+ # collect parameters that are included in param_shapes
+ param_names = []
+ for s in param_shapes:
+ for name in s.keys():
+ param_names.append(name)
+
+ # update with frozen parameters
+ frozen_param_shapes = state_dict.get(FROZEN_PARAM_SHAPES, None)
+ if frozen_param_shapes is not None:
+ if debug:
+ print(f"Found frozen_param_shapes: {frozen_param_shapes}")
+ param_names += list(frozen_param_shapes.keys())
+
+ # handle shared params
+ shared_params = [[k, v] for k, v in state_dict["shared_params"].items()]
+
+ ds_version = state_dict.get(DS_VERSION, None)
+
+ frozen_param_fragments = state_dict.get(FROZEN_PARAM_FRAGMENTS, None)
+
+ z_model_state = zero_model_state(buffers=buffers,
+ param_shapes=param_shapes,
+ shared_params=shared_params,
+ ds_version=ds_version,
+ frozen_param_shapes=frozen_param_shapes,
+ frozen_param_fragments=frozen_param_fragments)
+ zero_model_states.append(z_model_state)
+
+ return zero_model_states
+
+
+def parse_optim_states(files, ds_checkpoint_dir):
+ total_files = len(files)
+ state_dicts = []
+ for f in tqdm(files, desc='Loading checkpoint shards'):
+ state_dict = torch.load(f, map_location=device, mmap=True, weights_only=False)
+ # immediately discard the potentially huge 2 optimizer states as we only care for fp32 master weights
+ # and also handle the case where it was already removed by another helper script
+ state_dict["optimizer_state_dict"].pop("optimizer_state_dict", None)
+ state_dicts.append(state_dict)
+
+ if ZERO_STAGE not in state_dicts[0][OPTIMIZER_STATE_DICT]:
+ raise ValueError(f"{files[0]} is not a zero checkpoint")
+ zero_stage = state_dicts[0][OPTIMIZER_STATE_DICT][ZERO_STAGE]
+ world_size = state_dicts[0][OPTIMIZER_STATE_DICT][PARTITION_COUNT]
+
+ # For ZeRO-2 each param group can have different partition_count as data parallelism for expert
+ # parameters can be different from data parallelism for non-expert parameters. So we can just
+ # use the max of the partition_count to get the dp world_size.
+
+ if type(world_size) is list:
+ world_size = max(world_size)
+
+ if world_size != total_files:
+ raise ValueError(
+ f"Expected {world_size} of '*_optim_states.pt' under '{ds_checkpoint_dir}' but found {total_files} files. "
+ "Possibly due to an overwrite of an old checkpoint, or a checkpoint didn't get saved by one or more processes."
+ )
+
+ # the groups are named differently in each stage
+ if zero_stage <= 2:
+ fp32_groups_key = SINGLE_PARTITION_OF_FP32_GROUPS
+ elif zero_stage == 3:
+ fp32_groups_key = FP32_FLAT_GROUPS
+ else:
+ raise ValueError(f"unknown zero stage {zero_stage}")
+
+ fp32_flat_groups = [state_dicts[i][OPTIMIZER_STATE_DICT][fp32_groups_key] for i in range(len(state_dicts))]
+ param_alignment_paddings = state_dicts[0][OPTIMIZER_STATE_DICT].get(PARAM_ALIGNMENT_PADDINGS)
+ return zero_stage, world_size, fp32_flat_groups, param_alignment_paddings
+
+
+def _get_fp32_state_dict_from_zero_checkpoint(ds_checkpoint_dir, exclude_frozen_parameters):
+ """
+ Returns fp32 state_dict reconstructed from ds checkpoint
+
+ Args:
+ - ``ds_checkpoint_dir``: path to the deepspeed checkpoint folder (where the optimizer files are)
+
+ """
+ print(f"Processing zero checkpoint '{ds_checkpoint_dir}'")
+
+ # parse_model_states rejects AutoEP ZeRO-3 partition-native checkpoints
+ # before the expensive optimizer-shard load below.
+ model_files = get_model_state_files(ds_checkpoint_dir)
+ zero_model_states = parse_model_states(model_files)
+ print(f'Parsing checkpoint created by deepspeed=={zero_model_states[0].ds_version}')
+
+ optim_files = get_optim_files(ds_checkpoint_dir)
+ zero_stage, world_size, fp32_flat_groups, param_alignment_paddings = parse_optim_states(
+ optim_files, ds_checkpoint_dir)
+ print(f"Detected checkpoint of type zero stage {zero_stage}, world_size: {world_size}")
+
+ if zero_stage <= 2:
+ return _get_fp32_state_dict_from_zero2_checkpoint(world_size, fp32_flat_groups, zero_model_states,
+ exclude_frozen_parameters, param_alignment_paddings)
+ elif zero_stage == 3:
+ return _get_fp32_state_dict_from_zero3_checkpoint(world_size, fp32_flat_groups, zero_model_states,
+ exclude_frozen_parameters)
+
+
+def _zero2_merge_frozen_params(state_dict, zero_model_states):
+ if zero_model_states[0].frozen_param_shapes is None or len(zero_model_states[0].frozen_param_shapes) == 0:
+ return
+
+ frozen_param_shapes = zero_model_states[0].frozen_param_shapes
+ frozen_param_fragments = zero_model_states[0].frozen_param_fragments
+
+ if debug:
+ num_elem = sum(s.numel() for s in frozen_param_shapes.values())
+ print(f'rank 0: {FROZEN_PARAM_SHAPES}.numel = {num_elem}')
+
+ wanted_params = len(frozen_param_shapes)
+ wanted_numel = sum(s.numel() for s in frozen_param_shapes.values())
+ avail_numel = sum([p.numel() for p in frozen_param_fragments.values()])
+ print(f'Frozen params: Have {avail_numel} numels to process.')
+ print(f'Frozen params: Need {wanted_numel} numels in {wanted_params} params')
+
+ total_params = 0
+ total_numel = 0
+ for name, shape in frozen_param_shapes.items():
+ total_params += 1
+ unpartitioned_numel = shape.numel()
+ total_numel += unpartitioned_numel
+
+ state_dict[name] = frozen_param_fragments[name]
+
+ if debug:
+ print(f"{name} full shape: {shape} unpartitioned numel {unpartitioned_numel} ")
+
+ print(f"Reconstructed Frozen fp32 state dict with {total_params} params {total_numel} elements")
+
+
+def _has_callable(obj, fn):
+ attr = getattr(obj, fn, None)
+ return callable(attr)
+
+
+def _zero2_merge_trainable_params(state_dict,
+ world_size,
+ fp32_flat_groups,
+ zero_model_states,
+ param_alignment_paddings=None):
+ param_shapes = zero_model_states[0].param_shapes
+
+ # Reconstruction protocol:
+ #
+ # XXX: document this
+
+ if debug:
+ for i in range(world_size):
+ for j in range(len(fp32_flat_groups[0])):
+ print(f"{FP32_FLAT_GROUPS}[{i}][{j}].shape={fp32_flat_groups[i][j].shape}")
+
+ # XXX: memory usage doubles here (zero2)
+ num_param_groups = len(fp32_flat_groups[0])
+ merged_single_partition_of_fp32_groups = []
+ for i in range(num_param_groups):
+ merged_partitions = [sd[i] for sd in fp32_flat_groups]
+ full_single_fp32_vector = torch.cat(merged_partitions, 0)
+ merged_single_partition_of_fp32_groups.append(full_single_fp32_vector)
+ avail_numel = sum(
+ [full_single_fp32_vector.numel() for full_single_fp32_vector in merged_single_partition_of_fp32_groups])
+
+ if debug:
+ wanted_params = sum([len(shapes) for shapes in param_shapes])
+ wanted_numel = sum([sum(shape.numel() for shape in shapes.values()) for shapes in param_shapes])
+ # not asserting if there is a mismatch due to possible padding
+ print(f"Have {avail_numel} numels to process.")
+ print(f"Need {wanted_numel} numels in {wanted_params} params.")
+
+ # params
+ # XXX: for huge models that can't fit into the host's RAM we will have to recode this to support
+ # out-of-core computing solution
+ total_numel = 0
+ total_params = 0
+ for group_idx, (shapes,
+ full_single_fp32_vector) in enumerate(zip(param_shapes, merged_single_partition_of_fp32_groups)):
+ offset = 0
+ avail_numel = full_single_fp32_vector.numel()
+ group_alignment_paddings = None
+ if param_alignment_paddings is not None:
+ group_alignment_paddings = param_alignment_paddings[group_idx]
+ if len(group_alignment_paddings) != len(shapes):
+ raise ValueError(f"Expected {len(shapes)} parameter alignment paddings for group {group_idx}, "
+ f"but found {len(group_alignment_paddings)}")
+
+ for param_idx, (name, shape) in enumerate(shapes.items()):
+
+ unpartitioned_numel = shape.numel() if _has_callable(shape, 'numel') else math.prod(shape)
+ total_numel += unpartitioned_numel
+ total_params += 1
+
+ if debug:
+ print(f"{name} full shape: {shape} unpartitioned numel {unpartitioned_numel} ")
+ state_dict[name] = full_single_fp32_vector.narrow(0, offset, unpartitioned_numel).view(shape)
+ offset += unpartitioned_numel
+ if group_alignment_paddings is not None:
+ offset += group_alignment_paddings[param_idx]
+
+ # Z2 started to align to 2*world_size to improve nccl performance. Therefore both offset and
+ # avail_numel can differ by anywhere between 0..2*world_size. Due to two unrelated complex
+ # paddings performed in the code it's almost impossible to predict the exact numbers w/o the
+ # live optimizer object, so we are checking that the numbers are within the right range
+ align_to = 2 * world_size
+
+ def zero2_align(x):
+ return align_to * math.ceil(x / align_to)
+
+ if debug:
+ print(f"original offset={offset}, avail_numel={avail_numel}")
+
+ offset = zero2_align(offset)
+ avail_numel = zero2_align(avail_numel)
+
+ if debug:
+ print(f"aligned offset={offset}, avail_numel={avail_numel}")
+
+ # Sanity check
+ if offset != avail_numel:
+ raise ValueError(f"consumed {offset} numels out of {avail_numel} - something is wrong")
+
+ print(f"Reconstructed fp32 state dict with {total_params} params {total_numel} elements")
+
+
+def _get_fp32_state_dict_from_zero2_checkpoint(world_size,
+ fp32_flat_groups,
+ zero_model_states,
+ exclude_frozen_parameters,
+ param_alignment_paddings=None):
+ state_dict = OrderedDict()
+
+ # buffers
+ buffers = zero_model_states[0].buffers
+ state_dict.update(buffers)
+ if debug:
+ print(f"added {len(buffers)} buffers")
+
+ if not exclude_frozen_parameters:
+ _zero2_merge_frozen_params(state_dict, zero_model_states)
+
+ _zero2_merge_trainable_params(state_dict, world_size, fp32_flat_groups, zero_model_states,
+ param_alignment_paddings)
+
+ # recover shared parameters
+ for pair in zero_model_states[0].shared_params:
+ if pair[1] in state_dict:
+ state_dict[pair[0]] = state_dict[pair[1]]
+
+ return state_dict
+
+
+def zero3_partitioned_param_info(unpartitioned_numel, world_size):
+ remainder = unpartitioned_numel % world_size
+ padding_numel = (world_size - remainder) if remainder else 0
+ partitioned_numel = math.ceil(unpartitioned_numel / world_size)
+ return partitioned_numel, padding_numel
+
+
+def _zero3_merge_frozen_params(state_dict, world_size, zero_model_states):
+ if zero_model_states[0].frozen_param_shapes is None or len(zero_model_states[0].frozen_param_shapes) == 0:
+ return
+
+ if debug:
+ for i in range(world_size):
+ num_elem = sum(s.numel() for s in zero_model_states[i].frozen_param_fragments.values())
+ print(f'rank {i}: {FROZEN_PARAM_SHAPES}.numel = {num_elem}')
+
+ frozen_param_shapes = zero_model_states[0].frozen_param_shapes
+ wanted_params = len(frozen_param_shapes)
+ wanted_numel = sum(s.numel() for s in frozen_param_shapes.values())
+ avail_numel = sum([p.numel() for p in zero_model_states[0].frozen_param_fragments.values()]) * world_size
+ print(f'Frozen params: Have {avail_numel} numels to process.')
+ print(f'Frozen params: Need {wanted_numel} numels in {wanted_params} params')
+
+ total_params = 0
+ total_numel = 0
+ for name, shape in zero_model_states[0].frozen_param_shapes.items():
+ total_params += 1
+ unpartitioned_numel = shape.numel()
+ total_numel += unpartitioned_numel
+
+ param_frags = tuple(model_state.frozen_param_fragments[name] for model_state in zero_model_states)
+ state_dict[name] = torch.cat(param_frags, 0).narrow(0, 0, unpartitioned_numel).view(shape)
+
+ partitioned_numel, partitioned_padding_numel = zero3_partitioned_param_info(unpartitioned_numel, world_size)
+
+ if debug:
+ print(
+ f"Frozen params: {total_params} {name} full shape: {shape} partition0 numel={partitioned_numel} partitioned_padding_numel={partitioned_padding_numel}"
+ )
+
+ print(f"Reconstructed Frozen fp32 state dict with {total_params} params {total_numel} elements")
+
+
+class GatheredTensor:
+ """
+ A pseudo tensor that collects partitioned weights.
+ It is more memory efficient when there are multiple groups.
+ """
+
+ def __init__(self, flat_groups, flat_groups_offset, offset, partitioned_numel, shape):
+ self.flat_groups = flat_groups
+ self.flat_groups_offset = flat_groups_offset
+ self.offset = offset
+ self.partitioned_numel = partitioned_numel
+ self.shape = shape
+ self.dtype = self.flat_groups[0][0].dtype
+
+ def contiguous(self):
+ """
+ Merge partitioned weights from flat_groups into a single tensor.
+ """
+ end_idx = self.offset + self.partitioned_numel
+ world_size = len(self.flat_groups)
+ pad_flat_param_chunks = []
+
+ for rank_i in range(world_size):
+ # for each rank, we need to collect weights from related group/groups
+ flat_groups_at_rank_i = self.flat_groups[rank_i]
+ start_group_id = None
+ end_group_id = None
+ for group_id in range(len(self.flat_groups_offset)):
+ if self.flat_groups_offset[group_id] <= self.offset < self.flat_groups_offset[group_id + 1]:
+ start_group_id = group_id
+ if self.flat_groups_offset[group_id] < end_idx <= self.flat_groups_offset[group_id + 1]:
+ end_group_id = group_id
+ break
+ # collect weights from related group/groups
+ for group_id in range(start_group_id, end_group_id + 1):
+ flat_tensor = flat_groups_at_rank_i[group_id]
+ start_offset = self.offset - self.flat_groups_offset[group_id]
+ end_offset = min(end_idx, self.flat_groups_offset[group_id + 1]) - self.flat_groups_offset[group_id]
+ pad_flat_param_chunks.append(flat_tensor[start_offset:end_offset])
+
+ # collect weights from all ranks
+ pad_flat_param = torch.cat(pad_flat_param_chunks, dim=0)
+ param = pad_flat_param[:self.shape.numel()].view(self.shape).contiguous()
+ return param
+
+
+def _zero3_merge_trainable_params(state_dict, world_size, fp32_flat_groups, zero_model_states):
+ param_shapes = zero_model_states[0].param_shapes
+ avail_numel = sum([flat_group.numel() for flat_group in fp32_flat_groups[0]]) * world_size
+
+ # Reconstruction protocol: For zero3 we need to zip the partitions together at boundary of each
+ # param, re-consolidating each param, while dealing with padding if any
+
+ # merge list of dicts, preserving order
+ param_shapes = {k: v for d in param_shapes for k, v in d.items()}
+
+ if debug:
+ for i in range(world_size):
+ print(f"{FP32_FLAT_GROUPS}[{i}].shape={fp32_flat_groups[i].shape}")
+
+ wanted_params = len(param_shapes)
+ wanted_numel = sum(shape.numel() for shape in param_shapes.values())
+ # not asserting if there is a mismatch due to possible padding
+ avail_numel = fp32_flat_groups[0].numel() * world_size
+ print(f"Trainable params: Have {avail_numel} numels to process.")
+ print(f"Trainable params: Need {wanted_numel} numels in {wanted_params} params.")
+
+ # params
+ # XXX: for huge models that can't fit into the host's RAM we will have to recode this to support
+ # out-of-core computing solution
+ offset = 0
+ total_numel = 0
+ total_params = 0
+ flat_groups_offset = [0] + list(np.cumsum([flat_tensor.numel() for flat_tensor in fp32_flat_groups[0]]))
+ for name, shape in tqdm(param_shapes.items(), desc='Gathering sharded weights'):
+ unpartitioned_numel = shape.numel()
+ total_numel += unpartitioned_numel
+ total_params += 1
+ partitioned_numel, partitioned_padding_numel = zero3_partitioned_param_info(unpartitioned_numel, world_size)
+
+ if debug:
+ print(
+ f"Trainable params: {total_params} {name} full shape: {shape} partition0 numel={partitioned_numel} partitioned_padding_numel={partitioned_padding_numel}"
+ )
+
+ # memory efficient tensor
+ tensor = GatheredTensor(fp32_flat_groups, flat_groups_offset, offset, partitioned_numel, shape)
+ state_dict[name] = tensor
+ offset += partitioned_numel
+
+ offset *= world_size
+
+ # Sanity check
+ if offset != avail_numel:
+ raise ValueError(f"consumed {offset} numels out of {avail_numel} - something is wrong")
+
+ print(f"Reconstructed Trainable fp32 state dict with {total_params} params {total_numel} elements")
+
+
+def _get_fp32_state_dict_from_zero3_checkpoint(world_size, fp32_flat_groups, zero_model_states,
+ exclude_frozen_parameters):
+ state_dict = OrderedDict()
+
+ # buffers
+ buffers = zero_model_states[0].buffers
+ state_dict.update(buffers)
+ if debug:
+ print(f"added {len(buffers)} buffers")
+
+ if not exclude_frozen_parameters:
+ _zero3_merge_frozen_params(state_dict, world_size, zero_model_states)
+
+ _zero3_merge_trainable_params(state_dict, world_size, fp32_flat_groups, zero_model_states)
+
+ # recover shared parameters
+ for pair in zero_model_states[0].shared_params:
+ if pair[1] in state_dict:
+ state_dict[pair[0]] = state_dict[pair[1]]
+
+ return state_dict
+
+
+def to_torch_tensor(state_dict, return_empty_tensor=False, dtype=None):
+ """
+ Convert state_dict of GatheredTensor to torch tensor
+ """
+ if dtype is not None:
+ dtype = _resolve_output_dtype(dtype)
+
+ torch_state_dict = {}
+ converted_tensors = {}
+ for name, tensor in state_dict.items():
+ tensor_id = id(tensor)
+ if tensor_id in converted_tensors: # shared tensors
+ shared_tensor = torch_state_dict[converted_tensors[tensor_id]]
+ torch_state_dict[name] = shared_tensor
+ else:
+ converted_tensors[tensor_id] = name
+ if return_empty_tensor:
+ torch_state_dict[name] = torch.empty(tensor.shape, dtype=dtype or tensor.dtype)
+ else:
+ contiguous_tensor = tensor.contiguous()
+ torch_state_dict[name] = contiguous_tensor.to(dtype=dtype) if dtype else contiguous_tensor
+ return torch_state_dict
+
+
+def get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir,
+ tag=None,
+ exclude_frozen_parameters=False,
+ lazy_mode=False):
+ """
+ Convert ZeRO 2 or 3 checkpoint into a single fp32 consolidated state_dict that can be loaded with
+ ``load_state_dict()`` and used for training without DeepSpeed or shared with others, for example
+ via a model hub.
+
+ Args:
+ - ``checkpoint_dir``: path to the desired checkpoint folder
+ - ``tag``: checkpoint tag used as a unique identifier for checkpoint. If not provided will attempt to load tag in 'latest' file. e.g., ``global_step14``
+ - ``exclude_frozen_parameters``: exclude frozen parameters
+ - ``lazy_mode``: get state_dict in lazy mode. It returns a dict of pesduo tensor instead of torch tensor, which is more memory efficient.
+ Convert the pesduo tensor to torch tensor by ``.contiguous()``
+
+ Returns:
+ - pytorch ``state_dict``
+
+ A typical usage might be ::
+
+ from deepspeed.utils.zero_to_fp32 import get_fp32_state_dict_from_zero_checkpoint
+ # do the training and checkpoint saving
+ state_dict = get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir) # already on cpu
+ model = model.cpu() # move to cpu
+ model.load_state_dict(state_dict)
+ # submit to model hub or save the model to share with others
+
+ In this example the ``model`` will no longer be usable in the deepspeed context of the same
+ application. i.e. you will need to re-initialize the deepspeed engine, since
+ ``model.load_state_dict(state_dict)`` will remove all the deepspeed magic from it.
+
+ If you want it all done for you, use ``load_state_dict_from_zero_checkpoint`` instead.
+
+ Note: the above usage may not work if your application doesn't have sufficient free CPU memory.
+ You may need to use the offline approach using the ``zero_to_fp32.py`` script that is saved with
+ the checkpoint. Or you can load state_dict in lazy mode ::
+
+ from deepspeed.utils.zero_to_fp32 import get_fp32_state_dict_from_zero_checkpoint
+ state_dict = get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir, lazy_mode=True) # not on cpu
+ for name, lazy_tensor in state_dict.item():
+ tensor = lazy_tensor.contiguous() # to cpu
+ print(name, tensor)
+ # del tensor to release memory if it no longer in use
+ """
+ if tag is None:
+ latest_path = os.path.join(checkpoint_dir, 'latest')
+ if os.path.isfile(latest_path):
+ with open(latest_path, 'r') as fd:
+ tag = fd.read().strip()
+ else:
+ raise ValueError(f"Unable to find 'latest' file at {latest_path}")
+
+ ds_checkpoint_dir = os.path.join(checkpoint_dir, tag)
+
+ if not os.path.isdir(ds_checkpoint_dir):
+ raise FileNotFoundError(f"Directory '{ds_checkpoint_dir}' doesn't exist")
+
+ state_dict = _get_fp32_state_dict_from_zero_checkpoint(ds_checkpoint_dir, exclude_frozen_parameters)
+ if lazy_mode:
+ return state_dict
+ else:
+ return to_torch_tensor(state_dict)
+
+
+def convert_zero_checkpoint_to_state_dict(checkpoint_dir,
+ output_dir,
+ dtype=torch.float32,
+ max_shard_size="5GB",
+ safe_serialization=False,
+ tag=None,
+ exclude_frozen_parameters=False):
+ """
+ Convert ZeRO 2 or 3 checkpoint into a consolidated ``state_dict`` file that can be
+ loaded with ``torch.load(file)`` + ``load_state_dict()`` and used for training without DeepSpeed.
+
+ Args:
+ - ``checkpoint_dir``: path to the desired checkpoint folder. (one that contains the tag-folder, like ``global_step14``)
+ - ``output_dir``: directory for the PyTorch state_dict output files
+ - ``dtype``: output tensor dtype. Supports float32, float16, and bfloat16 as strings or torch dtypes.
+ - ``max_shard_size``: the maximum size for a checkpoint before being sharded, default value is 5GB
+ - ``safe_serialization``: whether to save the model using `safetensors` or the traditional PyTorch way (that uses `pickle`).
+ - ``tag``: checkpoint tag used as a unique identifier for checkpoint. If not provided will attempt to load tag in the file named ``latest`` in the checkpoint folder, e.g., ``global_step14``
+ - ``exclude_frozen_parameters``: exclude frozen parameters
+ """
+
+ dtype = _resolve_output_dtype(dtype)
+
+ # Dependency pre-check
+ if safe_serialization:
+ try:
+ from safetensors.torch import save_file
+ except ImportError:
+ print('If you want to use `safe_serialization`, please `pip install safetensors`')
+ raise
+ if max_shard_size is not None:
+ try:
+ from huggingface_hub import split_torch_state_dict_into_shards
+ except ImportError:
+ print('If you want to use `max_shard_size`, please `pip install huggingface_hub`')
+ raise
+
+ # Convert zero checkpoint to state_dict
+ state_dict = get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir,
+ tag,
+ exclude_frozen_parameters,
+ lazy_mode=True)
+
+ # Shard the model if it is too big.
+ weights_name = "model.safetensors" if safe_serialization else "pytorch_model.bin"
+ if max_shard_size is not None:
+ filename_pattern = weights_name.replace(".bin", "{suffix}.bin").replace(".safetensors", "{suffix}.safetensors")
+ # an memory-efficient approach for sharding
+ empty_state_dict = to_torch_tensor(state_dict, return_empty_tensor=True, dtype=dtype)
+ state_dict_split = split_torch_state_dict_into_shards(empty_state_dict,
+ filename_pattern=filename_pattern,
+ max_shard_size=max_shard_size)
+ else:
+ from collections import namedtuple
+ StateDictSplit = namedtuple("StateDictSplit", ["is_sharded", "filename_to_tensors"])
+ state_dict_split = StateDictSplit(is_sharded=False,
+ filename_to_tensors={weights_name: list(state_dict.keys())})
+
+ # Save the model by shard
+ os.makedirs(output_dir, exist_ok=True)
+ filename_to_tensors = state_dict_split.filename_to_tensors.items()
+ for shard_file, tensors in tqdm(filename_to_tensors, desc="Saving checkpoint shards"):
+ shard_state_dict = {tensor_name: state_dict[tensor_name] for tensor_name in tensors}
+ shard_state_dict = to_torch_tensor(shard_state_dict, dtype=dtype)
+ output_path = os.path.join(output_dir, shard_file)
+ if safe_serialization:
+ save_file(shard_state_dict, output_path, metadata={"format": "pt"})
+ else:
+ torch.save(shard_state_dict, output_path)
+ # release the memory of current shard
+ for tensor_name in list(shard_state_dict.keys()):
+ del state_dict[tensor_name]
+ del shard_state_dict[tensor_name]
+ del shard_state_dict
+ gc.collect()
+
+ # Save index if sharded
+ if state_dict_split.is_sharded:
+ index = {
+ "metadata": state_dict_split.metadata,
+ "weight_map": state_dict_split.tensor_to_filename,
+ }
+ save_index_file = "model.safetensors.index.json" if safe_serialization else "pytorch_model.bin.index.json"
+ save_index_file = os.path.join(output_dir, save_index_file)
+ with open(save_index_file, "w", encoding="utf-8") as f:
+ content = json.dumps(index, indent=2, sort_keys=True) + "\n"
+ f.write(content)
+
+
+def convert_zero_checkpoint_to_fp32_state_dict(checkpoint_dir,
+ output_dir,
+ max_shard_size="5GB",
+ safe_serialization=False,
+ tag=None,
+ exclude_frozen_parameters=False):
+ """Backward-compatible fp32 checkpoint conversion."""
+ return convert_zero_checkpoint_to_state_dict(checkpoint_dir,
+ output_dir,
+ dtype=torch.float32,
+ max_shard_size=max_shard_size,
+ safe_serialization=safe_serialization,
+ tag=tag,
+ exclude_frozen_parameters=exclude_frozen_parameters)
+
+
+def load_state_dict_from_zero_checkpoint(model, checkpoint_dir, tag=None):
+ """
+ 1. Put the provided model to cpu
+ 2. Convert ZeRO 2 or 3 checkpoint into a single fp32 consolidated ``state_dict``
+ 3. Load it into the provided model
+
+ Args:
+ - ``model``: the model object to update
+ - ``checkpoint_dir``: path to the desired checkpoint folder. (one that contains the tag-folder, like ``global_step14``)
+ - ``tag``: checkpoint tag used as a unique identifier for checkpoint. If not provided will attempt to load tag in the file named ``latest`` in the checkpoint folder, e.g., ``global_step14``
+
+ Returns:
+ - ``model`: modified model
+
+ Make sure you have plenty of CPU memory available before you call this function. If you don't
+ have enough use the ``zero_to_fp32.py`` utility to do the conversion. You will find it
+ conveniently placed for you in the checkpoint folder.
+
+ A typical usage might be ::
+
+ from deepspeed.utils.zero_to_fp32 import load_state_dict_from_zero_checkpoint
+ model = load_state_dict_from_zero_checkpoint(trainer.model, checkpoint_dir)
+ # submit to model hub or save the model to share with others
+
+ Note, that once this was run, the ``model`` will no longer be usable in the deepspeed context
+ of the same application. i.e. you will need to re-initialize the deepspeed engine, since
+ ``model.load_state_dict(state_dict)`` will remove all the deepspeed magic from it.
+
+ """
+ logger.info("Extracting fp32 weights")
+ state_dict = get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir, tag)
+
+ logger.info("Overwriting model with fp32 weights")
+ model = model.cpu()
+ model.load_state_dict(state_dict, strict=False)
+
+ return model
+
+
+if __name__ == "__main__":
+ parser = argparse.ArgumentParser()
+ parser.add_argument("checkpoint_dir",
+ type=str,
+ help="path to the desired checkpoint folder, e.g., path/checkpoint-12")
+ parser.add_argument("output_dir",
+ type=str,
+ help="directory to the pytorch fp32 state_dict output files"
+ "(e.g. path/checkpoint-12-output/)")
+ parser.add_argument(
+ "--max_shard_size",
+ type=str,
+ default="5GB",
+ help="The maximum size for a checkpoint before being sharded. Checkpoints shard will then be each of size"
+ "lower than this size. If expressed as a string, needs to be digits followed by a unit (like `5MB`"
+ "We default it to 5GB in order for models to be able to run easily on free-tier google colab instances"
+ "without CPU OOM issues.")
+ parser.add_argument(
+ "--safe_serialization",
+ default=False,
+ action='store_true',
+ help="Whether to save the model using `safetensors` or the traditional PyTorch way (that uses `pickle`).")
+ parser.add_argument("-t",
+ "--tag",
+ type=str,
+ default=None,
+ help="checkpoint tag used as a unique identifier for checkpoint. e.g., global_step1")
+ parser.add_argument("--exclude_frozen_parameters", action='store_true', help="exclude frozen parameters")
+ parser.add_argument("-d", "--debug", action='store_true', help="enable debug")
+ args = parser.parse_args()
+
+ debug = args.debug
+
+ convert_zero_checkpoint_to_fp32_state_dict(args.checkpoint_dir,
+ args.output_dir,
+ max_shard_size=args.max_shard_size,
+ safe_serialization=args.safe_serialization,
+ tag=args.tag,
+ exclude_frozen_parameters=args.exclude_frozen_parameters)
diff --git a/qwen3-1.7b-teutonic-5e6/checkpoint-2000/zero_to_torch.py b/qwen3-1.7b-teutonic-5e6/checkpoint-2000/zero_to_torch.py
new file mode 100644
index 0000000000000000000000000000000000000000..81312e252400d77f03cc1a88522f8f18ebe70ee7
--- /dev/null
+++ b/qwen3-1.7b-teutonic-5e6/checkpoint-2000/zero_to_torch.py
@@ -0,0 +1,54 @@
+#!/usr/bin/env python
+
+# SPDX-License-Identifier: Apache-2.0
+# DeepSpeed Team
+
+import argparse
+
+if __package__:
+ from . import zero_to_fp32
+else:
+ import zero_to_fp32
+
+
+def main(args=None):
+ parser = argparse.ArgumentParser(
+ description="Convert a DeepSpeed ZeRO checkpoint to float32, float16, or bfloat16 PyTorch weights.")
+ parser.add_argument("checkpoint_dir",
+ type=str,
+ help="path to the desired checkpoint folder, e.g., path/checkpoint-12")
+ parser.add_argument("output_dir", type=str, help="directory for the converted PyTorch state_dict files")
+ parser.add_argument("--dtype",
+ type=str,
+ choices=sorted(zero_to_fp32.OUTPUT_DTYPE_NAMES),
+ required=True,
+ help="output tensor dtype")
+ parser.add_argument("--max_shard_size",
+ type=str,
+ default="5GB",
+ help="maximum size of each checkpoint shard, such as 5GB or 500MB")
+ parser.add_argument("--safe_serialization",
+ default=False,
+ action='store_true',
+ help="save with safetensors instead of PyTorch pickle serialization")
+ parser.add_argument("-t",
+ "--tag",
+ type=str,
+ default=None,
+ help="checkpoint tag used as a unique identifier, e.g., global_step1")
+ parser.add_argument("--exclude_frozen_parameters", action='store_true', help="exclude frozen parameters")
+ parser.add_argument("-d", "--debug", action='store_true', help="enable debug output")
+ parsed_args = parser.parse_args(args)
+
+ zero_to_fp32.debug = parsed_args.debug
+ zero_to_fp32.convert_zero_checkpoint_to_state_dict(parsed_args.checkpoint_dir,
+ parsed_args.output_dir,
+ dtype=parsed_args.dtype,
+ max_shard_size=parsed_args.max_shard_size,
+ safe_serialization=parsed_args.safe_serialization,
+ tag=parsed_args.tag,
+ exclude_frozen_parameters=parsed_args.exclude_frozen_parameters)
+
+
+if __name__ == "__main__":
+ main()
diff --git a/qwen3-1.7b-teutonic-5e6/checkpoint-2475/chat_template.jinja b/qwen3-1.7b-teutonic-5e6/checkpoint-2475/chat_template.jinja
new file mode 100644
index 0000000000000000000000000000000000000000..01be9b307daa2d425f7c168c9fb145a286e0afb4
--- /dev/null
+++ b/qwen3-1.7b-teutonic-5e6/checkpoint-2475/chat_template.jinja
@@ -0,0 +1,89 @@
+{%- if tools %}
+ {{- '<|im_start|>system\n' }}
+ {%- if messages[0].role == 'system' %}
+ {{- messages[0].content + '\n\n' }}
+ {%- endif %}
+ {{- "# Tools\n\nYou may call one or more functions to assist with the user query.\n\nYou are provided with function signatures within XML tags:\n" }}
+ {%- for tool in tools %}
+ {{- "\n" }}
+ {{- tool | tojson }}
+ {%- endfor %}
+ {{- "\n\n\nFor each function call, return a json object with function name and arguments within XML tags:\n\n{\"name\": , \"arguments\": }\n<|im_end|>\n" }}
+{%- else %}
+ {%- if messages[0].role == 'system' %}
+ {{- '<|im_start|>system\n' + messages[0].content + '<|im_end|>\n' }}
+ {%- endif %}
+{%- endif %}
+{%- set ns = namespace(multi_step_tool=true, last_query_index=messages|length - 1) %}
+{%- for message in messages[::-1] %}
+ {%- set index = (messages|length - 1) - loop.index0 %}
+ {%- if ns.multi_step_tool and message.role == "user" and message.content is string and not(message.content.startswith('') and message.content.endswith('')) %}
+ {%- set ns.multi_step_tool = false %}
+ {%- set ns.last_query_index = index %}
+ {%- endif %}
+{%- endfor %}
+{%- for message in messages %}
+ {%- if message.content is string %}
+ {%- set content = message.content %}
+ {%- else %}
+ {%- set content = '' %}
+ {%- endif %}
+ {%- if (message.role == "user") or (message.role == "system" and not loop.first) %}
+ {{- '<|im_start|>' + message.role + '\n' + content + '<|im_end|>' + '\n' }}
+ {%- elif message.role == "assistant" %}
+ {%- set reasoning_content = '' %}
+ {%- if message.reasoning_content is string %}
+ {%- set reasoning_content = message.reasoning_content %}
+ {%- else %}
+ {%- if '' in content %}
+ {%- set reasoning_content = content.split('')[0].rstrip('\n').split('')[-1].lstrip('\n') %}
+ {%- set content = content.split('')[-1].lstrip('\n') %}
+ {%- endif %}
+ {%- endif %}
+ {%- if loop.index0 > ns.last_query_index %}
+ {%- if loop.last or (not loop.last and reasoning_content) %}
+ {{- '<|im_start|>' + message.role + '\n\n' + reasoning_content.strip('\n') + '\n\n\n' + content.lstrip('\n') }}
+ {%- else %}
+ {{- '<|im_start|>' + message.role + '\n' + content }}
+ {%- endif %}
+ {%- else %}
+ {{- '<|im_start|>' + message.role + '\n' + content }}
+ {%- endif %}
+ {%- if message.tool_calls %}
+ {%- for tool_call in message.tool_calls %}
+ {%- if (loop.first and content) or (not loop.first) %}
+ {{- '\n' }}
+ {%- endif %}
+ {%- if tool_call.function %}
+ {%- set tool_call = tool_call.function %}
+ {%- endif %}
+ {{- '\n{"name": "' }}
+ {{- tool_call.name }}
+ {{- '", "arguments": ' }}
+ {%- if tool_call.arguments is string %}
+ {{- tool_call.arguments }}
+ {%- else %}
+ {{- tool_call.arguments | tojson }}
+ {%- endif %}
+ {{- '}\n' }}
+ {%- endfor %}
+ {%- endif %}
+ {{- '<|im_end|>\n' }}
+ {%- elif message.role == "tool" %}
+ {%- if loop.first or (messages[loop.index0 - 1].role != "tool") %}
+ {{- '<|im_start|>user' }}
+ {%- endif %}
+ {{- '\n\n' }}
+ {{- content }}
+ {{- '\n' }}
+ {%- if loop.last or (messages[loop.index0 + 1].role != "tool") %}
+ {{- '<|im_end|>\n' }}
+ {%- endif %}
+ {%- endif %}
+{%- endfor %}
+{%- if add_generation_prompt %}
+ {{- '<|im_start|>assistant\n' }}
+ {%- if enable_thinking is defined and enable_thinking is false %}
+ {{- '\n\n\n\n' }}
+ {%- endif %}
+{%- endif %}
\ No newline at end of file
diff --git a/qwen3-1.7b-teutonic-5e6/checkpoint-2475/config.json b/qwen3-1.7b-teutonic-5e6/checkpoint-2475/config.json
new file mode 100644
index 0000000000000000000000000000000000000000..5d9cef07396baadff5390e4508eb33025967a029
--- /dev/null
+++ b/qwen3-1.7b-teutonic-5e6/checkpoint-2475/config.json
@@ -0,0 +1,63 @@
+{
+ "architectures": [
+ "Qwen3ForCausalLM"
+ ],
+ "attention_bias": false,
+ "attention_dropout": 0.0,
+ "bos_token_id": null,
+ "dtype": "bfloat16",
+ "eos_token_id": 151645,
+ "head_dim": 128,
+ "hidden_act": "silu",
+ "hidden_size": 2048,
+ "initializer_range": 0.02,
+ "intermediate_size": 6144,
+ "layer_types": [
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention"
+ ],
+ "max_position_embeddings": 40960,
+ "max_window_layers": 28,
+ "model_type": "qwen3",
+ "num_attention_heads": 16,
+ "num_hidden_layers": 28,
+ "num_key_value_heads": 8,
+ "pad_token_id": 151643,
+ "rms_norm_eps": 1e-06,
+ "rope_parameters": {
+ "rope_theta": 1000000,
+ "rope_type": "default"
+ },
+ "sliding_window": null,
+ "tie_word_embeddings": true,
+ "transformers_version": "5.17.0",
+ "use_cache": false,
+ "use_sliding_window": false,
+ "vocab_size": 151936
+}
diff --git a/qwen3-1.7b-teutonic-5e6/checkpoint-2475/generation_config.json b/qwen3-1.7b-teutonic-5e6/checkpoint-2475/generation_config.json
new file mode 100644
index 0000000000000000000000000000000000000000..e6941fe4b04f26113d6eef6255d005c4d7090bda
--- /dev/null
+++ b/qwen3-1.7b-teutonic-5e6/checkpoint-2475/generation_config.json
@@ -0,0 +1,12 @@
+{
+ "do_sample": true,
+ "eos_token_id": [
+ 151645,
+ 151643
+ ],
+ "pad_token_id": 151643,
+ "temperature": 0.6,
+ "top_k": 20,
+ "top_p": 0.95,
+ "transformers_version": "5.17.0"
+}
diff --git a/qwen3-1.7b-teutonic-5e6/checkpoint-2475/global_step2475/bf16_zero_pp_rank_0_mp_rank_00_optim_states.pt b/qwen3-1.7b-teutonic-5e6/checkpoint-2475/global_step2475/bf16_zero_pp_rank_0_mp_rank_00_optim_states.pt
new file mode 100644
index 0000000000000000000000000000000000000000..f7bef0b6f7a29272f6c6b35e9cef9e9a8c43c973
--- /dev/null
+++ b/qwen3-1.7b-teutonic-5e6/checkpoint-2475/global_step2475/bf16_zero_pp_rank_0_mp_rank_00_optim_states.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:5b7dc896c12572435eb9976837f5cce7f3f70d76572df2261f79b9ad7cb47055
+size 10323466465
diff --git a/qwen3-1.7b-teutonic-5e6/checkpoint-2475/global_step2475/bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt b/qwen3-1.7b-teutonic-5e6/checkpoint-2475/global_step2475/bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt
new file mode 100644
index 0000000000000000000000000000000000000000..ee078fe6c39eeb9ed6ef0f87320673f0b81b1ce1
--- /dev/null
+++ b/qwen3-1.7b-teutonic-5e6/checkpoint-2475/global_step2475/bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a309692649a69670b1edd7342b9e85ee0bd63b85a84d2b4980c5907407241776
+size 10323469601
diff --git a/qwen3-1.7b-teutonic-5e6/checkpoint-2475/global_step2475/mp_rank_00_model_states.pt b/qwen3-1.7b-teutonic-5e6/checkpoint-2475/global_step2475/mp_rank_00_model_states.pt
new file mode 100644
index 0000000000000000000000000000000000000000..60749262a949a122f52c61c4b393e5ed69dc39b9
--- /dev/null
+++ b/qwen3-1.7b-teutonic-5e6/checkpoint-2475/global_step2475/mp_rank_00_model_states.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c474b42b2049e044cb884a724f865ff3052d0a830e1bf02adcca1cf869d4855b
+size 3441239653
diff --git a/qwen3-1.7b-teutonic-5e6/checkpoint-2475/latest b/qwen3-1.7b-teutonic-5e6/checkpoint-2475/latest
new file mode 100644
index 0000000000000000000000000000000000000000..fbeedf03172d647e3cbe93eb80f563c2fd003077
--- /dev/null
+++ b/qwen3-1.7b-teutonic-5e6/checkpoint-2475/latest
@@ -0,0 +1 @@
+global_step2475
\ No newline at end of file
diff --git a/qwen3-1.7b-teutonic-5e6/checkpoint-2475/model.safetensors b/qwen3-1.7b-teutonic-5e6/checkpoint-2475/model.safetensors
new file mode 100644
index 0000000000000000000000000000000000000000..7a8918d1c9d62ceb14cea048b7d448374c01ead2
--- /dev/null
+++ b/qwen3-1.7b-teutonic-5e6/checkpoint-2475/model.safetensors
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:fa5d551b4af650e317b35a41dce4800011bdfe6d7e07b934fdd8441e44043ddb
+size 4063515640
diff --git a/qwen3-1.7b-teutonic-5e6/checkpoint-2475/rng_state_0.pth b/qwen3-1.7b-teutonic-5e6/checkpoint-2475/rng_state_0.pth
new file mode 100644
index 0000000000000000000000000000000000000000..06aaa9eb50010af39d8e52dfa7116959fb72976b
--- /dev/null
+++ b/qwen3-1.7b-teutonic-5e6/checkpoint-2475/rng_state_0.pth
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:948282bfdd138e468bab62957552a94f8947ff21a89bf81fbefc3360c5f0965e
+size 14917
diff --git a/qwen3-1.7b-teutonic-5e6/checkpoint-2475/rng_state_1.pth b/qwen3-1.7b-teutonic-5e6/checkpoint-2475/rng_state_1.pth
new file mode 100644
index 0000000000000000000000000000000000000000..a2eddcc2f1907f052432f2cbf01e9eeb0ac9c9e3
--- /dev/null
+++ b/qwen3-1.7b-teutonic-5e6/checkpoint-2475/rng_state_1.pth
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:47a0523bd1f56a1423a54522d445aad486271bb34e995627e44b9c4abdc454ed
+size 14917
diff --git a/qwen3-1.7b-teutonic-5e6/checkpoint-2475/scheduler.pt b/qwen3-1.7b-teutonic-5e6/checkpoint-2475/scheduler.pt
new file mode 100644
index 0000000000000000000000000000000000000000..bd98883ea1f8fcdc981a2fee26dc446d08222be6
--- /dev/null
+++ b/qwen3-1.7b-teutonic-5e6/checkpoint-2475/scheduler.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d2890c1ef6057f33ee4e0f15383f1f24eb5a02e92cd00aeb789ea714f44aa540
+size 1465
diff --git a/qwen3-1.7b-teutonic-5e6/checkpoint-2475/tokenizer.json b/qwen3-1.7b-teutonic-5e6/checkpoint-2475/tokenizer.json
new file mode 100644
index 0000000000000000000000000000000000000000..c7afbed2efcdf019f88ab0572ec29d3bf595dfe2
--- /dev/null
+++ b/qwen3-1.7b-teutonic-5e6/checkpoint-2475/tokenizer.json
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:be75606093db2094d7cd20f3c2f385c212750648bd6ea4fb2bf507a6a4c55506
+size 11422650
diff --git a/qwen3-1.7b-teutonic-5e6/checkpoint-2475/tokenizer_config.json b/qwen3-1.7b-teutonic-5e6/checkpoint-2475/tokenizer_config.json
new file mode 100644
index 0000000000000000000000000000000000000000..770e41d6c92519d525eede4cbcf3ba27f6425311
--- /dev/null
+++ b/qwen3-1.7b-teutonic-5e6/checkpoint-2475/tokenizer_config.json
@@ -0,0 +1,30 @@
+{
+ "add_prefix_space": false,
+ "backend": "tokenizers",
+ "bos_token": null,
+ "clean_up_tokenization_spaces": false,
+ "eos_token": "<|im_end|>",
+ "errors": "replace",
+ "extra_special_tokens": [
+ "<|im_start|>",
+ "<|im_end|>",
+ "<|object_ref_start|>",
+ "<|object_ref_end|>",
+ "<|box_start|>",
+ "<|box_end|>",
+ "<|quad_start|>",
+ "<|quad_end|>",
+ "<|vision_start|>",
+ "<|vision_end|>",
+ "<|vision_pad|>",
+ "<|image_pad|>",
+ "<|video_pad|>"
+ ],
+ "is_local": false,
+ "local_files_only": false,
+ "model_max_length": 131072,
+ "pad_token": "<|endoftext|>",
+ "split_special_tokens": false,
+ "tokenizer_class": "Qwen2Tokenizer",
+ "unk_token": null
+}
diff --git a/qwen3-1.7b-teutonic-5e6/checkpoint-2475/trainer_state.json b/qwen3-1.7b-teutonic-5e6/checkpoint-2475/trainer_state.json
new file mode 100644
index 0000000000000000000000000000000000000000..0bf4da376561fa99c96de8776bbb48b33719e469
--- /dev/null
+++ b/qwen3-1.7b-teutonic-5e6/checkpoint-2475/trainer_state.json
@@ -0,0 +1,25334 @@
+{
+ "best_global_step": 950,
+ "best_metric": 1.5272752046585083,
+ "best_model_checkpoint": "./checkpoints/qwen3-1.7b-teutonic-5e6/checkpoint-500",
+ "epoch": 5.0,
+ "eval_steps": 50,
+ "global_step": 2475,
+ "is_hyper_param_search": false,
+ "is_local_process_zero": true,
+ "is_world_process_zero": true,
+ "log_history": [
+ {
+ "entropy": 0.8295683860778809,
+ "epoch": 0.00202020202020202,
+ "grad_norm": 10.886772155761719,
+ "learning_rate": 5e-06,
+ "loss": 1.5185801982879639,
+ "mean_token_accuracy": 0.672874927520752,
+ "num_tokens": 16384.0,
+ "step": 1
+ },
+ {
+ "entropy": 1.1781222820281982,
+ "epoch": 0.00404040404040404,
+ "grad_norm": 13.80163288116455,
+ "learning_rate": 4.997979797979798e-06,
+ "loss": 2.18377685546875,
+ "mean_token_accuracy": 0.5700415372848511,
+ "num_tokens": 32768.0,
+ "step": 2
+ },
+ {
+ "entropy": 0.7972303032875061,
+ "epoch": 0.006060606060606061,
+ "grad_norm": 13.71261215209961,
+ "learning_rate": 4.995959595959596e-06,
+ "loss": 1.5810277462005615,
+ "mean_token_accuracy": 0.6782486438751221,
+ "num_tokens": 49152.0,
+ "step": 3
+ },
+ {
+ "entropy": 0.9709298610687256,
+ "epoch": 0.00808080808080808,
+ "grad_norm": 9.18797779083252,
+ "learning_rate": 4.993939393939394e-06,
+ "loss": 1.6923656463623047,
+ "mean_token_accuracy": 0.6447850465774536,
+ "num_tokens": 65536.0,
+ "step": 4
+ },
+ {
+ "entropy": 1.2161898612976074,
+ "epoch": 0.010101010101010102,
+ "grad_norm": 8.97508430480957,
+ "learning_rate": 4.991919191919192e-06,
+ "loss": 1.9361608028411865,
+ "mean_token_accuracy": 0.5922691822052002,
+ "num_tokens": 81920.0,
+ "step": 5
+ },
+ {
+ "entropy": 0.9950039982795715,
+ "epoch": 0.012121212121212121,
+ "grad_norm": 6.453213214874268,
+ "learning_rate": 4.98989898989899e-06,
+ "loss": 1.5012480020523071,
+ "mean_token_accuracy": 0.6671959161758423,
+ "num_tokens": 98304.0,
+ "step": 6
+ },
+ {
+ "entropy": 1.452682614326477,
+ "epoch": 0.014141414141414142,
+ "grad_norm": 8.50411605834961,
+ "learning_rate": 4.987878787878789e-06,
+ "loss": 2.1297783851623535,
+ "mean_token_accuracy": 0.5807889699935913,
+ "num_tokens": 114688.0,
+ "step": 7
+ },
+ {
+ "entropy": 1.4661014080047607,
+ "epoch": 0.01616161616161616,
+ "grad_norm": 4.290146827697754,
+ "learning_rate": 4.9858585858585865e-06,
+ "loss": 1.7391963005065918,
+ "mean_token_accuracy": 0.611993134021759,
+ "num_tokens": 131072.0,
+ "step": 8
+ },
+ {
+ "entropy": 1.9030903577804565,
+ "epoch": 0.01818181818181818,
+ "grad_norm": 4.503584384918213,
+ "learning_rate": 4.983838383838384e-06,
+ "loss": 2.283596992492676,
+ "mean_token_accuracy": 0.5261968970298767,
+ "num_tokens": 147456.0,
+ "step": 9
+ },
+ {
+ "entropy": 1.2784137725830078,
+ "epoch": 0.020202020202020204,
+ "grad_norm": 3.427410840988159,
+ "learning_rate": 4.981818181818182e-06,
+ "loss": 1.5032392740249634,
+ "mean_token_accuracy": 0.6578529477119446,
+ "num_tokens": 163840.0,
+ "step": 10
+ },
+ {
+ "entropy": 1.7487188577651978,
+ "epoch": 0.022222222222222223,
+ "grad_norm": 3.20536470413208,
+ "learning_rate": 4.97979797979798e-06,
+ "loss": 1.9184643030166626,
+ "mean_token_accuracy": 0.6093673706054688,
+ "num_tokens": 180224.0,
+ "step": 11
+ },
+ {
+ "entropy": 1.6585394144058228,
+ "epoch": 0.024242424242424242,
+ "grad_norm": 3.2463598251342773,
+ "learning_rate": 4.977777777777778e-06,
+ "loss": 1.8364639282226562,
+ "mean_token_accuracy": 0.6014899611473083,
+ "num_tokens": 196608.0,
+ "step": 12
+ },
+ {
+ "entropy": 1.4869519472122192,
+ "epoch": 0.026262626262626262,
+ "grad_norm": 2.8681719303131104,
+ "learning_rate": 4.975757575757576e-06,
+ "loss": 1.6356265544891357,
+ "mean_token_accuracy": 0.6361138224601746,
+ "num_tokens": 212992.0,
+ "step": 13
+ },
+ {
+ "entropy": 1.5179094076156616,
+ "epoch": 0.028282828282828285,
+ "grad_norm": 3.0470707416534424,
+ "learning_rate": 4.973737373737374e-06,
+ "loss": 1.6317660808563232,
+ "mean_token_accuracy": 0.6334269642829895,
+ "num_tokens": 229376.0,
+ "step": 14
+ },
+ {
+ "entropy": 1.7384330034255981,
+ "epoch": 0.030303030303030304,
+ "grad_norm": 2.6149742603302,
+ "learning_rate": 4.971717171717172e-06,
+ "loss": 1.67923903465271,
+ "mean_token_accuracy": 0.6469223499298096,
+ "num_tokens": 245760.0,
+ "step": 15
+ },
+ {
+ "entropy": 2.108799457550049,
+ "epoch": 0.03232323232323232,
+ "grad_norm": 3.694795608520508,
+ "learning_rate": 4.9696969696969696e-06,
+ "loss": 1.9052371978759766,
+ "mean_token_accuracy": 0.5927577018737793,
+ "num_tokens": 262144.0,
+ "step": 16
+ },
+ {
+ "entropy": 1.7532848119735718,
+ "epoch": 0.03434343434343434,
+ "grad_norm": 3.1510567665100098,
+ "learning_rate": 4.9676767676767675e-06,
+ "loss": 1.6382691860198975,
+ "mean_token_accuracy": 0.6311675906181335,
+ "num_tokens": 278528.0,
+ "step": 17
+ },
+ {
+ "entropy": 1.723548412322998,
+ "epoch": 0.03636363636363636,
+ "grad_norm": 3.093825340270996,
+ "learning_rate": 4.965656565656566e-06,
+ "loss": 1.6039624214172363,
+ "mean_token_accuracy": 0.6313507556915283,
+ "num_tokens": 294912.0,
+ "step": 18
+ },
+ {
+ "entropy": 1.4790765047073364,
+ "epoch": 0.03838383838383838,
+ "grad_norm": 2.7150535583496094,
+ "learning_rate": 4.963636363636364e-06,
+ "loss": 1.4173667430877686,
+ "mean_token_accuracy": 0.6627381443977356,
+ "num_tokens": 311296.0,
+ "step": 19
+ },
+ {
+ "entropy": 1.5827536582946777,
+ "epoch": 0.04040404040404041,
+ "grad_norm": 2.610966444015503,
+ "learning_rate": 4.961616161616162e-06,
+ "loss": 1.4604644775390625,
+ "mean_token_accuracy": 0.656509518623352,
+ "num_tokens": 327680.0,
+ "step": 20
+ },
+ {
+ "entropy": 2.054673194885254,
+ "epoch": 0.04242424242424243,
+ "grad_norm": 2.612940788269043,
+ "learning_rate": 4.95959595959596e-06,
+ "loss": 1.962029218673706,
+ "mean_token_accuracy": 0.5719956159591675,
+ "num_tokens": 344064.0,
+ "step": 21
+ },
+ {
+ "entropy": 1.6684198379516602,
+ "epoch": 0.044444444444444446,
+ "grad_norm": 2.5426013469696045,
+ "learning_rate": 4.957575757575758e-06,
+ "loss": 1.591176986694336,
+ "mean_token_accuracy": 0.6519907116889954,
+ "num_tokens": 360448.0,
+ "step": 22
+ },
+ {
+ "entropy": 1.9070855379104614,
+ "epoch": 0.046464646464646465,
+ "grad_norm": 2.5853357315063477,
+ "learning_rate": 4.9555555555555565e-06,
+ "loss": 1.930276870727539,
+ "mean_token_accuracy": 0.5895823240280151,
+ "num_tokens": 376832.0,
+ "step": 23
+ },
+ {
+ "entropy": 1.716689109802246,
+ "epoch": 0.048484848484848485,
+ "grad_norm": 2.7608494758605957,
+ "learning_rate": 4.953535353535354e-06,
+ "loss": 1.739159345626831,
+ "mean_token_accuracy": 0.6028944849967957,
+ "num_tokens": 393216.0,
+ "step": 24
+ },
+ {
+ "entropy": 1.5666695833206177,
+ "epoch": 0.050505050505050504,
+ "grad_norm": 2.5973074436187744,
+ "learning_rate": 4.951515151515152e-06,
+ "loss": 1.6016302108764648,
+ "mean_token_accuracy": 0.626099169254303,
+ "num_tokens": 409600.0,
+ "step": 25
+ },
+ {
+ "entropy": 1.4267652034759521,
+ "epoch": 0.052525252525252523,
+ "grad_norm": 2.163451910018921,
+ "learning_rate": 4.94949494949495e-06,
+ "loss": 1.5093598365783691,
+ "mean_token_accuracy": 0.6530288457870483,
+ "num_tokens": 425984.0,
+ "step": 26
+ },
+ {
+ "entropy": 1.8537150621414185,
+ "epoch": 0.05454545454545454,
+ "grad_norm": 2.6983258724212646,
+ "learning_rate": 4.947474747474748e-06,
+ "loss": 1.8831868171691895,
+ "mean_token_accuracy": 0.5802393555641174,
+ "num_tokens": 442368.0,
+ "step": 27
+ },
+ {
+ "entropy": 1.410015344619751,
+ "epoch": 0.05656565656565657,
+ "grad_norm": 2.4777796268463135,
+ "learning_rate": 4.945454545454546e-06,
+ "loss": 1.4835734367370605,
+ "mean_token_accuracy": 0.6519907116889954,
+ "num_tokens": 458752.0,
+ "step": 28
+ },
+ {
+ "entropy": 1.2399017810821533,
+ "epoch": 0.05858585858585859,
+ "grad_norm": 2.1676478385925293,
+ "learning_rate": 4.943434343434344e-06,
+ "loss": 1.330633282661438,
+ "mean_token_accuracy": 0.6802638173103333,
+ "num_tokens": 475136.0,
+ "step": 29
+ },
+ {
+ "entropy": 1.6478898525238037,
+ "epoch": 0.06060606060606061,
+ "grad_norm": 2.5325536727905273,
+ "learning_rate": 4.941414141414142e-06,
+ "loss": 1.771777868270874,
+ "mean_token_accuracy": 0.6128480434417725,
+ "num_tokens": 491520.0,
+ "step": 30
+ },
+ {
+ "entropy": 1.7915360927581787,
+ "epoch": 0.06262626262626263,
+ "grad_norm": 2.2419557571411133,
+ "learning_rate": 4.93939393939394e-06,
+ "loss": 1.870645523071289,
+ "mean_token_accuracy": 0.6074743270874023,
+ "num_tokens": 507904.0,
+ "step": 31
+ },
+ {
+ "entropy": 1.7638899087905884,
+ "epoch": 0.06464646464646465,
+ "grad_norm": 2.8063058853149414,
+ "learning_rate": 4.937373737373738e-06,
+ "loss": 1.83987557888031,
+ "mean_token_accuracy": 0.5836589932441711,
+ "num_tokens": 524288.0,
+ "step": 32
+ },
+ {
+ "entropy": 1.4097516536712646,
+ "epoch": 0.06666666666666667,
+ "grad_norm": 2.1857714653015137,
+ "learning_rate": 4.935353535353536e-06,
+ "loss": 1.477668285369873,
+ "mean_token_accuracy": 0.6507083773612976,
+ "num_tokens": 540672.0,
+ "step": 33
+ },
+ {
+ "entropy": 1.4447426795959473,
+ "epoch": 0.06868686868686869,
+ "grad_norm": 2.3771870136260986,
+ "learning_rate": 4.933333333333334e-06,
+ "loss": 1.536318063735962,
+ "mean_token_accuracy": 0.6383732557296753,
+ "num_tokens": 557056.0,
+ "step": 34
+ },
+ {
+ "entropy": 1.9658164978027344,
+ "epoch": 0.0707070707070707,
+ "grad_norm": 2.4784936904907227,
+ "learning_rate": 4.931313131313132e-06,
+ "loss": 2.0565035343170166,
+ "mean_token_accuracy": 0.5581949353218079,
+ "num_tokens": 573440.0,
+ "step": 35
+ },
+ {
+ "entropy": 1.6045317649841309,
+ "epoch": 0.07272727272727272,
+ "grad_norm": 2.1128504276275635,
+ "learning_rate": 4.92929292929293e-06,
+ "loss": 1.6234952211380005,
+ "mean_token_accuracy": 0.6293356418609619,
+ "num_tokens": 589824.0,
+ "step": 36
+ },
+ {
+ "entropy": 1.5491269826889038,
+ "epoch": 0.07474747474747474,
+ "grad_norm": 2.1672937870025635,
+ "learning_rate": 4.927272727272728e-06,
+ "loss": 1.5809822082519531,
+ "mean_token_accuracy": 0.6419760584831238,
+ "num_tokens": 606208.0,
+ "step": 37
+ },
+ {
+ "entropy": 1.545534610748291,
+ "epoch": 0.07676767676767676,
+ "grad_norm": 2.2526934146881104,
+ "learning_rate": 4.925252525252526e-06,
+ "loss": 1.5707473754882812,
+ "mean_token_accuracy": 0.6344650983810425,
+ "num_tokens": 622592.0,
+ "step": 38
+ },
+ {
+ "entropy": 1.579596757888794,
+ "epoch": 0.07878787878787878,
+ "grad_norm": 2.3988492488861084,
+ "learning_rate": 4.9232323232323235e-06,
+ "loss": 1.6725983619689941,
+ "mean_token_accuracy": 0.6243282556533813,
+ "num_tokens": 638976.0,
+ "step": 39
+ },
+ {
+ "entropy": 2.0438785552978516,
+ "epoch": 0.08080808080808081,
+ "grad_norm": 2.3314359188079834,
+ "learning_rate": 4.9212121212121214e-06,
+ "loss": 2.0424888134002686,
+ "mean_token_accuracy": 0.570713222026825,
+ "num_tokens": 655360.0,
+ "step": 40
+ },
+ {
+ "entropy": 1.8690773248672485,
+ "epoch": 0.08282828282828283,
+ "grad_norm": 2.130401134490967,
+ "learning_rate": 4.919191919191919e-06,
+ "loss": 1.8796970844268799,
+ "mean_token_accuracy": 0.5882999300956726,
+ "num_tokens": 671744.0,
+ "step": 41
+ },
+ {
+ "entropy": 1.756626009941101,
+ "epoch": 0.08484848484848485,
+ "grad_norm": 2.342318534851074,
+ "learning_rate": 4.917171717171717e-06,
+ "loss": 1.71901535987854,
+ "mean_token_accuracy": 0.6173668503761292,
+ "num_tokens": 688128.0,
+ "step": 42
+ },
+ {
+ "entropy": 1.4642508029937744,
+ "epoch": 0.08686868686868687,
+ "grad_norm": 1.993338704109192,
+ "learning_rate": 4.915151515151516e-06,
+ "loss": 1.4660165309906006,
+ "mean_token_accuracy": 0.6687225103378296,
+ "num_tokens": 704512.0,
+ "step": 43
+ },
+ {
+ "entropy": 1.790807843208313,
+ "epoch": 0.08888888888888889,
+ "grad_norm": 2.3186943531036377,
+ "learning_rate": 4.913131313131314e-06,
+ "loss": 1.7102060317993164,
+ "mean_token_accuracy": 0.6238397359848022,
+ "num_tokens": 720896.0,
+ "step": 44
+ },
+ {
+ "entropy": 1.5970485210418701,
+ "epoch": 0.09090909090909091,
+ "grad_norm": 2.299307346343994,
+ "learning_rate": 4.911111111111112e-06,
+ "loss": 1.5170013904571533,
+ "mean_token_accuracy": 0.652662456035614,
+ "num_tokens": 737280.0,
+ "step": 45
+ },
+ {
+ "entropy": 1.3955466747283936,
+ "epoch": 0.09292929292929293,
+ "grad_norm": 2.171952962875366,
+ "learning_rate": 4.90909090909091e-06,
+ "loss": 1.4059661626815796,
+ "mean_token_accuracy": 0.6654250025749207,
+ "num_tokens": 753664.0,
+ "step": 46
+ },
+ {
+ "entropy": 1.7198575735092163,
+ "epoch": 0.09494949494949495,
+ "grad_norm": 2.385092258453369,
+ "learning_rate": 4.9070707070707075e-06,
+ "loss": 1.730346918106079,
+ "mean_token_accuracy": 0.6191987991333008,
+ "num_tokens": 770048.0,
+ "step": 47
+ },
+ {
+ "entropy": 1.3542555570602417,
+ "epoch": 0.09696969696969697,
+ "grad_norm": 2.1463189125061035,
+ "learning_rate": 4.905050505050505e-06,
+ "loss": 1.346085786819458,
+ "mean_token_accuracy": 0.680446982383728,
+ "num_tokens": 786432.0,
+ "step": 48
+ },
+ {
+ "entropy": 1.8084443807601929,
+ "epoch": 0.09898989898989899,
+ "grad_norm": 2.1505849361419678,
+ "learning_rate": 4.903030303030303e-06,
+ "loss": 1.847151756286621,
+ "mean_token_accuracy": 0.5926355719566345,
+ "num_tokens": 802816.0,
+ "step": 49
+ },
+ {
+ "entropy": 1.751160979270935,
+ "epoch": 0.10101010101010101,
+ "grad_norm": 2.1436259746551514,
+ "learning_rate": 4.901010101010101e-06,
+ "loss": 1.7802404165267944,
+ "mean_token_accuracy": 0.5996580123901367,
+ "num_tokens": 819200.0,
+ "step": 50
+ },
+ {
+ "epoch": 0.10101010101010101,
+ "eval_entropy": 1.6292865045609013,
+ "eval_loss": 1.6725393533706665,
+ "eval_mean_token_accuracy": 0.6230050469598463,
+ "eval_num_tokens": 819200.0,
+ "eval_runtime": 43.9148,
+ "eval_samples_per_second": 22.544,
+ "eval_steps_per_second": 2.824,
+ "step": 50
+ },
+ {
+ "entropy": 1.4428319931030273,
+ "epoch": 0.10303030303030303,
+ "grad_norm": 2.0954954624176025,
+ "learning_rate": 4.898989898989899e-06,
+ "loss": 1.4927232265472412,
+ "mean_token_accuracy": 0.6522349715232849,
+ "num_tokens": 835584.0,
+ "step": 51
+ },
+ {
+ "entropy": 1.7493115663528442,
+ "epoch": 0.10505050505050505,
+ "grad_norm": 2.300030469894409,
+ "learning_rate": 4.896969696969697e-06,
+ "loss": 1.737417221069336,
+ "mean_token_accuracy": 0.6213361024856567,
+ "num_tokens": 851968.0,
+ "step": 52
+ },
+ {
+ "entropy": 1.7801647186279297,
+ "epoch": 0.10707070707070707,
+ "grad_norm": 2.3947081565856934,
+ "learning_rate": 4.894949494949495e-06,
+ "loss": 1.8372564315795898,
+ "mean_token_accuracy": 0.5931240916252136,
+ "num_tokens": 868352.0,
+ "step": 53
+ },
+ {
+ "entropy": 1.6934887170791626,
+ "epoch": 0.10909090909090909,
+ "grad_norm": 2.1981089115142822,
+ "learning_rate": 4.8929292929292936e-06,
+ "loss": 1.734646201133728,
+ "mean_token_accuracy": 0.611932098865509,
+ "num_tokens": 884736.0,
+ "step": 54
+ },
+ {
+ "entropy": 1.3401941061019897,
+ "epoch": 0.1111111111111111,
+ "grad_norm": 2.193511724472046,
+ "learning_rate": 4.8909090909090914e-06,
+ "loss": 1.399888038635254,
+ "mean_token_accuracy": 0.6745847463607788,
+ "num_tokens": 901120.0,
+ "step": 55
+ },
+ {
+ "entropy": 1.8244661092758179,
+ "epoch": 0.11313131313131314,
+ "grad_norm": 2.4358489513397217,
+ "learning_rate": 4.888888888888889e-06,
+ "loss": 1.8716282844543457,
+ "mean_token_accuracy": 0.5867122411727905,
+ "num_tokens": 917504.0,
+ "step": 56
+ },
+ {
+ "entropy": 1.5019619464874268,
+ "epoch": 0.11515151515151516,
+ "grad_norm": 2.1135261058807373,
+ "learning_rate": 4.886868686868687e-06,
+ "loss": 1.523103952407837,
+ "mean_token_accuracy": 0.6414265036582947,
+ "num_tokens": 933888.0,
+ "step": 57
+ },
+ {
+ "entropy": 1.6132855415344238,
+ "epoch": 0.11717171717171718,
+ "grad_norm": 2.026301145553589,
+ "learning_rate": 4.884848484848485e-06,
+ "loss": 1.6233609914779663,
+ "mean_token_accuracy": 0.6279311180114746,
+ "num_tokens": 950272.0,
+ "step": 58
+ },
+ {
+ "entropy": 1.738538384437561,
+ "epoch": 0.1191919191919192,
+ "grad_norm": 2.1539394855499268,
+ "learning_rate": 4.882828282828283e-06,
+ "loss": 1.8147332668304443,
+ "mean_token_accuracy": 0.6124816536903381,
+ "num_tokens": 966656.0,
+ "step": 59
+ },
+ {
+ "entropy": 1.5533764362335205,
+ "epoch": 0.12121212121212122,
+ "grad_norm": 2.2603628635406494,
+ "learning_rate": 4.880808080808081e-06,
+ "loss": 1.562873125076294,
+ "mean_token_accuracy": 0.6398388147354126,
+ "num_tokens": 983040.0,
+ "step": 60
+ },
+ {
+ "entropy": 1.722406268119812,
+ "epoch": 0.12323232323232323,
+ "grad_norm": 2.3630757331848145,
+ "learning_rate": 4.878787878787879e-06,
+ "loss": 1.7461689710617065,
+ "mean_token_accuracy": 0.6023448705673218,
+ "num_tokens": 999424.0,
+ "step": 61
+ },
+ {
+ "entropy": 1.6533517837524414,
+ "epoch": 0.12525252525252525,
+ "grad_norm": 2.2165415287017822,
+ "learning_rate": 4.876767676767677e-06,
+ "loss": 1.676560640335083,
+ "mean_token_accuracy": 0.6437469720840454,
+ "num_tokens": 1015808.0,
+ "step": 62
+ },
+ {
+ "entropy": 1.749995470046997,
+ "epoch": 0.12727272727272726,
+ "grad_norm": 2.1572678089141846,
+ "learning_rate": 4.8747474747474745e-06,
+ "loss": 1.784087896347046,
+ "mean_token_accuracy": 0.5884831547737122,
+ "num_tokens": 1032192.0,
+ "step": 63
+ },
+ {
+ "entropy": 1.442152738571167,
+ "epoch": 0.1292929292929293,
+ "grad_norm": 2.163490056991577,
+ "learning_rate": 4.872727272727273e-06,
+ "loss": 1.4307503700256348,
+ "mean_token_accuracy": 0.6618832349777222,
+ "num_tokens": 1048576.0,
+ "step": 64
+ },
+ {
+ "entropy": 1.2657029628753662,
+ "epoch": 0.13131313131313133,
+ "grad_norm": 2.1225337982177734,
+ "learning_rate": 4.870707070707071e-06,
+ "loss": 1.2731966972351074,
+ "mean_token_accuracy": 0.688568651676178,
+ "num_tokens": 1064960.0,
+ "step": 65
+ },
+ {
+ "entropy": 1.1613880395889282,
+ "epoch": 0.13333333333333333,
+ "grad_norm": 1.9527196884155273,
+ "learning_rate": 4.868686868686869e-06,
+ "loss": 1.13922119140625,
+ "mean_token_accuracy": 0.7389472126960754,
+ "num_tokens": 1081344.0,
+ "step": 66
+ },
+ {
+ "entropy": 1.6936380863189697,
+ "epoch": 0.13535353535353536,
+ "grad_norm": 2.004284620285034,
+ "learning_rate": 4.866666666666667e-06,
+ "loss": 1.6982238292694092,
+ "mean_token_accuracy": 0.6191987991333008,
+ "num_tokens": 1097728.0,
+ "step": 67
+ },
+ {
+ "entropy": 1.750565528869629,
+ "epoch": 0.13737373737373737,
+ "grad_norm": 2.225955009460449,
+ "learning_rate": 4.864646464646466e-06,
+ "loss": 1.796521782875061,
+ "mean_token_accuracy": 0.5934293866157532,
+ "num_tokens": 1114112.0,
+ "step": 68
+ },
+ {
+ "entropy": 1.6608220338821411,
+ "epoch": 0.1393939393939394,
+ "grad_norm": 2.127035617828369,
+ "learning_rate": 4.8626262626262636e-06,
+ "loss": 1.6633095741271973,
+ "mean_token_accuracy": 0.6356253027915955,
+ "num_tokens": 1130496.0,
+ "step": 69
+ },
+ {
+ "entropy": 1.4848661422729492,
+ "epoch": 0.1414141414141414,
+ "grad_norm": 2.0338215827941895,
+ "learning_rate": 4.8606060606060615e-06,
+ "loss": 1.4789674282073975,
+ "mean_token_accuracy": 0.6680508255958557,
+ "num_tokens": 1146880.0,
+ "step": 70
+ },
+ {
+ "entropy": 1.2310466766357422,
+ "epoch": 0.14343434343434344,
+ "grad_norm": 2.105898141860962,
+ "learning_rate": 4.858585858585859e-06,
+ "loss": 1.2301937341690063,
+ "mean_token_accuracy": 0.7040791511535645,
+ "num_tokens": 1163264.0,
+ "step": 71
+ },
+ {
+ "entropy": 1.6310514211654663,
+ "epoch": 0.14545454545454545,
+ "grad_norm": 2.152125358581543,
+ "learning_rate": 4.856565656565657e-06,
+ "loss": 1.643636703491211,
+ "mean_token_accuracy": 0.6221299171447754,
+ "num_tokens": 1179648.0,
+ "step": 72
+ },
+ {
+ "entropy": 1.4747940301895142,
+ "epoch": 0.14747474747474748,
+ "grad_norm": 2.096461296081543,
+ "learning_rate": 4.854545454545455e-06,
+ "loss": 1.42953360080719,
+ "mean_token_accuracy": 0.6651807427406311,
+ "num_tokens": 1196032.0,
+ "step": 73
+ },
+ {
+ "entropy": 1.819259524345398,
+ "epoch": 0.1494949494949495,
+ "grad_norm": 2.2852063179016113,
+ "learning_rate": 4.852525252525253e-06,
+ "loss": 1.845325231552124,
+ "mean_token_accuracy": 0.6036272644996643,
+ "num_tokens": 1212416.0,
+ "step": 74
+ },
+ {
+ "entropy": 1.4233598709106445,
+ "epoch": 0.15151515151515152,
+ "grad_norm": 2.1157381534576416,
+ "learning_rate": 4.850505050505051e-06,
+ "loss": 1.4565438032150269,
+ "mean_token_accuracy": 0.6607230305671692,
+ "num_tokens": 1228800.0,
+ "step": 75
+ },
+ {
+ "entropy": 1.6441459655761719,
+ "epoch": 0.15353535353535352,
+ "grad_norm": 2.2092180252075195,
+ "learning_rate": 4.848484848484849e-06,
+ "loss": 1.6467639207839966,
+ "mean_token_accuracy": 0.6285417675971985,
+ "num_tokens": 1245184.0,
+ "step": 76
+ },
+ {
+ "entropy": 1.6124308109283447,
+ "epoch": 0.15555555555555556,
+ "grad_norm": 2.231876850128174,
+ "learning_rate": 4.846464646464647e-06,
+ "loss": 1.617384672164917,
+ "mean_token_accuracy": 0.6340987086296082,
+ "num_tokens": 1261568.0,
+ "step": 77
+ },
+ {
+ "entropy": 1.828405499458313,
+ "epoch": 0.15757575757575756,
+ "grad_norm": 2.1372313499450684,
+ "learning_rate": 4.8444444444444446e-06,
+ "loss": 1.8333203792572021,
+ "mean_token_accuracy": 0.5979481935501099,
+ "num_tokens": 1277952.0,
+ "step": 78
+ },
+ {
+ "entropy": 1.855564832687378,
+ "epoch": 0.1595959595959596,
+ "grad_norm": 2.1422762870788574,
+ "learning_rate": 4.842424242424243e-06,
+ "loss": 1.9133609533309937,
+ "mean_token_accuracy": 0.570652186870575,
+ "num_tokens": 1294336.0,
+ "step": 79
+ },
+ {
+ "entropy": 1.7908183336257935,
+ "epoch": 0.16161616161616163,
+ "grad_norm": 2.172368049621582,
+ "learning_rate": 4.840404040404041e-06,
+ "loss": 1.8109419345855713,
+ "mean_token_accuracy": 0.60332190990448,
+ "num_tokens": 1310720.0,
+ "step": 80
+ },
+ {
+ "entropy": 1.9537721872329712,
+ "epoch": 0.16363636363636364,
+ "grad_norm": 2.199505090713501,
+ "learning_rate": 4.838383838383839e-06,
+ "loss": 2.0001401901245117,
+ "mean_token_accuracy": 0.5585613250732422,
+ "num_tokens": 1327104.0,
+ "step": 81
+ },
+ {
+ "entropy": 1.5365772247314453,
+ "epoch": 0.16565656565656567,
+ "grad_norm": 2.0778913497924805,
+ "learning_rate": 4.836363636363637e-06,
+ "loss": 1.575089931488037,
+ "mean_token_accuracy": 0.6375183463096619,
+ "num_tokens": 1343488.0,
+ "step": 82
+ },
+ {
+ "entropy": 1.7177143096923828,
+ "epoch": 0.16767676767676767,
+ "grad_norm": 2.1010894775390625,
+ "learning_rate": 4.834343434343435e-06,
+ "loss": 1.7428388595581055,
+ "mean_token_accuracy": 0.6113824844360352,
+ "num_tokens": 1359872.0,
+ "step": 83
+ },
+ {
+ "entropy": 1.5080527067184448,
+ "epoch": 0.1696969696969697,
+ "grad_norm": 2.021969795227051,
+ "learning_rate": 4.832323232323233e-06,
+ "loss": 1.5361201763153076,
+ "mean_token_accuracy": 0.656509518623352,
+ "num_tokens": 1376256.0,
+ "step": 84
+ },
+ {
+ "entropy": 1.5042120218276978,
+ "epoch": 0.1717171717171717,
+ "grad_norm": 2.121314287185669,
+ "learning_rate": 4.830303030303031e-06,
+ "loss": 1.4912033081054688,
+ "mean_token_accuracy": 0.6621274948120117,
+ "num_tokens": 1392640.0,
+ "step": 85
+ },
+ {
+ "entropy": 1.9523948431015015,
+ "epoch": 0.17373737373737375,
+ "grad_norm": 2.1702609062194824,
+ "learning_rate": 4.8282828282828285e-06,
+ "loss": 1.9369449615478516,
+ "mean_token_accuracy": 0.5727283954620361,
+ "num_tokens": 1409024.0,
+ "step": 86
+ },
+ {
+ "entropy": 1.6174770593643188,
+ "epoch": 0.17575757575757575,
+ "grad_norm": 2.222412109375,
+ "learning_rate": 4.826262626262626e-06,
+ "loss": 1.6794973611831665,
+ "mean_token_accuracy": 0.6151685118675232,
+ "num_tokens": 1425408.0,
+ "step": 87
+ },
+ {
+ "entropy": 1.1480307579040527,
+ "epoch": 0.17777777777777778,
+ "grad_norm": 1.98936128616333,
+ "learning_rate": 4.824242424242424e-06,
+ "loss": 1.1324063539505005,
+ "mean_token_accuracy": 0.7193453907966614,
+ "num_tokens": 1441792.0,
+ "step": 88
+ },
+ {
+ "entropy": 1.5290263891220093,
+ "epoch": 0.1797979797979798,
+ "grad_norm": 2.098860263824463,
+ "learning_rate": 4.822222222222222e-06,
+ "loss": 1.556044340133667,
+ "mean_token_accuracy": 0.6415486335754395,
+ "num_tokens": 1458176.0,
+ "step": 89
+ },
+ {
+ "entropy": 1.1366127729415894,
+ "epoch": 0.18181818181818182,
+ "grad_norm": 1.9387420415878296,
+ "learning_rate": 4.820202020202021e-06,
+ "loss": 1.1635141372680664,
+ "mean_token_accuracy": 0.7168416976928711,
+ "num_tokens": 1474560.0,
+ "step": 90
+ },
+ {
+ "entropy": 1.5142875909805298,
+ "epoch": 0.18383838383838383,
+ "grad_norm": 2.259131908416748,
+ "learning_rate": 4.818181818181819e-06,
+ "loss": 1.5329954624176025,
+ "mean_token_accuracy": 0.6471666097640991,
+ "num_tokens": 1490944.0,
+ "step": 91
+ },
+ {
+ "entropy": 1.5197135210037231,
+ "epoch": 0.18585858585858586,
+ "grad_norm": 2.1230807304382324,
+ "learning_rate": 4.816161616161617e-06,
+ "loss": 1.541062593460083,
+ "mean_token_accuracy": 0.6398388147354126,
+ "num_tokens": 1507328.0,
+ "step": 92
+ },
+ {
+ "entropy": 1.5469954013824463,
+ "epoch": 0.18787878787878787,
+ "grad_norm": 2.0568583011627197,
+ "learning_rate": 4.8141414141414146e-06,
+ "loss": 1.5564078092575073,
+ "mean_token_accuracy": 0.6384953856468201,
+ "num_tokens": 1523712.0,
+ "step": 93
+ },
+ {
+ "entropy": 1.5429692268371582,
+ "epoch": 0.1898989898989899,
+ "grad_norm": 2.200144052505493,
+ "learning_rate": 4.8121212121212125e-06,
+ "loss": 1.5292989015579224,
+ "mean_token_accuracy": 0.6373351216316223,
+ "num_tokens": 1540096.0,
+ "step": 94
+ },
+ {
+ "entropy": 1.4556076526641846,
+ "epoch": 0.1919191919191919,
+ "grad_norm": 2.250291585922241,
+ "learning_rate": 4.81010101010101e-06,
+ "loss": 1.426419734954834,
+ "mean_token_accuracy": 0.6604176759719849,
+ "num_tokens": 1556480.0,
+ "step": 95
+ },
+ {
+ "entropy": 1.4021440744400024,
+ "epoch": 0.19393939393939394,
+ "grad_norm": 2.287038803100586,
+ "learning_rate": 4.808080808080808e-06,
+ "loss": 1.4377000331878662,
+ "mean_token_accuracy": 0.6572422981262207,
+ "num_tokens": 1572864.0,
+ "step": 96
+ },
+ {
+ "entropy": 1.658683180809021,
+ "epoch": 0.19595959595959597,
+ "grad_norm": 2.1817963123321533,
+ "learning_rate": 4.806060606060606e-06,
+ "loss": 1.6514620780944824,
+ "mean_token_accuracy": 0.6278700828552246,
+ "num_tokens": 1589248.0,
+ "step": 97
+ },
+ {
+ "entropy": 1.5182844400405884,
+ "epoch": 0.19797979797979798,
+ "grad_norm": 2.144071340560913,
+ "learning_rate": 4.804040404040404e-06,
+ "loss": 1.5554628372192383,
+ "mean_token_accuracy": 0.6311064958572388,
+ "num_tokens": 1605632.0,
+ "step": 98
+ },
+ {
+ "entropy": 1.5997719764709473,
+ "epoch": 0.2,
+ "grad_norm": 2.0921664237976074,
+ "learning_rate": 4.802020202020202e-06,
+ "loss": 1.6185517311096191,
+ "mean_token_accuracy": 0.6286028623580933,
+ "num_tokens": 1622016.0,
+ "step": 99
+ },
+ {
+ "entropy": 1.4011279344558716,
+ "epoch": 0.20202020202020202,
+ "grad_norm": 2.2240700721740723,
+ "learning_rate": 4.800000000000001e-06,
+ "loss": 1.4112927913665771,
+ "mean_token_accuracy": 0.6696995496749878,
+ "num_tokens": 1638400.0,
+ "step": 100
+ },
+ {
+ "epoch": 0.20202020202020202,
+ "eval_entropy": 1.606662715634992,
+ "eval_loss": 1.6283859014511108,
+ "eval_mean_token_accuracy": 0.6293055717983553,
+ "eval_num_tokens": 1638400.0,
+ "eval_runtime": 43.7782,
+ "eval_samples_per_second": 22.614,
+ "eval_steps_per_second": 2.832,
+ "step": 100
+ },
+ {
+ "entropy": 1.6743865013122559,
+ "epoch": 0.20404040404040405,
+ "grad_norm": 2.025153398513794,
+ "learning_rate": 4.7979797979797985e-06,
+ "loss": 1.6404725313186646,
+ "mean_token_accuracy": 0.656509518623352,
+ "num_tokens": 1654784.0,
+ "step": 101
+ },
+ {
+ "entropy": 1.6196308135986328,
+ "epoch": 0.20606060606060606,
+ "grad_norm": 2.037229299545288,
+ "learning_rate": 4.795959595959596e-06,
+ "loss": 1.647303581237793,
+ "mean_token_accuracy": 0.6203590631484985,
+ "num_tokens": 1671168.0,
+ "step": 102
+ },
+ {
+ "entropy": 1.5600125789642334,
+ "epoch": 0.2080808080808081,
+ "grad_norm": 2.144221782684326,
+ "learning_rate": 4.793939393939394e-06,
+ "loss": 1.6199731826782227,
+ "mean_token_accuracy": 0.6249389052391052,
+ "num_tokens": 1687552.0,
+ "step": 103
+ },
+ {
+ "entropy": 2.064497947692871,
+ "epoch": 0.2101010101010101,
+ "grad_norm": 2.3956470489501953,
+ "learning_rate": 4.791919191919192e-06,
+ "loss": 2.048987627029419,
+ "mean_token_accuracy": 0.5507450103759766,
+ "num_tokens": 1703936.0,
+ "step": 104
+ },
+ {
+ "entropy": 1.7934812307357788,
+ "epoch": 0.21212121212121213,
+ "grad_norm": 2.487302780151367,
+ "learning_rate": 4.78989898989899e-06,
+ "loss": 1.8285956382751465,
+ "mean_token_accuracy": 0.5749877691268921,
+ "num_tokens": 1720320.0,
+ "step": 105
+ },
+ {
+ "entropy": 1.365216851234436,
+ "epoch": 0.21414141414141413,
+ "grad_norm": 2.0714964866638184,
+ "learning_rate": 4.787878787878788e-06,
+ "loss": 1.3870220184326172,
+ "mean_token_accuracy": 0.6692721247673035,
+ "num_tokens": 1736704.0,
+ "step": 106
+ },
+ {
+ "entropy": 1.4983874559402466,
+ "epoch": 0.21616161616161617,
+ "grad_norm": 1.9995853900909424,
+ "learning_rate": 4.785858585858586e-06,
+ "loss": 1.4949266910552979,
+ "mean_token_accuracy": 0.6554103493690491,
+ "num_tokens": 1753088.0,
+ "step": 107
+ },
+ {
+ "entropy": 2.0503509044647217,
+ "epoch": 0.21818181818181817,
+ "grad_norm": 2.190884590148926,
+ "learning_rate": 4.783838383838385e-06,
+ "loss": 2.079286813735962,
+ "mean_token_accuracy": 0.5657669901847839,
+ "num_tokens": 1769472.0,
+ "step": 108
+ },
+ {
+ "entropy": 1.5557374954223633,
+ "epoch": 0.2202020202020202,
+ "grad_norm": 2.044100761413574,
+ "learning_rate": 4.7818181818181825e-06,
+ "loss": 1.5735318660736084,
+ "mean_token_accuracy": 0.6359916925430298,
+ "num_tokens": 1785856.0,
+ "step": 109
+ },
+ {
+ "entropy": 1.5567635297775269,
+ "epoch": 0.2222222222222222,
+ "grad_norm": 2.3714160919189453,
+ "learning_rate": 4.77979797979798e-06,
+ "loss": 1.6026921272277832,
+ "mean_token_accuracy": 0.6290302872657776,
+ "num_tokens": 1802240.0,
+ "step": 110
+ },
+ {
+ "entropy": 1.8457536697387695,
+ "epoch": 0.22424242424242424,
+ "grad_norm": 2.202939987182617,
+ "learning_rate": 4.777777777777778e-06,
+ "loss": 1.7986081838607788,
+ "mean_token_accuracy": 0.5952613353729248,
+ "num_tokens": 1818624.0,
+ "step": 111
+ },
+ {
+ "entropy": 1.8822020292282104,
+ "epoch": 0.22626262626262628,
+ "grad_norm": 2.2674992084503174,
+ "learning_rate": 4.775757575757576e-06,
+ "loss": 1.9095954895019531,
+ "mean_token_accuracy": 0.5769418478012085,
+ "num_tokens": 1835008.0,
+ "step": 112
+ },
+ {
+ "entropy": 1.5445998907089233,
+ "epoch": 0.22828282828282828,
+ "grad_norm": 2.2848100662231445,
+ "learning_rate": 4.773737373737374e-06,
+ "loss": 1.5832195281982422,
+ "mean_token_accuracy": 0.6337933540344238,
+ "num_tokens": 1851392.0,
+ "step": 113
+ },
+ {
+ "entropy": 1.934509515762329,
+ "epoch": 0.23030303030303031,
+ "grad_norm": 2.3585174083709717,
+ "learning_rate": 4.771717171717172e-06,
+ "loss": 1.9809319972991943,
+ "mean_token_accuracy": 0.5789570212364197,
+ "num_tokens": 1867776.0,
+ "step": 114
+ },
+ {
+ "entropy": 1.7354214191436768,
+ "epoch": 0.23232323232323232,
+ "grad_norm": 2.13913631439209,
+ "learning_rate": 4.769696969696971e-06,
+ "loss": 1.7527806758880615,
+ "mean_token_accuracy": 0.6061308979988098,
+ "num_tokens": 1884160.0,
+ "step": 115
+ },
+ {
+ "entropy": 1.7714784145355225,
+ "epoch": 0.23434343434343435,
+ "grad_norm": 1.9915403127670288,
+ "learning_rate": 4.7676767676767685e-06,
+ "loss": 1.8065431118011475,
+ "mean_token_accuracy": 0.5986199378967285,
+ "num_tokens": 1900544.0,
+ "step": 116
+ },
+ {
+ "entropy": 1.5272125005722046,
+ "epoch": 0.23636363636363636,
+ "grad_norm": 2.004565715789795,
+ "learning_rate": 4.765656565656566e-06,
+ "loss": 1.5074517726898193,
+ "mean_token_accuracy": 0.6601123809814453,
+ "num_tokens": 1916928.0,
+ "step": 117
+ },
+ {
+ "entropy": 1.5393218994140625,
+ "epoch": 0.2383838383838384,
+ "grad_norm": 2.018089532852173,
+ "learning_rate": 4.763636363636364e-06,
+ "loss": 1.5607573986053467,
+ "mean_token_accuracy": 0.6540058851242065,
+ "num_tokens": 1933312.0,
+ "step": 118
+ },
+ {
+ "entropy": 1.7827534675598145,
+ "epoch": 0.2404040404040404,
+ "grad_norm": 2.062368392944336,
+ "learning_rate": 4.761616161616162e-06,
+ "loss": 1.7844001054763794,
+ "mean_token_accuracy": 0.6187102794647217,
+ "num_tokens": 1949696.0,
+ "step": 119
+ },
+ {
+ "entropy": 1.416417121887207,
+ "epoch": 0.24242424242424243,
+ "grad_norm": 2.0980024337768555,
+ "learning_rate": 4.75959595959596e-06,
+ "loss": 1.4041050672531128,
+ "mean_token_accuracy": 0.6561431288719177,
+ "num_tokens": 1966080.0,
+ "step": 120
+ },
+ {
+ "entropy": 1.3325153589248657,
+ "epoch": 0.24444444444444444,
+ "grad_norm": 1.9985002279281616,
+ "learning_rate": 4.757575757575758e-06,
+ "loss": 1.3197274208068848,
+ "mean_token_accuracy": 0.6806912422180176,
+ "num_tokens": 1982464.0,
+ "step": 121
+ },
+ {
+ "entropy": 1.7306798696517944,
+ "epoch": 0.24646464646464647,
+ "grad_norm": 2.2098441123962402,
+ "learning_rate": 4.755555555555556e-06,
+ "loss": 1.7595295906066895,
+ "mean_token_accuracy": 0.6110160946846008,
+ "num_tokens": 1998848.0,
+ "step": 122
+ },
+ {
+ "entropy": 1.3733264207839966,
+ "epoch": 0.24848484848484848,
+ "grad_norm": 1.9827327728271484,
+ "learning_rate": 4.753535353535354e-06,
+ "loss": 1.4026854038238525,
+ "mean_token_accuracy": 0.65486079454422,
+ "num_tokens": 2015232.0,
+ "step": 123
+ },
+ {
+ "entropy": 1.5910528898239136,
+ "epoch": 0.2505050505050505,
+ "grad_norm": 1.9615319967269897,
+ "learning_rate": 4.751515151515152e-06,
+ "loss": 1.6071114540100098,
+ "mean_token_accuracy": 0.648937463760376,
+ "num_tokens": 2031616.0,
+ "step": 124
+ },
+ {
+ "entropy": 1.3300938606262207,
+ "epoch": 0.25252525252525254,
+ "grad_norm": 1.9878504276275635,
+ "learning_rate": 4.7494949494949495e-06,
+ "loss": 1.3623383045196533,
+ "mean_token_accuracy": 0.6873473525047302,
+ "num_tokens": 2048000.0,
+ "step": 125
+ },
+ {
+ "entropy": 1.6087368726730347,
+ "epoch": 0.2545454545454545,
+ "grad_norm": 2.264647960662842,
+ "learning_rate": 4.747474747474748e-06,
+ "loss": 1.634058952331543,
+ "mean_token_accuracy": 0.6211528778076172,
+ "num_tokens": 2064384.0,
+ "step": 126
+ },
+ {
+ "entropy": 1.536401391029358,
+ "epoch": 0.25656565656565655,
+ "grad_norm": 1.9561539888381958,
+ "learning_rate": 4.745454545454546e-06,
+ "loss": 1.5699501037597656,
+ "mean_token_accuracy": 0.6248167753219604,
+ "num_tokens": 2080768.0,
+ "step": 127
+ },
+ {
+ "entropy": 1.656266212463379,
+ "epoch": 0.2585858585858586,
+ "grad_norm": 2.1124353408813477,
+ "learning_rate": 4.743434343434344e-06,
+ "loss": 1.6612167358398438,
+ "mean_token_accuracy": 0.623900830745697,
+ "num_tokens": 2097152.0,
+ "step": 128
+ },
+ {
+ "entropy": 1.170629620552063,
+ "epoch": 0.2606060606060606,
+ "grad_norm": 1.917840600013733,
+ "learning_rate": 4.741414141414142e-06,
+ "loss": 1.1992300748825073,
+ "mean_token_accuracy": 0.6995603442192078,
+ "num_tokens": 2113536.0,
+ "step": 129
+ },
+ {
+ "entropy": 1.4126694202423096,
+ "epoch": 0.26262626262626265,
+ "grad_norm": 1.9041539430618286,
+ "learning_rate": 4.73939393939394e-06,
+ "loss": 1.4334447383880615,
+ "mean_token_accuracy": 0.6637151837348938,
+ "num_tokens": 2129920.0,
+ "step": 130
+ },
+ {
+ "entropy": 1.8490277528762817,
+ "epoch": 0.26464646464646463,
+ "grad_norm": 2.1440138816833496,
+ "learning_rate": 4.737373737373738e-06,
+ "loss": 1.9226353168487549,
+ "mean_token_accuracy": 0.5805447101593018,
+ "num_tokens": 2146304.0,
+ "step": 131
+ },
+ {
+ "entropy": 1.5000964403152466,
+ "epoch": 0.26666666666666666,
+ "grad_norm": 1.9615508317947388,
+ "learning_rate": 4.735353535353536e-06,
+ "loss": 1.495596170425415,
+ "mean_token_accuracy": 0.6542501449584961,
+ "num_tokens": 2162688.0,
+ "step": 132
+ },
+ {
+ "entropy": 1.9371142387390137,
+ "epoch": 0.2686868686868687,
+ "grad_norm": 2.202200412750244,
+ "learning_rate": 4.7333333333333335e-06,
+ "loss": 1.94151771068573,
+ "mean_token_accuracy": 0.5810332298278809,
+ "num_tokens": 2179072.0,
+ "step": 133
+ },
+ {
+ "entropy": 1.4535126686096191,
+ "epoch": 0.27070707070707073,
+ "grad_norm": 1.9804027080535889,
+ "learning_rate": 4.731313131313131e-06,
+ "loss": 1.4401545524597168,
+ "mean_token_accuracy": 0.6585246920585632,
+ "num_tokens": 2195456.0,
+ "step": 134
+ },
+ {
+ "entropy": 2.0871424674987793,
+ "epoch": 0.2727272727272727,
+ "grad_norm": 2.1417081356048584,
+ "learning_rate": 4.729292929292929e-06,
+ "loss": 2.117375373840332,
+ "mean_token_accuracy": 0.5600268840789795,
+ "num_tokens": 2211840.0,
+ "step": 135
+ },
+ {
+ "entropy": 1.2986469268798828,
+ "epoch": 0.27474747474747474,
+ "grad_norm": 2.0299136638641357,
+ "learning_rate": 4.727272727272728e-06,
+ "loss": 1.3631991147994995,
+ "mean_token_accuracy": 0.6750732660293579,
+ "num_tokens": 2228224.0,
+ "step": 136
+ },
+ {
+ "entropy": 1.163429617881775,
+ "epoch": 0.2767676767676768,
+ "grad_norm": 1.8368210792541504,
+ "learning_rate": 4.725252525252526e-06,
+ "loss": 1.1603795289993286,
+ "mean_token_accuracy": 0.7150708436965942,
+ "num_tokens": 2244608.0,
+ "step": 137
+ },
+ {
+ "entropy": 1.4301519393920898,
+ "epoch": 0.2787878787878788,
+ "grad_norm": 1.996767520904541,
+ "learning_rate": 4.723232323232324e-06,
+ "loss": 1.422170639038086,
+ "mean_token_accuracy": 0.6816682815551758,
+ "num_tokens": 2260992.0,
+ "step": 138
+ },
+ {
+ "entropy": 1.5160106420516968,
+ "epoch": 0.2808080808080808,
+ "grad_norm": 2.004770278930664,
+ "learning_rate": 4.721212121212122e-06,
+ "loss": 1.5385751724243164,
+ "mean_token_accuracy": 0.640693724155426,
+ "num_tokens": 2277376.0,
+ "step": 139
+ },
+ {
+ "entropy": 1.2483867406845093,
+ "epoch": 0.2828282828282828,
+ "grad_norm": 1.8488364219665527,
+ "learning_rate": 4.7191919191919195e-06,
+ "loss": 1.2563843727111816,
+ "mean_token_accuracy": 0.6998046040534973,
+ "num_tokens": 2293760.0,
+ "step": 140
+ },
+ {
+ "entropy": 1.3214129209518433,
+ "epoch": 0.28484848484848485,
+ "grad_norm": 2.1334660053253174,
+ "learning_rate": 4.717171717171717e-06,
+ "loss": 1.3392664194107056,
+ "mean_token_accuracy": 0.6822789311408997,
+ "num_tokens": 2310144.0,
+ "step": 141
+ },
+ {
+ "entropy": 1.611849069595337,
+ "epoch": 0.2868686868686869,
+ "grad_norm": 2.0539803504943848,
+ "learning_rate": 4.715151515151515e-06,
+ "loss": 1.6335396766662598,
+ "mean_token_accuracy": 0.6235954761505127,
+ "num_tokens": 2326528.0,
+ "step": 142
+ },
+ {
+ "entropy": 1.5847896337509155,
+ "epoch": 0.28888888888888886,
+ "grad_norm": 2.1658759117126465,
+ "learning_rate": 4.713131313131313e-06,
+ "loss": 1.603764533996582,
+ "mean_token_accuracy": 0.633671224117279,
+ "num_tokens": 2342912.0,
+ "step": 143
+ },
+ {
+ "entropy": 1.5731406211853027,
+ "epoch": 0.2909090909090909,
+ "grad_norm": 2.0830390453338623,
+ "learning_rate": 4.711111111111111e-06,
+ "loss": 1.558933973312378,
+ "mean_token_accuracy": 0.6392892003059387,
+ "num_tokens": 2359296.0,
+ "step": 144
+ },
+ {
+ "entropy": 1.8901628255844116,
+ "epoch": 0.29292929292929293,
+ "grad_norm": 2.1436922550201416,
+ "learning_rate": 4.709090909090909e-06,
+ "loss": 1.878631353378296,
+ "mean_token_accuracy": 0.57333904504776,
+ "num_tokens": 2375680.0,
+ "step": 145
+ },
+ {
+ "entropy": 1.9344228506088257,
+ "epoch": 0.29494949494949496,
+ "grad_norm": 2.1245176792144775,
+ "learning_rate": 4.707070707070707e-06,
+ "loss": 1.9546704292297363,
+ "mean_token_accuracy": 0.5987420678138733,
+ "num_tokens": 2392064.0,
+ "step": 146
+ },
+ {
+ "entropy": 1.2455096244812012,
+ "epoch": 0.296969696969697,
+ "grad_norm": 1.896581768989563,
+ "learning_rate": 4.705050505050506e-06,
+ "loss": 1.2738728523254395,
+ "mean_token_accuracy": 0.7025524973869324,
+ "num_tokens": 2408448.0,
+ "step": 147
+ },
+ {
+ "entropy": 1.4235948324203491,
+ "epoch": 0.298989898989899,
+ "grad_norm": 2.522636890411377,
+ "learning_rate": 4.7030303030303035e-06,
+ "loss": 1.4524480104446411,
+ "mean_token_accuracy": 0.6441133618354797,
+ "num_tokens": 2424832.0,
+ "step": 148
+ },
+ {
+ "entropy": 1.764552116394043,
+ "epoch": 0.301010101010101,
+ "grad_norm": 2.0232255458831787,
+ "learning_rate": 4.701010101010101e-06,
+ "loss": 1.7811740636825562,
+ "mean_token_accuracy": 0.6229848265647888,
+ "num_tokens": 2441216.0,
+ "step": 149
+ },
+ {
+ "entropy": 1.74843168258667,
+ "epoch": 0.30303030303030304,
+ "grad_norm": 2.1589369773864746,
+ "learning_rate": 4.698989898989899e-06,
+ "loss": 1.7337679862976074,
+ "mean_token_accuracy": 0.5975207686424255,
+ "num_tokens": 2457600.0,
+ "step": 150
+ },
+ {
+ "epoch": 0.30303030303030304,
+ "eval_entropy": 1.6033287221385586,
+ "eval_loss": 1.6036633253097534,
+ "eval_mean_token_accuracy": 0.6329842450157288,
+ "eval_num_tokens": 2457600.0,
+ "eval_runtime": 43.7655,
+ "eval_samples_per_second": 22.621,
+ "eval_steps_per_second": 2.833,
+ "step": 150
+ },
+ {
+ "entropy": 2.0572290420532227,
+ "epoch": 0.30505050505050507,
+ "grad_norm": 2.0511579513549805,
+ "learning_rate": 4.696969696969698e-06,
+ "loss": 2.0196030139923096,
+ "mean_token_accuracy": 0.5519052147865295,
+ "num_tokens": 2473984.0,
+ "step": 151
+ },
+ {
+ "entropy": 1.5792397260665894,
+ "epoch": 0.30707070707070705,
+ "grad_norm": 2.048396587371826,
+ "learning_rate": 4.694949494949496e-06,
+ "loss": 1.575985074043274,
+ "mean_token_accuracy": 0.623961865901947,
+ "num_tokens": 2490368.0,
+ "step": 152
+ },
+ {
+ "entropy": 1.487528681755066,
+ "epoch": 0.3090909090909091,
+ "grad_norm": 2.1903791427612305,
+ "learning_rate": 4.692929292929294e-06,
+ "loss": 1.479973316192627,
+ "mean_token_accuracy": 0.6595017313957214,
+ "num_tokens": 2506752.0,
+ "step": 153
+ },
+ {
+ "entropy": 1.766797423362732,
+ "epoch": 0.3111111111111111,
+ "grad_norm": 2.1843011379241943,
+ "learning_rate": 4.690909090909092e-06,
+ "loss": 1.7993412017822266,
+ "mean_token_accuracy": 0.5965437293052673,
+ "num_tokens": 2523136.0,
+ "step": 154
+ },
+ {
+ "entropy": 1.7058254480361938,
+ "epoch": 0.31313131313131315,
+ "grad_norm": 2.204461097717285,
+ "learning_rate": 4.6888888888888895e-06,
+ "loss": 1.7346007823944092,
+ "mean_token_accuracy": 0.6077185869216919,
+ "num_tokens": 2539520.0,
+ "step": 155
+ },
+ {
+ "entropy": 1.4929591417312622,
+ "epoch": 0.3151515151515151,
+ "grad_norm": 2.1739237308502197,
+ "learning_rate": 4.6868686868686874e-06,
+ "loss": 1.4708621501922607,
+ "mean_token_accuracy": 0.6427088379859924,
+ "num_tokens": 2555904.0,
+ "step": 156
+ },
+ {
+ "entropy": 1.5797587633132935,
+ "epoch": 0.31717171717171716,
+ "grad_norm": 2.150561571121216,
+ "learning_rate": 4.684848484848485e-06,
+ "loss": 1.5921857357025146,
+ "mean_token_accuracy": 0.6262823939323425,
+ "num_tokens": 2572288.0,
+ "step": 157
+ },
+ {
+ "entropy": 1.6198230981826782,
+ "epoch": 0.3191919191919192,
+ "grad_norm": 2.061169385910034,
+ "learning_rate": 4.682828282828283e-06,
+ "loss": 1.6601009368896484,
+ "mean_token_accuracy": 0.619015634059906,
+ "num_tokens": 2588672.0,
+ "step": 158
+ },
+ {
+ "entropy": 1.4026126861572266,
+ "epoch": 0.3212121212121212,
+ "grad_norm": 2.066208839416504,
+ "learning_rate": 4.680808080808081e-06,
+ "loss": 1.4062483310699463,
+ "mean_token_accuracy": 0.6681729555130005,
+ "num_tokens": 2605056.0,
+ "step": 159
+ },
+ {
+ "entropy": 1.4608066082000732,
+ "epoch": 0.32323232323232326,
+ "grad_norm": 2.022627353668213,
+ "learning_rate": 4.678787878787879e-06,
+ "loss": 1.4644970893859863,
+ "mean_token_accuracy": 0.6533341407775879,
+ "num_tokens": 2621440.0,
+ "step": 160
+ },
+ {
+ "entropy": 1.222448468208313,
+ "epoch": 0.32525252525252524,
+ "grad_norm": 2.0399329662323,
+ "learning_rate": 4.676767676767677e-06,
+ "loss": 1.2547852993011475,
+ "mean_token_accuracy": 0.6925378441810608,
+ "num_tokens": 2637824.0,
+ "step": 161
+ },
+ {
+ "entropy": 1.8702255487442017,
+ "epoch": 0.32727272727272727,
+ "grad_norm": 2.177307367324829,
+ "learning_rate": 4.674747474747476e-06,
+ "loss": 1.9041712284088135,
+ "mean_token_accuracy": 0.5754152536392212,
+ "num_tokens": 2654208.0,
+ "step": 162
+ },
+ {
+ "entropy": 1.6861947774887085,
+ "epoch": 0.3292929292929293,
+ "grad_norm": 2.009544610977173,
+ "learning_rate": 4.6727272727272735e-06,
+ "loss": 1.7079355716705322,
+ "mean_token_accuracy": 0.615290641784668,
+ "num_tokens": 2670592.0,
+ "step": 163
+ },
+ {
+ "entropy": 1.3880327939987183,
+ "epoch": 0.33131313131313134,
+ "grad_norm": 2.16359543800354,
+ "learning_rate": 4.670707070707071e-06,
+ "loss": 1.4220571517944336,
+ "mean_token_accuracy": 0.6604787707328796,
+ "num_tokens": 2686976.0,
+ "step": 164
+ },
+ {
+ "entropy": 1.5760643482208252,
+ "epoch": 0.3333333333333333,
+ "grad_norm": 2.09773325920105,
+ "learning_rate": 4.668686868686869e-06,
+ "loss": 1.6036784648895264,
+ "mean_token_accuracy": 0.6267098188400269,
+ "num_tokens": 2703360.0,
+ "step": 165
+ },
+ {
+ "entropy": 1.078303575515747,
+ "epoch": 0.33535353535353535,
+ "grad_norm": 1.7760599851608276,
+ "learning_rate": 4.666666666666667e-06,
+ "loss": 1.0612695217132568,
+ "mean_token_accuracy": 0.7349169254302979,
+ "num_tokens": 2719744.0,
+ "step": 166
+ },
+ {
+ "entropy": 1.3722130060195923,
+ "epoch": 0.3373737373737374,
+ "grad_norm": 2.0816409587860107,
+ "learning_rate": 4.664646464646465e-06,
+ "loss": 1.3693504333496094,
+ "mean_token_accuracy": 0.6682950854301453,
+ "num_tokens": 2736128.0,
+ "step": 167
+ },
+ {
+ "entropy": 1.8752760887145996,
+ "epoch": 0.3393939393939394,
+ "grad_norm": 2.1670358180999756,
+ "learning_rate": 4.662626262626263e-06,
+ "loss": 1.8874578475952148,
+ "mean_token_accuracy": 0.5881778001785278,
+ "num_tokens": 2752512.0,
+ "step": 168
+ },
+ {
+ "entropy": 1.7384581565856934,
+ "epoch": 0.3414141414141414,
+ "grad_norm": 2.0072429180145264,
+ "learning_rate": 4.660606060606061e-06,
+ "loss": 1.7491583824157715,
+ "mean_token_accuracy": 0.6041157841682434,
+ "num_tokens": 2768896.0,
+ "step": 169
+ },
+ {
+ "entropy": 1.6651691198349,
+ "epoch": 0.3434343434343434,
+ "grad_norm": 2.0812578201293945,
+ "learning_rate": 4.658585858585859e-06,
+ "loss": 1.6808438301086426,
+ "mean_token_accuracy": 0.6273204684257507,
+ "num_tokens": 2785280.0,
+ "step": 170
+ },
+ {
+ "entropy": 1.3426616191864014,
+ "epoch": 0.34545454545454546,
+ "grad_norm": 1.9108822345733643,
+ "learning_rate": 4.656565656565657e-06,
+ "loss": 1.3404264450073242,
+ "mean_token_accuracy": 0.6703101992607117,
+ "num_tokens": 2801664.0,
+ "step": 171
+ },
+ {
+ "entropy": 1.631722092628479,
+ "epoch": 0.3474747474747475,
+ "grad_norm": 2.07317852973938,
+ "learning_rate": 4.654545454545455e-06,
+ "loss": 1.6545813083648682,
+ "mean_token_accuracy": 0.6143136024475098,
+ "num_tokens": 2818048.0,
+ "step": 172
+ },
+ {
+ "entropy": 1.3678289651870728,
+ "epoch": 0.34949494949494947,
+ "grad_norm": 1.8517454862594604,
+ "learning_rate": 4.652525252525253e-06,
+ "loss": 1.3759769201278687,
+ "mean_token_accuracy": 0.6693942546844482,
+ "num_tokens": 2834432.0,
+ "step": 173
+ },
+ {
+ "entropy": 1.3927773237228394,
+ "epoch": 0.3515151515151515,
+ "grad_norm": 2.1304867267608643,
+ "learning_rate": 4.650505050505051e-06,
+ "loss": 1.4104211330413818,
+ "mean_token_accuracy": 0.6612725853919983,
+ "num_tokens": 2850816.0,
+ "step": 174
+ },
+ {
+ "entropy": 1.3352863788604736,
+ "epoch": 0.35353535353535354,
+ "grad_norm": 2.2788777351379395,
+ "learning_rate": 4.648484848484849e-06,
+ "loss": 1.3665719032287598,
+ "mean_token_accuracy": 0.6640205383300781,
+ "num_tokens": 2867200.0,
+ "step": 175
+ },
+ {
+ "entropy": 1.6800754070281982,
+ "epoch": 0.35555555555555557,
+ "grad_norm": 2.030787944793701,
+ "learning_rate": 4.646464646464647e-06,
+ "loss": 1.7239181995391846,
+ "mean_token_accuracy": 0.6193209290504456,
+ "num_tokens": 2883584.0,
+ "step": 176
+ },
+ {
+ "entropy": 1.2852731943130493,
+ "epoch": 0.3575757575757576,
+ "grad_norm": 1.9148248434066772,
+ "learning_rate": 4.644444444444445e-06,
+ "loss": 1.2934880256652832,
+ "mean_token_accuracy": 0.6798363327980042,
+ "num_tokens": 2899968.0,
+ "step": 177
+ },
+ {
+ "entropy": 1.6366593837738037,
+ "epoch": 0.3595959595959596,
+ "grad_norm": 2.2264015674591064,
+ "learning_rate": 4.642424242424243e-06,
+ "loss": 1.632002592086792,
+ "mean_token_accuracy": 0.6180385947227478,
+ "num_tokens": 2916352.0,
+ "step": 178
+ },
+ {
+ "entropy": 1.3935565948486328,
+ "epoch": 0.3616161616161616,
+ "grad_norm": 2.0410475730895996,
+ "learning_rate": 4.6404040404040406e-06,
+ "loss": 1.407370686531067,
+ "mean_token_accuracy": 0.6722032427787781,
+ "num_tokens": 2932736.0,
+ "step": 179
+ },
+ {
+ "entropy": 1.0952510833740234,
+ "epoch": 0.36363636363636365,
+ "grad_norm": 1.937270164489746,
+ "learning_rate": 4.6383838383838384e-06,
+ "loss": 1.1101973056793213,
+ "mean_token_accuracy": 0.730947732925415,
+ "num_tokens": 2949120.0,
+ "step": 180
+ },
+ {
+ "entropy": 1.184926152229309,
+ "epoch": 0.3656565656565657,
+ "grad_norm": 1.9867528676986694,
+ "learning_rate": 4.636363636363636e-06,
+ "loss": 1.1932196617126465,
+ "mean_token_accuracy": 0.7085368633270264,
+ "num_tokens": 2965504.0,
+ "step": 181
+ },
+ {
+ "entropy": 1.4552072286605835,
+ "epoch": 0.36767676767676766,
+ "grad_norm": 1.9395766258239746,
+ "learning_rate": 4.634343434343434e-06,
+ "loss": 1.4559956789016724,
+ "mean_token_accuracy": 0.6534562706947327,
+ "num_tokens": 2981888.0,
+ "step": 182
+ },
+ {
+ "entropy": 1.6437443494796753,
+ "epoch": 0.3696969696969697,
+ "grad_norm": 2.1257567405700684,
+ "learning_rate": 4.632323232323233e-06,
+ "loss": 1.6260980367660522,
+ "mean_token_accuracy": 0.625,
+ "num_tokens": 2998272.0,
+ "step": 183
+ },
+ {
+ "entropy": 1.8264633417129517,
+ "epoch": 0.3717171717171717,
+ "grad_norm": 2.187041759490967,
+ "learning_rate": 4.630303030303031e-06,
+ "loss": 1.8431676626205444,
+ "mean_token_accuracy": 0.60332190990448,
+ "num_tokens": 3014656.0,
+ "step": 184
+ },
+ {
+ "entropy": 1.1121351718902588,
+ "epoch": 0.37373737373737376,
+ "grad_norm": 1.8460402488708496,
+ "learning_rate": 4.628282828282829e-06,
+ "loss": 1.1054425239562988,
+ "mean_token_accuracy": 0.7254518866539001,
+ "num_tokens": 3031040.0,
+ "step": 185
+ },
+ {
+ "entropy": 1.3531955480575562,
+ "epoch": 0.37575757575757573,
+ "grad_norm": 2.3463470935821533,
+ "learning_rate": 4.626262626262627e-06,
+ "loss": 1.359381914138794,
+ "mean_token_accuracy": 0.6610894203186035,
+ "num_tokens": 3047424.0,
+ "step": 186
+ },
+ {
+ "entropy": 1.2131577730178833,
+ "epoch": 0.37777777777777777,
+ "grad_norm": 1.9595171213150024,
+ "learning_rate": 4.6242424242424245e-06,
+ "loss": 1.2312005758285522,
+ "mean_token_accuracy": 0.6979115605354309,
+ "num_tokens": 3063808.0,
+ "step": 187
+ },
+ {
+ "entropy": 1.5441125631332397,
+ "epoch": 0.3797979797979798,
+ "grad_norm": 2.2971343994140625,
+ "learning_rate": 4.622222222222222e-06,
+ "loss": 1.5923339128494263,
+ "mean_token_accuracy": 0.6297020316123962,
+ "num_tokens": 3080192.0,
+ "step": 188
+ },
+ {
+ "entropy": 1.7998323440551758,
+ "epoch": 0.38181818181818183,
+ "grad_norm": 2.010552167892456,
+ "learning_rate": 4.62020202020202e-06,
+ "loss": 1.8257204294204712,
+ "mean_token_accuracy": 0.5997191071510315,
+ "num_tokens": 3096576.0,
+ "step": 189
+ },
+ {
+ "entropy": 1.389051914215088,
+ "epoch": 0.3838383838383838,
+ "grad_norm": 2.1183855533599854,
+ "learning_rate": 4.618181818181818e-06,
+ "loss": 1.431575059890747,
+ "mean_token_accuracy": 0.6537005305290222,
+ "num_tokens": 3112960.0,
+ "step": 190
+ },
+ {
+ "entropy": 2.0066142082214355,
+ "epoch": 0.38585858585858585,
+ "grad_norm": 2.061399221420288,
+ "learning_rate": 4.616161616161616e-06,
+ "loss": 2.0477406978607178,
+ "mean_token_accuracy": 0.5454934239387512,
+ "num_tokens": 3129344.0,
+ "step": 191
+ },
+ {
+ "entropy": 1.3063198328018188,
+ "epoch": 0.3878787878787879,
+ "grad_norm": 1.9490301609039307,
+ "learning_rate": 4.614141414141414e-06,
+ "loss": 1.3293564319610596,
+ "mean_token_accuracy": 0.6797142028808594,
+ "num_tokens": 3145728.0,
+ "step": 192
+ },
+ {
+ "entropy": 1.3636538982391357,
+ "epoch": 0.3898989898989899,
+ "grad_norm": 2.025458574295044,
+ "learning_rate": 4.612121212121212e-06,
+ "loss": 1.3736209869384766,
+ "mean_token_accuracy": 0.679286777973175,
+ "num_tokens": 3162112.0,
+ "step": 193
+ },
+ {
+ "entropy": 1.5669925212860107,
+ "epoch": 0.39191919191919194,
+ "grad_norm": 2.064033269882202,
+ "learning_rate": 4.6101010101010106e-06,
+ "loss": 1.5587083101272583,
+ "mean_token_accuracy": 0.6267709136009216,
+ "num_tokens": 3178496.0,
+ "step": 194
+ },
+ {
+ "entropy": 1.4489716291427612,
+ "epoch": 0.3939393939393939,
+ "grad_norm": 2.0174880027770996,
+ "learning_rate": 4.6080808080808085e-06,
+ "loss": 1.4331417083740234,
+ "mean_token_accuracy": 0.6571812629699707,
+ "num_tokens": 3194880.0,
+ "step": 195
+ },
+ {
+ "entropy": 2.026416778564453,
+ "epoch": 0.39595959595959596,
+ "grad_norm": 2.1820878982543945,
+ "learning_rate": 4.606060606060606e-06,
+ "loss": 2.029003143310547,
+ "mean_token_accuracy": 0.563568651676178,
+ "num_tokens": 3211264.0,
+ "step": 196
+ },
+ {
+ "entropy": 2.1844022274017334,
+ "epoch": 0.397979797979798,
+ "grad_norm": 2.175349473953247,
+ "learning_rate": 4.604040404040404e-06,
+ "loss": 2.2053825855255127,
+ "mean_token_accuracy": 0.5553248524665833,
+ "num_tokens": 3227648.0,
+ "step": 197
+ },
+ {
+ "entropy": 1.6750906705856323,
+ "epoch": 0.4,
+ "grad_norm": 1.8300689458847046,
+ "learning_rate": 4.602020202020203e-06,
+ "loss": 1.692289113998413,
+ "mean_token_accuracy": 0.6345261335372925,
+ "num_tokens": 3244032.0,
+ "step": 198
+ },
+ {
+ "entropy": 1.6192772388458252,
+ "epoch": 0.402020202020202,
+ "grad_norm": 1.9788341522216797,
+ "learning_rate": 4.600000000000001e-06,
+ "loss": 1.628842830657959,
+ "mean_token_accuracy": 0.615229606628418,
+ "num_tokens": 3260416.0,
+ "step": 199
+ },
+ {
+ "entropy": 1.444870948791504,
+ "epoch": 0.40404040404040403,
+ "grad_norm": 2.05141544342041,
+ "learning_rate": 4.597979797979799e-06,
+ "loss": 1.4365839958190918,
+ "mean_token_accuracy": 0.652723491191864,
+ "num_tokens": 3276800.0,
+ "step": 200
+ },
+ {
+ "epoch": 0.40404040404040403,
+ "eval_entropy": 1.5766179907706477,
+ "eval_loss": 1.5878442525863647,
+ "eval_mean_token_accuracy": 0.6355829551335304,
+ "eval_num_tokens": 3276800.0,
+ "eval_runtime": 43.7272,
+ "eval_samples_per_second": 22.64,
+ "eval_steps_per_second": 2.836,
+ "step": 200
+ },
+ {
+ "entropy": 1.3354029655456543,
+ "epoch": 0.40606060606060607,
+ "grad_norm": 2.012895345687866,
+ "learning_rate": 4.595959595959597e-06,
+ "loss": 1.3289990425109863,
+ "mean_token_accuracy": 0.685271143913269,
+ "num_tokens": 3293184.0,
+ "step": 201
+ },
+ {
+ "entropy": 1.1990011930465698,
+ "epoch": 0.4080808080808081,
+ "grad_norm": 1.9462409019470215,
+ "learning_rate": 4.5939393939393945e-06,
+ "loss": 1.224869728088379,
+ "mean_token_accuracy": 0.6976673007011414,
+ "num_tokens": 3309568.0,
+ "step": 202
+ },
+ {
+ "entropy": 1.6442710161209106,
+ "epoch": 0.4101010101010101,
+ "grad_norm": 2.064532995223999,
+ "learning_rate": 4.591919191919192e-06,
+ "loss": 1.6800963878631592,
+ "mean_token_accuracy": 0.6111993193626404,
+ "num_tokens": 3325952.0,
+ "step": 203
+ },
+ {
+ "entropy": 1.8336306810379028,
+ "epoch": 0.4121212121212121,
+ "grad_norm": 2.478731393814087,
+ "learning_rate": 4.58989898989899e-06,
+ "loss": 1.8518702983856201,
+ "mean_token_accuracy": 0.6021006107330322,
+ "num_tokens": 3342336.0,
+ "step": 204
+ },
+ {
+ "entropy": 1.6767220497131348,
+ "epoch": 0.41414141414141414,
+ "grad_norm": 2.0676512718200684,
+ "learning_rate": 4.587878787878788e-06,
+ "loss": 1.674522042274475,
+ "mean_token_accuracy": 0.6178553700447083,
+ "num_tokens": 3358720.0,
+ "step": 205
+ },
+ {
+ "entropy": 1.7236497402191162,
+ "epoch": 0.4161616161616162,
+ "grad_norm": 2.057074785232544,
+ "learning_rate": 4.585858585858586e-06,
+ "loss": 1.725832462310791,
+ "mean_token_accuracy": 0.620114803314209,
+ "num_tokens": 3375104.0,
+ "step": 206
+ },
+ {
+ "entropy": 1.6493065357208252,
+ "epoch": 0.41818181818181815,
+ "grad_norm": 2.0946099758148193,
+ "learning_rate": 4.583838383838384e-06,
+ "loss": 1.6826295852661133,
+ "mean_token_accuracy": 0.6251221299171448,
+ "num_tokens": 3391488.0,
+ "step": 207
+ },
+ {
+ "entropy": 1.3565926551818848,
+ "epoch": 0.4202020202020202,
+ "grad_norm": 1.9611284732818604,
+ "learning_rate": 4.581818181818183e-06,
+ "loss": 1.346787452697754,
+ "mean_token_accuracy": 0.685332179069519,
+ "num_tokens": 3407872.0,
+ "step": 208
+ },
+ {
+ "entropy": 1.4401211738586426,
+ "epoch": 0.4222222222222222,
+ "grad_norm": 1.8614766597747803,
+ "learning_rate": 4.579797979797981e-06,
+ "loss": 1.4717891216278076,
+ "mean_token_accuracy": 0.650036633014679,
+ "num_tokens": 3424256.0,
+ "step": 209
+ },
+ {
+ "entropy": 1.194276213645935,
+ "epoch": 0.42424242424242425,
+ "grad_norm": 2.0625810623168945,
+ "learning_rate": 4.5777777777777785e-06,
+ "loss": 1.2106804847717285,
+ "mean_token_accuracy": 0.700537383556366,
+ "num_tokens": 3440640.0,
+ "step": 210
+ },
+ {
+ "entropy": 1.7622313499450684,
+ "epoch": 0.4262626262626263,
+ "grad_norm": 2.330610752105713,
+ "learning_rate": 4.575757575757576e-06,
+ "loss": 1.7872710227966309,
+ "mean_token_accuracy": 0.598436713218689,
+ "num_tokens": 3457024.0,
+ "step": 211
+ },
+ {
+ "entropy": 1.409427523612976,
+ "epoch": 0.42828282828282827,
+ "grad_norm": 2.0988259315490723,
+ "learning_rate": 4.573737373737374e-06,
+ "loss": 1.416553020477295,
+ "mean_token_accuracy": 0.6682950854301453,
+ "num_tokens": 3473408.0,
+ "step": 212
+ },
+ {
+ "entropy": 1.6856107711791992,
+ "epoch": 0.4303030303030303,
+ "grad_norm": 2.341475009918213,
+ "learning_rate": 4.571717171717172e-06,
+ "loss": 1.692287802696228,
+ "mean_token_accuracy": 0.6024059653282166,
+ "num_tokens": 3489792.0,
+ "step": 213
+ },
+ {
+ "entropy": 1.997343897819519,
+ "epoch": 0.43232323232323233,
+ "grad_norm": 2.200498104095459,
+ "learning_rate": 4.56969696969697e-06,
+ "loss": 2.026289939880371,
+ "mean_token_accuracy": 0.5781631469726562,
+ "num_tokens": 3506176.0,
+ "step": 214
+ },
+ {
+ "entropy": 1.5175039768218994,
+ "epoch": 0.43434343434343436,
+ "grad_norm": 2.105257272720337,
+ "learning_rate": 4.567676767676768e-06,
+ "loss": 1.521841049194336,
+ "mean_token_accuracy": 0.6404494643211365,
+ "num_tokens": 3522560.0,
+ "step": 215
+ },
+ {
+ "entropy": 1.3616528511047363,
+ "epoch": 0.43636363636363634,
+ "grad_norm": 2.0035297870635986,
+ "learning_rate": 4.565656565656566e-06,
+ "loss": 1.3674360513687134,
+ "mean_token_accuracy": 0.6771494746208191,
+ "num_tokens": 3538944.0,
+ "step": 216
+ },
+ {
+ "entropy": 1.42499840259552,
+ "epoch": 0.4383838383838384,
+ "grad_norm": 1.9115629196166992,
+ "learning_rate": 4.563636363636364e-06,
+ "loss": 1.4129266738891602,
+ "mean_token_accuracy": 0.6686614751815796,
+ "num_tokens": 3555328.0,
+ "step": 217
+ },
+ {
+ "entropy": 1.2844709157943726,
+ "epoch": 0.4404040404040404,
+ "grad_norm": 2.3313121795654297,
+ "learning_rate": 4.5616161616161616e-06,
+ "loss": 1.315006971359253,
+ "mean_token_accuracy": 0.681546151638031,
+ "num_tokens": 3571712.0,
+ "step": 218
+ },
+ {
+ "entropy": 1.5227075815200806,
+ "epoch": 0.44242424242424244,
+ "grad_norm": 2.1849124431610107,
+ "learning_rate": 4.55959595959596e-06,
+ "loss": 1.517989158630371,
+ "mean_token_accuracy": 0.6466169953346252,
+ "num_tokens": 3588096.0,
+ "step": 219
+ },
+ {
+ "entropy": 1.494642734527588,
+ "epoch": 0.4444444444444444,
+ "grad_norm": 2.3109116554260254,
+ "learning_rate": 4.557575757575758e-06,
+ "loss": 1.525421142578125,
+ "mean_token_accuracy": 0.6392281651496887,
+ "num_tokens": 3604480.0,
+ "step": 220
+ },
+ {
+ "entropy": 1.6428948640823364,
+ "epoch": 0.44646464646464645,
+ "grad_norm": 2.1182680130004883,
+ "learning_rate": 4.555555555555556e-06,
+ "loss": 1.6634926795959473,
+ "mean_token_accuracy": 0.6207864880561829,
+ "num_tokens": 3620864.0,
+ "step": 221
+ },
+ {
+ "entropy": 1.7323675155639648,
+ "epoch": 0.4484848484848485,
+ "grad_norm": 2.2620837688446045,
+ "learning_rate": 4.553535353535354e-06,
+ "loss": 1.7312631607055664,
+ "mean_token_accuracy": 0.6138250827789307,
+ "num_tokens": 3637248.0,
+ "step": 222
+ },
+ {
+ "entropy": 1.7863894701004028,
+ "epoch": 0.4505050505050505,
+ "grad_norm": 2.1416971683502197,
+ "learning_rate": 4.551515151515152e-06,
+ "loss": 1.799318552017212,
+ "mean_token_accuracy": 0.5906203985214233,
+ "num_tokens": 3653632.0,
+ "step": 223
+ },
+ {
+ "entropy": 1.7404330968856812,
+ "epoch": 0.45252525252525255,
+ "grad_norm": 2.0814504623413086,
+ "learning_rate": 4.54949494949495e-06,
+ "loss": 1.742197036743164,
+ "mean_token_accuracy": 0.6050317287445068,
+ "num_tokens": 3670016.0,
+ "step": 224
+ },
+ {
+ "entropy": 1.4387869834899902,
+ "epoch": 0.45454545454545453,
+ "grad_norm": 2.1386022567749023,
+ "learning_rate": 4.547474747474748e-06,
+ "loss": 1.4602317810058594,
+ "mean_token_accuracy": 0.6502198576927185,
+ "num_tokens": 3686400.0,
+ "step": 225
+ },
+ {
+ "entropy": 1.2733348608016968,
+ "epoch": 0.45656565656565656,
+ "grad_norm": 2.02687668800354,
+ "learning_rate": 4.5454545454545455e-06,
+ "loss": 1.2683714628219604,
+ "mean_token_accuracy": 0.6966902613639832,
+ "num_tokens": 3702784.0,
+ "step": 226
+ },
+ {
+ "entropy": 1.4554595947265625,
+ "epoch": 0.4585858585858586,
+ "grad_norm": 2.169268846511841,
+ "learning_rate": 4.543434343434343e-06,
+ "loss": 1.4541833400726318,
+ "mean_token_accuracy": 0.6522960662841797,
+ "num_tokens": 3719168.0,
+ "step": 227
+ },
+ {
+ "entropy": 1.487573266029358,
+ "epoch": 0.46060606060606063,
+ "grad_norm": 1.9726165533065796,
+ "learning_rate": 4.541414141414141e-06,
+ "loss": 1.5024409294128418,
+ "mean_token_accuracy": 0.6509526371955872,
+ "num_tokens": 3735552.0,
+ "step": 228
+ },
+ {
+ "entropy": 1.3991138935089111,
+ "epoch": 0.4626262626262626,
+ "grad_norm": 2.0992283821105957,
+ "learning_rate": 4.539393939393939e-06,
+ "loss": 1.4016860723495483,
+ "mean_token_accuracy": 0.6639594435691833,
+ "num_tokens": 3751936.0,
+ "step": 229
+ },
+ {
+ "entropy": 1.6564134359359741,
+ "epoch": 0.46464646464646464,
+ "grad_norm": 2.047656297683716,
+ "learning_rate": 4.537373737373738e-06,
+ "loss": 1.6598721742630005,
+ "mean_token_accuracy": 0.6166951656341553,
+ "num_tokens": 3768320.0,
+ "step": 230
+ },
+ {
+ "entropy": 1.5165014266967773,
+ "epoch": 0.4666666666666667,
+ "grad_norm": 2.079996109008789,
+ "learning_rate": 4.535353535353536e-06,
+ "loss": 1.4980010986328125,
+ "mean_token_accuracy": 0.6370908617973328,
+ "num_tokens": 3784704.0,
+ "step": 231
+ },
+ {
+ "entropy": 1.5409225225448608,
+ "epoch": 0.4686868686868687,
+ "grad_norm": 2.0996923446655273,
+ "learning_rate": 4.533333333333334e-06,
+ "loss": 1.5504257678985596,
+ "mean_token_accuracy": 0.6465559601783752,
+ "num_tokens": 3801088.0,
+ "step": 232
+ },
+ {
+ "entropy": 1.3590043783187866,
+ "epoch": 0.4707070707070707,
+ "grad_norm": 2.14617919921875,
+ "learning_rate": 4.531313131313132e-06,
+ "loss": 1.3581812381744385,
+ "mean_token_accuracy": 0.6683561205863953,
+ "num_tokens": 3817472.0,
+ "step": 233
+ },
+ {
+ "entropy": 1.641785740852356,
+ "epoch": 0.4727272727272727,
+ "grad_norm": 1.9499926567077637,
+ "learning_rate": 4.5292929292929295e-06,
+ "loss": 1.6653470993041992,
+ "mean_token_accuracy": 0.6319614052772522,
+ "num_tokens": 3833856.0,
+ "step": 234
+ },
+ {
+ "entropy": 1.4700758457183838,
+ "epoch": 0.47474747474747475,
+ "grad_norm": 1.9411437511444092,
+ "learning_rate": 4.527272727272727e-06,
+ "loss": 1.4762463569641113,
+ "mean_token_accuracy": 0.6631656289100647,
+ "num_tokens": 3850240.0,
+ "step": 235
+ },
+ {
+ "entropy": 1.5418941974639893,
+ "epoch": 0.4767676767676768,
+ "grad_norm": 2.3191940784454346,
+ "learning_rate": 4.525252525252526e-06,
+ "loss": 1.5151214599609375,
+ "mean_token_accuracy": 0.6326331496238708,
+ "num_tokens": 3866624.0,
+ "step": 236
+ },
+ {
+ "entropy": 2.1102776527404785,
+ "epoch": 0.47878787878787876,
+ "grad_norm": 2.2287707328796387,
+ "learning_rate": 4.523232323232324e-06,
+ "loss": 2.106567859649658,
+ "mean_token_accuracy": 0.5506839156150818,
+ "num_tokens": 3883008.0,
+ "step": 237
+ },
+ {
+ "entropy": 1.7448827028274536,
+ "epoch": 0.4808080808080808,
+ "grad_norm": 2.2352893352508545,
+ "learning_rate": 4.521212121212122e-06,
+ "loss": 1.7495017051696777,
+ "mean_token_accuracy": 0.60326087474823,
+ "num_tokens": 3899392.0,
+ "step": 238
+ },
+ {
+ "entropy": 1.315240740776062,
+ "epoch": 0.48282828282828283,
+ "grad_norm": 1.9933631420135498,
+ "learning_rate": 4.51919191919192e-06,
+ "loss": 1.315302848815918,
+ "mean_token_accuracy": 0.686431348323822,
+ "num_tokens": 3915776.0,
+ "step": 239
+ },
+ {
+ "entropy": 1.5851637125015259,
+ "epoch": 0.48484848484848486,
+ "grad_norm": 2.153303623199463,
+ "learning_rate": 4.517171717171718e-06,
+ "loss": 1.5947662591934204,
+ "mean_token_accuracy": 0.6297020316123962,
+ "num_tokens": 3932160.0,
+ "step": 240
+ },
+ {
+ "entropy": 1.8050944805145264,
+ "epoch": 0.4868686868686869,
+ "grad_norm": 2.025022506713867,
+ "learning_rate": 4.5151515151515155e-06,
+ "loss": 1.8051010370254517,
+ "mean_token_accuracy": 0.6055202484130859,
+ "num_tokens": 3948544.0,
+ "step": 241
+ },
+ {
+ "entropy": 1.518296718597412,
+ "epoch": 0.4888888888888889,
+ "grad_norm": 2.1021833419799805,
+ "learning_rate": 4.513131313131313e-06,
+ "loss": 1.5396809577941895,
+ "mean_token_accuracy": 0.6359916925430298,
+ "num_tokens": 3964928.0,
+ "step": 242
+ },
+ {
+ "entropy": 1.518903136253357,
+ "epoch": 0.4909090909090909,
+ "grad_norm": 2.0960140228271484,
+ "learning_rate": 4.511111111111111e-06,
+ "loss": 1.5528055429458618,
+ "mean_token_accuracy": 0.6224963068962097,
+ "num_tokens": 3981312.0,
+ "step": 243
+ },
+ {
+ "entropy": 1.6960705518722534,
+ "epoch": 0.49292929292929294,
+ "grad_norm": 1.8580718040466309,
+ "learning_rate": 4.50909090909091e-06,
+ "loss": 1.7192862033843994,
+ "mean_token_accuracy": 0.6259770393371582,
+ "num_tokens": 3997696.0,
+ "step": 244
+ },
+ {
+ "entropy": 1.4633033275604248,
+ "epoch": 0.494949494949495,
+ "grad_norm": 1.8983049392700195,
+ "learning_rate": 4.507070707070708e-06,
+ "loss": 1.451701283454895,
+ "mean_token_accuracy": 0.6669516563415527,
+ "num_tokens": 4014080.0,
+ "step": 245
+ },
+ {
+ "entropy": 1.6508095264434814,
+ "epoch": 0.49696969696969695,
+ "grad_norm": 2.298679828643799,
+ "learning_rate": 4.505050505050506e-06,
+ "loss": 1.6596897840499878,
+ "mean_token_accuracy": 0.6115046143531799,
+ "num_tokens": 4030464.0,
+ "step": 246
+ },
+ {
+ "entropy": 1.313779354095459,
+ "epoch": 0.498989898989899,
+ "grad_norm": 1.9894335269927979,
+ "learning_rate": 4.503030303030304e-06,
+ "loss": 1.3285409212112427,
+ "mean_token_accuracy": 0.6802027225494385,
+ "num_tokens": 4046848.0,
+ "step": 247
+ },
+ {
+ "entropy": 1.5139521360397339,
+ "epoch": 0.501010101010101,
+ "grad_norm": 2.028320789337158,
+ "learning_rate": 4.501010101010102e-06,
+ "loss": 1.5366487503051758,
+ "mean_token_accuracy": 0.6416707634925842,
+ "num_tokens": 4063232.0,
+ "step": 248
+ },
+ {
+ "entropy": 1.5277045965194702,
+ "epoch": 0.503030303030303,
+ "grad_norm": 2.3160979747772217,
+ "learning_rate": 4.4989898989898995e-06,
+ "loss": 1.5571036338806152,
+ "mean_token_accuracy": 0.6436858773231506,
+ "num_tokens": 4079616.0,
+ "step": 249
+ },
+ {
+ "entropy": 1.6084188222885132,
+ "epoch": 0.5050505050505051,
+ "grad_norm": 2.20550274848938,
+ "learning_rate": 4.496969696969697e-06,
+ "loss": 1.622127652168274,
+ "mean_token_accuracy": 0.6284196376800537,
+ "num_tokens": 4096000.0,
+ "step": 250
+ },
+ {
+ "epoch": 0.5050505050505051,
+ "eval_entropy": 1.5547234262189558,
+ "eval_loss": 1.5764786005020142,
+ "eval_mean_token_accuracy": 0.6375306306346771,
+ "eval_num_tokens": 4096000.0,
+ "eval_runtime": 43.7607,
+ "eval_samples_per_second": 22.623,
+ "eval_steps_per_second": 2.834,
+ "step": 250
+ },
+ {
+ "entropy": 1.4309135675430298,
+ "epoch": 0.5070707070707071,
+ "grad_norm": 2.080864906311035,
+ "learning_rate": 4.494949494949495e-06,
+ "loss": 1.4481091499328613,
+ "mean_token_accuracy": 0.6694552898406982,
+ "num_tokens": 4112384.0,
+ "step": 251
+ },
+ {
+ "entropy": 1.4067270755767822,
+ "epoch": 0.509090909090909,
+ "grad_norm": 2.036475419998169,
+ "learning_rate": 4.492929292929293e-06,
+ "loss": 1.4035298824310303,
+ "mean_token_accuracy": 0.6547996997833252,
+ "num_tokens": 4128768.0,
+ "step": 252
+ },
+ {
+ "entropy": 2.1751608848571777,
+ "epoch": 0.5111111111111111,
+ "grad_norm": 1.953995943069458,
+ "learning_rate": 4.490909090909091e-06,
+ "loss": 2.1616692543029785,
+ "mean_token_accuracy": 0.5592941045761108,
+ "num_tokens": 4145152.0,
+ "step": 253
+ },
+ {
+ "entropy": 1.45353102684021,
+ "epoch": 0.5131313131313131,
+ "grad_norm": 2.084437370300293,
+ "learning_rate": 4.488888888888889e-06,
+ "loss": 1.4633586406707764,
+ "mean_token_accuracy": 0.6620664596557617,
+ "num_tokens": 4161536.0,
+ "step": 254
+ },
+ {
+ "entropy": 1.64310884475708,
+ "epoch": 0.5151515151515151,
+ "grad_norm": 2.0700111389160156,
+ "learning_rate": 4.486868686868688e-06,
+ "loss": 1.6868078708648682,
+ "mean_token_accuracy": 0.616328775882721,
+ "num_tokens": 4177920.0,
+ "step": 255
+ },
+ {
+ "entropy": 1.5113472938537598,
+ "epoch": 0.5171717171717172,
+ "grad_norm": 2.102180242538452,
+ "learning_rate": 4.4848484848484855e-06,
+ "loss": 1.5240120887756348,
+ "mean_token_accuracy": 0.638067901134491,
+ "num_tokens": 4194304.0,
+ "step": 256
+ },
+ {
+ "entropy": 1.5752851963043213,
+ "epoch": 0.5191919191919192,
+ "grad_norm": 2.026737689971924,
+ "learning_rate": 4.4828282828282834e-06,
+ "loss": 1.6041791439056396,
+ "mean_token_accuracy": 0.6275647282600403,
+ "num_tokens": 4210688.0,
+ "step": 257
+ },
+ {
+ "entropy": 1.548423171043396,
+ "epoch": 0.5212121212121212,
+ "grad_norm": 2.0578935146331787,
+ "learning_rate": 4.480808080808081e-06,
+ "loss": 1.563529372215271,
+ "mean_token_accuracy": 0.6348925232887268,
+ "num_tokens": 4227072.0,
+ "step": 258
+ },
+ {
+ "entropy": 1.5324457883834839,
+ "epoch": 0.5232323232323233,
+ "grad_norm": 2.216235637664795,
+ "learning_rate": 4.478787878787879e-06,
+ "loss": 1.5559678077697754,
+ "mean_token_accuracy": 0.6300073266029358,
+ "num_tokens": 4243456.0,
+ "step": 259
+ },
+ {
+ "entropy": 1.5275540351867676,
+ "epoch": 0.5252525252525253,
+ "grad_norm": 2.037306070327759,
+ "learning_rate": 4.476767676767677e-06,
+ "loss": 1.541567087173462,
+ "mean_token_accuracy": 0.6424035429954529,
+ "num_tokens": 4259840.0,
+ "step": 260
+ },
+ {
+ "entropy": 1.6842185258865356,
+ "epoch": 0.5272727272727272,
+ "grad_norm": 2.0241005420684814,
+ "learning_rate": 4.474747474747475e-06,
+ "loss": 1.6819056272506714,
+ "mean_token_accuracy": 0.6102222800254822,
+ "num_tokens": 4276224.0,
+ "step": 261
+ },
+ {
+ "entropy": 1.5656248331069946,
+ "epoch": 0.5292929292929293,
+ "grad_norm": 1.993054986000061,
+ "learning_rate": 4.472727272727273e-06,
+ "loss": 1.6132277250289917,
+ "mean_token_accuracy": 0.634709358215332,
+ "num_tokens": 4292608.0,
+ "step": 262
+ },
+ {
+ "entropy": 1.6354466676712036,
+ "epoch": 0.5313131313131313,
+ "grad_norm": 2.06508731842041,
+ "learning_rate": 4.470707070707071e-06,
+ "loss": 1.6475149393081665,
+ "mean_token_accuracy": 0.6121763586997986,
+ "num_tokens": 4308992.0,
+ "step": 263
+ },
+ {
+ "entropy": 1.5267232656478882,
+ "epoch": 0.5333333333333333,
+ "grad_norm": 2.1227569580078125,
+ "learning_rate": 4.468686868686869e-06,
+ "loss": 1.5468671321868896,
+ "mean_token_accuracy": 0.6365413069725037,
+ "num_tokens": 4325376.0,
+ "step": 264
+ },
+ {
+ "entropy": 1.2252761125564575,
+ "epoch": 0.5353535353535354,
+ "grad_norm": 2.0596134662628174,
+ "learning_rate": 4.4666666666666665e-06,
+ "loss": 1.2242389917373657,
+ "mean_token_accuracy": 0.6973620057106018,
+ "num_tokens": 4341760.0,
+ "step": 265
+ },
+ {
+ "entropy": 1.6007431745529175,
+ "epoch": 0.5373737373737374,
+ "grad_norm": 1.9341918230056763,
+ "learning_rate": 4.464646464646465e-06,
+ "loss": 1.5989094972610474,
+ "mean_token_accuracy": 0.6284196376800537,
+ "num_tokens": 4358144.0,
+ "step": 266
+ },
+ {
+ "entropy": 1.2708780765533447,
+ "epoch": 0.5393939393939394,
+ "grad_norm": 4.594091415405273,
+ "learning_rate": 4.462626262626263e-06,
+ "loss": 1.3484312295913696,
+ "mean_token_accuracy": 0.6790425181388855,
+ "num_tokens": 4374528.0,
+ "step": 267
+ },
+ {
+ "entropy": 1.5423723459243774,
+ "epoch": 0.5414141414141415,
+ "grad_norm": 1.933602213859558,
+ "learning_rate": 4.460606060606061e-06,
+ "loss": 1.594527244567871,
+ "mean_token_accuracy": 0.6386785507202148,
+ "num_tokens": 4390912.0,
+ "step": 268
+ },
+ {
+ "entropy": 1.7213952541351318,
+ "epoch": 0.5434343434343434,
+ "grad_norm": 2.195904016494751,
+ "learning_rate": 4.458585858585859e-06,
+ "loss": 1.724353551864624,
+ "mean_token_accuracy": 0.6008182764053345,
+ "num_tokens": 4407296.0,
+ "step": 269
+ },
+ {
+ "entropy": 1.4772557020187378,
+ "epoch": 0.5454545454545454,
+ "grad_norm": 2.0990796089172363,
+ "learning_rate": 4.456565656565657e-06,
+ "loss": 1.486825942993164,
+ "mean_token_accuracy": 0.6507083773612976,
+ "num_tokens": 4423680.0,
+ "step": 270
+ },
+ {
+ "entropy": 1.6876367330551147,
+ "epoch": 0.5474747474747474,
+ "grad_norm": 2.1944479942321777,
+ "learning_rate": 4.454545454545455e-06,
+ "loss": 1.71107816696167,
+ "mean_token_accuracy": 0.6027112603187561,
+ "num_tokens": 4440064.0,
+ "step": 271
+ },
+ {
+ "entropy": 1.5480726957321167,
+ "epoch": 0.5494949494949495,
+ "grad_norm": 2.1579341888427734,
+ "learning_rate": 4.452525252525253e-06,
+ "loss": 1.544647216796875,
+ "mean_token_accuracy": 0.6345261335372925,
+ "num_tokens": 4456448.0,
+ "step": 272
+ },
+ {
+ "entropy": 1.6161645650863647,
+ "epoch": 0.5515151515151515,
+ "grad_norm": 1.981154203414917,
+ "learning_rate": 4.4505050505050505e-06,
+ "loss": 1.6109068393707275,
+ "mean_token_accuracy": 0.6318392753601074,
+ "num_tokens": 4472832.0,
+ "step": 273
+ },
+ {
+ "entropy": 1.2576326131820679,
+ "epoch": 0.5535353535353535,
+ "grad_norm": 1.95668625831604,
+ "learning_rate": 4.448484848484848e-06,
+ "loss": 1.2614656686782837,
+ "mean_token_accuracy": 0.7025524973869324,
+ "num_tokens": 4489216.0,
+ "step": 274
+ },
+ {
+ "entropy": 1.405206561088562,
+ "epoch": 0.5555555555555556,
+ "grad_norm": 2.0302884578704834,
+ "learning_rate": 4.446464646464646e-06,
+ "loss": 1.416546106338501,
+ "mean_token_accuracy": 0.6753786206245422,
+ "num_tokens": 4505600.0,
+ "step": 275
+ },
+ {
+ "entropy": 1.9038625955581665,
+ "epoch": 0.5575757575757576,
+ "grad_norm": 2.06475567817688,
+ "learning_rate": 4.444444444444444e-06,
+ "loss": 1.9273614883422852,
+ "mean_token_accuracy": 0.5813996195793152,
+ "num_tokens": 4521984.0,
+ "step": 276
+ },
+ {
+ "entropy": 1.7208465337753296,
+ "epoch": 0.5595959595959596,
+ "grad_norm": 2.0136189460754395,
+ "learning_rate": 4.442424242424243e-06,
+ "loss": 1.7260158061981201,
+ "mean_token_accuracy": 0.6006350517272949,
+ "num_tokens": 4538368.0,
+ "step": 277
+ },
+ {
+ "entropy": 1.7570570707321167,
+ "epoch": 0.5616161616161616,
+ "grad_norm": 2.148594379425049,
+ "learning_rate": 4.440404040404041e-06,
+ "loss": 1.7799286842346191,
+ "mean_token_accuracy": 0.5943453907966614,
+ "num_tokens": 4554752.0,
+ "step": 278
+ },
+ {
+ "entropy": 1.6388157606124878,
+ "epoch": 0.5636363636363636,
+ "grad_norm": 2.1301987171173096,
+ "learning_rate": 4.438383838383839e-06,
+ "loss": 1.63419508934021,
+ "mean_token_accuracy": 0.6405715942382812,
+ "num_tokens": 4571136.0,
+ "step": 279
+ },
+ {
+ "entropy": 1.5902295112609863,
+ "epoch": 0.5656565656565656,
+ "grad_norm": 2.0526790618896484,
+ "learning_rate": 4.436363636363637e-06,
+ "loss": 1.6278276443481445,
+ "mean_token_accuracy": 0.615229606628418,
+ "num_tokens": 4587520.0,
+ "step": 280
+ },
+ {
+ "entropy": 1.5494027137756348,
+ "epoch": 0.5676767676767677,
+ "grad_norm": 1.9863859415054321,
+ "learning_rate": 4.434343434343435e-06,
+ "loss": 1.5622975826263428,
+ "mean_token_accuracy": 0.6261602640151978,
+ "num_tokens": 4603904.0,
+ "step": 281
+ },
+ {
+ "entropy": 1.2140685319900513,
+ "epoch": 0.5696969696969697,
+ "grad_norm": 2.4206902980804443,
+ "learning_rate": 4.432323232323233e-06,
+ "loss": 1.2389612197875977,
+ "mean_token_accuracy": 0.6943697929382324,
+ "num_tokens": 4620288.0,
+ "step": 282
+ },
+ {
+ "entropy": 1.4347270727157593,
+ "epoch": 0.5717171717171717,
+ "grad_norm": 2.0198237895965576,
+ "learning_rate": 4.430303030303031e-06,
+ "loss": 1.4648659229278564,
+ "mean_token_accuracy": 0.6573033928871155,
+ "num_tokens": 4636672.0,
+ "step": 283
+ },
+ {
+ "entropy": 1.3239331245422363,
+ "epoch": 0.5737373737373738,
+ "grad_norm": 1.9862704277038574,
+ "learning_rate": 4.428282828282829e-06,
+ "loss": 1.3520365953445435,
+ "mean_token_accuracy": 0.6852100491523743,
+ "num_tokens": 4653056.0,
+ "step": 284
+ },
+ {
+ "entropy": 1.7534631490707397,
+ "epoch": 0.5757575757575758,
+ "grad_norm": 2.0964176654815674,
+ "learning_rate": 4.426262626262627e-06,
+ "loss": 1.7659757137298584,
+ "mean_token_accuracy": 0.5975207686424255,
+ "num_tokens": 4669440.0,
+ "step": 285
+ },
+ {
+ "entropy": 1.1573433876037598,
+ "epoch": 0.5777777777777777,
+ "grad_norm": 2.0671567916870117,
+ "learning_rate": 4.424242424242425e-06,
+ "loss": 1.1479461193084717,
+ "mean_token_accuracy": 0.7143380641937256,
+ "num_tokens": 4685824.0,
+ "step": 286
+ },
+ {
+ "entropy": 1.2439504861831665,
+ "epoch": 0.5797979797979798,
+ "grad_norm": 2.0317327976226807,
+ "learning_rate": 4.422222222222223e-06,
+ "loss": 1.255782961845398,
+ "mean_token_accuracy": 0.691255509853363,
+ "num_tokens": 4702208.0,
+ "step": 287
+ },
+ {
+ "entropy": 1.5569473505020142,
+ "epoch": 0.5818181818181818,
+ "grad_norm": 2.0759670734405518,
+ "learning_rate": 4.4202020202020205e-06,
+ "loss": 1.5564974546432495,
+ "mean_token_accuracy": 0.6284807324409485,
+ "num_tokens": 4718592.0,
+ "step": 288
+ },
+ {
+ "entropy": 1.478676199913025,
+ "epoch": 0.5838383838383838,
+ "grad_norm": 1.9528205394744873,
+ "learning_rate": 4.418181818181818e-06,
+ "loss": 1.491654396057129,
+ "mean_token_accuracy": 0.6520518064498901,
+ "num_tokens": 4734976.0,
+ "step": 289
+ },
+ {
+ "entropy": 1.2454503774642944,
+ "epoch": 0.5858585858585859,
+ "grad_norm": 1.9008079767227173,
+ "learning_rate": 4.416161616161616e-06,
+ "loss": 1.253904938697815,
+ "mean_token_accuracy": 0.6971787810325623,
+ "num_tokens": 4751360.0,
+ "step": 290
+ },
+ {
+ "entropy": 1.1740810871124268,
+ "epoch": 0.5878787878787879,
+ "grad_norm": 1.8888787031173706,
+ "learning_rate": 4.414141414141415e-06,
+ "loss": 1.195070505142212,
+ "mean_token_accuracy": 0.7048119306564331,
+ "num_tokens": 4767744.0,
+ "step": 291
+ },
+ {
+ "entropy": 1.6064486503601074,
+ "epoch": 0.5898989898989899,
+ "grad_norm": 2.073559284210205,
+ "learning_rate": 4.412121212121213e-06,
+ "loss": 1.6180689334869385,
+ "mean_token_accuracy": 0.6242061257362366,
+ "num_tokens": 4784128.0,
+ "step": 292
+ },
+ {
+ "entropy": 1.6119383573532104,
+ "epoch": 0.591919191919192,
+ "grad_norm": 1.8773425817489624,
+ "learning_rate": 4.410101010101011e-06,
+ "loss": 1.5875662565231323,
+ "mean_token_accuracy": 0.647838294506073,
+ "num_tokens": 4800512.0,
+ "step": 293
+ },
+ {
+ "entropy": 1.392905592918396,
+ "epoch": 0.593939393939394,
+ "grad_norm": 2.1699368953704834,
+ "learning_rate": 4.408080808080809e-06,
+ "loss": 1.3949127197265625,
+ "mean_token_accuracy": 0.6538837552070618,
+ "num_tokens": 4816896.0,
+ "step": 294
+ },
+ {
+ "entropy": 1.5057002305984497,
+ "epoch": 0.5959595959595959,
+ "grad_norm": 2.2730712890625,
+ "learning_rate": 4.4060606060606066e-06,
+ "loss": 1.4755051136016846,
+ "mean_token_accuracy": 0.6498534679412842,
+ "num_tokens": 4833280.0,
+ "step": 295
+ },
+ {
+ "entropy": 1.5468902587890625,
+ "epoch": 0.597979797979798,
+ "grad_norm": 1.9911075830459595,
+ "learning_rate": 4.4040404040404044e-06,
+ "loss": 1.539963722229004,
+ "mean_token_accuracy": 0.6520518064498901,
+ "num_tokens": 4849664.0,
+ "step": 296
+ },
+ {
+ "entropy": 1.5196901559829712,
+ "epoch": 0.6,
+ "grad_norm": 1.9859540462493896,
+ "learning_rate": 4.402020202020202e-06,
+ "loss": 1.523442029953003,
+ "mean_token_accuracy": 0.6417317986488342,
+ "num_tokens": 4866048.0,
+ "step": 297
+ },
+ {
+ "entropy": 1.8251866102218628,
+ "epoch": 0.602020202020202,
+ "grad_norm": 2.2566516399383545,
+ "learning_rate": 4.4e-06,
+ "loss": 1.8667771816253662,
+ "mean_token_accuracy": 0.5872007608413696,
+ "num_tokens": 4882432.0,
+ "step": 298
+ },
+ {
+ "entropy": 1.6816744804382324,
+ "epoch": 0.604040404040404,
+ "grad_norm": 2.0107715129852295,
+ "learning_rate": 4.397979797979798e-06,
+ "loss": 1.6907548904418945,
+ "mean_token_accuracy": 0.6176722049713135,
+ "num_tokens": 4898816.0,
+ "step": 299
+ },
+ {
+ "entropy": 1.4992223978042603,
+ "epoch": 0.6060606060606061,
+ "grad_norm": 2.1236329078674316,
+ "learning_rate": 4.395959595959596e-06,
+ "loss": 1.5015790462493896,
+ "mean_token_accuracy": 0.6372129917144775,
+ "num_tokens": 4915200.0,
+ "step": 300
+ },
+ {
+ "epoch": 0.6060606060606061,
+ "eval_entropy": 1.5544315297757425,
+ "eval_loss": 1.5676765441894531,
+ "eval_mean_token_accuracy": 0.6389380799185845,
+ "eval_num_tokens": 4915200.0,
+ "eval_runtime": 43.7857,
+ "eval_samples_per_second": 22.61,
+ "eval_steps_per_second": 2.832,
+ "step": 300
+ },
+ {
+ "entropy": 1.518556833267212,
+ "epoch": 0.6080808080808081,
+ "grad_norm": 2.88371205329895,
+ "learning_rate": 4.393939393939394e-06,
+ "loss": 1.5310916900634766,
+ "mean_token_accuracy": 0.6551660895347595,
+ "num_tokens": 4931584.0,
+ "step": 301
+ },
+ {
+ "entropy": 1.4650386571884155,
+ "epoch": 0.6101010101010101,
+ "grad_norm": 2.1849780082702637,
+ "learning_rate": 4.391919191919193e-06,
+ "loss": 1.4405598640441895,
+ "mean_token_accuracy": 0.6531509757041931,
+ "num_tokens": 4947968.0,
+ "step": 302
+ },
+ {
+ "entropy": 1.5209505558013916,
+ "epoch": 0.6121212121212121,
+ "grad_norm": 2.1135010719299316,
+ "learning_rate": 4.3898989898989905e-06,
+ "loss": 1.5324647426605225,
+ "mean_token_accuracy": 0.6561431288719177,
+ "num_tokens": 4964352.0,
+ "step": 303
+ },
+ {
+ "entropy": 1.448391318321228,
+ "epoch": 0.6141414141414141,
+ "grad_norm": 1.9959306716918945,
+ "learning_rate": 4.387878787878788e-06,
+ "loss": 1.4721965789794922,
+ "mean_token_accuracy": 0.6578529477119446,
+ "num_tokens": 4980736.0,
+ "step": 304
+ },
+ {
+ "entropy": 1.4609358310699463,
+ "epoch": 0.6161616161616161,
+ "grad_norm": 2.1308915615081787,
+ "learning_rate": 4.385858585858586e-06,
+ "loss": 1.4707520008087158,
+ "mean_token_accuracy": 0.6482046842575073,
+ "num_tokens": 4997120.0,
+ "step": 305
+ },
+ {
+ "entropy": 1.7125155925750732,
+ "epoch": 0.6181818181818182,
+ "grad_norm": 2.2170841693878174,
+ "learning_rate": 4.383838383838384e-06,
+ "loss": 1.7630269527435303,
+ "mean_token_accuracy": 0.5999022722244263,
+ "num_tokens": 5013504.0,
+ "step": 306
+ },
+ {
+ "entropy": 1.6740268468856812,
+ "epoch": 0.6202020202020202,
+ "grad_norm": 1.983644723892212,
+ "learning_rate": 4.381818181818182e-06,
+ "loss": 1.6849563121795654,
+ "mean_token_accuracy": 0.6237176060676575,
+ "num_tokens": 5029888.0,
+ "step": 307
+ },
+ {
+ "entropy": 1.3754971027374268,
+ "epoch": 0.6222222222222222,
+ "grad_norm": 2.005277395248413,
+ "learning_rate": 4.37979797979798e-06,
+ "loss": 1.385213851928711,
+ "mean_token_accuracy": 0.6638984084129333,
+ "num_tokens": 5046272.0,
+ "step": 308
+ },
+ {
+ "entropy": 1.5476733446121216,
+ "epoch": 0.6242424242424243,
+ "grad_norm": 2.4317610263824463,
+ "learning_rate": 4.377777777777778e-06,
+ "loss": 1.5417041778564453,
+ "mean_token_accuracy": 0.6323888897895813,
+ "num_tokens": 5062656.0,
+ "step": 309
+ },
+ {
+ "entropy": 1.4437031745910645,
+ "epoch": 0.6262626262626263,
+ "grad_norm": 2.1464109420776367,
+ "learning_rate": 4.375757575757576e-06,
+ "loss": 1.4704627990722656,
+ "mean_token_accuracy": 0.6772105693817139,
+ "num_tokens": 5079040.0,
+ "step": 310
+ },
+ {
+ "entropy": 1.4859641790390015,
+ "epoch": 0.6282828282828283,
+ "grad_norm": 2.0800492763519287,
+ "learning_rate": 4.373737373737374e-06,
+ "loss": 1.5072834491729736,
+ "mean_token_accuracy": 0.658707857131958,
+ "num_tokens": 5095424.0,
+ "step": 311
+ },
+ {
+ "entropy": 1.8733419179916382,
+ "epoch": 0.6303030303030303,
+ "grad_norm": 2.120965003967285,
+ "learning_rate": 4.3717171717171715e-06,
+ "loss": 1.8567254543304443,
+ "mean_token_accuracy": 0.5847581624984741,
+ "num_tokens": 5111808.0,
+ "step": 312
+ },
+ {
+ "entropy": 1.4578243494033813,
+ "epoch": 0.6323232323232323,
+ "grad_norm": 1.9577823877334595,
+ "learning_rate": 4.36969696969697e-06,
+ "loss": 1.4758052825927734,
+ "mean_token_accuracy": 0.6590742468833923,
+ "num_tokens": 5128192.0,
+ "step": 313
+ },
+ {
+ "entropy": 1.8680084943771362,
+ "epoch": 0.6343434343434343,
+ "grad_norm": 2.170994520187378,
+ "learning_rate": 4.367676767676768e-06,
+ "loss": 1.904400110244751,
+ "mean_token_accuracy": 0.5857962965965271,
+ "num_tokens": 5144576.0,
+ "step": 314
+ },
+ {
+ "entropy": 1.6488304138183594,
+ "epoch": 0.6363636363636364,
+ "grad_norm": 1.959490418434143,
+ "learning_rate": 4.365656565656566e-06,
+ "loss": 1.6570477485656738,
+ "mean_token_accuracy": 0.6257327795028687,
+ "num_tokens": 5160960.0,
+ "step": 315
+ },
+ {
+ "entropy": 1.5038025379180908,
+ "epoch": 0.6383838383838384,
+ "grad_norm": 2.072697401046753,
+ "learning_rate": 4.363636363636364e-06,
+ "loss": 1.5234860181808472,
+ "mean_token_accuracy": 0.6494259834289551,
+ "num_tokens": 5177344.0,
+ "step": 316
+ },
+ {
+ "entropy": 1.4154654741287231,
+ "epoch": 0.6404040404040404,
+ "grad_norm": 1.894587755203247,
+ "learning_rate": 4.361616161616162e-06,
+ "loss": 1.4131592512130737,
+ "mean_token_accuracy": 0.6667073965072632,
+ "num_tokens": 5193728.0,
+ "step": 317
+ },
+ {
+ "entropy": 1.5813250541687012,
+ "epoch": 0.6424242424242425,
+ "grad_norm": 2.4088737964630127,
+ "learning_rate": 4.35959595959596e-06,
+ "loss": 1.6236622333526611,
+ "mean_token_accuracy": 0.6313507556915283,
+ "num_tokens": 5210112.0,
+ "step": 318
+ },
+ {
+ "entropy": 1.4140045642852783,
+ "epoch": 0.6444444444444445,
+ "grad_norm": 1.991557240486145,
+ "learning_rate": 4.3575757575757576e-06,
+ "loss": 1.4242517948150635,
+ "mean_token_accuracy": 0.6569980382919312,
+ "num_tokens": 5226496.0,
+ "step": 319
+ },
+ {
+ "entropy": 1.615628957748413,
+ "epoch": 0.6464646464646465,
+ "grad_norm": 1.8819077014923096,
+ "learning_rate": 4.3555555555555555e-06,
+ "loss": 1.6331532001495361,
+ "mean_token_accuracy": 0.6299462914466858,
+ "num_tokens": 5242880.0,
+ "step": 320
+ },
+ {
+ "entropy": 1.3118394613265991,
+ "epoch": 0.6484848484848484,
+ "grad_norm": 2.0589284896850586,
+ "learning_rate": 4.353535353535353e-06,
+ "loss": 1.2880034446716309,
+ "mean_token_accuracy": 0.6842941045761108,
+ "num_tokens": 5259264.0,
+ "step": 321
+ },
+ {
+ "entropy": 1.2786612510681152,
+ "epoch": 0.6505050505050505,
+ "grad_norm": 2.033839225769043,
+ "learning_rate": 4.351515151515152e-06,
+ "loss": 1.2829055786132812,
+ "mean_token_accuracy": 0.6910112500190735,
+ "num_tokens": 5275648.0,
+ "step": 322
+ },
+ {
+ "entropy": 1.5632988214492798,
+ "epoch": 0.6525252525252525,
+ "grad_norm": 2.1970419883728027,
+ "learning_rate": 4.34949494949495e-06,
+ "loss": 1.5865097045898438,
+ "mean_token_accuracy": 0.642892062664032,
+ "num_tokens": 5292032.0,
+ "step": 323
+ },
+ {
+ "entropy": 1.1542725563049316,
+ "epoch": 0.6545454545454545,
+ "grad_norm": 1.9750654697418213,
+ "learning_rate": 4.347474747474748e-06,
+ "loss": 1.1401035785675049,
+ "mean_token_accuracy": 0.7061553597450256,
+ "num_tokens": 5308416.0,
+ "step": 324
+ },
+ {
+ "entropy": 1.6879942417144775,
+ "epoch": 0.6565656565656566,
+ "grad_norm": 2.0022354125976562,
+ "learning_rate": 4.345454545454546e-06,
+ "loss": 1.70950448513031,
+ "mean_token_accuracy": 0.6089398860931396,
+ "num_tokens": 5324800.0,
+ "step": 325
+ },
+ {
+ "entropy": 1.7589699029922485,
+ "epoch": 0.6585858585858586,
+ "grad_norm": 1.925520420074463,
+ "learning_rate": 4.343434343434344e-06,
+ "loss": 1.7749860286712646,
+ "mean_token_accuracy": 0.6015510559082031,
+ "num_tokens": 5341184.0,
+ "step": 326
+ },
+ {
+ "entropy": 1.4720325469970703,
+ "epoch": 0.6606060606060606,
+ "grad_norm": 1.9487124681472778,
+ "learning_rate": 4.341414141414142e-06,
+ "loss": 1.4983797073364258,
+ "mean_token_accuracy": 0.6537005305290222,
+ "num_tokens": 5357568.0,
+ "step": 327
+ },
+ {
+ "entropy": 1.676164150238037,
+ "epoch": 0.6626262626262627,
+ "grad_norm": 2.13053035736084,
+ "learning_rate": 4.33939393939394e-06,
+ "loss": 1.699425458908081,
+ "mean_token_accuracy": 0.6100390553474426,
+ "num_tokens": 5373952.0,
+ "step": 328
+ },
+ {
+ "entropy": 1.3424437046051025,
+ "epoch": 0.6646464646464646,
+ "grad_norm": 2.0245954990386963,
+ "learning_rate": 4.337373737373738e-06,
+ "loss": 1.3393046855926514,
+ "mean_token_accuracy": 0.677760124206543,
+ "num_tokens": 5390336.0,
+ "step": 329
+ },
+ {
+ "entropy": 1.4926583766937256,
+ "epoch": 0.6666666666666666,
+ "grad_norm": 1.893343448638916,
+ "learning_rate": 4.335353535353536e-06,
+ "loss": 1.4779269695281982,
+ "mean_token_accuracy": 0.6713483333587646,
+ "num_tokens": 5406720.0,
+ "step": 330
+ },
+ {
+ "entropy": 1.5753449201583862,
+ "epoch": 0.6686868686868687,
+ "grad_norm": 2.051647424697876,
+ "learning_rate": 4.333333333333334e-06,
+ "loss": 1.622597336769104,
+ "mean_token_accuracy": 0.6436248421669006,
+ "num_tokens": 5423104.0,
+ "step": 331
+ },
+ {
+ "entropy": 1.4573988914489746,
+ "epoch": 0.6707070707070707,
+ "grad_norm": 1.8709567785263062,
+ "learning_rate": 4.331313131313132e-06,
+ "loss": 1.4902422428131104,
+ "mean_token_accuracy": 0.6682950854301453,
+ "num_tokens": 5439488.0,
+ "step": 332
+ },
+ {
+ "entropy": 1.378867745399475,
+ "epoch": 0.6727272727272727,
+ "grad_norm": 2.235928773880005,
+ "learning_rate": 4.32929292929293e-06,
+ "loss": 1.4067103862762451,
+ "mean_token_accuracy": 0.662432849407196,
+ "num_tokens": 5455872.0,
+ "step": 333
+ },
+ {
+ "entropy": 1.7553350925445557,
+ "epoch": 0.6747474747474748,
+ "grad_norm": 2.0335066318511963,
+ "learning_rate": 4.327272727272728e-06,
+ "loss": 1.772943139076233,
+ "mean_token_accuracy": 0.5953834652900696,
+ "num_tokens": 5472256.0,
+ "step": 334
+ },
+ {
+ "entropy": 1.1587097644805908,
+ "epoch": 0.6767676767676768,
+ "grad_norm": 1.8764615058898926,
+ "learning_rate": 4.3252525252525255e-06,
+ "loss": 1.1396210193634033,
+ "mean_token_accuracy": 0.7197117805480957,
+ "num_tokens": 5488640.0,
+ "step": 335
+ },
+ {
+ "entropy": 1.4819872379302979,
+ "epoch": 0.6787878787878788,
+ "grad_norm": 2.0907511711120605,
+ "learning_rate": 4.323232323232323e-06,
+ "loss": 1.460550308227539,
+ "mean_token_accuracy": 0.6520518064498901,
+ "num_tokens": 5505024.0,
+ "step": 336
+ },
+ {
+ "entropy": 1.0744472742080688,
+ "epoch": 0.6808080808080809,
+ "grad_norm": 2.227606773376465,
+ "learning_rate": 4.321212121212121e-06,
+ "loss": 1.0909301042556763,
+ "mean_token_accuracy": 0.7258182764053345,
+ "num_tokens": 5521408.0,
+ "step": 337
+ },
+ {
+ "entropy": 1.507999300956726,
+ "epoch": 0.6828282828282828,
+ "grad_norm": 2.0332415103912354,
+ "learning_rate": 4.31919191919192e-06,
+ "loss": 1.5173046588897705,
+ "mean_token_accuracy": 0.6374572515487671,
+ "num_tokens": 5537792.0,
+ "step": 338
+ },
+ {
+ "entropy": 1.6792720556259155,
+ "epoch": 0.6848484848484848,
+ "grad_norm": 2.0194997787475586,
+ "learning_rate": 4.317171717171718e-06,
+ "loss": 1.679445743560791,
+ "mean_token_accuracy": 0.6138250827789307,
+ "num_tokens": 5554176.0,
+ "step": 339
+ },
+ {
+ "entropy": 1.6470589637756348,
+ "epoch": 0.6868686868686869,
+ "grad_norm": 2.09116268157959,
+ "learning_rate": 4.315151515151516e-06,
+ "loss": 1.6386632919311523,
+ "mean_token_accuracy": 0.6146799921989441,
+ "num_tokens": 5570560.0,
+ "step": 340
+ },
+ {
+ "entropy": 1.3612488508224487,
+ "epoch": 0.6888888888888889,
+ "grad_norm": 2.1586217880249023,
+ "learning_rate": 4.313131313131314e-06,
+ "loss": 1.3698725700378418,
+ "mean_token_accuracy": 0.6696995496749878,
+ "num_tokens": 5586944.0,
+ "step": 341
+ },
+ {
+ "entropy": 1.400327205657959,
+ "epoch": 0.6909090909090909,
+ "grad_norm": 2.082094192504883,
+ "learning_rate": 4.3111111111111115e-06,
+ "loss": 1.396535873413086,
+ "mean_token_accuracy": 0.6780654788017273,
+ "num_tokens": 5603328.0,
+ "step": 342
+ },
+ {
+ "entropy": 1.389290690422058,
+ "epoch": 0.692929292929293,
+ "grad_norm": 2.0780303478240967,
+ "learning_rate": 4.309090909090909e-06,
+ "loss": 1.4275782108306885,
+ "mean_token_accuracy": 0.6656082272529602,
+ "num_tokens": 5619712.0,
+ "step": 343
+ },
+ {
+ "entropy": 1.5133870840072632,
+ "epoch": 0.694949494949495,
+ "grad_norm": 1.9819531440734863,
+ "learning_rate": 4.307070707070707e-06,
+ "loss": 1.5309596061706543,
+ "mean_token_accuracy": 0.6373351216316223,
+ "num_tokens": 5636096.0,
+ "step": 344
+ },
+ {
+ "entropy": 1.483219861984253,
+ "epoch": 0.696969696969697,
+ "grad_norm": 1.9794325828552246,
+ "learning_rate": 4.305050505050505e-06,
+ "loss": 1.4693087339401245,
+ "mean_token_accuracy": 0.6550439596176147,
+ "num_tokens": 5652480.0,
+ "step": 345
+ },
+ {
+ "entropy": 1.381489634513855,
+ "epoch": 0.6989898989898989,
+ "grad_norm": 2.0637686252593994,
+ "learning_rate": 4.303030303030303e-06,
+ "loss": 1.3862476348876953,
+ "mean_token_accuracy": 0.6631045341491699,
+ "num_tokens": 5668864.0,
+ "step": 346
+ },
+ {
+ "entropy": 1.9860024452209473,
+ "epoch": 0.701010101010101,
+ "grad_norm": 2.1626803874969482,
+ "learning_rate": 4.301010101010101e-06,
+ "loss": 1.9815950393676758,
+ "mean_token_accuracy": 0.5698583126068115,
+ "num_tokens": 5685248.0,
+ "step": 347
+ },
+ {
+ "entropy": 1.5044004917144775,
+ "epoch": 0.703030303030303,
+ "grad_norm": 2.060976266860962,
+ "learning_rate": 4.298989898989899e-06,
+ "loss": 1.4937127828598022,
+ "mean_token_accuracy": 0.6524792313575745,
+ "num_tokens": 5701632.0,
+ "step": 348
+ },
+ {
+ "entropy": 1.3397772312164307,
+ "epoch": 0.705050505050505,
+ "grad_norm": 2.0162901878356934,
+ "learning_rate": 4.296969696969698e-06,
+ "loss": 1.3358572721481323,
+ "mean_token_accuracy": 0.6949804425239563,
+ "num_tokens": 5718016.0,
+ "step": 349
+ },
+ {
+ "entropy": 1.145772099494934,
+ "epoch": 0.7070707070707071,
+ "grad_norm": 2.097634792327881,
+ "learning_rate": 4.2949494949494955e-06,
+ "loss": 1.1833416223526,
+ "mean_token_accuracy": 0.7104909420013428,
+ "num_tokens": 5734400.0,
+ "step": 350
+ },
+ {
+ "epoch": 0.7070707070707071,
+ "eval_entropy": 1.5476290602837839,
+ "eval_loss": 1.5603480339050293,
+ "eval_mean_token_accuracy": 0.640111118555069,
+ "eval_num_tokens": 5734400.0,
+ "eval_runtime": 43.7844,
+ "eval_samples_per_second": 22.611,
+ "eval_steps_per_second": 2.832,
+ "step": 350
+ },
+ {
+ "entropy": 1.4311926364898682,
+ "epoch": 0.7090909090909091,
+ "grad_norm": 2.002321720123291,
+ "learning_rate": 4.292929292929293e-06,
+ "loss": 1.4534825086593628,
+ "mean_token_accuracy": 0.6614558100700378,
+ "num_tokens": 5750784.0,
+ "step": 351
+ },
+ {
+ "entropy": 1.3983922004699707,
+ "epoch": 0.7111111111111111,
+ "grad_norm": 2.1724867820739746,
+ "learning_rate": 4.290909090909091e-06,
+ "loss": 1.3969402313232422,
+ "mean_token_accuracy": 0.6652418375015259,
+ "num_tokens": 5767168.0,
+ "step": 352
+ },
+ {
+ "entropy": 1.8218920230865479,
+ "epoch": 0.7131313131313132,
+ "grad_norm": 2.1629281044006348,
+ "learning_rate": 4.288888888888889e-06,
+ "loss": 1.8196980953216553,
+ "mean_token_accuracy": 0.6027112603187561,
+ "num_tokens": 5783552.0,
+ "step": 353
+ },
+ {
+ "entropy": 1.5322320461273193,
+ "epoch": 0.7151515151515152,
+ "grad_norm": 1.8486647605895996,
+ "learning_rate": 4.286868686868687e-06,
+ "loss": 1.5504610538482666,
+ "mean_token_accuracy": 0.6533341407775879,
+ "num_tokens": 5799936.0,
+ "step": 354
+ },
+ {
+ "entropy": 1.935966968536377,
+ "epoch": 0.7171717171717171,
+ "grad_norm": 2.252814292907715,
+ "learning_rate": 4.284848484848485e-06,
+ "loss": 1.9300384521484375,
+ "mean_token_accuracy": 0.5656448602676392,
+ "num_tokens": 5816320.0,
+ "step": 355
+ },
+ {
+ "entropy": 1.6711398363113403,
+ "epoch": 0.7191919191919192,
+ "grad_norm": 2.023379325866699,
+ "learning_rate": 4.282828282828283e-06,
+ "loss": 1.6584455966949463,
+ "mean_token_accuracy": 0.6235954761505127,
+ "num_tokens": 5832704.0,
+ "step": 356
+ },
+ {
+ "entropy": 1.3922803401947021,
+ "epoch": 0.7212121212121212,
+ "grad_norm": 2.0487611293792725,
+ "learning_rate": 4.280808080808081e-06,
+ "loss": 1.4220250844955444,
+ "mean_token_accuracy": 0.6591963768005371,
+ "num_tokens": 5849088.0,
+ "step": 357
+ },
+ {
+ "entropy": 1.59521484375,
+ "epoch": 0.7232323232323232,
+ "grad_norm": 1.8598614931106567,
+ "learning_rate": 4.278787878787879e-06,
+ "loss": 1.5979329347610474,
+ "mean_token_accuracy": 0.6237176060676575,
+ "num_tokens": 5865472.0,
+ "step": 358
+ },
+ {
+ "entropy": 1.5810115337371826,
+ "epoch": 0.7252525252525253,
+ "grad_norm": 2.140115737915039,
+ "learning_rate": 4.276767676767677e-06,
+ "loss": 1.5774705410003662,
+ "mean_token_accuracy": 0.6274425983428955,
+ "num_tokens": 5881856.0,
+ "step": 359
+ },
+ {
+ "entropy": 1.5839109420776367,
+ "epoch": 0.7272727272727273,
+ "grad_norm": 2.0947749614715576,
+ "learning_rate": 4.274747474747475e-06,
+ "loss": 1.601511001586914,
+ "mean_token_accuracy": 0.630984365940094,
+ "num_tokens": 5898240.0,
+ "step": 360
+ },
+ {
+ "entropy": 1.195770263671875,
+ "epoch": 0.7292929292929293,
+ "grad_norm": 1.8740363121032715,
+ "learning_rate": 4.272727272727273e-06,
+ "loss": 1.195652961730957,
+ "mean_token_accuracy": 0.7077430486679077,
+ "num_tokens": 5914624.0,
+ "step": 361
+ },
+ {
+ "entropy": 1.375698208808899,
+ "epoch": 0.7313131313131314,
+ "grad_norm": 1.9580703973770142,
+ "learning_rate": 4.270707070707071e-06,
+ "loss": 1.3746864795684814,
+ "mean_token_accuracy": 0.6775158643722534,
+ "num_tokens": 5931008.0,
+ "step": 362
+ },
+ {
+ "entropy": 1.7259451150894165,
+ "epoch": 0.7333333333333333,
+ "grad_norm": 2.3127365112304688,
+ "learning_rate": 4.268686868686869e-06,
+ "loss": 1.745998501777649,
+ "mean_token_accuracy": 0.6040546894073486,
+ "num_tokens": 5947392.0,
+ "step": 363
+ },
+ {
+ "entropy": 1.209228754043579,
+ "epoch": 0.7353535353535353,
+ "grad_norm": 1.9004793167114258,
+ "learning_rate": 4.266666666666668e-06,
+ "loss": 1.232427716255188,
+ "mean_token_accuracy": 0.6925989389419556,
+ "num_tokens": 5963776.0,
+ "step": 364
+ },
+ {
+ "entropy": 1.2811263799667358,
+ "epoch": 0.7373737373737373,
+ "grad_norm": 1.9568246603012085,
+ "learning_rate": 4.2646464646464655e-06,
+ "loss": 1.291853427886963,
+ "mean_token_accuracy": 0.6884465217590332,
+ "num_tokens": 5980160.0,
+ "step": 365
+ },
+ {
+ "entropy": 1.843044638633728,
+ "epoch": 0.7393939393939394,
+ "grad_norm": 2.132735252380371,
+ "learning_rate": 4.262626262626263e-06,
+ "loss": 1.8818857669830322,
+ "mean_token_accuracy": 0.5785295367240906,
+ "num_tokens": 5996544.0,
+ "step": 366
+ },
+ {
+ "entropy": 1.3353440761566162,
+ "epoch": 0.7414141414141414,
+ "grad_norm": 1.8893013000488281,
+ "learning_rate": 4.260606060606061e-06,
+ "loss": 1.354011058807373,
+ "mean_token_accuracy": 0.6797752976417542,
+ "num_tokens": 6012928.0,
+ "step": 367
+ },
+ {
+ "entropy": 1.3889728784561157,
+ "epoch": 0.7434343434343434,
+ "grad_norm": 1.980757236480713,
+ "learning_rate": 4.258585858585859e-06,
+ "loss": 1.3671875,
+ "mean_token_accuracy": 0.6627381443977356,
+ "num_tokens": 6029312.0,
+ "step": 368
+ },
+ {
+ "entropy": 1.4605348110198975,
+ "epoch": 0.7454545454545455,
+ "grad_norm": 2.1033780574798584,
+ "learning_rate": 4.256565656565657e-06,
+ "loss": 1.455024003982544,
+ "mean_token_accuracy": 0.6451514363288879,
+ "num_tokens": 6045696.0,
+ "step": 369
+ },
+ {
+ "entropy": 1.4382058382034302,
+ "epoch": 0.7474747474747475,
+ "grad_norm": 2.1068074703216553,
+ "learning_rate": 4.254545454545455e-06,
+ "loss": 1.4422768354415894,
+ "mean_token_accuracy": 0.6630434989929199,
+ "num_tokens": 6062080.0,
+ "step": 370
+ },
+ {
+ "entropy": 1.8046759366989136,
+ "epoch": 0.7494949494949495,
+ "grad_norm": 2.214466094970703,
+ "learning_rate": 4.252525252525253e-06,
+ "loss": 1.8247106075286865,
+ "mean_token_accuracy": 0.6003297567367554,
+ "num_tokens": 6078464.0,
+ "step": 371
+ },
+ {
+ "entropy": 1.671690583229065,
+ "epoch": 0.7515151515151515,
+ "grad_norm": 1.9790785312652588,
+ "learning_rate": 4.250505050505051e-06,
+ "loss": 1.6907753944396973,
+ "mean_token_accuracy": 0.6232290863990784,
+ "num_tokens": 6094848.0,
+ "step": 372
+ },
+ {
+ "entropy": 1.6881897449493408,
+ "epoch": 0.7535353535353535,
+ "grad_norm": 1.8638463020324707,
+ "learning_rate": 4.248484848484849e-06,
+ "loss": 1.7132716178894043,
+ "mean_token_accuracy": 0.6357474327087402,
+ "num_tokens": 6111232.0,
+ "step": 373
+ },
+ {
+ "entropy": 1.2555122375488281,
+ "epoch": 0.7555555555555555,
+ "grad_norm": 2.082378387451172,
+ "learning_rate": 4.246464646464647e-06,
+ "loss": 1.240688681602478,
+ "mean_token_accuracy": 0.6965070962905884,
+ "num_tokens": 6127616.0,
+ "step": 374
+ },
+ {
+ "entropy": 1.847135305404663,
+ "epoch": 0.7575757575757576,
+ "grad_norm": 2.142962694168091,
+ "learning_rate": 4.244444444444445e-06,
+ "loss": 1.8840911388397217,
+ "mean_token_accuracy": 0.5822545289993286,
+ "num_tokens": 6144000.0,
+ "step": 375
+ },
+ {
+ "entropy": 1.239328384399414,
+ "epoch": 0.7595959595959596,
+ "grad_norm": 2.0589468479156494,
+ "learning_rate": 4.242424242424243e-06,
+ "loss": 1.256324052810669,
+ "mean_token_accuracy": 0.6845383644104004,
+ "num_tokens": 6160384.0,
+ "step": 376
+ },
+ {
+ "entropy": 1.5047202110290527,
+ "epoch": 0.7616161616161616,
+ "grad_norm": 1.7951308488845825,
+ "learning_rate": 4.240404040404041e-06,
+ "loss": 1.5063304901123047,
+ "mean_token_accuracy": 0.6583414673805237,
+ "num_tokens": 6176768.0,
+ "step": 377
+ },
+ {
+ "entropy": 1.4072566032409668,
+ "epoch": 0.7636363636363637,
+ "grad_norm": 2.1670594215393066,
+ "learning_rate": 4.238383838383839e-06,
+ "loss": 1.4205389022827148,
+ "mean_token_accuracy": 0.658707857131958,
+ "num_tokens": 6193152.0,
+ "step": 378
+ },
+ {
+ "entropy": 1.3207885026931763,
+ "epoch": 0.7656565656565657,
+ "grad_norm": 1.9017083644866943,
+ "learning_rate": 4.236363636363637e-06,
+ "loss": 1.3169896602630615,
+ "mean_token_accuracy": 0.6954079270362854,
+ "num_tokens": 6209536.0,
+ "step": 379
+ },
+ {
+ "entropy": 1.4762436151504517,
+ "epoch": 0.7676767676767676,
+ "grad_norm": 2.0023069381713867,
+ "learning_rate": 4.234343434343435e-06,
+ "loss": 1.4725041389465332,
+ "mean_token_accuracy": 0.6620053648948669,
+ "num_tokens": 6225920.0,
+ "step": 380
+ },
+ {
+ "entropy": 1.4895304441452026,
+ "epoch": 0.7696969696969697,
+ "grad_norm": 1.9528017044067383,
+ "learning_rate": 4.2323232323232325e-06,
+ "loss": 1.49650239944458,
+ "mean_token_accuracy": 0.6508915424346924,
+ "num_tokens": 6242304.0,
+ "step": 381
+ },
+ {
+ "entropy": 1.3024516105651855,
+ "epoch": 0.7717171717171717,
+ "grad_norm": 1.9855588674545288,
+ "learning_rate": 4.2303030303030304e-06,
+ "loss": 1.34218168258667,
+ "mean_token_accuracy": 0.6838055849075317,
+ "num_tokens": 6258688.0,
+ "step": 382
+ },
+ {
+ "entropy": 1.6005626916885376,
+ "epoch": 0.7737373737373737,
+ "grad_norm": 2.028286933898926,
+ "learning_rate": 4.228282828282828e-06,
+ "loss": 1.6299283504486084,
+ "mean_token_accuracy": 0.6315950155258179,
+ "num_tokens": 6275072.0,
+ "step": 383
+ },
+ {
+ "entropy": 1.7050484418869019,
+ "epoch": 0.7757575757575758,
+ "grad_norm": 2.474047899246216,
+ "learning_rate": 4.226262626262626e-06,
+ "loss": 1.777151346206665,
+ "mean_token_accuracy": 0.5919027924537659,
+ "num_tokens": 6291456.0,
+ "step": 384
+ },
+ {
+ "entropy": 1.5014543533325195,
+ "epoch": 0.7777777777777778,
+ "grad_norm": 1.9866594076156616,
+ "learning_rate": 4.224242424242425e-06,
+ "loss": 1.5308949947357178,
+ "mean_token_accuracy": 0.639106035232544,
+ "num_tokens": 6307840.0,
+ "step": 385
+ },
+ {
+ "entropy": 1.6957359313964844,
+ "epoch": 0.7797979797979798,
+ "grad_norm": 2.229820966720581,
+ "learning_rate": 4.222222222222223e-06,
+ "loss": 1.728761911392212,
+ "mean_token_accuracy": 0.6242061257362366,
+ "num_tokens": 6324224.0,
+ "step": 386
+ },
+ {
+ "entropy": 1.6823521852493286,
+ "epoch": 0.7818181818181819,
+ "grad_norm": 2.033383369445801,
+ "learning_rate": 4.220202020202021e-06,
+ "loss": 1.7310154438018799,
+ "mean_token_accuracy": 0.6257938742637634,
+ "num_tokens": 6340608.0,
+ "step": 387
+ },
+ {
+ "entropy": 1.710307240486145,
+ "epoch": 0.7838383838383839,
+ "grad_norm": 2.061861038208008,
+ "learning_rate": 4.218181818181819e-06,
+ "loss": 1.7066943645477295,
+ "mean_token_accuracy": 0.6123595237731934,
+ "num_tokens": 6356992.0,
+ "step": 388
+ },
+ {
+ "entropy": 1.241638422012329,
+ "epoch": 0.7858585858585858,
+ "grad_norm": 1.9743852615356445,
+ "learning_rate": 4.2161616161616165e-06,
+ "loss": 1.2520272731781006,
+ "mean_token_accuracy": 0.692415714263916,
+ "num_tokens": 6373376.0,
+ "step": 389
+ },
+ {
+ "entropy": 1.5918776988983154,
+ "epoch": 0.7878787878787878,
+ "grad_norm": 2.156675100326538,
+ "learning_rate": 4.214141414141414e-06,
+ "loss": 1.6060255765914917,
+ "mean_token_accuracy": 0.6178553700447083,
+ "num_tokens": 6389760.0,
+ "step": 390
+ },
+ {
+ "entropy": 1.7307862043380737,
+ "epoch": 0.7898989898989899,
+ "grad_norm": 2.171247959136963,
+ "learning_rate": 4.212121212121212e-06,
+ "loss": 1.7327581644058228,
+ "mean_token_accuracy": 0.6502808928489685,
+ "num_tokens": 6406144.0,
+ "step": 391
+ },
+ {
+ "entropy": 1.6725553274154663,
+ "epoch": 0.7919191919191919,
+ "grad_norm": 2.020228624343872,
+ "learning_rate": 4.21010101010101e-06,
+ "loss": 1.6803646087646484,
+ "mean_token_accuracy": 0.6383732557296753,
+ "num_tokens": 6422528.0,
+ "step": 392
+ },
+ {
+ "entropy": 1.4120928049087524,
+ "epoch": 0.793939393939394,
+ "grad_norm": 1.9896639585494995,
+ "learning_rate": 4.208080808080808e-06,
+ "loss": 1.3974279165267944,
+ "mean_token_accuracy": 0.6661577820777893,
+ "num_tokens": 6438912.0,
+ "step": 393
+ },
+ {
+ "entropy": 0.933545708656311,
+ "epoch": 0.795959595959596,
+ "grad_norm": 1.6884223222732544,
+ "learning_rate": 4.206060606060606e-06,
+ "loss": 0.9287986755371094,
+ "mean_token_accuracy": 0.7636175155639648,
+ "num_tokens": 6455296.0,
+ "step": 394
+ },
+ {
+ "entropy": 1.419004201889038,
+ "epoch": 0.797979797979798,
+ "grad_norm": 2.2590551376342773,
+ "learning_rate": 4.204040404040405e-06,
+ "loss": 1.4652538299560547,
+ "mean_token_accuracy": 0.6589521169662476,
+ "num_tokens": 6471680.0,
+ "step": 395
+ },
+ {
+ "entropy": 1.750221610069275,
+ "epoch": 0.8,
+ "grad_norm": 2.195030450820923,
+ "learning_rate": 4.2020202020202026e-06,
+ "loss": 1.7328863143920898,
+ "mean_token_accuracy": 0.6008182764053345,
+ "num_tokens": 6488064.0,
+ "step": 396
+ },
+ {
+ "entropy": 1.7440015077590942,
+ "epoch": 0.802020202020202,
+ "grad_norm": 1.9833927154541016,
+ "learning_rate": 4.2000000000000004e-06,
+ "loss": 1.7617835998535156,
+ "mean_token_accuracy": 0.5992916226387024,
+ "num_tokens": 6504448.0,
+ "step": 397
+ },
+ {
+ "entropy": 1.1636452674865723,
+ "epoch": 0.804040404040404,
+ "grad_norm": 1.9506802558898926,
+ "learning_rate": 4.197979797979798e-06,
+ "loss": 1.1344032287597656,
+ "mean_token_accuracy": 0.7122618556022644,
+ "num_tokens": 6520832.0,
+ "step": 398
+ },
+ {
+ "entropy": 1.6173542737960815,
+ "epoch": 0.806060606060606,
+ "grad_norm": 2.0997231006622314,
+ "learning_rate": 4.195959595959596e-06,
+ "loss": 1.6303956508636475,
+ "mean_token_accuracy": 0.6215192675590515,
+ "num_tokens": 6537216.0,
+ "step": 399
+ },
+ {
+ "entropy": 1.4391542673110962,
+ "epoch": 0.8080808080808081,
+ "grad_norm": 2.356828212738037,
+ "learning_rate": 4.193939393939394e-06,
+ "loss": 1.4465923309326172,
+ "mean_token_accuracy": 0.6594406366348267,
+ "num_tokens": 6553600.0,
+ "step": 400
+ },
+ {
+ "epoch": 0.8080808080808081,
+ "eval_entropy": 1.5504949785047961,
+ "eval_loss": 1.5544542074203491,
+ "eval_mean_token_accuracy": 0.6409837569921247,
+ "eval_num_tokens": 6553600.0,
+ "eval_runtime": 43.7986,
+ "eval_samples_per_second": 22.603,
+ "eval_steps_per_second": 2.831,
+ "step": 400
+ },
+ {
+ "entropy": 1.60548996925354,
+ "epoch": 0.8101010101010101,
+ "grad_norm": 2.1894404888153076,
+ "learning_rate": 4.191919191919192e-06,
+ "loss": 1.6116085052490234,
+ "mean_token_accuracy": 0.6276868581771851,
+ "num_tokens": 6569984.0,
+ "step": 401
+ },
+ {
+ "entropy": 1.436041235923767,
+ "epoch": 0.8121212121212121,
+ "grad_norm": 2.1180264949798584,
+ "learning_rate": 4.18989898989899e-06,
+ "loss": 1.4203698635101318,
+ "mean_token_accuracy": 0.6554714441299438,
+ "num_tokens": 6586368.0,
+ "step": 402
+ },
+ {
+ "entropy": 1.1348106861114502,
+ "epoch": 0.8141414141414142,
+ "grad_norm": 2.0420851707458496,
+ "learning_rate": 4.187878787878788e-06,
+ "loss": 1.1424527168273926,
+ "mean_token_accuracy": 0.7154372334480286,
+ "num_tokens": 6602752.0,
+ "step": 403
+ },
+ {
+ "entropy": 1.4039727449417114,
+ "epoch": 0.8161616161616162,
+ "grad_norm": 2.148340940475464,
+ "learning_rate": 4.185858585858586e-06,
+ "loss": 1.4246132373809814,
+ "mean_token_accuracy": 0.6723864078521729,
+ "num_tokens": 6619136.0,
+ "step": 404
+ },
+ {
+ "entropy": 1.4222626686096191,
+ "epoch": 0.8181818181818182,
+ "grad_norm": 1.9436827898025513,
+ "learning_rate": 4.1838383838383835e-06,
+ "loss": 1.4235990047454834,
+ "mean_token_accuracy": 0.6759281754493713,
+ "num_tokens": 6635520.0,
+ "step": 405
+ },
+ {
+ "entropy": 1.4771310091018677,
+ "epoch": 0.8202020202020202,
+ "grad_norm": 2.0953726768493652,
+ "learning_rate": 4.181818181818182e-06,
+ "loss": 1.501934289932251,
+ "mean_token_accuracy": 0.6535173654556274,
+ "num_tokens": 6651904.0,
+ "step": 406
+ },
+ {
+ "entropy": 1.526256799697876,
+ "epoch": 0.8222222222222222,
+ "grad_norm": 2.245994806289673,
+ "learning_rate": 4.17979797979798e-06,
+ "loss": 1.539383888244629,
+ "mean_token_accuracy": 0.6417317986488342,
+ "num_tokens": 6668288.0,
+ "step": 407
+ },
+ {
+ "entropy": 1.1716097593307495,
+ "epoch": 0.8242424242424242,
+ "grad_norm": 1.8283652067184448,
+ "learning_rate": 4.177777777777778e-06,
+ "loss": 1.1798359155654907,
+ "mean_token_accuracy": 0.7123839855194092,
+ "num_tokens": 6684672.0,
+ "step": 408
+ },
+ {
+ "entropy": 1.4581540822982788,
+ "epoch": 0.8262626262626263,
+ "grad_norm": 1.9325168132781982,
+ "learning_rate": 4.175757575757576e-06,
+ "loss": 1.4526585340499878,
+ "mean_token_accuracy": 0.662432849407196,
+ "num_tokens": 6701056.0,
+ "step": 409
+ },
+ {
+ "entropy": 1.4073102474212646,
+ "epoch": 0.8282828282828283,
+ "grad_norm": 2.1543467044830322,
+ "learning_rate": 4.173737373737375e-06,
+ "loss": 1.437713861465454,
+ "mean_token_accuracy": 0.6725696325302124,
+ "num_tokens": 6717440.0,
+ "step": 410
+ },
+ {
+ "entropy": 1.5970062017440796,
+ "epoch": 0.8303030303030303,
+ "grad_norm": 2.1714792251586914,
+ "learning_rate": 4.1717171717171726e-06,
+ "loss": 1.613295078277588,
+ "mean_token_accuracy": 0.6229848265647888,
+ "num_tokens": 6733824.0,
+ "step": 411
+ },
+ {
+ "entropy": 1.449837327003479,
+ "epoch": 0.8323232323232324,
+ "grad_norm": 2.2499871253967285,
+ "learning_rate": 4.1696969696969705e-06,
+ "loss": 1.489758014678955,
+ "mean_token_accuracy": 0.645639955997467,
+ "num_tokens": 6750208.0,
+ "step": 412
+ },
+ {
+ "entropy": 1.7696173191070557,
+ "epoch": 0.8343434343434344,
+ "grad_norm": 2.3637073040008545,
+ "learning_rate": 4.167676767676768e-06,
+ "loss": 1.8147599697113037,
+ "mean_token_accuracy": 0.5821323990821838,
+ "num_tokens": 6766592.0,
+ "step": 413
+ },
+ {
+ "entropy": 1.99376380443573,
+ "epoch": 0.8363636363636363,
+ "grad_norm": 2.265683174133301,
+ "learning_rate": 4.165656565656566e-06,
+ "loss": 2.009024143218994,
+ "mean_token_accuracy": 0.5591108798980713,
+ "num_tokens": 6782976.0,
+ "step": 414
+ },
+ {
+ "entropy": 1.7276980876922607,
+ "epoch": 0.8383838383838383,
+ "grad_norm": 1.9407477378845215,
+ "learning_rate": 4.163636363636364e-06,
+ "loss": 1.7130229473114014,
+ "mean_token_accuracy": 0.6202979683876038,
+ "num_tokens": 6799360.0,
+ "step": 415
+ },
+ {
+ "entropy": 1.5418223142623901,
+ "epoch": 0.8404040404040404,
+ "grad_norm": 1.9765743017196655,
+ "learning_rate": 4.161616161616162e-06,
+ "loss": 1.5627632141113281,
+ "mean_token_accuracy": 0.6433194875717163,
+ "num_tokens": 6815744.0,
+ "step": 416
+ },
+ {
+ "entropy": 1.6040345430374146,
+ "epoch": 0.8424242424242424,
+ "grad_norm": 2.1942877769470215,
+ "learning_rate": 4.15959595959596e-06,
+ "loss": 1.6044622659683228,
+ "mean_token_accuracy": 0.6278700828552246,
+ "num_tokens": 6832128.0,
+ "step": 417
+ },
+ {
+ "entropy": 1.8461577892303467,
+ "epoch": 0.8444444444444444,
+ "grad_norm": 2.289196729660034,
+ "learning_rate": 4.157575757575758e-06,
+ "loss": 1.8679372072219849,
+ "mean_token_accuracy": 0.5887884497642517,
+ "num_tokens": 6848512.0,
+ "step": 418
+ },
+ {
+ "entropy": 1.4079350233078003,
+ "epoch": 0.8464646464646465,
+ "grad_norm": 1.9526047706604004,
+ "learning_rate": 4.155555555555556e-06,
+ "loss": 1.4145114421844482,
+ "mean_token_accuracy": 0.6731802821159363,
+ "num_tokens": 6864896.0,
+ "step": 419
+ },
+ {
+ "entropy": 1.1973973512649536,
+ "epoch": 0.8484848484848485,
+ "grad_norm": 2.0716679096221924,
+ "learning_rate": 4.1535353535353536e-06,
+ "loss": 1.260810136795044,
+ "mean_token_accuracy": 0.701697587966919,
+ "num_tokens": 6881280.0,
+ "step": 420
+ },
+ {
+ "entropy": 1.507702350616455,
+ "epoch": 0.8505050505050505,
+ "grad_norm": 2.0233314037323,
+ "learning_rate": 4.151515151515152e-06,
+ "loss": 1.513720154762268,
+ "mean_token_accuracy": 0.6497313380241394,
+ "num_tokens": 6897664.0,
+ "step": 421
+ },
+ {
+ "entropy": 1.54402756690979,
+ "epoch": 0.8525252525252526,
+ "grad_norm": 2.23366379737854,
+ "learning_rate": 4.14949494949495e-06,
+ "loss": 1.5434964895248413,
+ "mean_token_accuracy": 0.6502198576927185,
+ "num_tokens": 6914048.0,
+ "step": 422
+ },
+ {
+ "entropy": 1.820296049118042,
+ "epoch": 0.8545454545454545,
+ "grad_norm": 2.066905975341797,
+ "learning_rate": 4.147474747474748e-06,
+ "loss": 1.8356924057006836,
+ "mean_token_accuracy": 0.5886663198471069,
+ "num_tokens": 6930432.0,
+ "step": 423
+ },
+ {
+ "entropy": 1.4747720956802368,
+ "epoch": 0.8565656565656565,
+ "grad_norm": 2.040022850036621,
+ "learning_rate": 4.145454545454546e-06,
+ "loss": 1.4495999813079834,
+ "mean_token_accuracy": 0.6532731056213379,
+ "num_tokens": 6946816.0,
+ "step": 424
+ },
+ {
+ "entropy": 1.7536696195602417,
+ "epoch": 0.8585858585858586,
+ "grad_norm": 2.0884320735931396,
+ "learning_rate": 4.143434343434344e-06,
+ "loss": 1.7520158290863037,
+ "mean_token_accuracy": 0.6083903312683105,
+ "num_tokens": 6963200.0,
+ "step": 425
+ },
+ {
+ "entropy": 1.715582251548767,
+ "epoch": 0.8606060606060606,
+ "grad_norm": 2.2991514205932617,
+ "learning_rate": 4.141414141414142e-06,
+ "loss": 1.7138090133666992,
+ "mean_token_accuracy": 0.607107937335968,
+ "num_tokens": 6979584.0,
+ "step": 426
+ },
+ {
+ "entropy": 1.8116382360458374,
+ "epoch": 0.8626262626262626,
+ "grad_norm": 2.289909839630127,
+ "learning_rate": 4.13939393939394e-06,
+ "loss": 1.7755894660949707,
+ "mean_token_accuracy": 0.5776746273040771,
+ "num_tokens": 6995968.0,
+ "step": 427
+ },
+ {
+ "entropy": 1.487213373184204,
+ "epoch": 0.8646464646464647,
+ "grad_norm": 1.8834803104400635,
+ "learning_rate": 4.1373737373737375e-06,
+ "loss": 1.511157751083374,
+ "mean_token_accuracy": 0.6546165347099304,
+ "num_tokens": 7012352.0,
+ "step": 428
+ },
+ {
+ "entropy": 1.2769891023635864,
+ "epoch": 0.8666666666666667,
+ "grad_norm": 1.892616629600525,
+ "learning_rate": 4.135353535353535e-06,
+ "loss": 1.2772598266601562,
+ "mean_token_accuracy": 0.6984611749649048,
+ "num_tokens": 7028736.0,
+ "step": 429
+ },
+ {
+ "entropy": 1.579519271850586,
+ "epoch": 0.8686868686868687,
+ "grad_norm": 1.9801563024520874,
+ "learning_rate": 4.133333333333333e-06,
+ "loss": 1.5647528171539307,
+ "mean_token_accuracy": 0.6482046842575073,
+ "num_tokens": 7045120.0,
+ "step": 430
+ },
+ {
+ "entropy": 1.407600998878479,
+ "epoch": 0.8707070707070707,
+ "grad_norm": 2.1737446784973145,
+ "learning_rate": 4.131313131313132e-06,
+ "loss": 1.4184494018554688,
+ "mean_token_accuracy": 0.6640815734863281,
+ "num_tokens": 7061504.0,
+ "step": 431
+ },
+ {
+ "entropy": 1.9118441343307495,
+ "epoch": 0.8727272727272727,
+ "grad_norm": 1.9541205167770386,
+ "learning_rate": 4.12929292929293e-06,
+ "loss": 1.9581422805786133,
+ "mean_token_accuracy": 0.564667820930481,
+ "num_tokens": 7077888.0,
+ "step": 432
+ },
+ {
+ "entropy": 1.925604224205017,
+ "epoch": 0.8747474747474747,
+ "grad_norm": 1.9696223735809326,
+ "learning_rate": 4.127272727272728e-06,
+ "loss": 1.9226163625717163,
+ "mean_token_accuracy": 0.5870786309242249,
+ "num_tokens": 7094272.0,
+ "step": 433
+ },
+ {
+ "entropy": 1.3994735479354858,
+ "epoch": 0.8767676767676768,
+ "grad_norm": 2.003532886505127,
+ "learning_rate": 4.125252525252526e-06,
+ "loss": 1.4291752576828003,
+ "mean_token_accuracy": 0.669516384601593,
+ "num_tokens": 7110656.0,
+ "step": 434
+ },
+ {
+ "entropy": 1.5550929307937622,
+ "epoch": 0.8787878787878788,
+ "grad_norm": 2.2658586502075195,
+ "learning_rate": 4.1232323232323236e-06,
+ "loss": 1.5533335208892822,
+ "mean_token_accuracy": 0.6242672204971313,
+ "num_tokens": 7127040.0,
+ "step": 435
+ },
+ {
+ "entropy": 1.6622785329818726,
+ "epoch": 0.8808080808080808,
+ "grad_norm": 2.0748393535614014,
+ "learning_rate": 4.1212121212121215e-06,
+ "loss": 1.700000524520874,
+ "mean_token_accuracy": 0.6221299171447754,
+ "num_tokens": 7143424.0,
+ "step": 436
+ },
+ {
+ "entropy": 1.1223996877670288,
+ "epoch": 0.8828282828282829,
+ "grad_norm": 2.0348756313323975,
+ "learning_rate": 4.119191919191919e-06,
+ "loss": 1.143293857574463,
+ "mean_token_accuracy": 0.7045676708221436,
+ "num_tokens": 7159808.0,
+ "step": 437
+ },
+ {
+ "entropy": 1.7156380414962769,
+ "epoch": 0.8848484848484849,
+ "grad_norm": 2.306549549102783,
+ "learning_rate": 4.117171717171717e-06,
+ "loss": 1.7625794410705566,
+ "mean_token_accuracy": 0.6049706935882568,
+ "num_tokens": 7176192.0,
+ "step": 438
+ },
+ {
+ "entropy": 1.8507202863693237,
+ "epoch": 0.8868686868686869,
+ "grad_norm": 2.2955853939056396,
+ "learning_rate": 4.115151515151515e-06,
+ "loss": 1.8923052549362183,
+ "mean_token_accuracy": 0.5938568711280823,
+ "num_tokens": 7192576.0,
+ "step": 439
+ },
+ {
+ "entropy": 1.813754916191101,
+ "epoch": 0.8888888888888888,
+ "grad_norm": 2.095700263977051,
+ "learning_rate": 4.113131313131313e-06,
+ "loss": 1.8375307321548462,
+ "mean_token_accuracy": 0.5848192572593689,
+ "num_tokens": 7208960.0,
+ "step": 440
+ },
+ {
+ "entropy": 1.4185007810592651,
+ "epoch": 0.8909090909090909,
+ "grad_norm": 2.118213176727295,
+ "learning_rate": 4.111111111111111e-06,
+ "loss": 1.4548171758651733,
+ "mean_token_accuracy": 0.657608687877655,
+ "num_tokens": 7225344.0,
+ "step": 441
+ },
+ {
+ "entropy": 1.1041690111160278,
+ "epoch": 0.8929292929292929,
+ "grad_norm": 1.9638988971710205,
+ "learning_rate": 4.10909090909091e-06,
+ "loss": 1.1232322454452515,
+ "mean_token_accuracy": 0.7068881392478943,
+ "num_tokens": 7241728.0,
+ "step": 442
+ },
+ {
+ "entropy": 1.6423180103302002,
+ "epoch": 0.8949494949494949,
+ "grad_norm": 2.32987117767334,
+ "learning_rate": 4.1070707070707075e-06,
+ "loss": 1.665462613105774,
+ "mean_token_accuracy": 0.6133365631103516,
+ "num_tokens": 7258112.0,
+ "step": 443
+ },
+ {
+ "entropy": 1.4778310060501099,
+ "epoch": 0.896969696969697,
+ "grad_norm": 2.1247661113739014,
+ "learning_rate": 4.105050505050505e-06,
+ "loss": 1.4554922580718994,
+ "mean_token_accuracy": 0.6474108695983887,
+ "num_tokens": 7274496.0,
+ "step": 444
+ },
+ {
+ "entropy": 1.3777755498886108,
+ "epoch": 0.898989898989899,
+ "grad_norm": 1.9243263006210327,
+ "learning_rate": 4.103030303030303e-06,
+ "loss": 1.3942883014678955,
+ "mean_token_accuracy": 0.6656692624092102,
+ "num_tokens": 7290880.0,
+ "step": 445
+ },
+ {
+ "entropy": 1.1036415100097656,
+ "epoch": 0.901010101010101,
+ "grad_norm": 1.9711178541183472,
+ "learning_rate": 4.101010101010101e-06,
+ "loss": 1.1070351600646973,
+ "mean_token_accuracy": 0.7213605046272278,
+ "num_tokens": 7307264.0,
+ "step": 446
+ },
+ {
+ "entropy": 1.695487380027771,
+ "epoch": 0.9030303030303031,
+ "grad_norm": 2.1113150119781494,
+ "learning_rate": 4.098989898989899e-06,
+ "loss": 1.6993653774261475,
+ "mean_token_accuracy": 0.5992305874824524,
+ "num_tokens": 7323648.0,
+ "step": 447
+ },
+ {
+ "entropy": 1.496254801750183,
+ "epoch": 0.9050505050505051,
+ "grad_norm": 2.072986602783203,
+ "learning_rate": 4.096969696969697e-06,
+ "loss": 1.5046567916870117,
+ "mean_token_accuracy": 0.6558378338813782,
+ "num_tokens": 7340032.0,
+ "step": 448
+ },
+ {
+ "entropy": 2.1845033168792725,
+ "epoch": 0.907070707070707,
+ "grad_norm": 2.3246262073516846,
+ "learning_rate": 4.094949494949495e-06,
+ "loss": 2.1508708000183105,
+ "mean_token_accuracy": 0.5376160144805908,
+ "num_tokens": 7356416.0,
+ "step": 449
+ },
+ {
+ "entropy": 1.6329164505004883,
+ "epoch": 0.9090909090909091,
+ "grad_norm": 1.997310757637024,
+ "learning_rate": 4.092929292929294e-06,
+ "loss": 1.6422264575958252,
+ "mean_token_accuracy": 0.6232290863990784,
+ "num_tokens": 7372800.0,
+ "step": 450
+ },
+ {
+ "epoch": 0.9090909090909091,
+ "eval_entropy": 1.5515337480652718,
+ "eval_loss": 1.5485161542892456,
+ "eval_mean_token_accuracy": 0.6417716929028111,
+ "eval_num_tokens": 7372800.0,
+ "eval_runtime": 43.7901,
+ "eval_samples_per_second": 22.608,
+ "eval_steps_per_second": 2.832,
+ "step": 450
+ },
+ {
+ "entropy": 1.7854291200637817,
+ "epoch": 0.9111111111111111,
+ "grad_norm": 2.0212173461914062,
+ "learning_rate": 4.0909090909090915e-06,
+ "loss": 1.7920666933059692,
+ "mean_token_accuracy": 0.5950170755386353,
+ "num_tokens": 7389184.0,
+ "step": 451
+ },
+ {
+ "entropy": 1.216304898262024,
+ "epoch": 0.9131313131313131,
+ "grad_norm": 1.8800705671310425,
+ "learning_rate": 4.088888888888889e-06,
+ "loss": 1.2014267444610596,
+ "mean_token_accuracy": 0.7076819539070129,
+ "num_tokens": 7405568.0,
+ "step": 452
+ },
+ {
+ "entropy": 1.5593703985214233,
+ "epoch": 0.9151515151515152,
+ "grad_norm": 2.101381301879883,
+ "learning_rate": 4.086868686868687e-06,
+ "loss": 1.559610366821289,
+ "mean_token_accuracy": 0.6337933540344238,
+ "num_tokens": 7421952.0,
+ "step": 453
+ },
+ {
+ "entropy": 1.4199841022491455,
+ "epoch": 0.9171717171717172,
+ "grad_norm": 2.0038692951202393,
+ "learning_rate": 4.084848484848485e-06,
+ "loss": 1.4040653705596924,
+ "mean_token_accuracy": 0.660845160484314,
+ "num_tokens": 7438336.0,
+ "step": 454
+ },
+ {
+ "entropy": 1.2010453939437866,
+ "epoch": 0.9191919191919192,
+ "grad_norm": 2.004110336303711,
+ "learning_rate": 4.082828282828283e-06,
+ "loss": 1.194115400314331,
+ "mean_token_accuracy": 0.6955910921096802,
+ "num_tokens": 7454720.0,
+ "step": 455
+ },
+ {
+ "entropy": 1.2295804023742676,
+ "epoch": 0.9212121212121213,
+ "grad_norm": 2.130387544631958,
+ "learning_rate": 4.080808080808081e-06,
+ "loss": 1.2125787734985352,
+ "mean_token_accuracy": 0.6993771195411682,
+ "num_tokens": 7471104.0,
+ "step": 456
+ },
+ {
+ "entropy": 1.3706485033035278,
+ "epoch": 0.9232323232323232,
+ "grad_norm": 2.1667706966400146,
+ "learning_rate": 4.07878787878788e-06,
+ "loss": 1.3886957168579102,
+ "mean_token_accuracy": 0.6686614751815796,
+ "num_tokens": 7487488.0,
+ "step": 457
+ },
+ {
+ "entropy": 1.714059591293335,
+ "epoch": 0.9252525252525252,
+ "grad_norm": 2.0084264278411865,
+ "learning_rate": 4.0767676767676775e-06,
+ "loss": 1.724153757095337,
+ "mean_token_accuracy": 0.602161705493927,
+ "num_tokens": 7503872.0,
+ "step": 458
+ },
+ {
+ "entropy": 1.3170617818832397,
+ "epoch": 0.9272727272727272,
+ "grad_norm": 1.9923994541168213,
+ "learning_rate": 4.0747474747474754e-06,
+ "loss": 1.3518096208572388,
+ "mean_token_accuracy": 0.6801416873931885,
+ "num_tokens": 7520256.0,
+ "step": 459
+ },
+ {
+ "entropy": 1.7946380376815796,
+ "epoch": 0.9292929292929293,
+ "grad_norm": 2.3475260734558105,
+ "learning_rate": 4.072727272727273e-06,
+ "loss": 1.824514389038086,
+ "mean_token_accuracy": 0.6005740165710449,
+ "num_tokens": 7536640.0,
+ "step": 460
+ },
+ {
+ "entropy": 1.3695449829101562,
+ "epoch": 0.9313131313131313,
+ "grad_norm": 2.0983943939208984,
+ "learning_rate": 4.070707070707071e-06,
+ "loss": 1.3934273719787598,
+ "mean_token_accuracy": 0.6723253726959229,
+ "num_tokens": 7553024.0,
+ "step": 461
+ },
+ {
+ "entropy": 1.5890663862228394,
+ "epoch": 0.9333333333333333,
+ "grad_norm": 2.1465413570404053,
+ "learning_rate": 4.068686868686869e-06,
+ "loss": 1.611652135848999,
+ "mean_token_accuracy": 0.6408158540725708,
+ "num_tokens": 7569408.0,
+ "step": 462
+ },
+ {
+ "entropy": 1.2593727111816406,
+ "epoch": 0.9353535353535354,
+ "grad_norm": 1.7885241508483887,
+ "learning_rate": 4.066666666666667e-06,
+ "loss": 1.2781705856323242,
+ "mean_token_accuracy": 0.6946751475334167,
+ "num_tokens": 7585792.0,
+ "step": 463
+ },
+ {
+ "entropy": 1.5680429935455322,
+ "epoch": 0.9373737373737374,
+ "grad_norm": 2.1466636657714844,
+ "learning_rate": 4.064646464646465e-06,
+ "loss": 1.60218346118927,
+ "mean_token_accuracy": 0.6334269642829895,
+ "num_tokens": 7602176.0,
+ "step": 464
+ },
+ {
+ "entropy": 1.681670069694519,
+ "epoch": 0.9393939393939394,
+ "grad_norm": 2.2211875915527344,
+ "learning_rate": 4.062626262626263e-06,
+ "loss": 1.6819908618927002,
+ "mean_token_accuracy": 0.6177943348884583,
+ "num_tokens": 7618560.0,
+ "step": 465
+ },
+ {
+ "entropy": 1.2239924669265747,
+ "epoch": 0.9414141414141414,
+ "grad_norm": 1.9997022151947021,
+ "learning_rate": 4.060606060606061e-06,
+ "loss": 1.2383348941802979,
+ "mean_token_accuracy": 0.6922325491905212,
+ "num_tokens": 7634944.0,
+ "step": 466
+ },
+ {
+ "entropy": 1.291772723197937,
+ "epoch": 0.9434343434343434,
+ "grad_norm": 2.0868117809295654,
+ "learning_rate": 4.058585858585859e-06,
+ "loss": 1.2879221439361572,
+ "mean_token_accuracy": 0.6883854269981384,
+ "num_tokens": 7651328.0,
+ "step": 467
+ },
+ {
+ "entropy": 1.2860186100006104,
+ "epoch": 0.9454545454545454,
+ "grad_norm": 1.8586393594741821,
+ "learning_rate": 4.056565656565657e-06,
+ "loss": 1.2820203304290771,
+ "mean_token_accuracy": 0.6802638173103333,
+ "num_tokens": 7667712.0,
+ "step": 468
+ },
+ {
+ "entropy": 1.860795259475708,
+ "epoch": 0.9474747474747475,
+ "grad_norm": 2.364405393600464,
+ "learning_rate": 4.054545454545455e-06,
+ "loss": 1.8880727291107178,
+ "mean_token_accuracy": 0.5821323990821838,
+ "num_tokens": 7684096.0,
+ "step": 469
+ },
+ {
+ "entropy": 1.6017589569091797,
+ "epoch": 0.9494949494949495,
+ "grad_norm": 2.023054599761963,
+ "learning_rate": 4.052525252525253e-06,
+ "loss": 1.5993852615356445,
+ "mean_token_accuracy": 0.6366634368896484,
+ "num_tokens": 7700480.0,
+ "step": 470
+ },
+ {
+ "entropy": 1.8498003482818604,
+ "epoch": 0.9515151515151515,
+ "grad_norm": 2.4474003314971924,
+ "learning_rate": 4.050505050505051e-06,
+ "loss": 1.868700623512268,
+ "mean_token_accuracy": 0.5826209187507629,
+ "num_tokens": 7716864.0,
+ "step": 471
+ },
+ {
+ "entropy": 1.5625540018081665,
+ "epoch": 0.9535353535353536,
+ "grad_norm": 2.1018362045288086,
+ "learning_rate": 4.048484848484849e-06,
+ "loss": 1.571077585220337,
+ "mean_token_accuracy": 0.6433805823326111,
+ "num_tokens": 7733248.0,
+ "step": 472
+ },
+ {
+ "entropy": 1.8534579277038574,
+ "epoch": 0.9555555555555556,
+ "grad_norm": 2.407588243484497,
+ "learning_rate": 4.046464646464647e-06,
+ "loss": 1.8789153099060059,
+ "mean_token_accuracy": 0.5859794616699219,
+ "num_tokens": 7749632.0,
+ "step": 473
+ },
+ {
+ "entropy": 1.3264559507369995,
+ "epoch": 0.9575757575757575,
+ "grad_norm": 2.007199764251709,
+ "learning_rate": 4.044444444444445e-06,
+ "loss": 1.3489338159561157,
+ "mean_token_accuracy": 0.6650586128234863,
+ "num_tokens": 7766016.0,
+ "step": 474
+ },
+ {
+ "entropy": 1.1361761093139648,
+ "epoch": 0.9595959595959596,
+ "grad_norm": 1.9226998090744019,
+ "learning_rate": 4.0424242424242425e-06,
+ "loss": 1.156738519668579,
+ "mean_token_accuracy": 0.7226428985595703,
+ "num_tokens": 7782400.0,
+ "step": 475
+ },
+ {
+ "entropy": 1.550872802734375,
+ "epoch": 0.9616161616161616,
+ "grad_norm": 2.0082473754882812,
+ "learning_rate": 4.04040404040404e-06,
+ "loss": 1.5657379627227783,
+ "mean_token_accuracy": 0.642953097820282,
+ "num_tokens": 7798784.0,
+ "step": 476
+ },
+ {
+ "entropy": 1.0122721195220947,
+ "epoch": 0.9636363636363636,
+ "grad_norm": 1.9946776628494263,
+ "learning_rate": 4.038383838383838e-06,
+ "loss": 1.0301119089126587,
+ "mean_token_accuracy": 0.7361993193626404,
+ "num_tokens": 7815168.0,
+ "step": 477
+ },
+ {
+ "entropy": 1.61614990234375,
+ "epoch": 0.9656565656565657,
+ "grad_norm": 2.5009427070617676,
+ "learning_rate": 4.036363636363637e-06,
+ "loss": 1.6194994449615479,
+ "mean_token_accuracy": 0.620175838470459,
+ "num_tokens": 7831552.0,
+ "step": 478
+ },
+ {
+ "entropy": 1.5723443031311035,
+ "epoch": 0.9676767676767677,
+ "grad_norm": 1.9101917743682861,
+ "learning_rate": 4.034343434343435e-06,
+ "loss": 1.5586073398590088,
+ "mean_token_accuracy": 0.6223741769790649,
+ "num_tokens": 7847936.0,
+ "step": 479
+ },
+ {
+ "entropy": 1.271834373474121,
+ "epoch": 0.9696969696969697,
+ "grad_norm": 1.768438458442688,
+ "learning_rate": 4.032323232323233e-06,
+ "loss": 1.2466087341308594,
+ "mean_token_accuracy": 0.692354679107666,
+ "num_tokens": 7864320.0,
+ "step": 480
+ },
+ {
+ "entropy": 1.4550020694732666,
+ "epoch": 0.9717171717171718,
+ "grad_norm": 1.9753917455673218,
+ "learning_rate": 4.030303030303031e-06,
+ "loss": 1.4689980745315552,
+ "mean_token_accuracy": 0.6530288457870483,
+ "num_tokens": 7880704.0,
+ "step": 481
+ },
+ {
+ "entropy": 2.023750066757202,
+ "epoch": 0.9737373737373738,
+ "grad_norm": 2.3014674186706543,
+ "learning_rate": 4.0282828282828285e-06,
+ "loss": 2.0601019859313965,
+ "mean_token_accuracy": 0.5528212189674377,
+ "num_tokens": 7897088.0,
+ "step": 482
+ },
+ {
+ "entropy": 1.3225218057632446,
+ "epoch": 0.9757575757575757,
+ "grad_norm": 1.9937688112258911,
+ "learning_rate": 4.0262626262626264e-06,
+ "loss": 1.3186583518981934,
+ "mean_token_accuracy": 0.6751343607902527,
+ "num_tokens": 7913472.0,
+ "step": 483
+ },
+ {
+ "entropy": 1.4357911348342896,
+ "epoch": 0.9777777777777777,
+ "grad_norm": 1.9591492414474487,
+ "learning_rate": 4.024242424242424e-06,
+ "loss": 1.4157384634017944,
+ "mean_token_accuracy": 0.6618832349777222,
+ "num_tokens": 7929856.0,
+ "step": 484
+ },
+ {
+ "entropy": 1.470274567604065,
+ "epoch": 0.9797979797979798,
+ "grad_norm": 1.9318779706954956,
+ "learning_rate": 4.022222222222222e-06,
+ "loss": 1.4754853248596191,
+ "mean_token_accuracy": 0.6497923731803894,
+ "num_tokens": 7946240.0,
+ "step": 485
+ },
+ {
+ "entropy": 1.345750331878662,
+ "epoch": 0.9818181818181818,
+ "grad_norm": 1.9079619646072388,
+ "learning_rate": 4.02020202020202e-06,
+ "loss": 1.336526870727539,
+ "mean_token_accuracy": 0.6800805926322937,
+ "num_tokens": 7962624.0,
+ "step": 486
+ },
+ {
+ "entropy": 1.4641839265823364,
+ "epoch": 0.9838383838383838,
+ "grad_norm": 1.8413265943527222,
+ "learning_rate": 4.018181818181818e-06,
+ "loss": 1.4556326866149902,
+ "mean_token_accuracy": 0.6591963768005371,
+ "num_tokens": 7979008.0,
+ "step": 487
+ },
+ {
+ "entropy": 1.7966911792755127,
+ "epoch": 0.9858585858585859,
+ "grad_norm": 2.138899564743042,
+ "learning_rate": 4.016161616161616e-06,
+ "loss": 1.822898507118225,
+ "mean_token_accuracy": 0.5959941148757935,
+ "num_tokens": 7995392.0,
+ "step": 488
+ },
+ {
+ "entropy": 1.4334503412246704,
+ "epoch": 0.9878787878787879,
+ "grad_norm": 2.206122636795044,
+ "learning_rate": 4.014141414141415e-06,
+ "loss": 1.4127681255340576,
+ "mean_token_accuracy": 0.6609672904014587,
+ "num_tokens": 8011776.0,
+ "step": 489
+ },
+ {
+ "entropy": 1.684112548828125,
+ "epoch": 0.98989898989899,
+ "grad_norm": 2.0637400150299072,
+ "learning_rate": 4.0121212121212125e-06,
+ "loss": 1.6790317296981812,
+ "mean_token_accuracy": 0.6159012913703918,
+ "num_tokens": 8028160.0,
+ "step": 490
+ },
+ {
+ "entropy": 2.0716898441314697,
+ "epoch": 0.9919191919191919,
+ "grad_norm": 3.3912627696990967,
+ "learning_rate": 4.01010101010101e-06,
+ "loss": 2.0814826488494873,
+ "mean_token_accuracy": 0.5522105693817139,
+ "num_tokens": 8044544.0,
+ "step": 491
+ },
+ {
+ "entropy": 1.8764609098434448,
+ "epoch": 0.9939393939393939,
+ "grad_norm": 2.00569748878479,
+ "learning_rate": 4.008080808080808e-06,
+ "loss": 1.9150068759918213,
+ "mean_token_accuracy": 0.5823766589164734,
+ "num_tokens": 8060928.0,
+ "step": 492
+ },
+ {
+ "entropy": 1.630242943763733,
+ "epoch": 0.9959595959595959,
+ "grad_norm": 2.0648858547210693,
+ "learning_rate": 4.006060606060607e-06,
+ "loss": 1.6354784965515137,
+ "mean_token_accuracy": 0.621213972568512,
+ "num_tokens": 8077312.0,
+ "step": 493
+ },
+ {
+ "entropy": 1.7174078226089478,
+ "epoch": 0.997979797979798,
+ "grad_norm": 2.1376583576202393,
+ "learning_rate": 4.004040404040405e-06,
+ "loss": 1.7092773914337158,
+ "mean_token_accuracy": 0.6129701733589172,
+ "num_tokens": 8093696.0,
+ "step": 494
+ },
+ {
+ "entropy": 1.2112717628479004,
+ "epoch": 1.0,
+ "grad_norm": 1.9164371490478516,
+ "learning_rate": 4.002020202020203e-06,
+ "loss": 1.2106354236602783,
+ "mean_token_accuracy": 0.7040180563926697,
+ "num_tokens": 8110080.0,
+ "step": 495
+ },
+ {
+ "entropy": 1.2168891429901123,
+ "epoch": 1.002020202020202,
+ "grad_norm": 1.9845075607299805,
+ "learning_rate": 4.000000000000001e-06,
+ "loss": 1.1398870944976807,
+ "mean_token_accuracy": 0.7053004503250122,
+ "num_tokens": 8126464.0,
+ "step": 496
+ },
+ {
+ "entropy": 1.4670923948287964,
+ "epoch": 1.004040404040404,
+ "grad_norm": 2.0864014625549316,
+ "learning_rate": 3.9979797979797986e-06,
+ "loss": 1.4241242408752441,
+ "mean_token_accuracy": 0.6605398058891296,
+ "num_tokens": 8142848.0,
+ "step": 497
+ },
+ {
+ "entropy": 1.560593843460083,
+ "epoch": 1.006060606060606,
+ "grad_norm": 1.8882137537002563,
+ "learning_rate": 3.9959595959595964e-06,
+ "loss": 1.5164835453033447,
+ "mean_token_accuracy": 0.6610894203186035,
+ "num_tokens": 8159232.0,
+ "step": 498
+ },
+ {
+ "entropy": 1.2573609352111816,
+ "epoch": 1.0080808080808081,
+ "grad_norm": 2.011486530303955,
+ "learning_rate": 3.993939393939394e-06,
+ "loss": 1.2119636535644531,
+ "mean_token_accuracy": 0.6832559704780579,
+ "num_tokens": 8175616.0,
+ "step": 499
+ },
+ {
+ "entropy": 1.6119122505187988,
+ "epoch": 1.0101010101010102,
+ "grad_norm": 2.0089032649993896,
+ "learning_rate": 3.991919191919192e-06,
+ "loss": 1.5869622230529785,
+ "mean_token_accuracy": 0.6301905512809753,
+ "num_tokens": 8192000.0,
+ "step": 500
+ },
+ {
+ "epoch": 1.0101010101010102,
+ "eval_entropy": 1.4907484121860997,
+ "eval_loss": 1.5444872379302979,
+ "eval_mean_token_accuracy": 0.6426531960887294,
+ "eval_num_tokens": 8192000.0,
+ "eval_runtime": 43.729,
+ "eval_samples_per_second": 22.639,
+ "eval_steps_per_second": 2.836,
+ "step": 500
+ },
+ {
+ "entropy": 1.5645418167114258,
+ "epoch": 1.0121212121212122,
+ "grad_norm": 2.171133279800415,
+ "learning_rate": 3.98989898989899e-06,
+ "loss": 1.5252666473388672,
+ "mean_token_accuracy": 0.639594554901123,
+ "num_tokens": 8208384.0,
+ "step": 501
+ },
+ {
+ "entropy": 1.2985379695892334,
+ "epoch": 1.0141414141414142,
+ "grad_norm": 1.9075100421905518,
+ "learning_rate": 3.987878787878788e-06,
+ "loss": 1.2863168716430664,
+ "mean_token_accuracy": 0.6797142028808594,
+ "num_tokens": 8224768.0,
+ "step": 502
+ },
+ {
+ "entropy": 1.4055086374282837,
+ "epoch": 1.0161616161616163,
+ "grad_norm": 1.8511557579040527,
+ "learning_rate": 3.985858585858587e-06,
+ "loss": 1.440205454826355,
+ "mean_token_accuracy": 0.6671348214149475,
+ "num_tokens": 8241152.0,
+ "step": 503
+ },
+ {
+ "entropy": 1.0818580389022827,
+ "epoch": 1.018181818181818,
+ "grad_norm": 1.8912067413330078,
+ "learning_rate": 3.983838383838385e-06,
+ "loss": 1.0614542961120605,
+ "mean_token_accuracy": 0.7345505356788635,
+ "num_tokens": 8257536.0,
+ "step": 504
+ },
+ {
+ "entropy": 0.9176992177963257,
+ "epoch": 1.02020202020202,
+ "grad_norm": 1.7737077474594116,
+ "learning_rate": 3.9818181818181825e-06,
+ "loss": 0.9281337261199951,
+ "mean_token_accuracy": 0.7594040036201477,
+ "num_tokens": 8273920.0,
+ "step": 505
+ },
+ {
+ "entropy": 1.8801840543746948,
+ "epoch": 1.0222222222222221,
+ "grad_norm": 2.191689968109131,
+ "learning_rate": 3.97979797979798e-06,
+ "loss": 1.9127342700958252,
+ "mean_token_accuracy": 0.5732169151306152,
+ "num_tokens": 8290304.0,
+ "step": 506
+ },
+ {
+ "entropy": 1.2336418628692627,
+ "epoch": 1.0242424242424242,
+ "grad_norm": 2.0369555950164795,
+ "learning_rate": 3.977777777777778e-06,
+ "loss": 1.2513363361358643,
+ "mean_token_accuracy": 0.6852100491523743,
+ "num_tokens": 8306688.0,
+ "step": 507
+ },
+ {
+ "entropy": 1.8379974365234375,
+ "epoch": 1.0262626262626262,
+ "grad_norm": 2.176361083984375,
+ "learning_rate": 3.975757575757576e-06,
+ "loss": 1.8472888469696045,
+ "mean_token_accuracy": 0.5863458514213562,
+ "num_tokens": 8323072.0,
+ "step": 508
+ },
+ {
+ "entropy": 1.4421337842941284,
+ "epoch": 1.0282828282828282,
+ "grad_norm": 2.305441379547119,
+ "learning_rate": 3.973737373737374e-06,
+ "loss": 1.4835591316223145,
+ "mean_token_accuracy": 0.6530288457870483,
+ "num_tokens": 8339456.0,
+ "step": 509
+ },
+ {
+ "entropy": 1.535184621810913,
+ "epoch": 1.0303030303030303,
+ "grad_norm": 2.0329015254974365,
+ "learning_rate": 3.971717171717172e-06,
+ "loss": 1.547795057296753,
+ "mean_token_accuracy": 0.6268319487571716,
+ "num_tokens": 8355840.0,
+ "step": 510
+ },
+ {
+ "entropy": 1.295592188835144,
+ "epoch": 1.0323232323232323,
+ "grad_norm": 2.0321359634399414,
+ "learning_rate": 3.96969696969697e-06,
+ "loss": 1.2885974645614624,
+ "mean_token_accuracy": 0.681485116481781,
+ "num_tokens": 8372224.0,
+ "step": 511
+ },
+ {
+ "entropy": 1.345553994178772,
+ "epoch": 1.0343434343434343,
+ "grad_norm": 1.9151290655136108,
+ "learning_rate": 3.967676767676768e-06,
+ "loss": 1.3364320993423462,
+ "mean_token_accuracy": 0.6849657893180847,
+ "num_tokens": 8388608.0,
+ "step": 512
+ },
+ {
+ "entropy": 1.3150815963745117,
+ "epoch": 1.0363636363636364,
+ "grad_norm": 1.9379258155822754,
+ "learning_rate": 3.965656565656566e-06,
+ "loss": 1.317258596420288,
+ "mean_token_accuracy": 0.6873473525047302,
+ "num_tokens": 8404992.0,
+ "step": 513
+ },
+ {
+ "entropy": 1.46832275390625,
+ "epoch": 1.0383838383838384,
+ "grad_norm": 2.1009161472320557,
+ "learning_rate": 3.963636363636364e-06,
+ "loss": 1.4794366359710693,
+ "mean_token_accuracy": 0.6513800621032715,
+ "num_tokens": 8421376.0,
+ "step": 514
+ },
+ {
+ "entropy": 1.389290452003479,
+ "epoch": 1.0404040404040404,
+ "grad_norm": 1.8813941478729248,
+ "learning_rate": 3.961616161616162e-06,
+ "loss": 1.3930776119232178,
+ "mean_token_accuracy": 0.6799584627151489,
+ "num_tokens": 8437760.0,
+ "step": 515
+ },
+ {
+ "entropy": 1.2919796705245972,
+ "epoch": 1.0424242424242425,
+ "grad_norm": 2.0765745639801025,
+ "learning_rate": 3.95959595959596e-06,
+ "loss": 1.2943801879882812,
+ "mean_token_accuracy": 0.6929653286933899,
+ "num_tokens": 8454144.0,
+ "step": 516
+ },
+ {
+ "entropy": 1.172440767288208,
+ "epoch": 1.0444444444444445,
+ "grad_norm": 2.126800298690796,
+ "learning_rate": 3.957575757575758e-06,
+ "loss": 1.1923961639404297,
+ "mean_token_accuracy": 0.7098192572593689,
+ "num_tokens": 8470528.0,
+ "step": 517
+ },
+ {
+ "entropy": 1.8239458799362183,
+ "epoch": 1.0464646464646465,
+ "grad_norm": 1.96744966506958,
+ "learning_rate": 3.955555555555556e-06,
+ "loss": 1.840916633605957,
+ "mean_token_accuracy": 0.6033830046653748,
+ "num_tokens": 8486912.0,
+ "step": 518
+ },
+ {
+ "entropy": 1.4117076396942139,
+ "epoch": 1.0484848484848486,
+ "grad_norm": 2.0560126304626465,
+ "learning_rate": 3.953535353535354e-06,
+ "loss": 1.421530842781067,
+ "mean_token_accuracy": 0.6581583023071289,
+ "num_tokens": 8503296.0,
+ "step": 519
+ },
+ {
+ "entropy": 1.7085175514221191,
+ "epoch": 1.0505050505050506,
+ "grad_norm": 2.1386539936065674,
+ "learning_rate": 3.951515151515152e-06,
+ "loss": 1.7334365844726562,
+ "mean_token_accuracy": 0.6282975077629089,
+ "num_tokens": 8519680.0,
+ "step": 520
+ },
+ {
+ "entropy": 1.0607928037643433,
+ "epoch": 1.0525252525252524,
+ "grad_norm": 1.981020212173462,
+ "learning_rate": 3.9494949494949496e-06,
+ "loss": 1.0639690160751343,
+ "mean_token_accuracy": 0.722337543964386,
+ "num_tokens": 8536064.0,
+ "step": 521
+ },
+ {
+ "entropy": 1.2227518558502197,
+ "epoch": 1.0545454545454545,
+ "grad_norm": 1.9116015434265137,
+ "learning_rate": 3.9474747474747474e-06,
+ "loss": 1.2139748334884644,
+ "mean_token_accuracy": 0.704384446144104,
+ "num_tokens": 8552448.0,
+ "step": 522
+ },
+ {
+ "entropy": 1.2655551433563232,
+ "epoch": 1.0565656565656565,
+ "grad_norm": 2.176706075668335,
+ "learning_rate": 3.945454545454545e-06,
+ "loss": 1.2869982719421387,
+ "mean_token_accuracy": 0.678798258304596,
+ "num_tokens": 8568832.0,
+ "step": 523
+ },
+ {
+ "entropy": 1.436476707458496,
+ "epoch": 1.0585858585858585,
+ "grad_norm": 2.034846544265747,
+ "learning_rate": 3.943434343434343e-06,
+ "loss": 1.431445837020874,
+ "mean_token_accuracy": 0.6650586128234863,
+ "num_tokens": 8585216.0,
+ "step": 524
+ },
+ {
+ "entropy": 1.4563549757003784,
+ "epoch": 1.0606060606060606,
+ "grad_norm": 2.0301241874694824,
+ "learning_rate": 3.941414141414142e-06,
+ "loss": 1.468353033065796,
+ "mean_token_accuracy": 0.6550439596176147,
+ "num_tokens": 8601600.0,
+ "step": 525
+ },
+ {
+ "entropy": 1.4636993408203125,
+ "epoch": 1.0626262626262626,
+ "grad_norm": 2.092679023742676,
+ "learning_rate": 3.93939393939394e-06,
+ "loss": 1.4488987922668457,
+ "mean_token_accuracy": 0.652723491191864,
+ "num_tokens": 8617984.0,
+ "step": 526
+ },
+ {
+ "entropy": 1.1217654943466187,
+ "epoch": 1.0646464646464646,
+ "grad_norm": 1.953627347946167,
+ "learning_rate": 3.937373737373738e-06,
+ "loss": 1.1403257846832275,
+ "mean_token_accuracy": 0.7172080874443054,
+ "num_tokens": 8634368.0,
+ "step": 527
+ },
+ {
+ "entropy": 1.7785910367965698,
+ "epoch": 1.0666666666666667,
+ "grad_norm": 1.8496789932250977,
+ "learning_rate": 3.935353535353536e-06,
+ "loss": 1.7961615324020386,
+ "mean_token_accuracy": 0.6036272644996643,
+ "num_tokens": 8650752.0,
+ "step": 528
+ },
+ {
+ "entropy": 1.0998575687408447,
+ "epoch": 1.0686868686868687,
+ "grad_norm": 1.8732962608337402,
+ "learning_rate": 3.9333333333333335e-06,
+ "loss": 1.0960031747817993,
+ "mean_token_accuracy": 0.7223986387252808,
+ "num_tokens": 8667136.0,
+ "step": 529
+ },
+ {
+ "entropy": 1.6707724332809448,
+ "epoch": 1.0707070707070707,
+ "grad_norm": 2.220453977584839,
+ "learning_rate": 3.931313131313131e-06,
+ "loss": 1.6837453842163086,
+ "mean_token_accuracy": 0.6148021221160889,
+ "num_tokens": 8683520.0,
+ "step": 530
+ },
+ {
+ "entropy": 1.6108022928237915,
+ "epoch": 1.0727272727272728,
+ "grad_norm": 2.2464118003845215,
+ "learning_rate": 3.929292929292929e-06,
+ "loss": 1.633517861366272,
+ "mean_token_accuracy": 0.6195651888847351,
+ "num_tokens": 8699904.0,
+ "step": 531
+ },
+ {
+ "entropy": 1.1888173818588257,
+ "epoch": 1.0747474747474748,
+ "grad_norm": 1.8803986310958862,
+ "learning_rate": 3.927272727272727e-06,
+ "loss": 1.1904420852661133,
+ "mean_token_accuracy": 0.7134220600128174,
+ "num_tokens": 8716288.0,
+ "step": 532
+ },
+ {
+ "entropy": 1.0656445026397705,
+ "epoch": 1.0767676767676768,
+ "grad_norm": 2.038243532180786,
+ "learning_rate": 3.925252525252525e-06,
+ "loss": 1.0505216121673584,
+ "mean_token_accuracy": 0.7284440398216248,
+ "num_tokens": 8732672.0,
+ "step": 533
+ },
+ {
+ "entropy": 1.1473655700683594,
+ "epoch": 1.0787878787878789,
+ "grad_norm": 1.981107234954834,
+ "learning_rate": 3.923232323232323e-06,
+ "loss": 1.136179804801941,
+ "mean_token_accuracy": 0.7085368633270264,
+ "num_tokens": 8749056.0,
+ "step": 534
+ },
+ {
+ "entropy": 1.340027928352356,
+ "epoch": 1.0808080808080809,
+ "grad_norm": 2.2797436714172363,
+ "learning_rate": 3.921212121212122e-06,
+ "loss": 1.334214210510254,
+ "mean_token_accuracy": 0.6749511361122131,
+ "num_tokens": 8765440.0,
+ "step": 535
+ },
+ {
+ "entropy": 1.1464864015579224,
+ "epoch": 1.082828282828283,
+ "grad_norm": 1.9428092241287231,
+ "learning_rate": 3.9191919191919196e-06,
+ "loss": 1.1649314165115356,
+ "mean_token_accuracy": 0.72013920545578,
+ "num_tokens": 8781824.0,
+ "step": 536
+ },
+ {
+ "entropy": 1.784925103187561,
+ "epoch": 1.084848484848485,
+ "grad_norm": 2.157468795776367,
+ "learning_rate": 3.9171717171717175e-06,
+ "loss": 1.7725508213043213,
+ "mean_token_accuracy": 0.6014899611473083,
+ "num_tokens": 8798208.0,
+ "step": 537
+ },
+ {
+ "entropy": 1.362166166305542,
+ "epoch": 1.0868686868686868,
+ "grad_norm": 2.109646797180176,
+ "learning_rate": 3.915151515151515e-06,
+ "loss": 1.331969976425171,
+ "mean_token_accuracy": 0.6720200181007385,
+ "num_tokens": 8814592.0,
+ "step": 538
+ },
+ {
+ "entropy": 1.5614176988601685,
+ "epoch": 1.0888888888888888,
+ "grad_norm": 2.1047098636627197,
+ "learning_rate": 3.913131313131314e-06,
+ "loss": 1.534292459487915,
+ "mean_token_accuracy": 0.6446629166603088,
+ "num_tokens": 8830976.0,
+ "step": 539
+ },
+ {
+ "entropy": 1.5551490783691406,
+ "epoch": 1.0909090909090908,
+ "grad_norm": 2.10304594039917,
+ "learning_rate": 3.911111111111112e-06,
+ "loss": 1.5706363916397095,
+ "mean_token_accuracy": 0.6322056651115417,
+ "num_tokens": 8847360.0,
+ "step": 540
+ },
+ {
+ "entropy": 1.7433172464370728,
+ "epoch": 1.0929292929292929,
+ "grad_norm": 1.8621257543563843,
+ "learning_rate": 3.90909090909091e-06,
+ "loss": 1.7367552518844604,
+ "mean_token_accuracy": 0.6162066459655762,
+ "num_tokens": 8863744.0,
+ "step": 541
+ },
+ {
+ "entropy": 1.4073128700256348,
+ "epoch": 1.094949494949495,
+ "grad_norm": 1.939588189125061,
+ "learning_rate": 3.907070707070708e-06,
+ "loss": 1.3844857215881348,
+ "mean_token_accuracy": 0.6634709239006042,
+ "num_tokens": 8880128.0,
+ "step": 542
+ },
+ {
+ "entropy": 1.479836344718933,
+ "epoch": 1.096969696969697,
+ "grad_norm": 2.06921648979187,
+ "learning_rate": 3.905050505050506e-06,
+ "loss": 1.4933744668960571,
+ "mean_token_accuracy": 0.6510136723518372,
+ "num_tokens": 8896512.0,
+ "step": 543
+ },
+ {
+ "entropy": 1.3934576511383057,
+ "epoch": 1.098989898989899,
+ "grad_norm": 2.0697920322418213,
+ "learning_rate": 3.9030303030303035e-06,
+ "loss": 1.3886666297912598,
+ "mean_token_accuracy": 0.6775158643722534,
+ "num_tokens": 8912896.0,
+ "step": 544
+ },
+ {
+ "entropy": 1.5223709344863892,
+ "epoch": 1.101010101010101,
+ "grad_norm": 2.1627066135406494,
+ "learning_rate": 3.901010101010101e-06,
+ "loss": 1.5601061582565308,
+ "mean_token_accuracy": 0.6392892003059387,
+ "num_tokens": 8929280.0,
+ "step": 545
+ },
+ {
+ "entropy": 1.4462933540344238,
+ "epoch": 1.103030303030303,
+ "grad_norm": 1.9871046543121338,
+ "learning_rate": 3.898989898989899e-06,
+ "loss": 1.463792085647583,
+ "mean_token_accuracy": 0.6518075466156006,
+ "num_tokens": 8945664.0,
+ "step": 546
+ },
+ {
+ "entropy": 1.325921654701233,
+ "epoch": 1.105050505050505,
+ "grad_norm": 1.9081087112426758,
+ "learning_rate": 3.896969696969697e-06,
+ "loss": 1.3303362131118774,
+ "mean_token_accuracy": 0.7009648084640503,
+ "num_tokens": 8962048.0,
+ "step": 547
+ },
+ {
+ "entropy": 1.415048360824585,
+ "epoch": 1.107070707070707,
+ "grad_norm": 2.3306756019592285,
+ "learning_rate": 3.894949494949495e-06,
+ "loss": 1.4139145612716675,
+ "mean_token_accuracy": 0.6849047541618347,
+ "num_tokens": 8978432.0,
+ "step": 548
+ },
+ {
+ "entropy": 1.5863295793533325,
+ "epoch": 1.1090909090909091,
+ "grad_norm": 1.993159294128418,
+ "learning_rate": 3.892929292929293e-06,
+ "loss": 1.6010534763336182,
+ "mean_token_accuracy": 0.6424035429954529,
+ "num_tokens": 8994816.0,
+ "step": 549
+ },
+ {
+ "entropy": 1.5526721477508545,
+ "epoch": 1.1111111111111112,
+ "grad_norm": 2.0528876781463623,
+ "learning_rate": 3.890909090909092e-06,
+ "loss": 1.5465538501739502,
+ "mean_token_accuracy": 0.659807026386261,
+ "num_tokens": 9011200.0,
+ "step": 550
+ },
+ {
+ "epoch": 1.1111111111111112,
+ "eval_entropy": 1.4479231247978825,
+ "eval_loss": 1.5459802150726318,
+ "eval_mean_token_accuracy": 0.6425906530311031,
+ "eval_num_tokens": 9011200.0,
+ "eval_runtime": 43.7828,
+ "eval_samples_per_second": 22.612,
+ "eval_steps_per_second": 2.832,
+ "step": 550
+ },
+ {
+ "entropy": 1.4322376251220703,
+ "epoch": 1.1131313131313132,
+ "grad_norm": 1.933549404144287,
+ "learning_rate": 3.88888888888889e-06,
+ "loss": 1.4371025562286377,
+ "mean_token_accuracy": 0.670676589012146,
+ "num_tokens": 9027584.0,
+ "step": 551
+ },
+ {
+ "entropy": 1.5050671100616455,
+ "epoch": 1.1151515151515152,
+ "grad_norm": 1.9736099243164062,
+ "learning_rate": 3.8868686868686875e-06,
+ "loss": 1.5362370014190674,
+ "mean_token_accuracy": 0.651624321937561,
+ "num_tokens": 9043968.0,
+ "step": 552
+ },
+ {
+ "entropy": 1.4335887432098389,
+ "epoch": 1.1171717171717173,
+ "grad_norm": 1.8896421194076538,
+ "learning_rate": 3.884848484848485e-06,
+ "loss": 1.4439888000488281,
+ "mean_token_accuracy": 0.670615553855896,
+ "num_tokens": 9060352.0,
+ "step": 553
+ },
+ {
+ "entropy": 1.5979355573654175,
+ "epoch": 1.1191919191919193,
+ "grad_norm": 2.1710526943206787,
+ "learning_rate": 3.882828282828283e-06,
+ "loss": 1.629098653793335,
+ "mean_token_accuracy": 0.6226184368133545,
+ "num_tokens": 9076736.0,
+ "step": 554
+ },
+ {
+ "entropy": 1.2901698350906372,
+ "epoch": 1.121212121212121,
+ "grad_norm": 2.129443407058716,
+ "learning_rate": 3.880808080808081e-06,
+ "loss": 1.2824913263320923,
+ "mean_token_accuracy": 0.6816682815551758,
+ "num_tokens": 9093120.0,
+ "step": 555
+ },
+ {
+ "entropy": 1.403212308883667,
+ "epoch": 1.1232323232323231,
+ "grad_norm": 1.883240818977356,
+ "learning_rate": 3.878787878787879e-06,
+ "loss": 1.3960213661193848,
+ "mean_token_accuracy": 0.6651197075843811,
+ "num_tokens": 9109504.0,
+ "step": 556
+ },
+ {
+ "entropy": 1.3959479331970215,
+ "epoch": 1.1252525252525252,
+ "grad_norm": 2.1145691871643066,
+ "learning_rate": 3.876767676767677e-06,
+ "loss": 1.4046590328216553,
+ "mean_token_accuracy": 0.6607230305671692,
+ "num_tokens": 9125888.0,
+ "step": 557
+ },
+ {
+ "entropy": 1.0899873971939087,
+ "epoch": 1.1272727272727272,
+ "grad_norm": 1.8862192630767822,
+ "learning_rate": 3.874747474747475e-06,
+ "loss": 1.0965509414672852,
+ "mean_token_accuracy": 0.7245969772338867,
+ "num_tokens": 9142272.0,
+ "step": 558
+ },
+ {
+ "entropy": 1.0380184650421143,
+ "epoch": 1.1292929292929292,
+ "grad_norm": 1.946702241897583,
+ "learning_rate": 3.872727272727273e-06,
+ "loss": 1.0539875030517578,
+ "mean_token_accuracy": 0.7423058152198792,
+ "num_tokens": 9158656.0,
+ "step": 559
+ },
+ {
+ "entropy": 1.1173628568649292,
+ "epoch": 1.1313131313131313,
+ "grad_norm": 2.002847909927368,
+ "learning_rate": 3.8707070707070706e-06,
+ "loss": 1.128816843032837,
+ "mean_token_accuracy": 0.7112848162651062,
+ "num_tokens": 9175040.0,
+ "step": 560
+ },
+ {
+ "entropy": 1.6536206007003784,
+ "epoch": 1.1333333333333333,
+ "grad_norm": 2.212761640548706,
+ "learning_rate": 3.868686868686869e-06,
+ "loss": 1.6286437511444092,
+ "mean_token_accuracy": 0.6277479529380798,
+ "num_tokens": 9191424.0,
+ "step": 561
+ },
+ {
+ "entropy": 1.7187970876693726,
+ "epoch": 1.1353535353535353,
+ "grad_norm": 2.207310199737549,
+ "learning_rate": 3.866666666666667e-06,
+ "loss": 1.7072829008102417,
+ "mean_token_accuracy": 0.6180996298789978,
+ "num_tokens": 9207808.0,
+ "step": 562
+ },
+ {
+ "entropy": 1.7057682275772095,
+ "epoch": 1.1373737373737374,
+ "grad_norm": 2.1582961082458496,
+ "learning_rate": 3.864646464646465e-06,
+ "loss": 1.7216498851776123,
+ "mean_token_accuracy": 0.6105886697769165,
+ "num_tokens": 9224192.0,
+ "step": 563
+ },
+ {
+ "entropy": 1.8016951084136963,
+ "epoch": 1.1393939393939394,
+ "grad_norm": 2.035249948501587,
+ "learning_rate": 3.862626262626263e-06,
+ "loss": 1.8101240396499634,
+ "mean_token_accuracy": 0.6199315786361694,
+ "num_tokens": 9240576.0,
+ "step": 564
+ },
+ {
+ "entropy": 1.094889760017395,
+ "epoch": 1.1414141414141414,
+ "grad_norm": 2.119861364364624,
+ "learning_rate": 3.860606060606061e-06,
+ "loss": 1.0990344285964966,
+ "mean_token_accuracy": 0.7071323990821838,
+ "num_tokens": 9256960.0,
+ "step": 565
+ },
+ {
+ "entropy": 1.537480115890503,
+ "epoch": 1.1434343434343435,
+ "grad_norm": 2.167386770248413,
+ "learning_rate": 3.858585858585859e-06,
+ "loss": 1.5262541770935059,
+ "mean_token_accuracy": 0.6343429684638977,
+ "num_tokens": 9273344.0,
+ "step": 566
+ },
+ {
+ "entropy": 1.2940728664398193,
+ "epoch": 1.1454545454545455,
+ "grad_norm": 1.941097617149353,
+ "learning_rate": 3.856565656565657e-06,
+ "loss": 1.3145123720169067,
+ "mean_token_accuracy": 0.6822789311408997,
+ "num_tokens": 9289728.0,
+ "step": 567
+ },
+ {
+ "entropy": 1.3154000043869019,
+ "epoch": 1.1474747474747475,
+ "grad_norm": 2.088576078414917,
+ "learning_rate": 3.8545454545454545e-06,
+ "loss": 1.3154902458190918,
+ "mean_token_accuracy": 0.6703101992607117,
+ "num_tokens": 9306112.0,
+ "step": 568
+ },
+ {
+ "entropy": 1.3576427698135376,
+ "epoch": 1.1494949494949496,
+ "grad_norm": 1.9895246028900146,
+ "learning_rate": 3.852525252525252e-06,
+ "loss": 1.3295378684997559,
+ "mean_token_accuracy": 0.6762335300445557,
+ "num_tokens": 9322496.0,
+ "step": 569
+ },
+ {
+ "entropy": 1.6067878007888794,
+ "epoch": 1.1515151515151516,
+ "grad_norm": 2.021191358566284,
+ "learning_rate": 3.85050505050505e-06,
+ "loss": 1.6273870468139648,
+ "mean_token_accuracy": 0.6329995393753052,
+ "num_tokens": 9338880.0,
+ "step": 570
+ },
+ {
+ "entropy": 1.4831246137619019,
+ "epoch": 1.1535353535353536,
+ "grad_norm": 2.114612340927124,
+ "learning_rate": 3.848484848484848e-06,
+ "loss": 1.4925904273986816,
+ "mean_token_accuracy": 0.6566316485404968,
+ "num_tokens": 9355264.0,
+ "step": 571
+ },
+ {
+ "entropy": 1.4476258754730225,
+ "epoch": 1.1555555555555554,
+ "grad_norm": 2.3435230255126953,
+ "learning_rate": 3.846464646464647e-06,
+ "loss": 1.438629388809204,
+ "mean_token_accuracy": 0.656020998954773,
+ "num_tokens": 9371648.0,
+ "step": 572
+ },
+ {
+ "entropy": 1.7930315732955933,
+ "epoch": 1.1575757575757575,
+ "grad_norm": 2.1700010299682617,
+ "learning_rate": 3.844444444444445e-06,
+ "loss": 1.786928653717041,
+ "mean_token_accuracy": 0.6028333902359009,
+ "num_tokens": 9388032.0,
+ "step": 573
+ },
+ {
+ "entropy": 1.3800512552261353,
+ "epoch": 1.1595959595959595,
+ "grad_norm": 2.2036688327789307,
+ "learning_rate": 3.842424242424243e-06,
+ "loss": 1.3767224550247192,
+ "mean_token_accuracy": 0.6692110300064087,
+ "num_tokens": 9404416.0,
+ "step": 574
+ },
+ {
+ "entropy": 0.8562329411506653,
+ "epoch": 1.1616161616161615,
+ "grad_norm": 1.765388011932373,
+ "learning_rate": 3.840404040404041e-06,
+ "loss": 0.8366047143936157,
+ "mean_token_accuracy": 0.783707857131958,
+ "num_tokens": 9420800.0,
+ "step": 575
+ },
+ {
+ "entropy": 1.4267486333847046,
+ "epoch": 1.1636363636363636,
+ "grad_norm": 2.0768239498138428,
+ "learning_rate": 3.8383838383838385e-06,
+ "loss": 1.4165449142456055,
+ "mean_token_accuracy": 0.656020998954773,
+ "num_tokens": 9437184.0,
+ "step": 576
+ },
+ {
+ "entropy": 1.4123388528823853,
+ "epoch": 1.1656565656565656,
+ "grad_norm": 2.028716802597046,
+ "learning_rate": 3.836363636363636e-06,
+ "loss": 1.420403003692627,
+ "mean_token_accuracy": 0.6729360222816467,
+ "num_tokens": 9453568.0,
+ "step": 577
+ },
+ {
+ "entropy": 1.1549286842346191,
+ "epoch": 1.1676767676767676,
+ "grad_norm": 2.1142919063568115,
+ "learning_rate": 3.834343434343435e-06,
+ "loss": 1.1487064361572266,
+ "mean_token_accuracy": 0.7040180563926697,
+ "num_tokens": 9469952.0,
+ "step": 578
+ },
+ {
+ "entropy": 1.4779953956604004,
+ "epoch": 1.1696969696969697,
+ "grad_norm": 2.166046142578125,
+ "learning_rate": 3.832323232323233e-06,
+ "loss": 1.4788683652877808,
+ "mean_token_accuracy": 0.6436858773231506,
+ "num_tokens": 9486336.0,
+ "step": 579
+ },
+ {
+ "entropy": 1.568039059638977,
+ "epoch": 1.1717171717171717,
+ "grad_norm": 2.067704200744629,
+ "learning_rate": 3.830303030303031e-06,
+ "loss": 1.5830929279327393,
+ "mean_token_accuracy": 0.6461895704269409,
+ "num_tokens": 9502720.0,
+ "step": 580
+ },
+ {
+ "entropy": 1.1785298585891724,
+ "epoch": 1.1737373737373737,
+ "grad_norm": 1.9789953231811523,
+ "learning_rate": 3.828282828282829e-06,
+ "loss": 1.2083481550216675,
+ "mean_token_accuracy": 0.7120786309242249,
+ "num_tokens": 9519104.0,
+ "step": 581
+ },
+ {
+ "entropy": 1.3783847093582153,
+ "epoch": 1.1757575757575758,
+ "grad_norm": 2.1503992080688477,
+ "learning_rate": 3.826262626262627e-06,
+ "loss": 1.3721052408218384,
+ "mean_token_accuracy": 0.6669516563415527,
+ "num_tokens": 9535488.0,
+ "step": 582
+ },
+ {
+ "entropy": 1.120093584060669,
+ "epoch": 1.1777777777777778,
+ "grad_norm": 1.9822942018508911,
+ "learning_rate": 3.8242424242424245e-06,
+ "loss": 1.1215628385543823,
+ "mean_token_accuracy": 0.7122618556022644,
+ "num_tokens": 9551872.0,
+ "step": 583
+ },
+ {
+ "entropy": 1.4621506929397583,
+ "epoch": 1.1797979797979798,
+ "grad_norm": 2.159489631652832,
+ "learning_rate": 3.8222222222222224e-06,
+ "loss": 1.4614722728729248,
+ "mean_token_accuracy": 0.6535173654556274,
+ "num_tokens": 9568256.0,
+ "step": 584
+ },
+ {
+ "entropy": 1.015797734260559,
+ "epoch": 1.1818181818181819,
+ "grad_norm": 1.846848726272583,
+ "learning_rate": 3.82020202020202e-06,
+ "loss": 1.0082862377166748,
+ "mean_token_accuracy": 0.7394968271255493,
+ "num_tokens": 9584640.0,
+ "step": 585
+ },
+ {
+ "entropy": 1.364890217781067,
+ "epoch": 1.183838383838384,
+ "grad_norm": 2.175546646118164,
+ "learning_rate": 3.818181818181819e-06,
+ "loss": 1.3496818542480469,
+ "mean_token_accuracy": 0.6665241718292236,
+ "num_tokens": 9601024.0,
+ "step": 586
+ },
+ {
+ "entropy": 1.6056870222091675,
+ "epoch": 1.185858585858586,
+ "grad_norm": 2.1069748401641846,
+ "learning_rate": 3.816161616161617e-06,
+ "loss": 1.6048622131347656,
+ "mean_token_accuracy": 0.642892062664032,
+ "num_tokens": 9617408.0,
+ "step": 587
+ },
+ {
+ "entropy": 1.415107250213623,
+ "epoch": 1.187878787878788,
+ "grad_norm": 2.184544324874878,
+ "learning_rate": 3.8141414141414144e-06,
+ "loss": 1.4305826425552368,
+ "mean_token_accuracy": 0.6619443297386169,
+ "num_tokens": 9633792.0,
+ "step": 588
+ },
+ {
+ "entropy": 1.6554296016693115,
+ "epoch": 1.1898989898989898,
+ "grad_norm": 2.109793186187744,
+ "learning_rate": 3.8121212121212127e-06,
+ "loss": 1.6850776672363281,
+ "mean_token_accuracy": 0.6116267442703247,
+ "num_tokens": 9650176.0,
+ "step": 589
+ },
+ {
+ "entropy": 1.681472659111023,
+ "epoch": 1.1919191919191918,
+ "grad_norm": 2.3179714679718018,
+ "learning_rate": 3.8101010101010106e-06,
+ "loss": 1.699328064918518,
+ "mean_token_accuracy": 0.6105275750160217,
+ "num_tokens": 9666560.0,
+ "step": 590
+ },
+ {
+ "entropy": 1.113366961479187,
+ "epoch": 1.1939393939393939,
+ "grad_norm": 1.9885324239730835,
+ "learning_rate": 3.8080808080808085e-06,
+ "loss": 1.1357836723327637,
+ "mean_token_accuracy": 0.7157425284385681,
+ "num_tokens": 9682944.0,
+ "step": 591
+ },
+ {
+ "entropy": 1.1010088920593262,
+ "epoch": 1.195959595959596,
+ "grad_norm": 1.9745922088623047,
+ "learning_rate": 3.8060606060606064e-06,
+ "loss": 1.094293475151062,
+ "mean_token_accuracy": 0.7214215993881226,
+ "num_tokens": 9699328.0,
+ "step": 592
+ },
+ {
+ "entropy": 1.562911033630371,
+ "epoch": 1.197979797979798,
+ "grad_norm": 2.0526134967803955,
+ "learning_rate": 3.8040404040404043e-06,
+ "loss": 1.5747783184051514,
+ "mean_token_accuracy": 0.6342818737030029,
+ "num_tokens": 9715712.0,
+ "step": 593
+ },
+ {
+ "entropy": 1.6541608572006226,
+ "epoch": 1.2,
+ "grad_norm": 2.1268227100372314,
+ "learning_rate": 3.8020202020202026e-06,
+ "loss": 1.696983814239502,
+ "mean_token_accuracy": 0.6127259135246277,
+ "num_tokens": 9732096.0,
+ "step": 594
+ },
+ {
+ "entropy": 1.5259482860565186,
+ "epoch": 1.202020202020202,
+ "grad_norm": 2.240692377090454,
+ "learning_rate": 3.8000000000000005e-06,
+ "loss": 1.5582221746444702,
+ "mean_token_accuracy": 0.644052267074585,
+ "num_tokens": 9748480.0,
+ "step": 595
+ },
+ {
+ "entropy": 1.455372929573059,
+ "epoch": 1.204040404040404,
+ "grad_norm": 2.0934698581695557,
+ "learning_rate": 3.7979797979797984e-06,
+ "loss": 1.464172124862671,
+ "mean_token_accuracy": 0.6555935740470886,
+ "num_tokens": 9764864.0,
+ "step": 596
+ },
+ {
+ "entropy": 1.5074836015701294,
+ "epoch": 1.206060606060606,
+ "grad_norm": 1.9779568910598755,
+ "learning_rate": 3.7959595959595962e-06,
+ "loss": 1.5280466079711914,
+ "mean_token_accuracy": 0.6460063457489014,
+ "num_tokens": 9781248.0,
+ "step": 597
+ },
+ {
+ "entropy": 1.3286548852920532,
+ "epoch": 1.208080808080808,
+ "grad_norm": 1.9357717037200928,
+ "learning_rate": 3.793939393939394e-06,
+ "loss": 1.3448848724365234,
+ "mean_token_accuracy": 0.6816072463989258,
+ "num_tokens": 9797632.0,
+ "step": 598
+ },
+ {
+ "entropy": 1.5161375999450684,
+ "epoch": 1.2101010101010101,
+ "grad_norm": 2.224217414855957,
+ "learning_rate": 3.791919191919192e-06,
+ "loss": 1.5168235301971436,
+ "mean_token_accuracy": 0.6453346610069275,
+ "num_tokens": 9814016.0,
+ "step": 599
+ },
+ {
+ "entropy": 1.3337587118148804,
+ "epoch": 1.2121212121212122,
+ "grad_norm": 1.9308290481567383,
+ "learning_rate": 3.7898989898989903e-06,
+ "loss": 1.3429791927337646,
+ "mean_token_accuracy": 0.6827064156532288,
+ "num_tokens": 9830400.0,
+ "step": 600
+ },
+ {
+ "epoch": 1.2121212121212122,
+ "eval_entropy": 1.4592116455877981,
+ "eval_loss": 1.5424447059631348,
+ "eval_mean_token_accuracy": 0.6431668299821115,
+ "eval_num_tokens": 9830400.0,
+ "eval_runtime": 43.8289,
+ "eval_samples_per_second": 22.588,
+ "eval_steps_per_second": 2.829,
+ "step": 600
+ },
+ {
+ "entropy": 1.6697322130203247,
+ "epoch": 1.2141414141414142,
+ "grad_norm": 1.9462894201278687,
+ "learning_rate": 3.7878787878787882e-06,
+ "loss": 1.6434354782104492,
+ "mean_token_accuracy": 0.635808527469635,
+ "num_tokens": 9846784.0,
+ "step": 601
+ },
+ {
+ "entropy": 1.427184820175171,
+ "epoch": 1.2161616161616162,
+ "grad_norm": 2.196017026901245,
+ "learning_rate": 3.785858585858586e-06,
+ "loss": 1.4024749994277954,
+ "mean_token_accuracy": 0.6720200181007385,
+ "num_tokens": 9863168.0,
+ "step": 602
+ },
+ {
+ "entropy": 1.4623222351074219,
+ "epoch": 1.2181818181818183,
+ "grad_norm": 2.160585403442383,
+ "learning_rate": 3.783838383838384e-06,
+ "loss": 1.4455416202545166,
+ "mean_token_accuracy": 0.6557767391204834,
+ "num_tokens": 9879552.0,
+ "step": 603
+ },
+ {
+ "entropy": 1.348331093788147,
+ "epoch": 1.2202020202020203,
+ "grad_norm": 2.079458713531494,
+ "learning_rate": 3.781818181818182e-06,
+ "loss": 1.3672473430633545,
+ "mean_token_accuracy": 0.6834391951560974,
+ "num_tokens": 9895936.0,
+ "step": 604
+ },
+ {
+ "entropy": 1.362277865409851,
+ "epoch": 1.2222222222222223,
+ "grad_norm": 2.10718035697937,
+ "learning_rate": 3.77979797979798e-06,
+ "loss": 1.3567216396331787,
+ "mean_token_accuracy": 0.6659135222434998,
+ "num_tokens": 9912320.0,
+ "step": 605
+ },
+ {
+ "entropy": 1.2743226289749146,
+ "epoch": 1.2242424242424241,
+ "grad_norm": 2.116549491882324,
+ "learning_rate": 3.777777777777778e-06,
+ "loss": 1.303470253944397,
+ "mean_token_accuracy": 0.6791035532951355,
+ "num_tokens": 9928704.0,
+ "step": 606
+ },
+ {
+ "entropy": 1.079649567604065,
+ "epoch": 1.2262626262626264,
+ "grad_norm": 1.8429855108261108,
+ "learning_rate": 3.775757575757576e-06,
+ "loss": 1.0616915225982666,
+ "mean_token_accuracy": 0.7321690320968628,
+ "num_tokens": 9945088.0,
+ "step": 607
+ },
+ {
+ "entropy": 1.363136649131775,
+ "epoch": 1.2282828282828282,
+ "grad_norm": 2.171295166015625,
+ "learning_rate": 3.773737373737374e-06,
+ "loss": 1.3525331020355225,
+ "mean_token_accuracy": 0.6732413172721863,
+ "num_tokens": 9961472.0,
+ "step": 608
+ },
+ {
+ "entropy": 1.324924349784851,
+ "epoch": 1.2303030303030302,
+ "grad_norm": 2.0872693061828613,
+ "learning_rate": 3.7717171717171717e-06,
+ "loss": 1.3282644748687744,
+ "mean_token_accuracy": 0.674462616443634,
+ "num_tokens": 9977856.0,
+ "step": 609
+ },
+ {
+ "entropy": 1.0672967433929443,
+ "epoch": 1.2323232323232323,
+ "grad_norm": 2.0819947719573975,
+ "learning_rate": 3.76969696969697e-06,
+ "loss": 1.0929367542266846,
+ "mean_token_accuracy": 0.715254008769989,
+ "num_tokens": 9994240.0,
+ "step": 610
+ },
+ {
+ "entropy": 1.4465869665145874,
+ "epoch": 1.2343434343434343,
+ "grad_norm": 2.1786868572235107,
+ "learning_rate": 3.767676767676768e-06,
+ "loss": 1.436307668685913,
+ "mean_token_accuracy": 0.6642037034034729,
+ "num_tokens": 10010624.0,
+ "step": 611
+ },
+ {
+ "entropy": 1.6424144506454468,
+ "epoch": 1.2363636363636363,
+ "grad_norm": 2.2582032680511475,
+ "learning_rate": 3.765656565656566e-06,
+ "loss": 1.6621832847595215,
+ "mean_token_accuracy": 0.6218246221542358,
+ "num_tokens": 10027008.0,
+ "step": 612
+ },
+ {
+ "entropy": 1.546712875366211,
+ "epoch": 1.2383838383838384,
+ "grad_norm": 2.1685879230499268,
+ "learning_rate": 3.7636363636363637e-06,
+ "loss": 1.5617464780807495,
+ "mean_token_accuracy": 0.6325110197067261,
+ "num_tokens": 10043392.0,
+ "step": 613
+ },
+ {
+ "entropy": 1.5758557319641113,
+ "epoch": 1.2404040404040404,
+ "grad_norm": 2.0911128520965576,
+ "learning_rate": 3.7616161616161616e-06,
+ "loss": 1.5830271244049072,
+ "mean_token_accuracy": 0.6349536180496216,
+ "num_tokens": 10059776.0,
+ "step": 614
+ },
+ {
+ "entropy": 1.5223995447158813,
+ "epoch": 1.2424242424242424,
+ "grad_norm": 2.0333359241485596,
+ "learning_rate": 3.7595959595959595e-06,
+ "loss": 1.5132982730865479,
+ "mean_token_accuracy": 0.6518075466156006,
+ "num_tokens": 10076160.0,
+ "step": 615
+ },
+ {
+ "entropy": 1.0950185060501099,
+ "epoch": 1.2444444444444445,
+ "grad_norm": 2.0048129558563232,
+ "learning_rate": 3.757575757575758e-06,
+ "loss": 1.0901896953582764,
+ "mean_token_accuracy": 0.723375678062439,
+ "num_tokens": 10092544.0,
+ "step": 616
+ },
+ {
+ "entropy": 1.340641736984253,
+ "epoch": 1.2464646464646465,
+ "grad_norm": 2.079256534576416,
+ "learning_rate": 3.7555555555555557e-06,
+ "loss": 1.3583415746688843,
+ "mean_token_accuracy": 0.671775758266449,
+ "num_tokens": 10108928.0,
+ "step": 617
+ },
+ {
+ "entropy": 1.1836222410202026,
+ "epoch": 1.2484848484848485,
+ "grad_norm": 1.9266347885131836,
+ "learning_rate": 3.7535353535353536e-06,
+ "loss": 1.171565294265747,
+ "mean_token_accuracy": 0.7046287059783936,
+ "num_tokens": 10125312.0,
+ "step": 618
+ },
+ {
+ "entropy": 1.6007641553878784,
+ "epoch": 1.2505050505050506,
+ "grad_norm": 2.25252103805542,
+ "learning_rate": 3.7515151515151515e-06,
+ "loss": 1.5955660343170166,
+ "mean_token_accuracy": 0.6306790709495544,
+ "num_tokens": 10141696.0,
+ "step": 619
+ },
+ {
+ "entropy": 1.8159959316253662,
+ "epoch": 1.2525252525252526,
+ "grad_norm": 2.255959987640381,
+ "learning_rate": 3.74949494949495e-06,
+ "loss": 1.8223116397857666,
+ "mean_token_accuracy": 0.5867122411727905,
+ "num_tokens": 10158080.0,
+ "step": 620
+ },
+ {
+ "entropy": 1.6469950675964355,
+ "epoch": 1.2545454545454544,
+ "grad_norm": 1.9183777570724487,
+ "learning_rate": 3.747474747474748e-06,
+ "loss": 1.6623587608337402,
+ "mean_token_accuracy": 0.6283586025238037,
+ "num_tokens": 10174464.0,
+ "step": 621
+ },
+ {
+ "entropy": 1.3507136106491089,
+ "epoch": 1.2565656565656567,
+ "grad_norm": 1.9393805265426636,
+ "learning_rate": 3.745454545454546e-06,
+ "loss": 1.3371413946151733,
+ "mean_token_accuracy": 0.6808744668960571,
+ "num_tokens": 10190848.0,
+ "step": 622
+ },
+ {
+ "entropy": 1.5176761150360107,
+ "epoch": 1.2585858585858585,
+ "grad_norm": 2.0721757411956787,
+ "learning_rate": 3.743434343434344e-06,
+ "loss": 1.5010406970977783,
+ "mean_token_accuracy": 0.6603566408157349,
+ "num_tokens": 10207232.0,
+ "step": 623
+ },
+ {
+ "entropy": 1.1409586668014526,
+ "epoch": 1.2606060606060607,
+ "grad_norm": 1.90805983543396,
+ "learning_rate": 3.7414141414141418e-06,
+ "loss": 1.1339551210403442,
+ "mean_token_accuracy": 0.7151318788528442,
+ "num_tokens": 10223616.0,
+ "step": 624
+ },
+ {
+ "entropy": 1.2258127927780151,
+ "epoch": 1.2626262626262625,
+ "grad_norm": 1.9276432991027832,
+ "learning_rate": 3.73939393939394e-06,
+ "loss": 1.2232249975204468,
+ "mean_token_accuracy": 0.7053614854812622,
+ "num_tokens": 10240000.0,
+ "step": 625
+ },
+ {
+ "entropy": 1.236358642578125,
+ "epoch": 1.2646464646464646,
+ "grad_norm": 2.0792832374572754,
+ "learning_rate": 3.737373737373738e-06,
+ "loss": 1.2570738792419434,
+ "mean_token_accuracy": 0.6869809627532959,
+ "num_tokens": 10256384.0,
+ "step": 626
+ },
+ {
+ "entropy": 1.0995185375213623,
+ "epoch": 1.2666666666666666,
+ "grad_norm": 2.035024881362915,
+ "learning_rate": 3.735353535353536e-06,
+ "loss": 1.10158109664917,
+ "mean_token_accuracy": 0.7248412370681763,
+ "num_tokens": 10272768.0,
+ "step": 627
+ },
+ {
+ "entropy": 1.8980706930160522,
+ "epoch": 1.2686868686868686,
+ "grad_norm": 2.040414333343506,
+ "learning_rate": 3.7333333333333337e-06,
+ "loss": 1.9260807037353516,
+ "mean_token_accuracy": 0.5953834652900696,
+ "num_tokens": 10289152.0,
+ "step": 628
+ },
+ {
+ "entropy": 1.4265168905258179,
+ "epoch": 1.2707070707070707,
+ "grad_norm": 2.418179750442505,
+ "learning_rate": 3.7313131313131316e-06,
+ "loss": 1.4415385723114014,
+ "mean_token_accuracy": 0.6535784006118774,
+ "num_tokens": 10305536.0,
+ "step": 629
+ },
+ {
+ "entropy": 1.217471957206726,
+ "epoch": 1.2727272727272727,
+ "grad_norm": 2.072441577911377,
+ "learning_rate": 3.72929292929293e-06,
+ "loss": 1.2361960411071777,
+ "mean_token_accuracy": 0.6947972774505615,
+ "num_tokens": 10321920.0,
+ "step": 630
+ },
+ {
+ "entropy": 1.0714137554168701,
+ "epoch": 1.2747474747474747,
+ "grad_norm": 3.0428647994995117,
+ "learning_rate": 3.727272727272728e-06,
+ "loss": 1.112711787223816,
+ "mean_token_accuracy": 0.7243527173995972,
+ "num_tokens": 10338304.0,
+ "step": 631
+ },
+ {
+ "entropy": 1.5873851776123047,
+ "epoch": 1.2767676767676768,
+ "grad_norm": 1.945098638534546,
+ "learning_rate": 3.7252525252525257e-06,
+ "loss": 1.595362663269043,
+ "mean_token_accuracy": 0.639594554901123,
+ "num_tokens": 10354688.0,
+ "step": 632
+ },
+ {
+ "entropy": 1.9688408374786377,
+ "epoch": 1.2787878787878788,
+ "grad_norm": 2.0450353622436523,
+ "learning_rate": 3.7232323232323236e-06,
+ "loss": 1.9989259243011475,
+ "mean_token_accuracy": 0.5685759782791138,
+ "num_tokens": 10371072.0,
+ "step": 633
+ },
+ {
+ "entropy": 1.2927459478378296,
+ "epoch": 1.2808080808080808,
+ "grad_norm": 1.959108591079712,
+ "learning_rate": 3.7212121212121215e-06,
+ "loss": 1.3025646209716797,
+ "mean_token_accuracy": 0.6853932738304138,
+ "num_tokens": 10387456.0,
+ "step": 634
+ },
+ {
+ "entropy": 1.6645994186401367,
+ "epoch": 1.2828282828282829,
+ "grad_norm": 2.041923999786377,
+ "learning_rate": 3.7191919191919194e-06,
+ "loss": 1.6959854364395142,
+ "mean_token_accuracy": 0.6319003701210022,
+ "num_tokens": 10403840.0,
+ "step": 635
+ },
+ {
+ "entropy": 1.401423692703247,
+ "epoch": 1.284848484848485,
+ "grad_norm": 2.040456533432007,
+ "learning_rate": 3.7171717171717177e-06,
+ "loss": 1.3859784603118896,
+ "mean_token_accuracy": 0.6740962266921997,
+ "num_tokens": 10420224.0,
+ "step": 636
+ },
+ {
+ "entropy": 1.0597256422042847,
+ "epoch": 1.286868686868687,
+ "grad_norm": 2.053690195083618,
+ "learning_rate": 3.7151515151515156e-06,
+ "loss": 1.052672266960144,
+ "mean_token_accuracy": 0.7290546894073486,
+ "num_tokens": 10436608.0,
+ "step": 637
+ },
+ {
+ "entropy": 1.280874252319336,
+ "epoch": 1.2888888888888888,
+ "grad_norm": 2.0566959381103516,
+ "learning_rate": 3.7131313131313135e-06,
+ "loss": 1.2883412837982178,
+ "mean_token_accuracy": 0.6822178959846497,
+ "num_tokens": 10452992.0,
+ "step": 638
+ },
+ {
+ "entropy": 1.299896240234375,
+ "epoch": 1.290909090909091,
+ "grad_norm": 1.8896070718765259,
+ "learning_rate": 3.7111111111111113e-06,
+ "loss": 1.3038103580474854,
+ "mean_token_accuracy": 0.6712872385978699,
+ "num_tokens": 10469376.0,
+ "step": 639
+ },
+ {
+ "entropy": 1.1713769435882568,
+ "epoch": 1.2929292929292928,
+ "grad_norm": 2.1309149265289307,
+ "learning_rate": 3.7090909090909092e-06,
+ "loss": 1.1899120807647705,
+ "mean_token_accuracy": 0.700537383556366,
+ "num_tokens": 10485760.0,
+ "step": 640
+ },
+ {
+ "entropy": 1.3056974411010742,
+ "epoch": 1.294949494949495,
+ "grad_norm": 1.9990391731262207,
+ "learning_rate": 3.7070707070707075e-06,
+ "loss": 1.3287105560302734,
+ "mean_token_accuracy": 0.6773326992988586,
+ "num_tokens": 10502144.0,
+ "step": 641
+ },
+ {
+ "entropy": 1.7352585792541504,
+ "epoch": 1.2969696969696969,
+ "grad_norm": 2.100043535232544,
+ "learning_rate": 3.7050505050505054e-06,
+ "loss": 1.7806944847106934,
+ "mean_token_accuracy": 0.60332190990448,
+ "num_tokens": 10518528.0,
+ "step": 642
+ },
+ {
+ "entropy": 1.3748377561569214,
+ "epoch": 1.298989898989899,
+ "grad_norm": 2.1280689239501953,
+ "learning_rate": 3.7030303030303033e-06,
+ "loss": 1.379891037940979,
+ "mean_token_accuracy": 0.6617611050605774,
+ "num_tokens": 10534912.0,
+ "step": 643
+ },
+ {
+ "entropy": 0.9991571307182312,
+ "epoch": 1.301010101010101,
+ "grad_norm": 1.7343134880065918,
+ "learning_rate": 3.701010101010101e-06,
+ "loss": 1.0082780122756958,
+ "mean_token_accuracy": 0.740229606628418,
+ "num_tokens": 10551296.0,
+ "step": 644
+ },
+ {
+ "entropy": 1.6313605308532715,
+ "epoch": 1.303030303030303,
+ "grad_norm": 2.216167449951172,
+ "learning_rate": 3.698989898989899e-06,
+ "loss": 1.6183781623840332,
+ "mean_token_accuracy": 0.6164509057998657,
+ "num_tokens": 10567680.0,
+ "step": 645
+ },
+ {
+ "entropy": 1.4174962043762207,
+ "epoch": 1.305050505050505,
+ "grad_norm": 2.0921854972839355,
+ "learning_rate": 3.6969696969696974e-06,
+ "loss": 1.4087945222854614,
+ "mean_token_accuracy": 0.6705544590950012,
+ "num_tokens": 10584064.0,
+ "step": 646
+ },
+ {
+ "entropy": 1.8302873373031616,
+ "epoch": 1.307070707070707,
+ "grad_norm": 2.0742204189300537,
+ "learning_rate": 3.6949494949494953e-06,
+ "loss": 1.8456642627716064,
+ "mean_token_accuracy": 0.5914142727851868,
+ "num_tokens": 10600448.0,
+ "step": 647
+ },
+ {
+ "entropy": 1.29628324508667,
+ "epoch": 1.309090909090909,
+ "grad_norm": 2.1446709632873535,
+ "learning_rate": 3.692929292929293e-06,
+ "loss": 1.3066375255584717,
+ "mean_token_accuracy": 0.6798363327980042,
+ "num_tokens": 10616832.0,
+ "step": 648
+ },
+ {
+ "entropy": 1.396378993988037,
+ "epoch": 1.3111111111111111,
+ "grad_norm": 1.9578899145126343,
+ "learning_rate": 3.690909090909091e-06,
+ "loss": 1.3943686485290527,
+ "mean_token_accuracy": 0.6772105693817139,
+ "num_tokens": 10633216.0,
+ "step": 649
+ },
+ {
+ "entropy": 1.6457722187042236,
+ "epoch": 1.3131313131313131,
+ "grad_norm": 1.9999926090240479,
+ "learning_rate": 3.688888888888889e-06,
+ "loss": 1.6266872882843018,
+ "mean_token_accuracy": 0.630923330783844,
+ "num_tokens": 10649600.0,
+ "step": 650
+ },
+ {
+ "epoch": 1.3131313131313131,
+ "eval_entropy": 1.456459879875183,
+ "eval_loss": 1.539737582206726,
+ "eval_mean_token_accuracy": 0.643713459853203,
+ "eval_num_tokens": 10649600.0,
+ "eval_runtime": 43.7637,
+ "eval_samples_per_second": 22.621,
+ "eval_steps_per_second": 2.833,
+ "step": 650
+ },
+ {
+ "entropy": 1.1152859926223755,
+ "epoch": 1.3151515151515152,
+ "grad_norm": 2.9232723712921143,
+ "learning_rate": 3.686868686868687e-06,
+ "loss": 1.099653959274292,
+ "mean_token_accuracy": 0.7227039337158203,
+ "num_tokens": 10665984.0,
+ "step": 651
+ },
+ {
+ "entropy": 1.6141386032104492,
+ "epoch": 1.3171717171717172,
+ "grad_norm": 2.1557929515838623,
+ "learning_rate": 3.684848484848485e-06,
+ "loss": 1.6156055927276611,
+ "mean_token_accuracy": 0.6288471221923828,
+ "num_tokens": 10682368.0,
+ "step": 652
+ },
+ {
+ "entropy": 1.282158374786377,
+ "epoch": 1.3191919191919192,
+ "grad_norm": 2.061983346939087,
+ "learning_rate": 3.682828282828283e-06,
+ "loss": 1.2867733240127563,
+ "mean_token_accuracy": 0.6869198679924011,
+ "num_tokens": 10698752.0,
+ "step": 653
+ },
+ {
+ "entropy": 0.9986366033554077,
+ "epoch": 1.3212121212121213,
+ "grad_norm": 2.0446274280548096,
+ "learning_rate": 3.680808080808081e-06,
+ "loss": 1.019629716873169,
+ "mean_token_accuracy": 0.7427943348884583,
+ "num_tokens": 10715136.0,
+ "step": 654
+ },
+ {
+ "entropy": 1.4256011247634888,
+ "epoch": 1.3232323232323233,
+ "grad_norm": 2.1024749279022217,
+ "learning_rate": 3.678787878787879e-06,
+ "loss": 1.4269766807556152,
+ "mean_token_accuracy": 0.6674401760101318,
+ "num_tokens": 10731520.0,
+ "step": 655
+ },
+ {
+ "entropy": 1.4913274049758911,
+ "epoch": 1.3252525252525253,
+ "grad_norm": 2.068432569503784,
+ "learning_rate": 3.6767676767676767e-06,
+ "loss": 1.4930779933929443,
+ "mean_token_accuracy": 0.6474719047546387,
+ "num_tokens": 10747904.0,
+ "step": 656
+ },
+ {
+ "entropy": 1.2957285642623901,
+ "epoch": 1.3272727272727272,
+ "grad_norm": 2.2480008602142334,
+ "learning_rate": 3.674747474747475e-06,
+ "loss": 1.3001320362091064,
+ "mean_token_accuracy": 0.679286777973175,
+ "num_tokens": 10764288.0,
+ "step": 657
+ },
+ {
+ "entropy": 1.328946828842163,
+ "epoch": 1.3292929292929294,
+ "grad_norm": 2.311176300048828,
+ "learning_rate": 3.672727272727273e-06,
+ "loss": 1.3286024332046509,
+ "mean_token_accuracy": 0.662432849407196,
+ "num_tokens": 10780672.0,
+ "step": 658
+ },
+ {
+ "entropy": 1.458139419555664,
+ "epoch": 1.3313131313131312,
+ "grad_norm": 2.1967899799346924,
+ "learning_rate": 3.670707070707071e-06,
+ "loss": 1.4384279251098633,
+ "mean_token_accuracy": 0.6435637474060059,
+ "num_tokens": 10797056.0,
+ "step": 659
+ },
+ {
+ "entropy": 1.43666672706604,
+ "epoch": 1.3333333333333333,
+ "grad_norm": 2.163083076477051,
+ "learning_rate": 3.6686868686868687e-06,
+ "loss": 1.4396371841430664,
+ "mean_token_accuracy": 0.6531509757041931,
+ "num_tokens": 10813440.0,
+ "step": 660
+ },
+ {
+ "entropy": 1.6210501194000244,
+ "epoch": 1.3353535353535353,
+ "grad_norm": 2.3601770401000977,
+ "learning_rate": 3.6666666666666666e-06,
+ "loss": 1.6370227336883545,
+ "mean_token_accuracy": 0.6149242520332336,
+ "num_tokens": 10829824.0,
+ "step": 661
+ },
+ {
+ "entropy": 0.9602832794189453,
+ "epoch": 1.3373737373737373,
+ "grad_norm": 1.8452314138412476,
+ "learning_rate": 3.664646464646465e-06,
+ "loss": 0.9445997476577759,
+ "mean_token_accuracy": 0.759709358215332,
+ "num_tokens": 10846208.0,
+ "step": 662
+ },
+ {
+ "entropy": 1.2215200662612915,
+ "epoch": 1.3393939393939394,
+ "grad_norm": 2.019989252090454,
+ "learning_rate": 3.662626262626263e-06,
+ "loss": 1.2451637983322144,
+ "mean_token_accuracy": 0.7051172256469727,
+ "num_tokens": 10862592.0,
+ "step": 663
+ },
+ {
+ "entropy": 1.3253310918807983,
+ "epoch": 1.3414141414141414,
+ "grad_norm": 1.9700425863265991,
+ "learning_rate": 3.660606060606061e-06,
+ "loss": 1.3407385349273682,
+ "mean_token_accuracy": 0.6988885998725891,
+ "num_tokens": 10878976.0,
+ "step": 664
+ },
+ {
+ "entropy": 1.1397737264633179,
+ "epoch": 1.3434343434343434,
+ "grad_norm": 2.1194040775299072,
+ "learning_rate": 3.658585858585859e-06,
+ "loss": 1.1545679569244385,
+ "mean_token_accuracy": 0.7071323990821838,
+ "num_tokens": 10895360.0,
+ "step": 665
+ },
+ {
+ "entropy": 1.1714493036270142,
+ "epoch": 1.3454545454545455,
+ "grad_norm": 2.078051805496216,
+ "learning_rate": 3.6565656565656573e-06,
+ "loss": 1.1751537322998047,
+ "mean_token_accuracy": 0.6963238716125488,
+ "num_tokens": 10911744.0,
+ "step": 666
+ },
+ {
+ "entropy": 1.2954860925674438,
+ "epoch": 1.3474747474747475,
+ "grad_norm": 1.9348593950271606,
+ "learning_rate": 3.654545454545455e-06,
+ "loss": 1.2961801290512085,
+ "mean_token_accuracy": 0.6831338405609131,
+ "num_tokens": 10928128.0,
+ "step": 667
+ },
+ {
+ "entropy": 1.401628851890564,
+ "epoch": 1.3494949494949495,
+ "grad_norm": 2.1757614612579346,
+ "learning_rate": 3.652525252525253e-06,
+ "loss": 1.4025013446807861,
+ "mean_token_accuracy": 0.6590132117271423,
+ "num_tokens": 10944512.0,
+ "step": 668
+ },
+ {
+ "entropy": 1.4164557456970215,
+ "epoch": 1.3515151515151516,
+ "grad_norm": 2.0192055702209473,
+ "learning_rate": 3.650505050505051e-06,
+ "loss": 1.43379545211792,
+ "mean_token_accuracy": 0.6554714441299438,
+ "num_tokens": 10960896.0,
+ "step": 669
+ },
+ {
+ "entropy": 1.4888815879821777,
+ "epoch": 1.3535353535353536,
+ "grad_norm": 2.2166643142700195,
+ "learning_rate": 3.648484848484849e-06,
+ "loss": 1.494372844696045,
+ "mean_token_accuracy": 0.6402052044868469,
+ "num_tokens": 10977280.0,
+ "step": 670
+ },
+ {
+ "entropy": 1.215625286102295,
+ "epoch": 1.3555555555555556,
+ "grad_norm": 1.8059836626052856,
+ "learning_rate": 3.6464646464646467e-06,
+ "loss": 1.2074902057647705,
+ "mean_token_accuracy": 0.7123228907585144,
+ "num_tokens": 10993664.0,
+ "step": 671
+ },
+ {
+ "entropy": 0.9577685594558716,
+ "epoch": 1.3575757575757577,
+ "grad_norm": 1.8803631067276,
+ "learning_rate": 3.644444444444445e-06,
+ "loss": 0.956764817237854,
+ "mean_token_accuracy": 0.759709358215332,
+ "num_tokens": 11010048.0,
+ "step": 672
+ },
+ {
+ "entropy": 1.7215807437896729,
+ "epoch": 1.3595959595959597,
+ "grad_norm": 2.1421561241149902,
+ "learning_rate": 3.642424242424243e-06,
+ "loss": 1.7220706939697266,
+ "mean_token_accuracy": 0.6187102794647217,
+ "num_tokens": 11026432.0,
+ "step": 673
+ },
+ {
+ "entropy": 1.057277798652649,
+ "epoch": 1.3616161616161615,
+ "grad_norm": 1.9014254808425903,
+ "learning_rate": 3.640404040404041e-06,
+ "loss": 1.0582892894744873,
+ "mean_token_accuracy": 0.7327185869216919,
+ "num_tokens": 11042816.0,
+ "step": 674
+ },
+ {
+ "entropy": 1.220097541809082,
+ "epoch": 1.3636363636363638,
+ "grad_norm": 2.1406402587890625,
+ "learning_rate": 3.6383838383838387e-06,
+ "loss": 1.2152808904647827,
+ "mean_token_accuracy": 0.6976673007011414,
+ "num_tokens": 11059200.0,
+ "step": 675
+ },
+ {
+ "entropy": 1.4171489477157593,
+ "epoch": 1.3656565656565656,
+ "grad_norm": 2.1809890270233154,
+ "learning_rate": 3.6363636363636366e-06,
+ "loss": 1.4846035242080688,
+ "mean_token_accuracy": 0.6681118607521057,
+ "num_tokens": 11075584.0,
+ "step": 676
+ },
+ {
+ "entropy": 1.2915124893188477,
+ "epoch": 1.3676767676767676,
+ "grad_norm": 2.0533218383789062,
+ "learning_rate": 3.634343434343435e-06,
+ "loss": 1.3094089031219482,
+ "mean_token_accuracy": 0.6841719746589661,
+ "num_tokens": 11091968.0,
+ "step": 677
+ },
+ {
+ "entropy": 1.4717603921890259,
+ "epoch": 1.3696969696969696,
+ "grad_norm": 2.125450849533081,
+ "learning_rate": 3.6323232323232328e-06,
+ "loss": 1.479896903038025,
+ "mean_token_accuracy": 0.6601734161376953,
+ "num_tokens": 11108352.0,
+ "step": 678
+ },
+ {
+ "entropy": 1.0841981172561646,
+ "epoch": 1.3717171717171717,
+ "grad_norm": 1.936905026435852,
+ "learning_rate": 3.6303030303030307e-06,
+ "loss": 1.0895963907241821,
+ "mean_token_accuracy": 0.7307645082473755,
+ "num_tokens": 11124736.0,
+ "step": 679
+ },
+ {
+ "entropy": 0.85239177942276,
+ "epoch": 1.3737373737373737,
+ "grad_norm": 1.7117112874984741,
+ "learning_rate": 3.6282828282828286e-06,
+ "loss": 0.8716775178909302,
+ "mean_token_accuracy": 0.7721666097640991,
+ "num_tokens": 11141120.0,
+ "step": 680
+ },
+ {
+ "entropy": 1.4737738370895386,
+ "epoch": 1.3757575757575757,
+ "grad_norm": 2.0140671730041504,
+ "learning_rate": 3.6262626262626264e-06,
+ "loss": 1.5029723644256592,
+ "mean_token_accuracy": 0.653822660446167,
+ "num_tokens": 11157504.0,
+ "step": 681
+ },
+ {
+ "entropy": 1.260698914527893,
+ "epoch": 1.3777777777777778,
+ "grad_norm": 1.9868851900100708,
+ "learning_rate": 3.6242424242424248e-06,
+ "loss": 1.256290316581726,
+ "mean_token_accuracy": 0.6891182065010071,
+ "num_tokens": 11173888.0,
+ "step": 682
+ },
+ {
+ "entropy": 1.7418453693389893,
+ "epoch": 1.3797979797979798,
+ "grad_norm": 2.0905439853668213,
+ "learning_rate": 3.6222222222222226e-06,
+ "loss": 1.7568984031677246,
+ "mean_token_accuracy": 0.6291524171829224,
+ "num_tokens": 11190272.0,
+ "step": 683
+ },
+ {
+ "entropy": 1.5552523136138916,
+ "epoch": 1.3818181818181818,
+ "grad_norm": 2.2071683406829834,
+ "learning_rate": 3.6202020202020205e-06,
+ "loss": 1.555542230606079,
+ "mean_token_accuracy": 0.6403883695602417,
+ "num_tokens": 11206656.0,
+ "step": 684
+ },
+ {
+ "entropy": 1.4404902458190918,
+ "epoch": 1.3838383838383839,
+ "grad_norm": 2.276245594024658,
+ "learning_rate": 3.6181818181818184e-06,
+ "loss": 1.4155148267745972,
+ "mean_token_accuracy": 0.6630434989929199,
+ "num_tokens": 11223040.0,
+ "step": 685
+ },
+ {
+ "entropy": 1.1282018423080444,
+ "epoch": 1.385858585858586,
+ "grad_norm": 1.9824057817459106,
+ "learning_rate": 3.6161616161616163e-06,
+ "loss": 1.099952220916748,
+ "mean_token_accuracy": 0.7234978079795837,
+ "num_tokens": 11239424.0,
+ "step": 686
+ },
+ {
+ "entropy": 1.5559885501861572,
+ "epoch": 1.387878787878788,
+ "grad_norm": 2.243121862411499,
+ "learning_rate": 3.614141414141414e-06,
+ "loss": 1.520850658416748,
+ "mean_token_accuracy": 0.6340987086296082,
+ "num_tokens": 11255808.0,
+ "step": 687
+ },
+ {
+ "entropy": 1.322407841682434,
+ "epoch": 1.38989898989899,
+ "grad_norm": 2.049233913421631,
+ "learning_rate": 3.6121212121212125e-06,
+ "loss": 1.322263479232788,
+ "mean_token_accuracy": 0.6778212189674377,
+ "num_tokens": 11272192.0,
+ "step": 688
+ },
+ {
+ "entropy": 1.3112859725952148,
+ "epoch": 1.391919191919192,
+ "grad_norm": 2.0666465759277344,
+ "learning_rate": 3.6101010101010104e-06,
+ "loss": 1.3103243112564087,
+ "mean_token_accuracy": 0.6828285455703735,
+ "num_tokens": 11288576.0,
+ "step": 689
+ },
+ {
+ "entropy": 1.3321104049682617,
+ "epoch": 1.393939393939394,
+ "grad_norm": 2.5464279651641846,
+ "learning_rate": 3.6080808080808083e-06,
+ "loss": 1.3247697353363037,
+ "mean_token_accuracy": 0.6785539984703064,
+ "num_tokens": 11304960.0,
+ "step": 690
+ },
+ {
+ "entropy": 1.0779680013656616,
+ "epoch": 1.3959595959595958,
+ "grad_norm": 2.1386117935180664,
+ "learning_rate": 3.606060606060606e-06,
+ "loss": 1.0994013547897339,
+ "mean_token_accuracy": 0.7129946351051331,
+ "num_tokens": 11321344.0,
+ "step": 691
+ },
+ {
+ "entropy": 1.0701864957809448,
+ "epoch": 1.397979797979798,
+ "grad_norm": 1.9387527704238892,
+ "learning_rate": 3.604040404040404e-06,
+ "loss": 1.0633184909820557,
+ "mean_token_accuracy": 0.7378480434417725,
+ "num_tokens": 11337728.0,
+ "step": 692
+ },
+ {
+ "entropy": 1.2258752584457397,
+ "epoch": 1.4,
+ "grad_norm": 2.0486111640930176,
+ "learning_rate": 3.6020202020202024e-06,
+ "loss": 1.2098207473754883,
+ "mean_token_accuracy": 0.6999877691268921,
+ "num_tokens": 11354112.0,
+ "step": 693
+ },
+ {
+ "entropy": 1.4652774333953857,
+ "epoch": 1.402020202020202,
+ "grad_norm": 2.043079376220703,
+ "learning_rate": 3.6000000000000003e-06,
+ "loss": 1.4780974388122559,
+ "mean_token_accuracy": 0.6681729555130005,
+ "num_tokens": 11370496.0,
+ "step": 694
+ },
+ {
+ "entropy": 1.3215711116790771,
+ "epoch": 1.404040404040404,
+ "grad_norm": 2.1327438354492188,
+ "learning_rate": 3.597979797979798e-06,
+ "loss": 1.3116662502288818,
+ "mean_token_accuracy": 0.6757450103759766,
+ "num_tokens": 11386880.0,
+ "step": 695
+ },
+ {
+ "entropy": 1.624266266822815,
+ "epoch": 1.406060606060606,
+ "grad_norm": 2.114398717880249,
+ "learning_rate": 3.595959595959596e-06,
+ "loss": 1.628610372543335,
+ "mean_token_accuracy": 0.6487542986869812,
+ "num_tokens": 11403264.0,
+ "step": 696
+ },
+ {
+ "entropy": 1.6059489250183105,
+ "epoch": 1.408080808080808,
+ "grad_norm": 1.952991008758545,
+ "learning_rate": 3.593939393939394e-06,
+ "loss": 1.6145906448364258,
+ "mean_token_accuracy": 0.6257938742637634,
+ "num_tokens": 11419648.0,
+ "step": 697
+ },
+ {
+ "entropy": 1.0662391185760498,
+ "epoch": 1.41010101010101,
+ "grad_norm": 2.0804851055145264,
+ "learning_rate": 3.5919191919191922e-06,
+ "loss": 1.0772000551223755,
+ "mean_token_accuracy": 0.7187347412109375,
+ "num_tokens": 11436032.0,
+ "step": 698
+ },
+ {
+ "entropy": 1.221283197402954,
+ "epoch": 1.412121212121212,
+ "grad_norm": 1.9502744674682617,
+ "learning_rate": 3.58989898989899e-06,
+ "loss": 1.2352337837219238,
+ "mean_token_accuracy": 0.6918050646781921,
+ "num_tokens": 11452416.0,
+ "step": 699
+ },
+ {
+ "entropy": 1.4205158948898315,
+ "epoch": 1.4141414141414141,
+ "grad_norm": 2.2222518920898438,
+ "learning_rate": 3.587878787878788e-06,
+ "loss": 1.451093077659607,
+ "mean_token_accuracy": 0.6648754477500916,
+ "num_tokens": 11468800.0,
+ "step": 700
+ },
+ {
+ "epoch": 1.4141414141414141,
+ "eval_entropy": 1.4370074204860195,
+ "eval_loss": 1.5383458137512207,
+ "eval_mean_token_accuracy": 0.6441497793120723,
+ "eval_num_tokens": 11468800.0,
+ "eval_runtime": 43.744,
+ "eval_samples_per_second": 22.632,
+ "eval_steps_per_second": 2.835,
+ "step": 700
+ },
+ {
+ "entropy": 1.5432499647140503,
+ "epoch": 1.4161616161616162,
+ "grad_norm": 2.084463596343994,
+ "learning_rate": 3.585858585858586e-06,
+ "loss": 1.5958356857299805,
+ "mean_token_accuracy": 0.6377015113830566,
+ "num_tokens": 11485184.0,
+ "step": 701
+ },
+ {
+ "entropy": 1.297119379043579,
+ "epoch": 1.4181818181818182,
+ "grad_norm": 2.1996524333953857,
+ "learning_rate": 3.5838383838383838e-06,
+ "loss": 1.3346054553985596,
+ "mean_token_accuracy": 0.6801416873931885,
+ "num_tokens": 11501568.0,
+ "step": 702
+ },
+ {
+ "entropy": 1.3905766010284424,
+ "epoch": 1.4202020202020202,
+ "grad_norm": 2.1683545112609863,
+ "learning_rate": 3.5818181818181817e-06,
+ "loss": 1.40826416015625,
+ "mean_token_accuracy": 0.6653639674186707,
+ "num_tokens": 11517952.0,
+ "step": 703
+ },
+ {
+ "entropy": 0.9598695635795593,
+ "epoch": 1.4222222222222223,
+ "grad_norm": 1.9240537881851196,
+ "learning_rate": 3.57979797979798e-06,
+ "loss": 0.9487459659576416,
+ "mean_token_accuracy": 0.7556179761886597,
+ "num_tokens": 11534336.0,
+ "step": 704
+ },
+ {
+ "entropy": 1.2008687257766724,
+ "epoch": 1.4242424242424243,
+ "grad_norm": 1.9687083959579468,
+ "learning_rate": 3.577777777777778e-06,
+ "loss": 1.2234617471694946,
+ "mean_token_accuracy": 0.6982779502868652,
+ "num_tokens": 11550720.0,
+ "step": 705
+ },
+ {
+ "entropy": 1.1168856620788574,
+ "epoch": 1.4262626262626263,
+ "grad_norm": 2.1796388626098633,
+ "learning_rate": 3.575757575757576e-06,
+ "loss": 1.1320589780807495,
+ "mean_token_accuracy": 0.7057889699935913,
+ "num_tokens": 11567104.0,
+ "step": 706
+ },
+ {
+ "entropy": 1.3408682346343994,
+ "epoch": 1.4282828282828284,
+ "grad_norm": 1.9048689603805542,
+ "learning_rate": 3.573737373737374e-06,
+ "loss": 1.3476686477661133,
+ "mean_token_accuracy": 0.6839887499809265,
+ "num_tokens": 11583488.0,
+ "step": 707
+ },
+ {
+ "entropy": 1.4392427206039429,
+ "epoch": 1.4303030303030302,
+ "grad_norm": 2.183016777038574,
+ "learning_rate": 3.5717171717171724e-06,
+ "loss": 1.4443378448486328,
+ "mean_token_accuracy": 0.6546775698661804,
+ "num_tokens": 11599872.0,
+ "step": 708
+ },
+ {
+ "entropy": 1.1428641080856323,
+ "epoch": 1.4323232323232324,
+ "grad_norm": 2.0477187633514404,
+ "learning_rate": 3.5696969696969703e-06,
+ "loss": 1.1438124179840088,
+ "mean_token_accuracy": 0.7126282453536987,
+ "num_tokens": 11616256.0,
+ "step": 709
+ },
+ {
+ "entropy": 1.6266967058181763,
+ "epoch": 1.4343434343434343,
+ "grad_norm": 2.322098731994629,
+ "learning_rate": 3.567676767676768e-06,
+ "loss": 1.6161916255950928,
+ "mean_token_accuracy": 0.6202979683876038,
+ "num_tokens": 11632640.0,
+ "step": 710
+ },
+ {
+ "entropy": 1.349686622619629,
+ "epoch": 1.4363636363636363,
+ "grad_norm": 2.021871328353882,
+ "learning_rate": 3.565656565656566e-06,
+ "loss": 1.3713490962982178,
+ "mean_token_accuracy": 0.6596238613128662,
+ "num_tokens": 11649024.0,
+ "step": 711
+ },
+ {
+ "entropy": 1.4476866722106934,
+ "epoch": 1.4383838383838383,
+ "grad_norm": 2.194718837738037,
+ "learning_rate": 3.563636363636364e-06,
+ "loss": 1.4593651294708252,
+ "mean_token_accuracy": 0.6476551294326782,
+ "num_tokens": 11665408.0,
+ "step": 712
+ },
+ {
+ "entropy": 1.625157356262207,
+ "epoch": 1.4404040404040404,
+ "grad_norm": 2.0966169834136963,
+ "learning_rate": 3.5616161616161622e-06,
+ "loss": 1.6374845504760742,
+ "mean_token_accuracy": 0.6293356418609619,
+ "num_tokens": 11681792.0,
+ "step": 713
+ },
+ {
+ "entropy": 1.985729455947876,
+ "epoch": 1.4424242424242424,
+ "grad_norm": 2.129556655883789,
+ "learning_rate": 3.55959595959596e-06,
+ "loss": 2.0454561710357666,
+ "mean_token_accuracy": 0.5728505253791809,
+ "num_tokens": 11698176.0,
+ "step": 714
+ },
+ {
+ "entropy": 1.546321153640747,
+ "epoch": 1.4444444444444444,
+ "grad_norm": 2.1565628051757812,
+ "learning_rate": 3.557575757575758e-06,
+ "loss": 1.5467555522918701,
+ "mean_token_accuracy": 0.6351978778839111,
+ "num_tokens": 11714560.0,
+ "step": 715
+ },
+ {
+ "entropy": 1.3690776824951172,
+ "epoch": 1.4464646464646465,
+ "grad_norm": 2.206984519958496,
+ "learning_rate": 3.555555555555556e-06,
+ "loss": 1.3877894878387451,
+ "mean_token_accuracy": 0.6911333799362183,
+ "num_tokens": 11730944.0,
+ "step": 716
+ },
+ {
+ "entropy": 1.3643333911895752,
+ "epoch": 1.4484848484848485,
+ "grad_norm": 2.0049662590026855,
+ "learning_rate": 3.553535353535354e-06,
+ "loss": 1.364513635635376,
+ "mean_token_accuracy": 0.6781876087188721,
+ "num_tokens": 11747328.0,
+ "step": 717
+ },
+ {
+ "entropy": 1.337018370628357,
+ "epoch": 1.4505050505050505,
+ "grad_norm": 2.0919127464294434,
+ "learning_rate": 3.551515151515152e-06,
+ "loss": 1.3578065633773804,
+ "mean_token_accuracy": 0.672874927520752,
+ "num_tokens": 11763712.0,
+ "step": 718
+ },
+ {
+ "entropy": 1.3421508073806763,
+ "epoch": 1.4525252525252526,
+ "grad_norm": 2.0651159286499023,
+ "learning_rate": 3.54949494949495e-06,
+ "loss": 1.33928382396698,
+ "mean_token_accuracy": 0.6866145730018616,
+ "num_tokens": 11780096.0,
+ "step": 719
+ },
+ {
+ "entropy": 1.2841005325317383,
+ "epoch": 1.4545454545454546,
+ "grad_norm": 2.124370813369751,
+ "learning_rate": 3.547474747474748e-06,
+ "loss": 1.258314609527588,
+ "mean_token_accuracy": 0.6806302070617676,
+ "num_tokens": 11796480.0,
+ "step": 720
+ },
+ {
+ "entropy": 1.56289803981781,
+ "epoch": 1.4565656565656566,
+ "grad_norm": 1.9271100759506226,
+ "learning_rate": 3.5454545454545458e-06,
+ "loss": 1.5867946147918701,
+ "mean_token_accuracy": 0.6430752277374268,
+ "num_tokens": 11812864.0,
+ "step": 721
+ },
+ {
+ "entropy": 1.3374838829040527,
+ "epoch": 1.4585858585858587,
+ "grad_norm": 2.1113364696502686,
+ "learning_rate": 3.5434343434343437e-06,
+ "loss": 1.327169418334961,
+ "mean_token_accuracy": 0.6806912422180176,
+ "num_tokens": 11829248.0,
+ "step": 722
+ },
+ {
+ "entropy": 1.7233415842056274,
+ "epoch": 1.4606060606060607,
+ "grad_norm": 2.1862399578094482,
+ "learning_rate": 3.5414141414141416e-06,
+ "loss": 1.7290821075439453,
+ "mean_token_accuracy": 0.6056423783302307,
+ "num_tokens": 11845632.0,
+ "step": 723
+ },
+ {
+ "entropy": 1.28484046459198,
+ "epoch": 1.4626262626262627,
+ "grad_norm": 2.0059030055999756,
+ "learning_rate": 3.53939393939394e-06,
+ "loss": 1.2958555221557617,
+ "mean_token_accuracy": 0.6774548292160034,
+ "num_tokens": 11862016.0,
+ "step": 724
+ },
+ {
+ "entropy": 1.2592295408248901,
+ "epoch": 1.4646464646464645,
+ "grad_norm": 1.9422065019607544,
+ "learning_rate": 3.5373737373737378e-06,
+ "loss": 1.2501411437988281,
+ "mean_token_accuracy": 0.700537383556366,
+ "num_tokens": 11878400.0,
+ "step": 725
+ },
+ {
+ "entropy": 1.3708571195602417,
+ "epoch": 1.4666666666666668,
+ "grad_norm": 2.164499044418335,
+ "learning_rate": 3.5353535353535356e-06,
+ "loss": 1.3901662826538086,
+ "mean_token_accuracy": 0.6618832349777222,
+ "num_tokens": 11894784.0,
+ "step": 726
+ },
+ {
+ "entropy": 1.239045262336731,
+ "epoch": 1.4686868686868686,
+ "grad_norm": 1.9581060409545898,
+ "learning_rate": 3.5333333333333335e-06,
+ "loss": 1.2436164617538452,
+ "mean_token_accuracy": 0.6939423680305481,
+ "num_tokens": 11911168.0,
+ "step": 727
+ },
+ {
+ "entropy": 1.5965995788574219,
+ "epoch": 1.4707070707070706,
+ "grad_norm": 2.343669891357422,
+ "learning_rate": 3.5313131313131314e-06,
+ "loss": 1.5918347835540771,
+ "mean_token_accuracy": 0.6274425983428955,
+ "num_tokens": 11927552.0,
+ "step": 728
+ },
+ {
+ "entropy": 1.1196811199188232,
+ "epoch": 1.4727272727272727,
+ "grad_norm": 2.181840181350708,
+ "learning_rate": 3.5292929292929297e-06,
+ "loss": 1.1142621040344238,
+ "mean_token_accuracy": 0.7192842960357666,
+ "num_tokens": 11943936.0,
+ "step": 729
+ },
+ {
+ "entropy": 1.129361629486084,
+ "epoch": 1.4747474747474747,
+ "grad_norm": 2.065730571746826,
+ "learning_rate": 3.5272727272727276e-06,
+ "loss": 1.143613576889038,
+ "mean_token_accuracy": 0.7145212292671204,
+ "num_tokens": 11960320.0,
+ "step": 730
+ },
+ {
+ "entropy": 1.4940218925476074,
+ "epoch": 1.4767676767676767,
+ "grad_norm": 2.083453893661499,
+ "learning_rate": 3.5252525252525255e-06,
+ "loss": 1.4957321882247925,
+ "mean_token_accuracy": 0.6481436491012573,
+ "num_tokens": 11976704.0,
+ "step": 731
+ },
+ {
+ "entropy": 1.3427786827087402,
+ "epoch": 1.4787878787878788,
+ "grad_norm": 2.0897183418273926,
+ "learning_rate": 3.5232323232323234e-06,
+ "loss": 1.354066014289856,
+ "mean_token_accuracy": 0.6660356521606445,
+ "num_tokens": 11993088.0,
+ "step": 732
+ },
+ {
+ "entropy": 1.3300065994262695,
+ "epoch": 1.4808080808080808,
+ "grad_norm": 2.0537912845611572,
+ "learning_rate": 3.5212121212121213e-06,
+ "loss": 1.3332751989364624,
+ "mean_token_accuracy": 0.6817293763160706,
+ "num_tokens": 12009472.0,
+ "step": 733
+ },
+ {
+ "entropy": 1.0657827854156494,
+ "epoch": 1.4828282828282828,
+ "grad_norm": 1.9099130630493164,
+ "learning_rate": 3.5191919191919196e-06,
+ "loss": 1.067232608795166,
+ "mean_token_accuracy": 0.734245240688324,
+ "num_tokens": 12025856.0,
+ "step": 734
+ },
+ {
+ "entropy": 1.5265862941741943,
+ "epoch": 1.4848484848484849,
+ "grad_norm": 2.100236177444458,
+ "learning_rate": 3.5171717171717175e-06,
+ "loss": 1.5169599056243896,
+ "mean_token_accuracy": 0.6382511258125305,
+ "num_tokens": 12042240.0,
+ "step": 735
+ },
+ {
+ "entropy": 1.4428536891937256,
+ "epoch": 1.486868686868687,
+ "grad_norm": 1.9902187585830688,
+ "learning_rate": 3.5151515151515154e-06,
+ "loss": 1.4398434162139893,
+ "mean_token_accuracy": 0.675500750541687,
+ "num_tokens": 12058624.0,
+ "step": 736
+ },
+ {
+ "entropy": 1.528032660484314,
+ "epoch": 1.488888888888889,
+ "grad_norm": 2.043238878250122,
+ "learning_rate": 3.5131313131313133e-06,
+ "loss": 1.5280101299285889,
+ "mean_token_accuracy": 0.6431363224983215,
+ "num_tokens": 12075008.0,
+ "step": 737
+ },
+ {
+ "entropy": 1.4250329732894897,
+ "epoch": 1.490909090909091,
+ "grad_norm": 1.791640281677246,
+ "learning_rate": 3.511111111111111e-06,
+ "loss": 1.426466464996338,
+ "mean_token_accuracy": 0.6765388250350952,
+ "num_tokens": 12091392.0,
+ "step": 738
+ },
+ {
+ "entropy": 1.1264820098876953,
+ "epoch": 1.492929292929293,
+ "grad_norm": 2.0549607276916504,
+ "learning_rate": 3.509090909090909e-06,
+ "loss": 1.1271553039550781,
+ "mean_token_accuracy": 0.6996213793754578,
+ "num_tokens": 12107776.0,
+ "step": 739
+ },
+ {
+ "entropy": 1.538375735282898,
+ "epoch": 1.494949494949495,
+ "grad_norm": 2.22322940826416,
+ "learning_rate": 3.5070707070707073e-06,
+ "loss": 1.5588887929916382,
+ "mean_token_accuracy": 0.6384953856468201,
+ "num_tokens": 12124160.0,
+ "step": 740
+ },
+ {
+ "entropy": 1.2698255777359009,
+ "epoch": 1.496969696969697,
+ "grad_norm": 2.17710542678833,
+ "learning_rate": 3.5050505050505052e-06,
+ "loss": 1.29567289352417,
+ "mean_token_accuracy": 0.6762945652008057,
+ "num_tokens": 12140544.0,
+ "step": 741
+ },
+ {
+ "entropy": 1.523453950881958,
+ "epoch": 1.4989898989898989,
+ "grad_norm": 2.0350148677825928,
+ "learning_rate": 3.503030303030303e-06,
+ "loss": 1.5341038703918457,
+ "mean_token_accuracy": 0.6367855668067932,
+ "num_tokens": 12156928.0,
+ "step": 742
+ },
+ {
+ "entropy": 1.4250539541244507,
+ "epoch": 1.5010101010101011,
+ "grad_norm": 2.1306707859039307,
+ "learning_rate": 3.501010101010101e-06,
+ "loss": 1.4353611469268799,
+ "mean_token_accuracy": 0.6626160144805908,
+ "num_tokens": 12173312.0,
+ "step": 743
+ },
+ {
+ "entropy": 1.1440296173095703,
+ "epoch": 1.503030303030303,
+ "grad_norm": 2.053969383239746,
+ "learning_rate": 3.498989898989899e-06,
+ "loss": 1.1151821613311768,
+ "mean_token_accuracy": 0.7080483436584473,
+ "num_tokens": 12189696.0,
+ "step": 744
+ },
+ {
+ "entropy": 1.5290073156356812,
+ "epoch": 1.5050505050505052,
+ "grad_norm": 2.0550003051757812,
+ "learning_rate": 3.496969696969697e-06,
+ "loss": 1.5260624885559082,
+ "mean_token_accuracy": 0.6493038535118103,
+ "num_tokens": 12206080.0,
+ "step": 745
+ },
+ {
+ "entropy": 1.1096605062484741,
+ "epoch": 1.507070707070707,
+ "grad_norm": 1.8131766319274902,
+ "learning_rate": 3.494949494949495e-06,
+ "loss": 1.1221846342086792,
+ "mean_token_accuracy": 0.7263678312301636,
+ "num_tokens": 12222464.0,
+ "step": 746
+ },
+ {
+ "entropy": 1.282676339149475,
+ "epoch": 1.509090909090909,
+ "grad_norm": 2.3051960468292236,
+ "learning_rate": 3.492929292929293e-06,
+ "loss": 1.2863187789916992,
+ "mean_token_accuracy": 0.6765388250350952,
+ "num_tokens": 12238848.0,
+ "step": 747
+ },
+ {
+ "entropy": 1.2971303462982178,
+ "epoch": 1.511111111111111,
+ "grad_norm": 2.0721139907836914,
+ "learning_rate": 3.4909090909090913e-06,
+ "loss": 1.3010127544403076,
+ "mean_token_accuracy": 0.6753175258636475,
+ "num_tokens": 12255232.0,
+ "step": 748
+ },
+ {
+ "entropy": 1.3711751699447632,
+ "epoch": 1.513131313131313,
+ "grad_norm": 1.9657210111618042,
+ "learning_rate": 3.4888888888888896e-06,
+ "loss": 1.3703930377960205,
+ "mean_token_accuracy": 0.6686614751815796,
+ "num_tokens": 12271616.0,
+ "step": 749
+ },
+ {
+ "entropy": 1.512154459953308,
+ "epoch": 1.5151515151515151,
+ "grad_norm": 2.214172124862671,
+ "learning_rate": 3.4868686868686875e-06,
+ "loss": 1.5243256092071533,
+ "mean_token_accuracy": 0.6485100388526917,
+ "num_tokens": 12288000.0,
+ "step": 750
+ },
+ {
+ "epoch": 1.5151515151515151,
+ "eval_entropy": 1.4458443743567313,
+ "eval_loss": 1.535510540008545,
+ "eval_mean_token_accuracy": 0.6444821915318889,
+ "eval_num_tokens": 12288000.0,
+ "eval_runtime": 43.8043,
+ "eval_samples_per_second": 22.601,
+ "eval_steps_per_second": 2.831,
+ "step": 750
+ },
+ {
+ "entropy": 1.2052820920944214,
+ "epoch": 1.5171717171717172,
+ "grad_norm": 2.104870319366455,
+ "learning_rate": 3.4848484848484854e-06,
+ "loss": 1.2207554578781128,
+ "mean_token_accuracy": 0.695713222026825,
+ "num_tokens": 12304384.0,
+ "step": 751
+ },
+ {
+ "entropy": 1.732282280921936,
+ "epoch": 1.5191919191919192,
+ "grad_norm": 2.101760149002075,
+ "learning_rate": 3.4828282828282833e-06,
+ "loss": 1.7461241483688354,
+ "mean_token_accuracy": 0.6505251526832581,
+ "num_tokens": 12320768.0,
+ "step": 752
+ },
+ {
+ "entropy": 1.5805878639221191,
+ "epoch": 1.5212121212121212,
+ "grad_norm": 2.139244318008423,
+ "learning_rate": 3.480808080808081e-06,
+ "loss": 1.5813164710998535,
+ "mean_token_accuracy": 0.6281143426895142,
+ "num_tokens": 12337152.0,
+ "step": 753
+ },
+ {
+ "entropy": 1.1882269382476807,
+ "epoch": 1.5232323232323233,
+ "grad_norm": 2.187206745147705,
+ "learning_rate": 3.4787878787878795e-06,
+ "loss": 1.1732902526855469,
+ "mean_token_accuracy": 0.7063385248184204,
+ "num_tokens": 12353536.0,
+ "step": 754
+ },
+ {
+ "entropy": 1.4743424654006958,
+ "epoch": 1.5252525252525253,
+ "grad_norm": 2.12687349319458,
+ "learning_rate": 3.4767676767676774e-06,
+ "loss": 1.4681663513183594,
+ "mean_token_accuracy": 0.6558378338813782,
+ "num_tokens": 12369920.0,
+ "step": 755
+ },
+ {
+ "entropy": 1.5533041954040527,
+ "epoch": 1.5272727272727273,
+ "grad_norm": 2.1306610107421875,
+ "learning_rate": 3.4747474747474752e-06,
+ "loss": 1.5599431991577148,
+ "mean_token_accuracy": 0.6399609446525574,
+ "num_tokens": 12386304.0,
+ "step": 756
+ },
+ {
+ "entropy": 0.8933209180831909,
+ "epoch": 1.5292929292929291,
+ "grad_norm": 2.425602912902832,
+ "learning_rate": 3.472727272727273e-06,
+ "loss": 0.8840203285217285,
+ "mean_token_accuracy": 0.7638006806373596,
+ "num_tokens": 12402688.0,
+ "step": 757
+ },
+ {
+ "entropy": 1.6764189004898071,
+ "epoch": 1.5313131313131314,
+ "grad_norm": 2.0684356689453125,
+ "learning_rate": 3.470707070707071e-06,
+ "loss": 1.6948609352111816,
+ "mean_token_accuracy": 0.6087567210197449,
+ "num_tokens": 12419072.0,
+ "step": 758
+ },
+ {
+ "entropy": 1.1875263452529907,
+ "epoch": 1.5333333333333332,
+ "grad_norm": 2.0625953674316406,
+ "learning_rate": 3.468686868686869e-06,
+ "loss": 1.1840041875839233,
+ "mean_token_accuracy": 0.7067660093307495,
+ "num_tokens": 12435456.0,
+ "step": 759
+ },
+ {
+ "entropy": 1.1589192152023315,
+ "epoch": 1.5353535353535355,
+ "grad_norm": 1.9212846755981445,
+ "learning_rate": 3.4666666666666672e-06,
+ "loss": 1.1537678241729736,
+ "mean_token_accuracy": 0.7107962965965271,
+ "num_tokens": 12451840.0,
+ "step": 760
+ },
+ {
+ "entropy": 0.9687032699584961,
+ "epoch": 1.5373737373737373,
+ "grad_norm": 1.9402192831039429,
+ "learning_rate": 3.464646464646465e-06,
+ "loss": 0.9954442977905273,
+ "mean_token_accuracy": 0.7412676811218262,
+ "num_tokens": 12468224.0,
+ "step": 761
+ },
+ {
+ "entropy": 1.4129019975662231,
+ "epoch": 1.5393939393939395,
+ "grad_norm": 2.0903310775756836,
+ "learning_rate": 3.462626262626263e-06,
+ "loss": 1.4131944179534912,
+ "mean_token_accuracy": 0.6676844358444214,
+ "num_tokens": 12484608.0,
+ "step": 762
+ },
+ {
+ "entropy": 1.5170351266860962,
+ "epoch": 1.5414141414141413,
+ "grad_norm": 2.12807035446167,
+ "learning_rate": 3.460606060606061e-06,
+ "loss": 1.5342246294021606,
+ "mean_token_accuracy": 0.6621885895729065,
+ "num_tokens": 12500992.0,
+ "step": 763
+ },
+ {
+ "entropy": 1.1815718412399292,
+ "epoch": 1.5434343434343434,
+ "grad_norm": 2.2491719722747803,
+ "learning_rate": 3.4585858585858588e-06,
+ "loss": 1.1669261455535889,
+ "mean_token_accuracy": 0.694614052772522,
+ "num_tokens": 12517376.0,
+ "step": 764
+ },
+ {
+ "entropy": 1.547598958015442,
+ "epoch": 1.5454545454545454,
+ "grad_norm": 2.136096954345703,
+ "learning_rate": 3.456565656565657e-06,
+ "loss": 1.5515470504760742,
+ "mean_token_accuracy": 0.648876428604126,
+ "num_tokens": 12533760.0,
+ "step": 765
+ },
+ {
+ "entropy": 0.7156143188476562,
+ "epoch": 1.5474747474747474,
+ "grad_norm": 1.8139498233795166,
+ "learning_rate": 3.454545454545455e-06,
+ "loss": 0.7052675485610962,
+ "mean_token_accuracy": 0.8103932738304138,
+ "num_tokens": 12550144.0,
+ "step": 766
+ },
+ {
+ "entropy": 1.400156855583191,
+ "epoch": 1.5494949494949495,
+ "grad_norm": 2.14924693107605,
+ "learning_rate": 3.452525252525253e-06,
+ "loss": 1.4047740697860718,
+ "mean_token_accuracy": 0.6719589829444885,
+ "num_tokens": 12566528.0,
+ "step": 767
+ },
+ {
+ "entropy": 1.8254657983779907,
+ "epoch": 1.5515151515151515,
+ "grad_norm": 2.1882269382476807,
+ "learning_rate": 3.4505050505050507e-06,
+ "loss": 1.8615440130233765,
+ "mean_token_accuracy": 0.5776746273040771,
+ "num_tokens": 12582912.0,
+ "step": 768
+ },
+ {
+ "entropy": 1.5947710275650024,
+ "epoch": 1.5535353535353535,
+ "grad_norm": 2.3221426010131836,
+ "learning_rate": 3.4484848484848486e-06,
+ "loss": 1.6178712844848633,
+ "mean_token_accuracy": 0.6210918426513672,
+ "num_tokens": 12599296.0,
+ "step": 769
+ },
+ {
+ "entropy": 1.2678714990615845,
+ "epoch": 1.5555555555555556,
+ "grad_norm": 1.970064640045166,
+ "learning_rate": 3.446464646464647e-06,
+ "loss": 1.2646088600158691,
+ "mean_token_accuracy": 0.6990107297897339,
+ "num_tokens": 12615680.0,
+ "step": 770
+ },
+ {
+ "entropy": 1.3723604679107666,
+ "epoch": 1.5575757575757576,
+ "grad_norm": 2.171534776687622,
+ "learning_rate": 3.444444444444445e-06,
+ "loss": 1.3890197277069092,
+ "mean_token_accuracy": 0.6709819436073303,
+ "num_tokens": 12632064.0,
+ "step": 771
+ },
+ {
+ "entropy": 1.0273096561431885,
+ "epoch": 1.5595959595959596,
+ "grad_norm": 2.072798490524292,
+ "learning_rate": 3.4424242424242427e-06,
+ "loss": 1.029193639755249,
+ "mean_token_accuracy": 0.7348558902740479,
+ "num_tokens": 12648448.0,
+ "step": 772
+ },
+ {
+ "entropy": 1.4004037380218506,
+ "epoch": 1.5616161616161617,
+ "grad_norm": 2.111480474472046,
+ "learning_rate": 3.4404040404040406e-06,
+ "loss": 1.4213340282440186,
+ "mean_token_accuracy": 0.660906195640564,
+ "num_tokens": 12664832.0,
+ "step": 773
+ },
+ {
+ "entropy": 1.533424735069275,
+ "epoch": 1.5636363636363635,
+ "grad_norm": 2.0949435234069824,
+ "learning_rate": 3.4383838383838385e-06,
+ "loss": 1.5259253978729248,
+ "mean_token_accuracy": 0.6544333100318909,
+ "num_tokens": 12681216.0,
+ "step": 774
+ },
+ {
+ "entropy": 1.5620275735855103,
+ "epoch": 1.5656565656565657,
+ "grad_norm": 2.339731454849243,
+ "learning_rate": 3.4363636363636364e-06,
+ "loss": 1.5596071481704712,
+ "mean_token_accuracy": 0.6290302872657776,
+ "num_tokens": 12697600.0,
+ "step": 775
+ },
+ {
+ "entropy": 1.3272082805633545,
+ "epoch": 1.5676767676767676,
+ "grad_norm": 2.0871188640594482,
+ "learning_rate": 3.4343434343434347e-06,
+ "loss": 1.3413164615631104,
+ "mean_token_accuracy": 0.6809965968132019,
+ "num_tokens": 12713984.0,
+ "step": 776
+ },
+ {
+ "entropy": 1.202813982963562,
+ "epoch": 1.5696969696969698,
+ "grad_norm": 2.0538578033447266,
+ "learning_rate": 3.4323232323232326e-06,
+ "loss": 1.1947424411773682,
+ "mean_token_accuracy": 0.6969956159591675,
+ "num_tokens": 12730368.0,
+ "step": 777
+ },
+ {
+ "entropy": 1.3332773447036743,
+ "epoch": 1.5717171717171716,
+ "grad_norm": 1.9941202402114868,
+ "learning_rate": 3.4303030303030305e-06,
+ "loss": 1.3345956802368164,
+ "mean_token_accuracy": 0.6761724352836609,
+ "num_tokens": 12746752.0,
+ "step": 778
+ },
+ {
+ "entropy": 1.1724364757537842,
+ "epoch": 1.5737373737373739,
+ "grad_norm": 2.213365316390991,
+ "learning_rate": 3.4282828282828284e-06,
+ "loss": 1.1479787826538086,
+ "mean_token_accuracy": 0.7132999300956726,
+ "num_tokens": 12763136.0,
+ "step": 779
+ },
+ {
+ "entropy": 1.4718358516693115,
+ "epoch": 1.5757575757575757,
+ "grad_norm": 1.9743510484695435,
+ "learning_rate": 3.4262626262626262e-06,
+ "loss": 1.4844720363616943,
+ "mean_token_accuracy": 0.656448483467102,
+ "num_tokens": 12779520.0,
+ "step": 780
+ },
+ {
+ "entropy": 1.362882375717163,
+ "epoch": 1.5777777777777777,
+ "grad_norm": 2.370483875274658,
+ "learning_rate": 3.4242424242424246e-06,
+ "loss": 1.350801706314087,
+ "mean_token_accuracy": 0.6707376837730408,
+ "num_tokens": 12795904.0,
+ "step": 781
+ },
+ {
+ "entropy": 1.8302308320999146,
+ "epoch": 1.5797979797979798,
+ "grad_norm": 2.19045090675354,
+ "learning_rate": 3.4222222222222224e-06,
+ "loss": 1.8371320962905884,
+ "mean_token_accuracy": 0.5929408669471741,
+ "num_tokens": 12812288.0,
+ "step": 782
+ },
+ {
+ "entropy": 1.3435529470443726,
+ "epoch": 1.5818181818181818,
+ "grad_norm": 2.1254351139068604,
+ "learning_rate": 3.4202020202020203e-06,
+ "loss": 1.3621928691864014,
+ "mean_token_accuracy": 0.6867367029190063,
+ "num_tokens": 12828672.0,
+ "step": 783
+ },
+ {
+ "entropy": 1.8402602672576904,
+ "epoch": 1.5838383838383838,
+ "grad_norm": 2.123666524887085,
+ "learning_rate": 3.4181818181818182e-06,
+ "loss": 1.8564848899841309,
+ "mean_token_accuracy": 0.5871397256851196,
+ "num_tokens": 12845056.0,
+ "step": 784
+ },
+ {
+ "entropy": 1.441917896270752,
+ "epoch": 1.5858585858585859,
+ "grad_norm": 2.3286895751953125,
+ "learning_rate": 3.416161616161616e-06,
+ "loss": 1.4724184274673462,
+ "mean_token_accuracy": 0.6438080072402954,
+ "num_tokens": 12861440.0,
+ "step": 785
+ },
+ {
+ "entropy": 1.3071452379226685,
+ "epoch": 1.587878787878788,
+ "grad_norm": 2.149143934249878,
+ "learning_rate": 3.414141414141414e-06,
+ "loss": 1.331390142440796,
+ "mean_token_accuracy": 0.679347813129425,
+ "num_tokens": 12877824.0,
+ "step": 786
+ },
+ {
+ "entropy": 1.3547145128250122,
+ "epoch": 1.58989898989899,
+ "grad_norm": 2.2531180381774902,
+ "learning_rate": 3.4121212121212123e-06,
+ "loss": 1.361851453781128,
+ "mean_token_accuracy": 0.6767830848693848,
+ "num_tokens": 12894208.0,
+ "step": 787
+ },
+ {
+ "entropy": 0.9811291098594666,
+ "epoch": 1.591919191919192,
+ "grad_norm": 2.1053760051727295,
+ "learning_rate": 3.41010101010101e-06,
+ "loss": 0.9885507822036743,
+ "mean_token_accuracy": 0.7480459213256836,
+ "num_tokens": 12910592.0,
+ "step": 788
+ },
+ {
+ "entropy": 1.4940402507781982,
+ "epoch": 1.593939393939394,
+ "grad_norm": 2.240074634552002,
+ "learning_rate": 3.408080808080808e-06,
+ "loss": 1.5014076232910156,
+ "mean_token_accuracy": 0.6441743969917297,
+ "num_tokens": 12926976.0,
+ "step": 789
+ },
+ {
+ "entropy": 1.8134219646453857,
+ "epoch": 1.595959595959596,
+ "grad_norm": 2.040239095687866,
+ "learning_rate": 3.406060606060606e-06,
+ "loss": 1.8437519073486328,
+ "mean_token_accuracy": 0.5904372334480286,
+ "num_tokens": 12943360.0,
+ "step": 790
+ },
+ {
+ "entropy": 1.7233681678771973,
+ "epoch": 1.5979797979797978,
+ "grad_norm": 2.381786823272705,
+ "learning_rate": 3.4040404040404047e-06,
+ "loss": 1.7328217029571533,
+ "mean_token_accuracy": 0.6004518866539001,
+ "num_tokens": 12959744.0,
+ "step": 791
+ },
+ {
+ "entropy": 1.3966271877288818,
+ "epoch": 1.6,
+ "grad_norm": 2.17006778717041,
+ "learning_rate": 3.4020202020202026e-06,
+ "loss": 1.4065337181091309,
+ "mean_token_accuracy": 0.6629824042320251,
+ "num_tokens": 12976128.0,
+ "step": 792
+ },
+ {
+ "entropy": 1.5002557039260864,
+ "epoch": 1.602020202020202,
+ "grad_norm": 2.558037281036377,
+ "learning_rate": 3.4000000000000005e-06,
+ "loss": 1.4841307401657104,
+ "mean_token_accuracy": 0.647838294506073,
+ "num_tokens": 12992512.0,
+ "step": 793
+ },
+ {
+ "entropy": 1.7432687282562256,
+ "epoch": 1.6040404040404042,
+ "grad_norm": 2.134734630584717,
+ "learning_rate": 3.3979797979797984e-06,
+ "loss": 1.7539207935333252,
+ "mean_token_accuracy": 0.6028333902359009,
+ "num_tokens": 13008896.0,
+ "step": 794
+ },
+ {
+ "entropy": 1.4490517377853394,
+ "epoch": 1.606060606060606,
+ "grad_norm": 2.1590232849121094,
+ "learning_rate": 3.3959595959595963e-06,
+ "loss": 1.468411922454834,
+ "mean_token_accuracy": 0.656509518623352,
+ "num_tokens": 13025280.0,
+ "step": 795
+ },
+ {
+ "entropy": 1.3823497295379639,
+ "epoch": 1.6080808080808082,
+ "grad_norm": 2.1508285999298096,
+ "learning_rate": 3.3939393939393946e-06,
+ "loss": 1.4011330604553223,
+ "mean_token_accuracy": 0.6530898809432983,
+ "num_tokens": 13041664.0,
+ "step": 796
+ },
+ {
+ "entropy": 1.6982436180114746,
+ "epoch": 1.61010101010101,
+ "grad_norm": 2.077131986618042,
+ "learning_rate": 3.3919191919191925e-06,
+ "loss": 1.727889060974121,
+ "mean_token_accuracy": 0.6232290863990784,
+ "num_tokens": 13058048.0,
+ "step": 797
+ },
+ {
+ "entropy": 1.7268915176391602,
+ "epoch": 1.612121212121212,
+ "grad_norm": 2.0703542232513428,
+ "learning_rate": 3.3898989898989903e-06,
+ "loss": 1.7333730459213257,
+ "mean_token_accuracy": 0.6110160946846008,
+ "num_tokens": 13074432.0,
+ "step": 798
+ },
+ {
+ "entropy": 1.6253595352172852,
+ "epoch": 1.614141414141414,
+ "grad_norm": 2.241154432296753,
+ "learning_rate": 3.3878787878787882e-06,
+ "loss": 1.6275582313537598,
+ "mean_token_accuracy": 0.6198094487190247,
+ "num_tokens": 13090816.0,
+ "step": 799
+ },
+ {
+ "entropy": 1.8784692287445068,
+ "epoch": 1.6161616161616161,
+ "grad_norm": 2.505871057510376,
+ "learning_rate": 3.385858585858586e-06,
+ "loss": 1.8784008026123047,
+ "mean_token_accuracy": 0.5944064259529114,
+ "num_tokens": 13107200.0,
+ "step": 800
+ },
+ {
+ "epoch": 1.6161616161616161,
+ "eval_entropy": 1.4455043437980837,
+ "eval_loss": 1.5330992937088013,
+ "eval_mean_token_accuracy": 0.6448845310557273,
+ "eval_num_tokens": 13107200.0,
+ "eval_runtime": 43.766,
+ "eval_samples_per_second": 22.62,
+ "eval_steps_per_second": 2.833,
+ "step": 800
+ },
+ {
+ "entropy": 1.921440601348877,
+ "epoch": 1.6181818181818182,
+ "grad_norm": 2.1760783195495605,
+ "learning_rate": 3.3838383838383844e-06,
+ "loss": 1.9410110712051392,
+ "mean_token_accuracy": 0.5776746273040771,
+ "num_tokens": 13123584.0,
+ "step": 801
+ },
+ {
+ "entropy": 1.4233160018920898,
+ "epoch": 1.6202020202020202,
+ "grad_norm": 2.2687666416168213,
+ "learning_rate": 3.3818181818181823e-06,
+ "loss": 1.4465446472167969,
+ "mean_token_accuracy": 0.6488153338432312,
+ "num_tokens": 13139968.0,
+ "step": 802
+ },
+ {
+ "entropy": 1.697847843170166,
+ "epoch": 1.6222222222222222,
+ "grad_norm": 2.0649187564849854,
+ "learning_rate": 3.37979797979798e-06,
+ "loss": 1.7235385179519653,
+ "mean_token_accuracy": 0.6196262836456299,
+ "num_tokens": 13156352.0,
+ "step": 803
+ },
+ {
+ "entropy": 1.2027701139450073,
+ "epoch": 1.6242424242424243,
+ "grad_norm": 2.0115890502929688,
+ "learning_rate": 3.377777777777778e-06,
+ "loss": 1.1890676021575928,
+ "mean_token_accuracy": 0.6928431987762451,
+ "num_tokens": 13172736.0,
+ "step": 804
+ },
+ {
+ "entropy": 1.4847105741500854,
+ "epoch": 1.6262626262626263,
+ "grad_norm": 2.145098924636841,
+ "learning_rate": 3.375757575757576e-06,
+ "loss": 1.4817116260528564,
+ "mean_token_accuracy": 0.6415486335754395,
+ "num_tokens": 13189120.0,
+ "step": 805
+ },
+ {
+ "entropy": 1.3548893928527832,
+ "epoch": 1.6282828282828283,
+ "grad_norm": 2.3037521839141846,
+ "learning_rate": 3.3737373737373743e-06,
+ "loss": 1.3433952331542969,
+ "mean_token_accuracy": 0.6681729555130005,
+ "num_tokens": 13205504.0,
+ "step": 806
+ },
+ {
+ "entropy": 1.375793218612671,
+ "epoch": 1.6303030303030304,
+ "grad_norm": 2.0814852714538574,
+ "learning_rate": 3.371717171717172e-06,
+ "loss": 1.3506405353546143,
+ "mean_token_accuracy": 0.6634098887443542,
+ "num_tokens": 13221888.0,
+ "step": 807
+ },
+ {
+ "entropy": 1.1417323350906372,
+ "epoch": 1.6323232323232322,
+ "grad_norm": 2.0166866779327393,
+ "learning_rate": 3.36969696969697e-06,
+ "loss": 1.1731860637664795,
+ "mean_token_accuracy": 0.7123839855194092,
+ "num_tokens": 13238272.0,
+ "step": 808
+ },
+ {
+ "entropy": 1.3159022331237793,
+ "epoch": 1.6343434343434344,
+ "grad_norm": 1.9626281261444092,
+ "learning_rate": 3.367676767676768e-06,
+ "loss": 1.3286499977111816,
+ "mean_token_accuracy": 0.675561785697937,
+ "num_tokens": 13254656.0,
+ "step": 809
+ },
+ {
+ "entropy": 1.325775384902954,
+ "epoch": 1.6363636363636362,
+ "grad_norm": 2.1239242553710938,
+ "learning_rate": 3.365656565656566e-06,
+ "loss": 1.345514178276062,
+ "mean_token_accuracy": 0.6780043840408325,
+ "num_tokens": 13271040.0,
+ "step": 810
+ },
+ {
+ "entropy": 1.8307870626449585,
+ "epoch": 1.6383838383838385,
+ "grad_norm": 2.208296298980713,
+ "learning_rate": 3.3636363636363637e-06,
+ "loss": 1.8402234315872192,
+ "mean_token_accuracy": 0.5958719849586487,
+ "num_tokens": 13287424.0,
+ "step": 811
+ },
+ {
+ "entropy": 1.2596150636672974,
+ "epoch": 1.6404040404040403,
+ "grad_norm": 2.0936427116394043,
+ "learning_rate": 3.361616161616162e-06,
+ "loss": 1.2552804946899414,
+ "mean_token_accuracy": 0.6874083876609802,
+ "num_tokens": 13303808.0,
+ "step": 812
+ },
+ {
+ "entropy": 1.2575111389160156,
+ "epoch": 1.6424242424242426,
+ "grad_norm": 2.0232772827148438,
+ "learning_rate": 3.35959595959596e-06,
+ "loss": 1.2805989980697632,
+ "mean_token_accuracy": 0.6916829347610474,
+ "num_tokens": 13320192.0,
+ "step": 813
+ },
+ {
+ "entropy": 1.0270155668258667,
+ "epoch": 1.6444444444444444,
+ "grad_norm": 1.9090715646743774,
+ "learning_rate": 3.357575757575758e-06,
+ "loss": 1.0277824401855469,
+ "mean_token_accuracy": 0.7379091382026672,
+ "num_tokens": 13336576.0,
+ "step": 814
+ },
+ {
+ "entropy": 1.2992243766784668,
+ "epoch": 1.6464646464646466,
+ "grad_norm": 2.1718814373016357,
+ "learning_rate": 3.3555555555555557e-06,
+ "loss": 1.3278638124465942,
+ "mean_token_accuracy": 0.6764777898788452,
+ "num_tokens": 13352960.0,
+ "step": 815
+ },
+ {
+ "entropy": 1.5365362167358398,
+ "epoch": 1.6484848484848484,
+ "grad_norm": 2.0298495292663574,
+ "learning_rate": 3.3535353535353536e-06,
+ "loss": 1.5257508754730225,
+ "mean_token_accuracy": 0.6511358022689819,
+ "num_tokens": 13369344.0,
+ "step": 816
+ },
+ {
+ "entropy": 1.4384456872940063,
+ "epoch": 1.6505050505050505,
+ "grad_norm": 2.375277042388916,
+ "learning_rate": 3.351515151515152e-06,
+ "loss": 1.4287049770355225,
+ "mean_token_accuracy": 0.645639955997467,
+ "num_tokens": 13385728.0,
+ "step": 817
+ },
+ {
+ "entropy": 1.7104265689849854,
+ "epoch": 1.6525252525252525,
+ "grad_norm": 2.052535057067871,
+ "learning_rate": 3.34949494949495e-06,
+ "loss": 1.6971676349639893,
+ "mean_token_accuracy": 0.6159623861312866,
+ "num_tokens": 13402112.0,
+ "step": 818
+ },
+ {
+ "entropy": 1.5908539295196533,
+ "epoch": 1.6545454545454545,
+ "grad_norm": 2.047088861465454,
+ "learning_rate": 3.3474747474747477e-06,
+ "loss": 1.5640246868133545,
+ "mean_token_accuracy": 0.640754759311676,
+ "num_tokens": 13418496.0,
+ "step": 819
+ },
+ {
+ "entropy": 1.3660207986831665,
+ "epoch": 1.6565656565656566,
+ "grad_norm": 2.2711079120635986,
+ "learning_rate": 3.3454545454545456e-06,
+ "loss": 1.3601036071777344,
+ "mean_token_accuracy": 0.6577919125556946,
+ "num_tokens": 13434880.0,
+ "step": 820
+ },
+ {
+ "entropy": 1.5897634029388428,
+ "epoch": 1.6585858585858586,
+ "grad_norm": 2.0869712829589844,
+ "learning_rate": 3.3434343434343435e-06,
+ "loss": 1.5801420211791992,
+ "mean_token_accuracy": 0.6312897205352783,
+ "num_tokens": 13451264.0,
+ "step": 821
+ },
+ {
+ "entropy": 1.428406834602356,
+ "epoch": 1.6606060606060606,
+ "grad_norm": 2.049370765686035,
+ "learning_rate": 3.3414141414141413e-06,
+ "loss": 1.4145596027374268,
+ "mean_token_accuracy": 0.6649975776672363,
+ "num_tokens": 13467648.0,
+ "step": 822
+ },
+ {
+ "entropy": 1.5026675462722778,
+ "epoch": 1.6626262626262627,
+ "grad_norm": 2.157031297683716,
+ "learning_rate": 3.3393939393939397e-06,
+ "loss": 1.4902830123901367,
+ "mean_token_accuracy": 0.6524181962013245,
+ "num_tokens": 13484032.0,
+ "step": 823
+ },
+ {
+ "entropy": 1.4907052516937256,
+ "epoch": 1.6646464646464647,
+ "grad_norm": 2.1476478576660156,
+ "learning_rate": 3.3373737373737375e-06,
+ "loss": 1.5102436542510986,
+ "mean_token_accuracy": 0.6485100388526917,
+ "num_tokens": 13500416.0,
+ "step": 824
+ },
+ {
+ "entropy": 1.4277782440185547,
+ "epoch": 1.6666666666666665,
+ "grad_norm": 2.2880661487579346,
+ "learning_rate": 3.3353535353535354e-06,
+ "loss": 1.4358019828796387,
+ "mean_token_accuracy": 0.6522960662841797,
+ "num_tokens": 13516800.0,
+ "step": 825
+ },
+ {
+ "entropy": 1.5560128688812256,
+ "epoch": 1.6686868686868688,
+ "grad_norm": 2.048403024673462,
+ "learning_rate": 3.3333333333333333e-06,
+ "loss": 1.5472501516342163,
+ "mean_token_accuracy": 0.6488153338432312,
+ "num_tokens": 13533184.0,
+ "step": 826
+ },
+ {
+ "entropy": 1.665900707244873,
+ "epoch": 1.6707070707070706,
+ "grad_norm": 2.0828492641448975,
+ "learning_rate": 3.331313131313131e-06,
+ "loss": 1.6905372142791748,
+ "mean_token_accuracy": 0.629885196685791,
+ "num_tokens": 13549568.0,
+ "step": 827
+ },
+ {
+ "entropy": 1.3484426736831665,
+ "epoch": 1.6727272727272728,
+ "grad_norm": 2.1873531341552734,
+ "learning_rate": 3.3292929292929295e-06,
+ "loss": 1.3510258197784424,
+ "mean_token_accuracy": 0.6721421480178833,
+ "num_tokens": 13565952.0,
+ "step": 828
+ },
+ {
+ "entropy": 1.7577860355377197,
+ "epoch": 1.6747474747474747,
+ "grad_norm": 2.0467135906219482,
+ "learning_rate": 3.3272727272727274e-06,
+ "loss": 1.7791969776153564,
+ "mean_token_accuracy": 0.6028944849967957,
+ "num_tokens": 13582336.0,
+ "step": 829
+ },
+ {
+ "entropy": 1.4549124240875244,
+ "epoch": 1.676767676767677,
+ "grad_norm": 2.2004988193511963,
+ "learning_rate": 3.3252525252525253e-06,
+ "loss": 1.4512457847595215,
+ "mean_token_accuracy": 0.6521128416061401,
+ "num_tokens": 13598720.0,
+ "step": 830
+ },
+ {
+ "entropy": 1.2226758003234863,
+ "epoch": 1.6787878787878787,
+ "grad_norm": 2.0866682529449463,
+ "learning_rate": 3.323232323232323e-06,
+ "loss": 1.226635456085205,
+ "mean_token_accuracy": 0.6969345211982727,
+ "num_tokens": 13615104.0,
+ "step": 831
+ },
+ {
+ "entropy": 0.9826563000679016,
+ "epoch": 1.680808080808081,
+ "grad_norm": 1.9865771532058716,
+ "learning_rate": 3.321212121212121e-06,
+ "loss": 1.009648323059082,
+ "mean_token_accuracy": 0.7507327795028687,
+ "num_tokens": 13631488.0,
+ "step": 832
+ },
+ {
+ "entropy": 1.620038390159607,
+ "epoch": 1.6828282828282828,
+ "grad_norm": 2.1745760440826416,
+ "learning_rate": 3.3191919191919194e-06,
+ "loss": 1.615234136581421,
+ "mean_token_accuracy": 0.6305569410324097,
+ "num_tokens": 13647872.0,
+ "step": 833
+ },
+ {
+ "entropy": 1.4803876876831055,
+ "epoch": 1.6848484848484848,
+ "grad_norm": 2.328029155731201,
+ "learning_rate": 3.3171717171717177e-06,
+ "loss": 1.509049892425537,
+ "mean_token_accuracy": 0.6353810429573059,
+ "num_tokens": 13664256.0,
+ "step": 834
+ },
+ {
+ "entropy": 1.2290453910827637,
+ "epoch": 1.6868686868686869,
+ "grad_norm": 2.1777281761169434,
+ "learning_rate": 3.3151515151515156e-06,
+ "loss": 1.222726583480835,
+ "mean_token_accuracy": 0.6842330098152161,
+ "num_tokens": 13680640.0,
+ "step": 835
+ },
+ {
+ "entropy": 1.1965430974960327,
+ "epoch": 1.6888888888888889,
+ "grad_norm": 2.211397886276245,
+ "learning_rate": 3.3131313131313135e-06,
+ "loss": 1.2027852535247803,
+ "mean_token_accuracy": 0.7028578519821167,
+ "num_tokens": 13697024.0,
+ "step": 836
+ },
+ {
+ "entropy": 1.5326080322265625,
+ "epoch": 1.690909090909091,
+ "grad_norm": 2.8396589756011963,
+ "learning_rate": 3.3111111111111118e-06,
+ "loss": 1.5819613933563232,
+ "mean_token_accuracy": 0.625,
+ "num_tokens": 13713408.0,
+ "step": 837
+ },
+ {
+ "entropy": 1.2514865398406982,
+ "epoch": 1.692929292929293,
+ "grad_norm": 1.923771858215332,
+ "learning_rate": 3.3090909090909097e-06,
+ "loss": 1.2518126964569092,
+ "mean_token_accuracy": 0.6955300569534302,
+ "num_tokens": 13729792.0,
+ "step": 838
+ },
+ {
+ "entropy": 1.3717304468154907,
+ "epoch": 1.694949494949495,
+ "grad_norm": 2.0393056869506836,
+ "learning_rate": 3.3070707070707076e-06,
+ "loss": 1.400814175605774,
+ "mean_token_accuracy": 0.6618832349777222,
+ "num_tokens": 13746176.0,
+ "step": 839
+ },
+ {
+ "entropy": 1.2871394157409668,
+ "epoch": 1.696969696969697,
+ "grad_norm": 2.114823579788208,
+ "learning_rate": 3.3050505050505054e-06,
+ "loss": 1.302154541015625,
+ "mean_token_accuracy": 0.6790425181388855,
+ "num_tokens": 13762560.0,
+ "step": 840
+ },
+ {
+ "entropy": 1.518404483795166,
+ "epoch": 1.698989898989899,
+ "grad_norm": 2.186727523803711,
+ "learning_rate": 3.3030303030303033e-06,
+ "loss": 1.5067068338394165,
+ "mean_token_accuracy": 0.638006865978241,
+ "num_tokens": 13778944.0,
+ "step": 841
+ },
+ {
+ "entropy": 1.5698254108428955,
+ "epoch": 1.7010101010101009,
+ "grad_norm": 2.159721851348877,
+ "learning_rate": 3.3010101010101016e-06,
+ "loss": 1.5568101406097412,
+ "mean_token_accuracy": 0.6398998498916626,
+ "num_tokens": 13795328.0,
+ "step": 842
+ },
+ {
+ "entropy": 1.5159670114517212,
+ "epoch": 1.7030303030303031,
+ "grad_norm": 2.0655386447906494,
+ "learning_rate": 3.2989898989898995e-06,
+ "loss": 1.5152846574783325,
+ "mean_token_accuracy": 0.6441133618354797,
+ "num_tokens": 13811712.0,
+ "step": 843
+ },
+ {
+ "entropy": 1.231048822402954,
+ "epoch": 1.705050505050505,
+ "grad_norm": 1.99764084815979,
+ "learning_rate": 3.2969696969696974e-06,
+ "loss": 1.2419885396957397,
+ "mean_token_accuracy": 0.6936370134353638,
+ "num_tokens": 13828096.0,
+ "step": 844
+ },
+ {
+ "entropy": 1.1431403160095215,
+ "epoch": 1.7070707070707072,
+ "grad_norm": 2.0046544075012207,
+ "learning_rate": 3.2949494949494953e-06,
+ "loss": 1.147315502166748,
+ "mean_token_accuracy": 0.7155593633651733,
+ "num_tokens": 13844480.0,
+ "step": 845
+ },
+ {
+ "entropy": 1.903164267539978,
+ "epoch": 1.709090909090909,
+ "grad_norm": 2.068286418914795,
+ "learning_rate": 3.292929292929293e-06,
+ "loss": 1.9086098670959473,
+ "mean_token_accuracy": 0.5785295367240906,
+ "num_tokens": 13860864.0,
+ "step": 846
+ },
+ {
+ "entropy": 1.028931736946106,
+ "epoch": 1.7111111111111112,
+ "grad_norm": 2.008197546005249,
+ "learning_rate": 3.290909090909091e-06,
+ "loss": 1.0251559019088745,
+ "mean_token_accuracy": 0.7308866381645203,
+ "num_tokens": 13877248.0,
+ "step": 847
+ },
+ {
+ "entropy": 1.8438282012939453,
+ "epoch": 1.713131313131313,
+ "grad_norm": 2.180811882019043,
+ "learning_rate": 3.2888888888888894e-06,
+ "loss": 1.8572561740875244,
+ "mean_token_accuracy": 0.5788959264755249,
+ "num_tokens": 13893632.0,
+ "step": 848
+ },
+ {
+ "entropy": 1.437190294265747,
+ "epoch": 1.7151515151515153,
+ "grad_norm": 2.1013762950897217,
+ "learning_rate": 3.2868686868686873e-06,
+ "loss": 1.4337413311004639,
+ "mean_token_accuracy": 0.6489985585212708,
+ "num_tokens": 13910016.0,
+ "step": 849
+ },
+ {
+ "entropy": 1.482985496520996,
+ "epoch": 1.7171717171717171,
+ "grad_norm": 2.290910243988037,
+ "learning_rate": 3.284848484848485e-06,
+ "loss": 1.4959741830825806,
+ "mean_token_accuracy": 0.6447850465774536,
+ "num_tokens": 13926400.0,
+ "step": 850
+ },
+ {
+ "epoch": 1.7171717171717171,
+ "eval_entropy": 1.447427170411233,
+ "eval_loss": 1.5309957265853882,
+ "eval_mean_token_accuracy": 0.6452888370521607,
+ "eval_num_tokens": 13926400.0,
+ "eval_runtime": 43.7619,
+ "eval_samples_per_second": 22.622,
+ "eval_steps_per_second": 2.834,
+ "step": 850
+ },
+ {
+ "entropy": 1.4470106363296509,
+ "epoch": 1.7191919191919192,
+ "grad_norm": 2.2015624046325684,
+ "learning_rate": 3.282828282828283e-06,
+ "loss": 1.462576150894165,
+ "mean_token_accuracy": 0.643991231918335,
+ "num_tokens": 13942784.0,
+ "step": 851
+ },
+ {
+ "entropy": 1.4063832759857178,
+ "epoch": 1.7212121212121212,
+ "grad_norm": 2.1743109226226807,
+ "learning_rate": 3.280808080808081e-06,
+ "loss": 1.402491569519043,
+ "mean_token_accuracy": 0.6533952355384827,
+ "num_tokens": 13959168.0,
+ "step": 852
+ },
+ {
+ "entropy": 1.3735251426696777,
+ "epoch": 1.7232323232323232,
+ "grad_norm": 2.1350960731506348,
+ "learning_rate": 3.2787878787878793e-06,
+ "loss": 1.383405327796936,
+ "mean_token_accuracy": 0.6767830848693848,
+ "num_tokens": 13975552.0,
+ "step": 853
+ },
+ {
+ "entropy": 1.359702229499817,
+ "epoch": 1.7252525252525253,
+ "grad_norm": 2.1547393798828125,
+ "learning_rate": 3.276767676767677e-06,
+ "loss": 1.3454554080963135,
+ "mean_token_accuracy": 0.6712262034416199,
+ "num_tokens": 13991936.0,
+ "step": 854
+ },
+ {
+ "entropy": 1.0641778707504272,
+ "epoch": 1.7272727272727273,
+ "grad_norm": 2.0301826000213623,
+ "learning_rate": 3.274747474747475e-06,
+ "loss": 1.0626115798950195,
+ "mean_token_accuracy": 0.7188568711280823,
+ "num_tokens": 14008320.0,
+ "step": 855
+ },
+ {
+ "entropy": 1.598792552947998,
+ "epoch": 1.7292929292929293,
+ "grad_norm": 2.3324217796325684,
+ "learning_rate": 3.272727272727273e-06,
+ "loss": 1.6028941869735718,
+ "mean_token_accuracy": 0.6462506055831909,
+ "num_tokens": 14024704.0,
+ "step": 856
+ },
+ {
+ "entropy": 1.6304137706756592,
+ "epoch": 1.7313131313131314,
+ "grad_norm": 1.9968358278274536,
+ "learning_rate": 3.270707070707071e-06,
+ "loss": 1.6449415683746338,
+ "mean_token_accuracy": 0.6322056651115417,
+ "num_tokens": 14041088.0,
+ "step": 857
+ },
+ {
+ "entropy": 1.3219174146652222,
+ "epoch": 1.7333333333333334,
+ "grad_norm": 2.059927225112915,
+ "learning_rate": 3.2686868686868687e-06,
+ "loss": 1.3487168550491333,
+ "mean_token_accuracy": 0.6769052147865295,
+ "num_tokens": 14057472.0,
+ "step": 858
+ },
+ {
+ "entropy": 1.4645651578903198,
+ "epoch": 1.7353535353535352,
+ "grad_norm": 2.068962812423706,
+ "learning_rate": 3.266666666666667e-06,
+ "loss": 1.4480202198028564,
+ "mean_token_accuracy": 0.6521128416061401,
+ "num_tokens": 14073856.0,
+ "step": 859
+ },
+ {
+ "entropy": 1.5005300045013428,
+ "epoch": 1.7373737373737375,
+ "grad_norm": 2.228736162185669,
+ "learning_rate": 3.264646464646465e-06,
+ "loss": 1.5110201835632324,
+ "mean_token_accuracy": 0.6482657790184021,
+ "num_tokens": 14090240.0,
+ "step": 860
+ },
+ {
+ "entropy": 1.1641852855682373,
+ "epoch": 1.7393939393939393,
+ "grad_norm": 2.0611488819122314,
+ "learning_rate": 3.262626262626263e-06,
+ "loss": 1.1905086040496826,
+ "mean_token_accuracy": 0.7104909420013428,
+ "num_tokens": 14106624.0,
+ "step": 861
+ },
+ {
+ "entropy": 1.4315950870513916,
+ "epoch": 1.7414141414141415,
+ "grad_norm": 1.9914237260818481,
+ "learning_rate": 3.2606060606060607e-06,
+ "loss": 1.467012882232666,
+ "mean_token_accuracy": 0.657608687877655,
+ "num_tokens": 14123008.0,
+ "step": 862
+ },
+ {
+ "entropy": 1.5196973085403442,
+ "epoch": 1.7434343434343433,
+ "grad_norm": 2.3258447647094727,
+ "learning_rate": 3.2585858585858586e-06,
+ "loss": 1.5382301807403564,
+ "mean_token_accuracy": 0.6370298266410828,
+ "num_tokens": 14139392.0,
+ "step": 863
+ },
+ {
+ "entropy": 1.5098228454589844,
+ "epoch": 1.7454545454545456,
+ "grad_norm": 2.146327495574951,
+ "learning_rate": 3.256565656565657e-06,
+ "loss": 1.540649652481079,
+ "mean_token_accuracy": 0.6439301371574402,
+ "num_tokens": 14155776.0,
+ "step": 864
+ },
+ {
+ "entropy": 1.128011703491211,
+ "epoch": 1.7474747474747474,
+ "grad_norm": 1.967443823814392,
+ "learning_rate": 3.2545454545454548e-06,
+ "loss": 1.1707985401153564,
+ "mean_token_accuracy": 0.7298485636711121,
+ "num_tokens": 14172160.0,
+ "step": 865
+ },
+ {
+ "entropy": 1.9117931127548218,
+ "epoch": 1.7494949494949497,
+ "grad_norm": 2.097781181335449,
+ "learning_rate": 3.2525252525252527e-06,
+ "loss": 1.9291374683380127,
+ "mean_token_accuracy": 0.580483615398407,
+ "num_tokens": 14188544.0,
+ "step": 866
+ },
+ {
+ "entropy": 1.9053187370300293,
+ "epoch": 1.7515151515151515,
+ "grad_norm": 1.9865039587020874,
+ "learning_rate": 3.2505050505050505e-06,
+ "loss": 1.932790756225586,
+ "mean_token_accuracy": 0.5770639777183533,
+ "num_tokens": 14204928.0,
+ "step": 867
+ },
+ {
+ "entropy": 1.2647870779037476,
+ "epoch": 1.7535353535353535,
+ "grad_norm": 2.099891185760498,
+ "learning_rate": 3.2484848484848484e-06,
+ "loss": 1.281562328338623,
+ "mean_token_accuracy": 0.6938812732696533,
+ "num_tokens": 14221312.0,
+ "step": 868
+ },
+ {
+ "entropy": 1.5446428060531616,
+ "epoch": 1.7555555555555555,
+ "grad_norm": 1.9044189453125,
+ "learning_rate": 3.2464646464646467e-06,
+ "loss": 1.5440425872802734,
+ "mean_token_accuracy": 0.6547996997833252,
+ "num_tokens": 14237696.0,
+ "step": 869
+ },
+ {
+ "entropy": 1.179898977279663,
+ "epoch": 1.7575757575757576,
+ "grad_norm": 2.0257933139801025,
+ "learning_rate": 3.2444444444444446e-06,
+ "loss": 1.1607933044433594,
+ "mean_token_accuracy": 0.7053614854812622,
+ "num_tokens": 14254080.0,
+ "step": 870
+ },
+ {
+ "entropy": 1.0462855100631714,
+ "epoch": 1.7595959595959596,
+ "grad_norm": 1.8995661735534668,
+ "learning_rate": 3.2424242424242425e-06,
+ "loss": 1.0395057201385498,
+ "mean_token_accuracy": 0.7405959963798523,
+ "num_tokens": 14270464.0,
+ "step": 871
+ },
+ {
+ "entropy": 0.8866081833839417,
+ "epoch": 1.7616161616161616,
+ "grad_norm": 1.7807546854019165,
+ "learning_rate": 3.2404040404040404e-06,
+ "loss": 0.8795047998428345,
+ "mean_token_accuracy": 0.7721666097640991,
+ "num_tokens": 14286848.0,
+ "step": 872
+ },
+ {
+ "entropy": 1.9576365947723389,
+ "epoch": 1.7636363636363637,
+ "grad_norm": 2.1867618560791016,
+ "learning_rate": 3.2383838383838383e-06,
+ "loss": 1.9605103731155396,
+ "mean_token_accuracy": 0.5809110999107361,
+ "num_tokens": 14303232.0,
+ "step": 873
+ },
+ {
+ "entropy": 1.010751485824585,
+ "epoch": 1.7656565656565657,
+ "grad_norm": 1.9643300771713257,
+ "learning_rate": 3.236363636363636e-06,
+ "loss": 0.9990894794464111,
+ "mean_token_accuracy": 0.7386419177055359,
+ "num_tokens": 14319616.0,
+ "step": 874
+ },
+ {
+ "entropy": 1.3587232828140259,
+ "epoch": 1.7676767676767677,
+ "grad_norm": 2.2632040977478027,
+ "learning_rate": 3.2343434343434345e-06,
+ "loss": 1.3631021976470947,
+ "mean_token_accuracy": 0.665791392326355,
+ "num_tokens": 14336000.0,
+ "step": 875
+ },
+ {
+ "entropy": 1.6577962636947632,
+ "epoch": 1.7696969696969695,
+ "grad_norm": 2.3661980628967285,
+ "learning_rate": 3.232323232323233e-06,
+ "loss": 1.6802719831466675,
+ "mean_token_accuracy": 0.6024059653282166,
+ "num_tokens": 14352384.0,
+ "step": 876
+ },
+ {
+ "entropy": 1.4332902431488037,
+ "epoch": 1.7717171717171718,
+ "grad_norm": 2.2296342849731445,
+ "learning_rate": 3.2303030303030307e-06,
+ "loss": 1.4129014015197754,
+ "mean_token_accuracy": 0.6511358022689819,
+ "num_tokens": 14368768.0,
+ "step": 877
+ },
+ {
+ "entropy": 1.2260215282440186,
+ "epoch": 1.7737373737373736,
+ "grad_norm": 2.0659983158111572,
+ "learning_rate": 3.228282828282829e-06,
+ "loss": 1.2417192459106445,
+ "mean_token_accuracy": 0.693514883518219,
+ "num_tokens": 14385152.0,
+ "step": 878
+ },
+ {
+ "entropy": 1.5402687788009644,
+ "epoch": 1.7757575757575759,
+ "grad_norm": 2.2024054527282715,
+ "learning_rate": 3.226262626262627e-06,
+ "loss": 1.5535321235656738,
+ "mean_token_accuracy": 0.6331827044487,
+ "num_tokens": 14401536.0,
+ "step": 879
+ },
+ {
+ "entropy": 1.2205268144607544,
+ "epoch": 1.7777777777777777,
+ "grad_norm": 2.0056662559509277,
+ "learning_rate": 3.2242424242424248e-06,
+ "loss": 1.2244641780853271,
+ "mean_token_accuracy": 0.7037127614021301,
+ "num_tokens": 14417920.0,
+ "step": 880
+ },
+ {
+ "entropy": 1.542244791984558,
+ "epoch": 1.77979797979798,
+ "grad_norm": 2.0935254096984863,
+ "learning_rate": 3.2222222222222227e-06,
+ "loss": 1.5638453960418701,
+ "mean_token_accuracy": 0.648937463760376,
+ "num_tokens": 14434304.0,
+ "step": 881
+ },
+ {
+ "entropy": 1.4249905347824097,
+ "epoch": 1.7818181818181817,
+ "grad_norm": 1.9853826761245728,
+ "learning_rate": 3.2202020202020206e-06,
+ "loss": 1.4453399181365967,
+ "mean_token_accuracy": 0.6869809627532959,
+ "num_tokens": 14450688.0,
+ "step": 882
+ },
+ {
+ "entropy": 1.4777165651321411,
+ "epoch": 1.783838383838384,
+ "grad_norm": 2.064542055130005,
+ "learning_rate": 3.2181818181818184e-06,
+ "loss": 1.4756921529769897,
+ "mean_token_accuracy": 0.6640205383300781,
+ "num_tokens": 14467072.0,
+ "step": 883
+ },
+ {
+ "entropy": 1.3442282676696777,
+ "epoch": 1.7858585858585858,
+ "grad_norm": 2.043712854385376,
+ "learning_rate": 3.2161616161616168e-06,
+ "loss": 1.3412827253341675,
+ "mean_token_accuracy": 0.6821568012237549,
+ "num_tokens": 14483456.0,
+ "step": 884
+ },
+ {
+ "entropy": 1.5201579332351685,
+ "epoch": 1.7878787878787878,
+ "grad_norm": 2.060661792755127,
+ "learning_rate": 3.2141414141414146e-06,
+ "loss": 1.5367834568023682,
+ "mean_token_accuracy": 0.6442354917526245,
+ "num_tokens": 14499840.0,
+ "step": 885
+ },
+ {
+ "entropy": 1.1536649465560913,
+ "epoch": 1.7898989898989899,
+ "grad_norm": 2.1190037727355957,
+ "learning_rate": 3.2121212121212125e-06,
+ "loss": 1.1588165760040283,
+ "mean_token_accuracy": 0.7039570212364197,
+ "num_tokens": 14516224.0,
+ "step": 886
+ },
+ {
+ "entropy": 1.109966516494751,
+ "epoch": 1.791919191919192,
+ "grad_norm": 2.0320141315460205,
+ "learning_rate": 3.2101010101010104e-06,
+ "loss": 1.1113003492355347,
+ "mean_token_accuracy": 0.7123228907585144,
+ "num_tokens": 14532608.0,
+ "step": 887
+ },
+ {
+ "entropy": 1.5709099769592285,
+ "epoch": 1.793939393939394,
+ "grad_norm": 2.0467336177825928,
+ "learning_rate": 3.2080808080808083e-06,
+ "loss": 1.5823577642440796,
+ "mean_token_accuracy": 0.6404494643211365,
+ "num_tokens": 14548992.0,
+ "step": 888
+ },
+ {
+ "entropy": 1.4305545091629028,
+ "epoch": 1.795959595959596,
+ "grad_norm": 2.0764269828796387,
+ "learning_rate": 3.2060606060606066e-06,
+ "loss": 1.438415765762329,
+ "mean_token_accuracy": 0.6574255228042603,
+ "num_tokens": 14565376.0,
+ "step": 889
+ },
+ {
+ "entropy": 1.3523615598678589,
+ "epoch": 1.797979797979798,
+ "grad_norm": 2.167036294937134,
+ "learning_rate": 3.2040404040404045e-06,
+ "loss": 1.3782477378845215,
+ "mean_token_accuracy": 0.6645090579986572,
+ "num_tokens": 14581760.0,
+ "step": 890
+ },
+ {
+ "entropy": 1.491297721862793,
+ "epoch": 1.8,
+ "grad_norm": 2.3090412616729736,
+ "learning_rate": 3.2020202020202024e-06,
+ "loss": 1.4965641498565674,
+ "mean_token_accuracy": 0.636907696723938,
+ "num_tokens": 14598144.0,
+ "step": 891
+ },
+ {
+ "entropy": 1.2584501504898071,
+ "epoch": 1.802020202020202,
+ "grad_norm": 2.1699554920196533,
+ "learning_rate": 3.2000000000000003e-06,
+ "loss": 1.2622835636138916,
+ "mean_token_accuracy": 0.6901563405990601,
+ "num_tokens": 14614528.0,
+ "step": 892
+ },
+ {
+ "entropy": 2.149350643157959,
+ "epoch": 1.8040404040404039,
+ "grad_norm": 2.2669718265533447,
+ "learning_rate": 3.197979797979798e-06,
+ "loss": 2.167891263961792,
+ "mean_token_accuracy": 0.5497679710388184,
+ "num_tokens": 14630912.0,
+ "step": 893
+ },
+ {
+ "entropy": 1.1999561786651611,
+ "epoch": 1.8060606060606061,
+ "grad_norm": 2.163228988647461,
+ "learning_rate": 3.195959595959596e-06,
+ "loss": 1.2024329900741577,
+ "mean_token_accuracy": 0.7035906314849854,
+ "num_tokens": 14647296.0,
+ "step": 894
+ },
+ {
+ "entropy": 1.770652413368225,
+ "epoch": 1.808080808080808,
+ "grad_norm": 2.1631994247436523,
+ "learning_rate": 3.1939393939393944e-06,
+ "loss": 1.7568567991256714,
+ "mean_token_accuracy": 0.6030776500701904,
+ "num_tokens": 14663680.0,
+ "step": 895
+ },
+ {
+ "entropy": 1.2487529516220093,
+ "epoch": 1.8101010101010102,
+ "grad_norm": 1.987622857093811,
+ "learning_rate": 3.1919191919191923e-06,
+ "loss": 1.255242109298706,
+ "mean_token_accuracy": 0.6929042339324951,
+ "num_tokens": 14680064.0,
+ "step": 896
+ },
+ {
+ "entropy": 1.138524055480957,
+ "epoch": 1.812121212121212,
+ "grad_norm": 2.0367984771728516,
+ "learning_rate": 3.18989898989899e-06,
+ "loss": 1.1514946222305298,
+ "mean_token_accuracy": 0.7093917727470398,
+ "num_tokens": 14696448.0,
+ "step": 897
+ },
+ {
+ "entropy": 1.3337515592575073,
+ "epoch": 1.8141414141414143,
+ "grad_norm": 1.9483140707015991,
+ "learning_rate": 3.187878787878788e-06,
+ "loss": 1.3251439332962036,
+ "mean_token_accuracy": 0.6861260533332825,
+ "num_tokens": 14712832.0,
+ "step": 898
+ },
+ {
+ "entropy": 1.047120451927185,
+ "epoch": 1.816161616161616,
+ "grad_norm": 2.0481390953063965,
+ "learning_rate": 3.185858585858586e-06,
+ "loss": 1.071772575378418,
+ "mean_token_accuracy": 0.7275891304016113,
+ "num_tokens": 14729216.0,
+ "step": 899
+ },
+ {
+ "entropy": 1.3311363458633423,
+ "epoch": 1.8181818181818183,
+ "grad_norm": 2.3203413486480713,
+ "learning_rate": 3.1838383838383842e-06,
+ "loss": 1.334134817123413,
+ "mean_token_accuracy": 0.6643869280815125,
+ "num_tokens": 14745600.0,
+ "step": 900
+ },
+ {
+ "epoch": 1.8181818181818183,
+ "eval_entropy": 1.4476436400605786,
+ "eval_loss": 1.5290467739105225,
+ "eval_mean_token_accuracy": 0.6456143543604882,
+ "eval_num_tokens": 14745600.0,
+ "eval_runtime": 43.8113,
+ "eval_samples_per_second": 22.597,
+ "eval_steps_per_second": 2.83,
+ "step": 900
+ },
+ {
+ "entropy": 1.6842114925384521,
+ "epoch": 1.8202020202020202,
+ "grad_norm": 2.1709823608398438,
+ "learning_rate": 3.181818181818182e-06,
+ "loss": 1.6816682815551758,
+ "mean_token_accuracy": 0.6033830046653748,
+ "num_tokens": 14761984.0,
+ "step": 901
+ },
+ {
+ "entropy": 1.2661798000335693,
+ "epoch": 1.8222222222222222,
+ "grad_norm": 2.113783836364746,
+ "learning_rate": 3.17979797979798e-06,
+ "loss": 1.249183177947998,
+ "mean_token_accuracy": 0.6865534782409668,
+ "num_tokens": 14778368.0,
+ "step": 902
+ },
+ {
+ "entropy": 1.2666339874267578,
+ "epoch": 1.8242424242424242,
+ "grad_norm": 2.0210978984832764,
+ "learning_rate": 3.177777777777778e-06,
+ "loss": 1.2548515796661377,
+ "mean_token_accuracy": 0.686431348323822,
+ "num_tokens": 14794752.0,
+ "step": 903
+ },
+ {
+ "entropy": 1.1660066843032837,
+ "epoch": 1.8262626262626263,
+ "grad_norm": 2.0998034477233887,
+ "learning_rate": 3.1757575757575758e-06,
+ "loss": 1.1654324531555176,
+ "mean_token_accuracy": 0.695713222026825,
+ "num_tokens": 14811136.0,
+ "step": 904
+ },
+ {
+ "entropy": 1.2902735471725464,
+ "epoch": 1.8282828282828283,
+ "grad_norm": 2.1510303020477295,
+ "learning_rate": 3.173737373737374e-06,
+ "loss": 1.2879812717437744,
+ "mean_token_accuracy": 0.6693942546844482,
+ "num_tokens": 14827520.0,
+ "step": 905
+ },
+ {
+ "entropy": 1.1441799402236938,
+ "epoch": 1.8303030303030303,
+ "grad_norm": 1.989449143409729,
+ "learning_rate": 3.171717171717172e-06,
+ "loss": 1.1552281379699707,
+ "mean_token_accuracy": 0.7127503752708435,
+ "num_tokens": 14843904.0,
+ "step": 906
+ },
+ {
+ "entropy": 1.08464515209198,
+ "epoch": 1.8323232323232324,
+ "grad_norm": 2.094696044921875,
+ "learning_rate": 3.16969696969697e-06,
+ "loss": 1.0758531093597412,
+ "mean_token_accuracy": 0.7164142727851868,
+ "num_tokens": 14860288.0,
+ "step": 907
+ },
+ {
+ "entropy": 1.3847618103027344,
+ "epoch": 1.8343434343434344,
+ "grad_norm": 2.207976818084717,
+ "learning_rate": 3.1676767676767678e-06,
+ "loss": 1.3722931146621704,
+ "mean_token_accuracy": 0.6741573214530945,
+ "num_tokens": 14876672.0,
+ "step": 908
+ },
+ {
+ "entropy": 1.3558534383773804,
+ "epoch": 1.8363636363636364,
+ "grad_norm": 2.166674852371216,
+ "learning_rate": 3.1656565656565656e-06,
+ "loss": 1.3612618446350098,
+ "mean_token_accuracy": 0.6669516563415527,
+ "num_tokens": 14893056.0,
+ "step": 909
+ },
+ {
+ "entropy": 1.4160822629928589,
+ "epoch": 1.8383838383838382,
+ "grad_norm": 2.2242727279663086,
+ "learning_rate": 3.1636363636363635e-06,
+ "loss": 1.408278226852417,
+ "mean_token_accuracy": 0.6634098887443542,
+ "num_tokens": 14909440.0,
+ "step": 910
+ },
+ {
+ "entropy": 1.1337573528289795,
+ "epoch": 1.8404040404040405,
+ "grad_norm": 2.1882550716400146,
+ "learning_rate": 3.161616161616162e-06,
+ "loss": 1.1503205299377441,
+ "mean_token_accuracy": 0.7059721350669861,
+ "num_tokens": 14925824.0,
+ "step": 911
+ },
+ {
+ "entropy": 1.4373403787612915,
+ "epoch": 1.8424242424242423,
+ "grad_norm": 2.1595654487609863,
+ "learning_rate": 3.1595959595959597e-06,
+ "loss": 1.4344165325164795,
+ "mean_token_accuracy": 0.6599902510643005,
+ "num_tokens": 14942208.0,
+ "step": 912
+ },
+ {
+ "entropy": 1.0798827409744263,
+ "epoch": 1.8444444444444446,
+ "grad_norm": 2.101088047027588,
+ "learning_rate": 3.1575757575757576e-06,
+ "loss": 1.0873513221740723,
+ "mean_token_accuracy": 0.7169027924537659,
+ "num_tokens": 14958592.0,
+ "step": 913
+ },
+ {
+ "entropy": 1.7024599313735962,
+ "epoch": 1.8464646464646464,
+ "grad_norm": 2.291907548904419,
+ "learning_rate": 3.1555555555555555e-06,
+ "loss": 1.7014999389648438,
+ "mean_token_accuracy": 0.6105275750160217,
+ "num_tokens": 14974976.0,
+ "step": 914
+ },
+ {
+ "entropy": 1.7765953540802002,
+ "epoch": 1.8484848484848486,
+ "grad_norm": 2.1465439796447754,
+ "learning_rate": 3.1535353535353534e-06,
+ "loss": 1.788269281387329,
+ "mean_token_accuracy": 0.6066194176673889,
+ "num_tokens": 14991360.0,
+ "step": 915
+ },
+ {
+ "entropy": 1.5223884582519531,
+ "epoch": 1.8505050505050504,
+ "grad_norm": 2.313291549682617,
+ "learning_rate": 3.1515151515151517e-06,
+ "loss": 1.55397367477417,
+ "mean_token_accuracy": 0.6377626061439514,
+ "num_tokens": 15007744.0,
+ "step": 916
+ },
+ {
+ "entropy": 1.5820955038070679,
+ "epoch": 1.8525252525252527,
+ "grad_norm": 2.094886541366577,
+ "learning_rate": 3.1494949494949496e-06,
+ "loss": 1.58372962474823,
+ "mean_token_accuracy": 0.6339154839515686,
+ "num_tokens": 15024128.0,
+ "step": 917
+ },
+ {
+ "entropy": 1.7399076223373413,
+ "epoch": 1.8545454545454545,
+ "grad_norm": 2.0311105251312256,
+ "learning_rate": 3.1474747474747475e-06,
+ "loss": 1.7362079620361328,
+ "mean_token_accuracy": 0.604421079158783,
+ "num_tokens": 15040512.0,
+ "step": 918
+ },
+ {
+ "entropy": 1.4311537742614746,
+ "epoch": 1.8565656565656565,
+ "grad_norm": 2.13740873336792,
+ "learning_rate": 3.145454545454546e-06,
+ "loss": 1.4574000835418701,
+ "mean_token_accuracy": 0.648876428604126,
+ "num_tokens": 15056896.0,
+ "step": 919
+ },
+ {
+ "entropy": 1.6997709274291992,
+ "epoch": 1.8585858585858586,
+ "grad_norm": 2.1604959964752197,
+ "learning_rate": 3.143434343434344e-06,
+ "loss": 1.694093942642212,
+ "mean_token_accuracy": 0.6144357323646545,
+ "num_tokens": 15073280.0,
+ "step": 920
+ },
+ {
+ "entropy": 1.5796921253204346,
+ "epoch": 1.8606060606060606,
+ "grad_norm": 2.3852932453155518,
+ "learning_rate": 3.141414141414142e-06,
+ "loss": 1.5799579620361328,
+ "mean_token_accuracy": 0.6322667598724365,
+ "num_tokens": 15089664.0,
+ "step": 921
+ },
+ {
+ "entropy": 1.6185977458953857,
+ "epoch": 1.8626262626262626,
+ "grad_norm": 2.0631208419799805,
+ "learning_rate": 3.13939393939394e-06,
+ "loss": 1.6541715860366821,
+ "mean_token_accuracy": 0.632083535194397,
+ "num_tokens": 15106048.0,
+ "step": 922
+ },
+ {
+ "entropy": 1.5872372388839722,
+ "epoch": 1.8646464646464647,
+ "grad_norm": 2.154554843902588,
+ "learning_rate": 3.1373737373737378e-06,
+ "loss": 1.5946767330169678,
+ "mean_token_accuracy": 0.6294577717781067,
+ "num_tokens": 15122432.0,
+ "step": 923
+ },
+ {
+ "entropy": 1.6059986352920532,
+ "epoch": 1.8666666666666667,
+ "grad_norm": 1.9893742799758911,
+ "learning_rate": 3.1353535353535357e-06,
+ "loss": 1.6153039932250977,
+ "mean_token_accuracy": 0.6373962163925171,
+ "num_tokens": 15138816.0,
+ "step": 924
+ },
+ {
+ "entropy": 1.3143881559371948,
+ "epoch": 1.8686868686868687,
+ "grad_norm": 1.8989832401275635,
+ "learning_rate": 3.133333333333334e-06,
+ "loss": 1.3332319259643555,
+ "mean_token_accuracy": 0.6842330098152161,
+ "num_tokens": 15155200.0,
+ "step": 925
+ },
+ {
+ "entropy": 1.0474469661712646,
+ "epoch": 1.8707070707070708,
+ "grad_norm": 1.9129916429519653,
+ "learning_rate": 3.131313131313132e-06,
+ "loss": 1.0238264799118042,
+ "mean_token_accuracy": 0.7459697127342224,
+ "num_tokens": 15171584.0,
+ "step": 926
+ },
+ {
+ "entropy": 1.581508994102478,
+ "epoch": 1.8727272727272726,
+ "grad_norm": 2.0047285556793213,
+ "learning_rate": 3.1292929292929297e-06,
+ "loss": 1.6087100505828857,
+ "mean_token_accuracy": 0.6282364726066589,
+ "num_tokens": 15187968.0,
+ "step": 927
+ },
+ {
+ "entropy": 1.0985661745071411,
+ "epoch": 1.8747474747474748,
+ "grad_norm": 2.1207540035247803,
+ "learning_rate": 3.1272727272727276e-06,
+ "loss": 1.1158297061920166,
+ "mean_token_accuracy": 0.7111626863479614,
+ "num_tokens": 15204352.0,
+ "step": 928
+ },
+ {
+ "entropy": 1.4591490030288696,
+ "epoch": 1.8767676767676766,
+ "grad_norm": 2.2879691123962402,
+ "learning_rate": 3.1252525252525255e-06,
+ "loss": 1.4775316715240479,
+ "mean_token_accuracy": 0.6477161645889282,
+ "num_tokens": 15220736.0,
+ "step": 929
+ },
+ {
+ "entropy": 1.5469937324523926,
+ "epoch": 1.878787878787879,
+ "grad_norm": 2.0972726345062256,
+ "learning_rate": 3.1232323232323234e-06,
+ "loss": 1.5949021577835083,
+ "mean_token_accuracy": 0.6547996997833252,
+ "num_tokens": 15237120.0,
+ "step": 930
+ },
+ {
+ "entropy": 1.363472819328308,
+ "epoch": 1.8808080808080807,
+ "grad_norm": 2.149606943130493,
+ "learning_rate": 3.1212121212121217e-06,
+ "loss": 1.358612298965454,
+ "mean_token_accuracy": 0.6780654788017273,
+ "num_tokens": 15253504.0,
+ "step": 931
+ },
+ {
+ "entropy": 1.189648985862732,
+ "epoch": 1.882828282828283,
+ "grad_norm": 2.2575035095214844,
+ "learning_rate": 3.1191919191919196e-06,
+ "loss": 1.209303379058838,
+ "mean_token_accuracy": 0.7002320289611816,
+ "num_tokens": 15269888.0,
+ "step": 932
+ },
+ {
+ "entropy": 1.3054684400558472,
+ "epoch": 1.8848484848484848,
+ "grad_norm": 2.243915319442749,
+ "learning_rate": 3.1171717171717175e-06,
+ "loss": 1.333367109298706,
+ "mean_token_accuracy": 0.6703712940216064,
+ "num_tokens": 15286272.0,
+ "step": 933
+ },
+ {
+ "entropy": 1.6298167705535889,
+ "epoch": 1.886868686868687,
+ "grad_norm": 1.9654396772384644,
+ "learning_rate": 3.1151515151515154e-06,
+ "loss": 1.6496453285217285,
+ "mean_token_accuracy": 0.6248778700828552,
+ "num_tokens": 15302656.0,
+ "step": 934
+ },
+ {
+ "entropy": 1.1228219270706177,
+ "epoch": 1.8888888888888888,
+ "grad_norm": 2.2492377758026123,
+ "learning_rate": 3.1131313131313133e-06,
+ "loss": 1.1560063362121582,
+ "mean_token_accuracy": 0.699499249458313,
+ "num_tokens": 15319040.0,
+ "step": 935
+ },
+ {
+ "entropy": 1.7481759786605835,
+ "epoch": 1.8909090909090909,
+ "grad_norm": 2.1635665893554688,
+ "learning_rate": 3.1111111111111116e-06,
+ "loss": 1.737868070602417,
+ "mean_token_accuracy": 0.6030776500701904,
+ "num_tokens": 15335424.0,
+ "step": 936
+ },
+ {
+ "entropy": 1.3964993953704834,
+ "epoch": 1.892929292929293,
+ "grad_norm": 2.262946367263794,
+ "learning_rate": 3.1090909090909095e-06,
+ "loss": 1.3927817344665527,
+ "mean_token_accuracy": 0.6546775698661804,
+ "num_tokens": 15351808.0,
+ "step": 937
+ },
+ {
+ "entropy": 1.2503907680511475,
+ "epoch": 1.894949494949495,
+ "grad_norm": 1.9837188720703125,
+ "learning_rate": 3.1070707070707074e-06,
+ "loss": 1.2529189586639404,
+ "mean_token_accuracy": 0.6965681314468384,
+ "num_tokens": 15368192.0,
+ "step": 938
+ },
+ {
+ "entropy": 0.9774144291877747,
+ "epoch": 1.896969696969697,
+ "grad_norm": 1.9201257228851318,
+ "learning_rate": 3.1050505050505052e-06,
+ "loss": 0.9864459037780762,
+ "mean_token_accuracy": 0.7534807324409485,
+ "num_tokens": 15384576.0,
+ "step": 939
+ },
+ {
+ "entropy": 1.4065781831741333,
+ "epoch": 1.898989898989899,
+ "grad_norm": 2.357865571975708,
+ "learning_rate": 3.103030303030303e-06,
+ "loss": 1.428115725517273,
+ "mean_token_accuracy": 0.6493649482727051,
+ "num_tokens": 15400960.0,
+ "step": 940
+ },
+ {
+ "entropy": 1.1632903814315796,
+ "epoch": 1.901010101010101,
+ "grad_norm": 1.8824918270111084,
+ "learning_rate": 3.1010101010101014e-06,
+ "loss": 1.1478252410888672,
+ "mean_token_accuracy": 0.7158036231994629,
+ "num_tokens": 15417344.0,
+ "step": 941
+ },
+ {
+ "entropy": 1.7470908164978027,
+ "epoch": 1.903030303030303,
+ "grad_norm": 2.2958669662475586,
+ "learning_rate": 3.0989898989898993e-06,
+ "loss": 1.7697877883911133,
+ "mean_token_accuracy": 0.5892769694328308,
+ "num_tokens": 15433728.0,
+ "step": 942
+ },
+ {
+ "entropy": 1.2897151708602905,
+ "epoch": 1.905050505050505,
+ "grad_norm": 2.155731439590454,
+ "learning_rate": 3.0969696969696972e-06,
+ "loss": 1.2971919775009155,
+ "mean_token_accuracy": 0.6731802821159363,
+ "num_tokens": 15450112.0,
+ "step": 943
+ },
+ {
+ "entropy": 1.2687026262283325,
+ "epoch": 1.907070707070707,
+ "grad_norm": 2.0896284580230713,
+ "learning_rate": 3.094949494949495e-06,
+ "loss": 1.2423906326293945,
+ "mean_token_accuracy": 0.6988885998725891,
+ "num_tokens": 15466496.0,
+ "step": 944
+ },
+ {
+ "entropy": 1.5710495710372925,
+ "epoch": 1.9090909090909092,
+ "grad_norm": 2.200758457183838,
+ "learning_rate": 3.092929292929293e-06,
+ "loss": 1.5329574346542358,
+ "mean_token_accuracy": 0.6560820937156677,
+ "num_tokens": 15482880.0,
+ "step": 945
+ },
+ {
+ "entropy": 1.1555253267288208,
+ "epoch": 1.911111111111111,
+ "grad_norm": 2.278230905532837,
+ "learning_rate": 3.090909090909091e-06,
+ "loss": 1.1469731330871582,
+ "mean_token_accuracy": 0.7220932841300964,
+ "num_tokens": 15499264.0,
+ "step": 946
+ },
+ {
+ "entropy": 1.2870018482208252,
+ "epoch": 1.9131313131313132,
+ "grad_norm": 1.8973854780197144,
+ "learning_rate": 3.088888888888889e-06,
+ "loss": 1.2787907123565674,
+ "mean_token_accuracy": 0.6885075569152832,
+ "num_tokens": 15515648.0,
+ "step": 947
+ },
+ {
+ "entropy": 1.324872612953186,
+ "epoch": 1.915151515151515,
+ "grad_norm": 2.1889214515686035,
+ "learning_rate": 3.086868686868687e-06,
+ "loss": 1.3286981582641602,
+ "mean_token_accuracy": 0.6743404865264893,
+ "num_tokens": 15532032.0,
+ "step": 948
+ },
+ {
+ "entropy": 1.5601656436920166,
+ "epoch": 1.9171717171717173,
+ "grad_norm": 2.304993152618408,
+ "learning_rate": 3.084848484848485e-06,
+ "loss": 1.5766493082046509,
+ "mean_token_accuracy": 0.6172447204589844,
+ "num_tokens": 15548416.0,
+ "step": 949
+ },
+ {
+ "entropy": 1.4633477926254272,
+ "epoch": 1.9191919191919191,
+ "grad_norm": 1.9083963632583618,
+ "learning_rate": 3.082828282828283e-06,
+ "loss": 1.4554777145385742,
+ "mean_token_accuracy": 0.6667073965072632,
+ "num_tokens": 15564800.0,
+ "step": 950
+ },
+ {
+ "epoch": 1.9191919191919191,
+ "eval_entropy": 1.4443931824737979,
+ "eval_loss": 1.5272752046585083,
+ "eval_mean_token_accuracy": 0.6459201723337173,
+ "eval_num_tokens": 15564800.0,
+ "eval_runtime": 43.7243,
+ "eval_samples_per_second": 22.642,
+ "eval_steps_per_second": 2.836,
+ "step": 950
+ },
+ {
+ "entropy": 1.3350679874420166,
+ "epoch": 1.9212121212121214,
+ "grad_norm": 2.139172315597534,
+ "learning_rate": 3.0808080808080807e-06,
+ "loss": 1.3285927772521973,
+ "mean_token_accuracy": 0.6833781003952026,
+ "num_tokens": 15581184.0,
+ "step": 951
+ },
+ {
+ "entropy": 1.2373814582824707,
+ "epoch": 1.9232323232323232,
+ "grad_norm": 2.1068115234375,
+ "learning_rate": 3.078787878787879e-06,
+ "loss": 1.2527921199798584,
+ "mean_token_accuracy": 0.6969345211982727,
+ "num_tokens": 15597568.0,
+ "step": 952
+ },
+ {
+ "entropy": 1.2182631492614746,
+ "epoch": 1.9252525252525252,
+ "grad_norm": 2.1326634883880615,
+ "learning_rate": 3.076767676767677e-06,
+ "loss": 1.2174904346466064,
+ "mean_token_accuracy": 0.6905227303504944,
+ "num_tokens": 15613952.0,
+ "step": 953
+ },
+ {
+ "entropy": 0.9768888354301453,
+ "epoch": 1.9272727272727272,
+ "grad_norm": 1.9833085536956787,
+ "learning_rate": 3.074747474747475e-06,
+ "loss": 0.9956569671630859,
+ "mean_token_accuracy": 0.7401685118675232,
+ "num_tokens": 15630336.0,
+ "step": 954
+ },
+ {
+ "entropy": 0.7818430066108704,
+ "epoch": 1.9292929292929293,
+ "grad_norm": 1.9219205379486084,
+ "learning_rate": 3.0727272727272727e-06,
+ "loss": 0.7922182083129883,
+ "mean_token_accuracy": 0.7827919125556946,
+ "num_tokens": 15646720.0,
+ "step": 955
+ },
+ {
+ "entropy": 1.6172020435333252,
+ "epoch": 1.9313131313131313,
+ "grad_norm": 1.9390695095062256,
+ "learning_rate": 3.0707070707070706e-06,
+ "loss": 1.6629729270935059,
+ "mean_token_accuracy": 0.6192598938941956,
+ "num_tokens": 15663104.0,
+ "step": 956
+ },
+ {
+ "entropy": 1.291207194328308,
+ "epoch": 1.9333333333333333,
+ "grad_norm": 1.9242857694625854,
+ "learning_rate": 3.068686868686869e-06,
+ "loss": 1.2721552848815918,
+ "mean_token_accuracy": 0.6867367029190063,
+ "num_tokens": 15679488.0,
+ "step": 957
+ },
+ {
+ "entropy": 1.3730298280715942,
+ "epoch": 1.9353535353535354,
+ "grad_norm": 1.9483041763305664,
+ "learning_rate": 3.066666666666667e-06,
+ "loss": 1.3835055828094482,
+ "mean_token_accuracy": 0.6745237112045288,
+ "num_tokens": 15695872.0,
+ "step": 958
+ },
+ {
+ "entropy": 1.1266423463821411,
+ "epoch": 1.9373737373737374,
+ "grad_norm": 2.1663992404937744,
+ "learning_rate": 3.0646464646464647e-06,
+ "loss": 1.1091173887252808,
+ "mean_token_accuracy": 0.7151318788528442,
+ "num_tokens": 15712256.0,
+ "step": 959
+ },
+ {
+ "entropy": 1.298812747001648,
+ "epoch": 1.9393939393939394,
+ "grad_norm": 2.0564448833465576,
+ "learning_rate": 3.0626262626262626e-06,
+ "loss": 1.3167932033538818,
+ "mean_token_accuracy": 0.6812408566474915,
+ "num_tokens": 15728640.0,
+ "step": 960
+ },
+ {
+ "entropy": 1.7007938623428345,
+ "epoch": 1.9414141414141413,
+ "grad_norm": 2.1043670177459717,
+ "learning_rate": 3.0606060606060605e-06,
+ "loss": 1.6956250667572021,
+ "mean_token_accuracy": 0.6194430589675903,
+ "num_tokens": 15745024.0,
+ "step": 961
+ },
+ {
+ "entropy": 1.408955693244934,
+ "epoch": 1.9434343434343435,
+ "grad_norm": 2.0432121753692627,
+ "learning_rate": 3.058585858585859e-06,
+ "loss": 1.3905062675476074,
+ "mean_token_accuracy": 0.6662799119949341,
+ "num_tokens": 15761408.0,
+ "step": 962
+ },
+ {
+ "entropy": 1.938867211341858,
+ "epoch": 1.9454545454545453,
+ "grad_norm": 2.3359744548797607,
+ "learning_rate": 3.056565656565657e-06,
+ "loss": 1.9670305252075195,
+ "mean_token_accuracy": 0.5845139026641846,
+ "num_tokens": 15777792.0,
+ "step": 963
+ },
+ {
+ "entropy": 1.5049448013305664,
+ "epoch": 1.9474747474747476,
+ "grad_norm": 2.1907997131347656,
+ "learning_rate": 3.054545454545455e-06,
+ "loss": 1.534651756286621,
+ "mean_token_accuracy": 0.652723491191864,
+ "num_tokens": 15794176.0,
+ "step": 964
+ },
+ {
+ "entropy": 1.1916700601577759,
+ "epoch": 1.9494949494949494,
+ "grad_norm": 2.098696231842041,
+ "learning_rate": 3.052525252525253e-06,
+ "loss": 1.191361904144287,
+ "mean_token_accuracy": 0.715192973613739,
+ "num_tokens": 15810560.0,
+ "step": 965
+ },
+ {
+ "entropy": 1.6707743406295776,
+ "epoch": 1.9515151515151516,
+ "grad_norm": 2.106846570968628,
+ "learning_rate": 3.0505050505050508e-06,
+ "loss": 1.6789159774780273,
+ "mean_token_accuracy": 0.6133365631103516,
+ "num_tokens": 15826944.0,
+ "step": 966
+ },
+ {
+ "entropy": 1.4679991006851196,
+ "epoch": 1.9535353535353535,
+ "grad_norm": 2.1568050384521484,
+ "learning_rate": 3.048484848484849e-06,
+ "loss": 1.4427556991577148,
+ "mean_token_accuracy": 0.6526013612747192,
+ "num_tokens": 15843328.0,
+ "step": 967
+ },
+ {
+ "entropy": 0.9881705641746521,
+ "epoch": 1.9555555555555557,
+ "grad_norm": 2.029680013656616,
+ "learning_rate": 3.046464646464647e-06,
+ "loss": 0.9961811304092407,
+ "mean_token_accuracy": 0.7408402562141418,
+ "num_tokens": 15859712.0,
+ "step": 968
+ },
+ {
+ "entropy": 1.0721628665924072,
+ "epoch": 1.9575757575757575,
+ "grad_norm": 1.9763140678405762,
+ "learning_rate": 3.044444444444445e-06,
+ "loss": 1.067029356956482,
+ "mean_token_accuracy": 0.733146071434021,
+ "num_tokens": 15876096.0,
+ "step": 969
+ },
+ {
+ "entropy": 1.7053834199905396,
+ "epoch": 1.9595959595959596,
+ "grad_norm": 2.3659849166870117,
+ "learning_rate": 3.0424242424242427e-06,
+ "loss": 1.7629203796386719,
+ "mean_token_accuracy": 0.6105275750160217,
+ "num_tokens": 15892480.0,
+ "step": 970
+ },
+ {
+ "entropy": 1.8083471059799194,
+ "epoch": 1.9616161616161616,
+ "grad_norm": 2.081869602203369,
+ "learning_rate": 3.0404040404040406e-06,
+ "loss": 1.8430367708206177,
+ "mean_token_accuracy": 0.6009404063224792,
+ "num_tokens": 15908864.0,
+ "step": 971
+ },
+ {
+ "entropy": 1.2576755285263062,
+ "epoch": 1.9636363636363636,
+ "grad_norm": 2.128230571746826,
+ "learning_rate": 3.038383838383839e-06,
+ "loss": 1.2493822574615479,
+ "mean_token_accuracy": 0.6913776397705078,
+ "num_tokens": 15925248.0,
+ "step": 972
+ },
+ {
+ "entropy": 0.8995128273963928,
+ "epoch": 1.9656565656565657,
+ "grad_norm": 2.0216031074523926,
+ "learning_rate": 3.036363636363637e-06,
+ "loss": 0.9069615602493286,
+ "mean_token_accuracy": 0.7620908617973328,
+ "num_tokens": 15941632.0,
+ "step": 973
+ },
+ {
+ "entropy": 1.615832805633545,
+ "epoch": 1.9676767676767677,
+ "grad_norm": 2.26552152633667,
+ "learning_rate": 3.0343434343434347e-06,
+ "loss": 1.6284873485565186,
+ "mean_token_accuracy": 0.6193209290504456,
+ "num_tokens": 15958016.0,
+ "step": 974
+ },
+ {
+ "entropy": 1.404674768447876,
+ "epoch": 1.9696969696969697,
+ "grad_norm": 2.1790690422058105,
+ "learning_rate": 3.0323232323232326e-06,
+ "loss": 1.4130914211273193,
+ "mean_token_accuracy": 0.6594406366348267,
+ "num_tokens": 15974400.0,
+ "step": 975
+ },
+ {
+ "entropy": 1.3440049886703491,
+ "epoch": 1.9717171717171718,
+ "grad_norm": 2.121338367462158,
+ "learning_rate": 3.0303030303030305e-06,
+ "loss": 1.3652238845825195,
+ "mean_token_accuracy": 0.6794699430465698,
+ "num_tokens": 15990784.0,
+ "step": 976
+ },
+ {
+ "entropy": 1.0894689559936523,
+ "epoch": 1.9737373737373738,
+ "grad_norm": 2.0155327320098877,
+ "learning_rate": 3.028282828282829e-06,
+ "loss": 1.0642526149749756,
+ "mean_token_accuracy": 0.7223986387252808,
+ "num_tokens": 16007168.0,
+ "step": 977
+ },
+ {
+ "entropy": 1.469588041305542,
+ "epoch": 1.9757575757575756,
+ "grad_norm": 2.2047550678253174,
+ "learning_rate": 3.0262626262626267e-06,
+ "loss": 1.498295545578003,
+ "mean_token_accuracy": 0.6441133618354797,
+ "num_tokens": 16023552.0,
+ "step": 978
+ },
+ {
+ "entropy": 1.6617093086242676,
+ "epoch": 1.9777777777777779,
+ "grad_norm": 1.9770722389221191,
+ "learning_rate": 3.0242424242424246e-06,
+ "loss": 1.6753556728363037,
+ "mean_token_accuracy": 0.6284807324409485,
+ "num_tokens": 16039936.0,
+ "step": 979
+ },
+ {
+ "entropy": 1.8029206991195679,
+ "epoch": 1.9797979797979797,
+ "grad_norm": 2.2365968227386475,
+ "learning_rate": 3.0222222222222225e-06,
+ "loss": 1.7957369089126587,
+ "mean_token_accuracy": 0.601062536239624,
+ "num_tokens": 16056320.0,
+ "step": 980
+ },
+ {
+ "entropy": 1.6331384181976318,
+ "epoch": 1.981818181818182,
+ "grad_norm": 2.0565884113311768,
+ "learning_rate": 3.0202020202020203e-06,
+ "loss": 1.673048973083496,
+ "mean_token_accuracy": 0.6203590631484985,
+ "num_tokens": 16072704.0,
+ "step": 981
+ },
+ {
+ "entropy": 1.1730273962020874,
+ "epoch": 1.9838383838383837,
+ "grad_norm": 2.0347228050231934,
+ "learning_rate": 3.0181818181818182e-06,
+ "loss": 1.1709872484207153,
+ "mean_token_accuracy": 0.7086589932441711,
+ "num_tokens": 16089088.0,
+ "step": 982
+ },
+ {
+ "entropy": 1.1109111309051514,
+ "epoch": 1.985858585858586,
+ "grad_norm": 1.9992051124572754,
+ "learning_rate": 3.0161616161616165e-06,
+ "loss": 1.097399353981018,
+ "mean_token_accuracy": 0.7245969772338867,
+ "num_tokens": 16105472.0,
+ "step": 983
+ },
+ {
+ "entropy": 1.2580686807632446,
+ "epoch": 1.9878787878787878,
+ "grad_norm": 2.1748640537261963,
+ "learning_rate": 3.0141414141414144e-06,
+ "loss": 1.2503113746643066,
+ "mean_token_accuracy": 0.6883854269981384,
+ "num_tokens": 16121856.0,
+ "step": 984
+ },
+ {
+ "entropy": 1.4884002208709717,
+ "epoch": 1.98989898989899,
+ "grad_norm": 2.3283042907714844,
+ "learning_rate": 3.0121212121212123e-06,
+ "loss": 1.4856922626495361,
+ "mean_token_accuracy": 0.6461895704269409,
+ "num_tokens": 16138240.0,
+ "step": 985
+ },
+ {
+ "entropy": 1.6224603652954102,
+ "epoch": 1.9919191919191919,
+ "grad_norm": 2.0596823692321777,
+ "learning_rate": 3.0101010101010102e-06,
+ "loss": 1.6202584505081177,
+ "mean_token_accuracy": 0.6388617753982544,
+ "num_tokens": 16154624.0,
+ "step": 986
+ },
+ {
+ "entropy": 1.559553623199463,
+ "epoch": 1.993939393939394,
+ "grad_norm": 2.100667953491211,
+ "learning_rate": 3.008080808080808e-06,
+ "loss": 1.5781179666519165,
+ "mean_token_accuracy": 0.6409379839897156,
+ "num_tokens": 16171008.0,
+ "step": 987
+ },
+ {
+ "entropy": 1.299710750579834,
+ "epoch": 1.995959595959596,
+ "grad_norm": 2.2846767902374268,
+ "learning_rate": 3.0060606060606064e-06,
+ "loss": 1.2758262157440186,
+ "mean_token_accuracy": 0.6882632970809937,
+ "num_tokens": 16187392.0,
+ "step": 988
+ },
+ {
+ "entropy": 1.3966935873031616,
+ "epoch": 1.997979797979798,
+ "grad_norm": 2.1244044303894043,
+ "learning_rate": 3.0040404040404043e-06,
+ "loss": 1.420769214630127,
+ "mean_token_accuracy": 0.6670737862586975,
+ "num_tokens": 16203776.0,
+ "step": 989
+ },
+ {
+ "entropy": 1.5851658582687378,
+ "epoch": 2.0,
+ "grad_norm": 2.277179002761841,
+ "learning_rate": 3.002020202020202e-06,
+ "loss": 1.5963382720947266,
+ "mean_token_accuracy": 0.6276868581771851,
+ "num_tokens": 16220160.0,
+ "step": 990
+ },
+ {
+ "entropy": 1.9146819114685059,
+ "epoch": 2.002020202020202,
+ "grad_norm": 2.0008981227874756,
+ "learning_rate": 3e-06,
+ "loss": 1.8820562362670898,
+ "mean_token_accuracy": 0.5895212292671204,
+ "num_tokens": 16236544.0,
+ "step": 991
+ },
+ {
+ "entropy": 1.3451478481292725,
+ "epoch": 2.004040404040404,
+ "grad_norm": 2.236388921737671,
+ "learning_rate": 2.997979797979798e-06,
+ "loss": 1.2709381580352783,
+ "mean_token_accuracy": 0.6848436594009399,
+ "num_tokens": 16252928.0,
+ "step": 992
+ },
+ {
+ "entropy": 1.4454528093338013,
+ "epoch": 2.006060606060606,
+ "grad_norm": 2.220324754714966,
+ "learning_rate": 2.9959595959595963e-06,
+ "loss": 1.387961506843567,
+ "mean_token_accuracy": 0.6642647981643677,
+ "num_tokens": 16269312.0,
+ "step": 993
+ },
+ {
+ "entropy": 1.2083219289779663,
+ "epoch": 2.008080808080808,
+ "grad_norm": 2.092430830001831,
+ "learning_rate": 2.993939393939394e-06,
+ "loss": 1.1450953483581543,
+ "mean_token_accuracy": 0.7159257531166077,
+ "num_tokens": 16285696.0,
+ "step": 994
+ },
+ {
+ "entropy": 1.3030426502227783,
+ "epoch": 2.01010101010101,
+ "grad_norm": 2.0722665786743164,
+ "learning_rate": 2.991919191919192e-06,
+ "loss": 1.2463462352752686,
+ "mean_token_accuracy": 0.6780654788017273,
+ "num_tokens": 16302080.0,
+ "step": 995
+ },
+ {
+ "entropy": 1.3418742418289185,
+ "epoch": 2.012121212121212,
+ "grad_norm": 1.9004424810409546,
+ "learning_rate": 2.98989898989899e-06,
+ "loss": 1.3043954372406006,
+ "mean_token_accuracy": 0.6850268840789795,
+ "num_tokens": 16318464.0,
+ "step": 996
+ },
+ {
+ "entropy": 1.6736825704574585,
+ "epoch": 2.014141414141414,
+ "grad_norm": 2.132983446121216,
+ "learning_rate": 2.987878787878788e-06,
+ "loss": 1.631580114364624,
+ "mean_token_accuracy": 0.619076669216156,
+ "num_tokens": 16334848.0,
+ "step": 997
+ },
+ {
+ "entropy": 1.435375690460205,
+ "epoch": 2.0161616161616163,
+ "grad_norm": 2.2508206367492676,
+ "learning_rate": 2.9858585858585857e-06,
+ "loss": 1.3867809772491455,
+ "mean_token_accuracy": 0.662432849407196,
+ "num_tokens": 16351232.0,
+ "step": 998
+ },
+ {
+ "entropy": 1.050812005996704,
+ "epoch": 2.018181818181818,
+ "grad_norm": 1.878668189048767,
+ "learning_rate": 2.983838383838384e-06,
+ "loss": 1.0428903102874756,
+ "mean_token_accuracy": 0.7329018115997314,
+ "num_tokens": 16367616.0,
+ "step": 999
+ },
+ {
+ "entropy": 1.325859546661377,
+ "epoch": 2.0202020202020203,
+ "grad_norm": 1.8450349569320679,
+ "learning_rate": 2.981818181818182e-06,
+ "loss": 1.302099347114563,
+ "mean_token_accuracy": 0.6926599740982056,
+ "num_tokens": 16384000.0,
+ "step": 1000
+ },
+ {
+ "epoch": 2.0202020202020203,
+ "eval_entropy": 1.3899660999736478,
+ "eval_loss": 1.5310550928115845,
+ "eval_mean_token_accuracy": 0.6458596015168775,
+ "eval_num_tokens": 16384000.0,
+ "eval_runtime": 43.8845,
+ "eval_samples_per_second": 22.559,
+ "eval_steps_per_second": 2.826,
+ "step": 1000
+ },
+ {
+ "entropy": 1.5226235389709473,
+ "epoch": 2.022222222222222,
+ "grad_norm": 2.0676803588867188,
+ "learning_rate": 2.97979797979798e-06,
+ "loss": 1.4971344470977783,
+ "mean_token_accuracy": 0.6517464518547058,
+ "num_tokens": 16400384.0,
+ "step": 1001
+ },
+ {
+ "entropy": 0.9506940841674805,
+ "epoch": 2.0242424242424244,
+ "grad_norm": 1.972718596458435,
+ "learning_rate": 2.9777777777777777e-06,
+ "loss": 0.9229776859283447,
+ "mean_token_accuracy": 0.7554958462715149,
+ "num_tokens": 16416768.0,
+ "step": 1002
+ },
+ {
+ "entropy": 1.8095428943634033,
+ "epoch": 2.026262626262626,
+ "grad_norm": 2.119502305984497,
+ "learning_rate": 2.9757575757575756e-06,
+ "loss": 1.8333206176757812,
+ "mean_token_accuracy": 0.5888495445251465,
+ "num_tokens": 16433152.0,
+ "step": 1003
+ },
+ {
+ "entropy": 1.110617756843567,
+ "epoch": 2.0282828282828285,
+ "grad_norm": 2.1480863094329834,
+ "learning_rate": 2.9737373737373743e-06,
+ "loss": 1.0999674797058105,
+ "mean_token_accuracy": 0.7213605046272278,
+ "num_tokens": 16449536.0,
+ "step": 1004
+ },
+ {
+ "entropy": 1.1889066696166992,
+ "epoch": 2.0303030303030303,
+ "grad_norm": 2.1086478233337402,
+ "learning_rate": 2.971717171717172e-06,
+ "loss": 1.2093305587768555,
+ "mean_token_accuracy": 0.7071323990821838,
+ "num_tokens": 16465920.0,
+ "step": 1005
+ },
+ {
+ "entropy": 1.2134279012680054,
+ "epoch": 2.0323232323232325,
+ "grad_norm": 2.020108461380005,
+ "learning_rate": 2.96969696969697e-06,
+ "loss": 1.2015900611877441,
+ "mean_token_accuracy": 0.7037127614021301,
+ "num_tokens": 16482304.0,
+ "step": 1006
+ },
+ {
+ "entropy": 1.1034245491027832,
+ "epoch": 2.0343434343434343,
+ "grad_norm": 2.1722943782806396,
+ "learning_rate": 2.967676767676768e-06,
+ "loss": 1.1243445873260498,
+ "mean_token_accuracy": 0.7051172256469727,
+ "num_tokens": 16498688.0,
+ "step": 1007
+ },
+ {
+ "entropy": 1.6888902187347412,
+ "epoch": 2.036363636363636,
+ "grad_norm": 2.1574206352233887,
+ "learning_rate": 2.9656565656565663e-06,
+ "loss": 1.7164931297302246,
+ "mean_token_accuracy": 0.6174889802932739,
+ "num_tokens": 16515072.0,
+ "step": 1008
+ },
+ {
+ "entropy": 0.8055605292320251,
+ "epoch": 2.0383838383838384,
+ "grad_norm": 1.8896156549453735,
+ "learning_rate": 2.963636363636364e-06,
+ "loss": 0.7961650490760803,
+ "mean_token_accuracy": 0.7895700931549072,
+ "num_tokens": 16531456.0,
+ "step": 1009
+ },
+ {
+ "entropy": 1.7698009014129639,
+ "epoch": 2.04040404040404,
+ "grad_norm": 2.1700897216796875,
+ "learning_rate": 2.961616161616162e-06,
+ "loss": 1.7871267795562744,
+ "mean_token_accuracy": 0.6139472126960754,
+ "num_tokens": 16547840.0,
+ "step": 1010
+ },
+ {
+ "entropy": 1.414292335510254,
+ "epoch": 2.0424242424242425,
+ "grad_norm": 2.0185277462005615,
+ "learning_rate": 2.95959595959596e-06,
+ "loss": 1.4295077323913574,
+ "mean_token_accuracy": 0.6670126914978027,
+ "num_tokens": 16564224.0,
+ "step": 1011
+ },
+ {
+ "entropy": 1.4013340473175049,
+ "epoch": 2.0444444444444443,
+ "grad_norm": 2.1133840084075928,
+ "learning_rate": 2.957575757575758e-06,
+ "loss": 1.4482200145721436,
+ "mean_token_accuracy": 0.6780654788017273,
+ "num_tokens": 16580608.0,
+ "step": 1012
+ },
+ {
+ "entropy": 1.342740774154663,
+ "epoch": 2.0464646464646465,
+ "grad_norm": 2.0520355701446533,
+ "learning_rate": 2.955555555555556e-06,
+ "loss": 1.3550419807434082,
+ "mean_token_accuracy": 0.6850879192352295,
+ "num_tokens": 16596992.0,
+ "step": 1013
+ },
+ {
+ "entropy": 1.2301024198532104,
+ "epoch": 2.0484848484848484,
+ "grad_norm": 2.1948137283325195,
+ "learning_rate": 2.953535353535354e-06,
+ "loss": 1.243154525756836,
+ "mean_token_accuracy": 0.6838055849075317,
+ "num_tokens": 16613376.0,
+ "step": 1014
+ },
+ {
+ "entropy": 1.3916606903076172,
+ "epoch": 2.0505050505050506,
+ "grad_norm": 2.241595506668091,
+ "learning_rate": 2.951515151515152e-06,
+ "loss": 1.4011318683624268,
+ "mean_token_accuracy": 0.664631187915802,
+ "num_tokens": 16629760.0,
+ "step": 1015
+ },
+ {
+ "entropy": 1.4715481996536255,
+ "epoch": 2.0525252525252524,
+ "grad_norm": 2.224256753921509,
+ "learning_rate": 2.94949494949495e-06,
+ "loss": 1.5011882781982422,
+ "mean_token_accuracy": 0.6493038535118103,
+ "num_tokens": 16646144.0,
+ "step": 1016
+ },
+ {
+ "entropy": 1.1261032819747925,
+ "epoch": 2.0545454545454547,
+ "grad_norm": 2.2784500122070312,
+ "learning_rate": 2.9474747474747477e-06,
+ "loss": 1.1250882148742676,
+ "mean_token_accuracy": 0.7178798317909241,
+ "num_tokens": 16662528.0,
+ "step": 1017
+ },
+ {
+ "entropy": 1.782008171081543,
+ "epoch": 2.0565656565656565,
+ "grad_norm": 2.1711416244506836,
+ "learning_rate": 2.9454545454545456e-06,
+ "loss": 1.8037612438201904,
+ "mean_token_accuracy": 0.6113824844360352,
+ "num_tokens": 16678912.0,
+ "step": 1018
+ },
+ {
+ "entropy": 1.521897315979004,
+ "epoch": 2.0585858585858587,
+ "grad_norm": 2.1903202533721924,
+ "learning_rate": 2.943434343434344e-06,
+ "loss": 1.5377610921859741,
+ "mean_token_accuracy": 0.642953097820282,
+ "num_tokens": 16695296.0,
+ "step": 1019
+ },
+ {
+ "entropy": 1.3112438917160034,
+ "epoch": 2.0606060606060606,
+ "grad_norm": 2.0299105644226074,
+ "learning_rate": 2.941414141414142e-06,
+ "loss": 1.3218681812286377,
+ "mean_token_accuracy": 0.6842941045761108,
+ "num_tokens": 16711680.0,
+ "step": 1020
+ },
+ {
+ "entropy": 1.3634895086288452,
+ "epoch": 2.062626262626263,
+ "grad_norm": 2.2818808555603027,
+ "learning_rate": 2.9393939393939397e-06,
+ "loss": 1.3611259460449219,
+ "mean_token_accuracy": 0.6758060455322266,
+ "num_tokens": 16728064.0,
+ "step": 1021
+ },
+ {
+ "entropy": 1.361467957496643,
+ "epoch": 2.0646464646464646,
+ "grad_norm": 2.290698289871216,
+ "learning_rate": 2.9373737373737376e-06,
+ "loss": 1.3082963228225708,
+ "mean_token_accuracy": 0.6809965968132019,
+ "num_tokens": 16744448.0,
+ "step": 1022
+ },
+ {
+ "entropy": 1.633399486541748,
+ "epoch": 2.066666666666667,
+ "grad_norm": 2.1775712966918945,
+ "learning_rate": 2.9353535353535355e-06,
+ "loss": 1.6423053741455078,
+ "mean_token_accuracy": 0.639655590057373,
+ "num_tokens": 16760832.0,
+ "step": 1023
+ },
+ {
+ "entropy": 1.4717872142791748,
+ "epoch": 2.0686868686868687,
+ "grad_norm": 2.2070701122283936,
+ "learning_rate": 2.9333333333333338e-06,
+ "loss": 1.508943796157837,
+ "mean_token_accuracy": 0.6469223499298096,
+ "num_tokens": 16777216.0,
+ "step": 1024
+ },
+ {
+ "entropy": 1.1494653224945068,
+ "epoch": 2.0707070707070705,
+ "grad_norm": 2.0239098072052,
+ "learning_rate": 2.9313131313131317e-06,
+ "loss": 1.1367695331573486,
+ "mean_token_accuracy": 0.7148265838623047,
+ "num_tokens": 16793600.0,
+ "step": 1025
+ },
+ {
+ "entropy": 1.0235252380371094,
+ "epoch": 2.0727272727272728,
+ "grad_norm": 2.071176052093506,
+ "learning_rate": 2.9292929292929295e-06,
+ "loss": 1.0369395017623901,
+ "mean_token_accuracy": 0.7396799921989441,
+ "num_tokens": 16809984.0,
+ "step": 1026
+ },
+ {
+ "entropy": 1.2325578927993774,
+ "epoch": 2.0747474747474746,
+ "grad_norm": 2.212671995162964,
+ "learning_rate": 2.9272727272727274e-06,
+ "loss": 1.2429314851760864,
+ "mean_token_accuracy": 0.688568651676178,
+ "num_tokens": 16826368.0,
+ "step": 1027
+ },
+ {
+ "entropy": 1.2879300117492676,
+ "epoch": 2.076767676767677,
+ "grad_norm": 2.0366690158843994,
+ "learning_rate": 2.9252525252525253e-06,
+ "loss": 1.2931241989135742,
+ "mean_token_accuracy": 0.6813018918037415,
+ "num_tokens": 16842752.0,
+ "step": 1028
+ },
+ {
+ "entropy": 0.9473312497138977,
+ "epoch": 2.0787878787878786,
+ "grad_norm": 2.0870468616485596,
+ "learning_rate": 2.9232323232323236e-06,
+ "loss": 0.9292516708374023,
+ "mean_token_accuracy": 0.7525036931037903,
+ "num_tokens": 16859136.0,
+ "step": 1029
+ },
+ {
+ "entropy": 1.3882765769958496,
+ "epoch": 2.080808080808081,
+ "grad_norm": 2.1285617351531982,
+ "learning_rate": 2.9212121212121215e-06,
+ "loss": 1.3798363208770752,
+ "mean_token_accuracy": 0.6768441796302795,
+ "num_tokens": 16875520.0,
+ "step": 1030
+ },
+ {
+ "entropy": 1.3588547706604004,
+ "epoch": 2.0828282828282827,
+ "grad_norm": 2.2275242805480957,
+ "learning_rate": 2.9191919191919194e-06,
+ "loss": 1.3538521528244019,
+ "mean_token_accuracy": 0.6615168452262878,
+ "num_tokens": 16891904.0,
+ "step": 1031
+ },
+ {
+ "entropy": 1.0415440797805786,
+ "epoch": 2.084848484848485,
+ "grad_norm": 2.2782297134399414,
+ "learning_rate": 2.9171717171717173e-06,
+ "loss": 1.03328537940979,
+ "mean_token_accuracy": 0.7577552795410156,
+ "num_tokens": 16908288.0,
+ "step": 1032
+ },
+ {
+ "entropy": 1.241741418838501,
+ "epoch": 2.0868686868686868,
+ "grad_norm": 2.141939163208008,
+ "learning_rate": 2.915151515151515e-06,
+ "loss": 1.2248635292053223,
+ "mean_token_accuracy": 0.6950415372848511,
+ "num_tokens": 16924672.0,
+ "step": 1033
+ },
+ {
+ "entropy": 1.1192725896835327,
+ "epoch": 2.088888888888889,
+ "grad_norm": 1.9980862140655518,
+ "learning_rate": 2.913131313131313e-06,
+ "loss": 1.1176412105560303,
+ "mean_token_accuracy": 0.7272838354110718,
+ "num_tokens": 16941056.0,
+ "step": 1034
+ },
+ {
+ "entropy": 1.1871322393417358,
+ "epoch": 2.090909090909091,
+ "grad_norm": 2.1212220191955566,
+ "learning_rate": 2.9111111111111114e-06,
+ "loss": 1.1670379638671875,
+ "mean_token_accuracy": 0.7048119306564331,
+ "num_tokens": 16957440.0,
+ "step": 1035
+ },
+ {
+ "entropy": 1.5194982290267944,
+ "epoch": 2.092929292929293,
+ "grad_norm": 2.2593677043914795,
+ "learning_rate": 2.9090909090909093e-06,
+ "loss": 1.50520920753479,
+ "mean_token_accuracy": 0.6475940346717834,
+ "num_tokens": 16973824.0,
+ "step": 1036
+ },
+ {
+ "entropy": 1.212785243988037,
+ "epoch": 2.094949494949495,
+ "grad_norm": 2.221132278442383,
+ "learning_rate": 2.907070707070707e-06,
+ "loss": 1.206315040588379,
+ "mean_token_accuracy": 0.7046898007392883,
+ "num_tokens": 16990208.0,
+ "step": 1037
+ },
+ {
+ "entropy": 1.2913068532943726,
+ "epoch": 2.096969696969697,
+ "grad_norm": 2.0525012016296387,
+ "learning_rate": 2.905050505050505e-06,
+ "loss": 1.2987749576568604,
+ "mean_token_accuracy": 0.6894235610961914,
+ "num_tokens": 17006592.0,
+ "step": 1038
+ },
+ {
+ "entropy": 1.139485239982605,
+ "epoch": 2.098989898989899,
+ "grad_norm": 2.093764543533325,
+ "learning_rate": 2.903030303030303e-06,
+ "loss": 1.1460036039352417,
+ "mean_token_accuracy": 0.7205055952072144,
+ "num_tokens": 17022976.0,
+ "step": 1039
+ },
+ {
+ "entropy": 1.3781968355178833,
+ "epoch": 2.101010101010101,
+ "grad_norm": 2.095311164855957,
+ "learning_rate": 2.9010101010101012e-06,
+ "loss": 1.3901969194412231,
+ "mean_token_accuracy": 0.679286777973175,
+ "num_tokens": 17039360.0,
+ "step": 1040
+ },
+ {
+ "entropy": 1.1370116472244263,
+ "epoch": 2.103030303030303,
+ "grad_norm": 2.139094352722168,
+ "learning_rate": 2.898989898989899e-06,
+ "loss": 1.1491400003433228,
+ "mean_token_accuracy": 0.72007817029953,
+ "num_tokens": 17055744.0,
+ "step": 1041
+ },
+ {
+ "entropy": 1.0845732688903809,
+ "epoch": 2.105050505050505,
+ "grad_norm": 2.180278778076172,
+ "learning_rate": 2.896969696969697e-06,
+ "loss": 1.0764763355255127,
+ "mean_token_accuracy": 0.7365657091140747,
+ "num_tokens": 17072128.0,
+ "step": 1042
+ },
+ {
+ "entropy": 1.4869064092636108,
+ "epoch": 2.107070707070707,
+ "grad_norm": 2.053727388381958,
+ "learning_rate": 2.894949494949495e-06,
+ "loss": 1.4862562417984009,
+ "mean_token_accuracy": 0.6643258333206177,
+ "num_tokens": 17088512.0,
+ "step": 1043
+ },
+ {
+ "entropy": 1.048888921737671,
+ "epoch": 2.109090909090909,
+ "grad_norm": 2.1302237510681152,
+ "learning_rate": 2.892929292929293e-06,
+ "loss": 1.0331683158874512,
+ "mean_token_accuracy": 0.7388250827789307,
+ "num_tokens": 17104896.0,
+ "step": 1044
+ },
+ {
+ "entropy": 1.2839856147766113,
+ "epoch": 2.111111111111111,
+ "grad_norm": 2.8510677814483643,
+ "learning_rate": 2.8909090909090907e-06,
+ "loss": 1.3016979694366455,
+ "mean_token_accuracy": 0.6904616355895996,
+ "num_tokens": 17121280.0,
+ "step": 1045
+ },
+ {
+ "entropy": 1.050045371055603,
+ "epoch": 2.113131313131313,
+ "grad_norm": 2.2457358837127686,
+ "learning_rate": 2.888888888888889e-06,
+ "loss": 1.0580981969833374,
+ "mean_token_accuracy": 0.7221543788909912,
+ "num_tokens": 17137664.0,
+ "step": 1046
+ },
+ {
+ "entropy": 1.3087437152862549,
+ "epoch": 2.1151515151515152,
+ "grad_norm": 2.0929996967315674,
+ "learning_rate": 2.8868686868686873e-06,
+ "loss": 1.2914986610412598,
+ "mean_token_accuracy": 0.6939423680305481,
+ "num_tokens": 17154048.0,
+ "step": 1047
+ },
+ {
+ "entropy": 1.7247086763381958,
+ "epoch": 2.117171717171717,
+ "grad_norm": 2.2230091094970703,
+ "learning_rate": 2.884848484848485e-06,
+ "loss": 1.7394956350326538,
+ "mean_token_accuracy": 0.6131533980369568,
+ "num_tokens": 17170432.0,
+ "step": 1048
+ },
+ {
+ "entropy": 1.5816357135772705,
+ "epoch": 2.1191919191919193,
+ "grad_norm": 2.196422576904297,
+ "learning_rate": 2.8828282828282835e-06,
+ "loss": 1.6226823329925537,
+ "mean_token_accuracy": 0.6375183463096619,
+ "num_tokens": 17186816.0,
+ "step": 1049
+ },
+ {
+ "entropy": 1.2316261529922485,
+ "epoch": 2.121212121212121,
+ "grad_norm": 2.173909902572632,
+ "learning_rate": 2.8808080808080814e-06,
+ "loss": 1.2228440046310425,
+ "mean_token_accuracy": 0.6940034031867981,
+ "num_tokens": 17203200.0,
+ "step": 1050
+ },
+ {
+ "epoch": 2.121212121212121,
+ "eval_entropy": 1.3722701356295617,
+ "eval_loss": 1.5385833978652954,
+ "eval_mean_token_accuracy": 0.6450051809510877,
+ "eval_num_tokens": 17203200.0,
+ "eval_runtime": 43.9186,
+ "eval_samples_per_second": 22.542,
+ "eval_steps_per_second": 2.823,
+ "step": 1050
+ },
+ {
+ "entropy": 1.3023537397384644,
+ "epoch": 2.1232323232323234,
+ "grad_norm": 2.1490018367767334,
+ "learning_rate": 2.8787878787878793e-06,
+ "loss": 1.284008264541626,
+ "mean_token_accuracy": 0.7057278752326965,
+ "num_tokens": 17219584.0,
+ "step": 1051
+ },
+ {
+ "entropy": 1.3198270797729492,
+ "epoch": 2.125252525252525,
+ "grad_norm": 2.582298994064331,
+ "learning_rate": 2.876767676767677e-06,
+ "loss": 1.3185001611709595,
+ "mean_token_accuracy": 0.6772105693817139,
+ "num_tokens": 17235968.0,
+ "step": 1052
+ },
+ {
+ "entropy": 0.8889825940132141,
+ "epoch": 2.1272727272727274,
+ "grad_norm": 2.0617194175720215,
+ "learning_rate": 2.874747474747475e-06,
+ "loss": 0.889773964881897,
+ "mean_token_accuracy": 0.767892062664032,
+ "num_tokens": 17252352.0,
+ "step": 1053
+ },
+ {
+ "entropy": 1.5485399961471558,
+ "epoch": 2.1292929292929292,
+ "grad_norm": 2.2752370834350586,
+ "learning_rate": 2.872727272727273e-06,
+ "loss": 1.5542051792144775,
+ "mean_token_accuracy": 0.6397166848182678,
+ "num_tokens": 17268736.0,
+ "step": 1054
+ },
+ {
+ "entropy": 1.3949565887451172,
+ "epoch": 2.1313131313131315,
+ "grad_norm": 2.0112996101379395,
+ "learning_rate": 2.8707070707070713e-06,
+ "loss": 1.4111721515655518,
+ "mean_token_accuracy": 0.6659746170043945,
+ "num_tokens": 17285120.0,
+ "step": 1055
+ },
+ {
+ "entropy": 1.0529720783233643,
+ "epoch": 2.1333333333333333,
+ "grad_norm": 2.2219948768615723,
+ "learning_rate": 2.868686868686869e-06,
+ "loss": 1.0484942197799683,
+ "mean_token_accuracy": 0.7224596738815308,
+ "num_tokens": 17301504.0,
+ "step": 1056
+ },
+ {
+ "entropy": 1.5362433195114136,
+ "epoch": 2.1353535353535356,
+ "grad_norm": 2.4097044467926025,
+ "learning_rate": 2.866666666666667e-06,
+ "loss": 1.5809426307678223,
+ "mean_token_accuracy": 0.6266487836837769,
+ "num_tokens": 17317888.0,
+ "step": 1057
+ },
+ {
+ "entropy": 1.6682685613632202,
+ "epoch": 2.1373737373737374,
+ "grad_norm": 2.121086359024048,
+ "learning_rate": 2.864646464646465e-06,
+ "loss": 1.6636815071105957,
+ "mean_token_accuracy": 0.6317782402038574,
+ "num_tokens": 17334272.0,
+ "step": 1058
+ },
+ {
+ "entropy": 1.5480221509933472,
+ "epoch": 2.1393939393939396,
+ "grad_norm": 2.2226176261901855,
+ "learning_rate": 2.862626262626263e-06,
+ "loss": 1.535983681678772,
+ "mean_token_accuracy": 0.6623717546463013,
+ "num_tokens": 17350656.0,
+ "step": 1059
+ },
+ {
+ "entropy": 1.3323101997375488,
+ "epoch": 2.1414141414141414,
+ "grad_norm": 2.2644572257995605,
+ "learning_rate": 2.860606060606061e-06,
+ "loss": 1.3316677808761597,
+ "mean_token_accuracy": 0.6741573214530945,
+ "num_tokens": 17367040.0,
+ "step": 1060
+ },
+ {
+ "entropy": 1.5107028484344482,
+ "epoch": 2.1434343434343432,
+ "grad_norm": 2.1291613578796387,
+ "learning_rate": 2.858585858585859e-06,
+ "loss": 1.4996232986450195,
+ "mean_token_accuracy": 0.6537005305290222,
+ "num_tokens": 17383424.0,
+ "step": 1061
+ },
+ {
+ "entropy": 1.5927387475967407,
+ "epoch": 2.1454545454545455,
+ "grad_norm": 2.1977617740631104,
+ "learning_rate": 2.856565656565657e-06,
+ "loss": 1.582688331604004,
+ "mean_token_accuracy": 0.6306790709495544,
+ "num_tokens": 17399808.0,
+ "step": 1062
+ },
+ {
+ "entropy": 1.0717219114303589,
+ "epoch": 2.1474747474747473,
+ "grad_norm": 2.2494494915008545,
+ "learning_rate": 2.8545454545454548e-06,
+ "loss": 1.0780384540557861,
+ "mean_token_accuracy": 0.7214826345443726,
+ "num_tokens": 17416192.0,
+ "step": 1063
+ },
+ {
+ "entropy": 1.484608769416809,
+ "epoch": 2.1494949494949496,
+ "grad_norm": 2.20664644241333,
+ "learning_rate": 2.8525252525252527e-06,
+ "loss": 1.5074207782745361,
+ "mean_token_accuracy": 0.6502808928489685,
+ "num_tokens": 17432576.0,
+ "step": 1064
+ },
+ {
+ "entropy": 1.6607396602630615,
+ "epoch": 2.1515151515151514,
+ "grad_norm": 2.123538017272949,
+ "learning_rate": 2.850505050505051e-06,
+ "loss": 1.659307837486267,
+ "mean_token_accuracy": 0.619076669216156,
+ "num_tokens": 17448960.0,
+ "step": 1065
+ },
+ {
+ "entropy": 1.1195216178894043,
+ "epoch": 2.1535353535353536,
+ "grad_norm": 2.037261486053467,
+ "learning_rate": 2.848484848484849e-06,
+ "loss": 1.127397894859314,
+ "mean_token_accuracy": 0.7209330797195435,
+ "num_tokens": 17465344.0,
+ "step": 1066
+ },
+ {
+ "entropy": 1.0512653589248657,
+ "epoch": 2.1555555555555554,
+ "grad_norm": 2.209764003753662,
+ "learning_rate": 2.8464646464646468e-06,
+ "loss": 1.073946475982666,
+ "mean_token_accuracy": 0.717391312122345,
+ "num_tokens": 17481728.0,
+ "step": 1067
+ },
+ {
+ "entropy": 1.131946325302124,
+ "epoch": 2.1575757575757577,
+ "grad_norm": 1.9188926219940186,
+ "learning_rate": 2.8444444444444446e-06,
+ "loss": 1.11629056930542,
+ "mean_token_accuracy": 0.7220322489738464,
+ "num_tokens": 17498112.0,
+ "step": 1068
+ },
+ {
+ "entropy": 1.4745967388153076,
+ "epoch": 2.1595959595959595,
+ "grad_norm": 2.285099983215332,
+ "learning_rate": 2.8424242424242425e-06,
+ "loss": 1.4795483350753784,
+ "mean_token_accuracy": 0.6466169953346252,
+ "num_tokens": 17514496.0,
+ "step": 1069
+ },
+ {
+ "entropy": 1.2614645957946777,
+ "epoch": 2.1616161616161618,
+ "grad_norm": 2.4019627571105957,
+ "learning_rate": 2.8404040404040404e-06,
+ "loss": 1.2659732103347778,
+ "mean_token_accuracy": 0.6797142028808594,
+ "num_tokens": 17530880.0,
+ "step": 1070
+ },
+ {
+ "entropy": 1.4247770309448242,
+ "epoch": 2.1636363636363636,
+ "grad_norm": 2.350632905960083,
+ "learning_rate": 2.8383838383838387e-06,
+ "loss": 1.415325403213501,
+ "mean_token_accuracy": 0.6655471324920654,
+ "num_tokens": 17547264.0,
+ "step": 1071
+ },
+ {
+ "entropy": 1.5746078491210938,
+ "epoch": 2.165656565656566,
+ "grad_norm": 2.18448543548584,
+ "learning_rate": 2.8363636363636366e-06,
+ "loss": 1.6022642850875854,
+ "mean_token_accuracy": 0.6324499249458313,
+ "num_tokens": 17563648.0,
+ "step": 1072
+ },
+ {
+ "entropy": 1.5869743824005127,
+ "epoch": 2.1676767676767676,
+ "grad_norm": 2.3553452491760254,
+ "learning_rate": 2.8343434343434345e-06,
+ "loss": 1.6163790225982666,
+ "mean_token_accuracy": 0.6199926733970642,
+ "num_tokens": 17580032.0,
+ "step": 1073
+ },
+ {
+ "entropy": 1.2412667274475098,
+ "epoch": 2.16969696969697,
+ "grad_norm": 2.209017038345337,
+ "learning_rate": 2.8323232323232324e-06,
+ "loss": 1.2491700649261475,
+ "mean_token_accuracy": 0.6911333799362183,
+ "num_tokens": 17596416.0,
+ "step": 1074
+ },
+ {
+ "entropy": 0.9127920866012573,
+ "epoch": 2.1717171717171717,
+ "grad_norm": 1.9761381149291992,
+ "learning_rate": 2.8303030303030303e-06,
+ "loss": 0.8961890339851379,
+ "mean_token_accuracy": 0.7523815631866455,
+ "num_tokens": 17612800.0,
+ "step": 1075
+ },
+ {
+ "entropy": 1.2798824310302734,
+ "epoch": 2.1737373737373735,
+ "grad_norm": 2.397031784057617,
+ "learning_rate": 2.8282828282828286e-06,
+ "loss": 1.286920189857483,
+ "mean_token_accuracy": 0.6805080771446228,
+ "num_tokens": 17629184.0,
+ "step": 1076
+ },
+ {
+ "entropy": 1.7863305807113647,
+ "epoch": 2.175757575757576,
+ "grad_norm": 2.2798852920532227,
+ "learning_rate": 2.8262626262626265e-06,
+ "loss": 1.7654080390930176,
+ "mean_token_accuracy": 0.5806057453155518,
+ "num_tokens": 17645568.0,
+ "step": 1077
+ },
+ {
+ "entropy": 1.327653408050537,
+ "epoch": 2.1777777777777776,
+ "grad_norm": 2.3473095893859863,
+ "learning_rate": 2.8242424242424244e-06,
+ "loss": 1.3175032138824463,
+ "mean_token_accuracy": 0.6750732660293579,
+ "num_tokens": 17661952.0,
+ "step": 1078
+ },
+ {
+ "entropy": 1.4917371273040771,
+ "epoch": 2.17979797979798,
+ "grad_norm": 2.181729793548584,
+ "learning_rate": 2.8222222222222223e-06,
+ "loss": 1.4738472700119019,
+ "mean_token_accuracy": 0.6619443297386169,
+ "num_tokens": 17678336.0,
+ "step": 1079
+ },
+ {
+ "entropy": 1.4644533395767212,
+ "epoch": 2.1818181818181817,
+ "grad_norm": 2.2558112144470215,
+ "learning_rate": 2.82020202020202e-06,
+ "loss": 1.4563480615615845,
+ "mean_token_accuracy": 0.6484489440917969,
+ "num_tokens": 17694720.0,
+ "step": 1080
+ },
+ {
+ "entropy": 0.7626141905784607,
+ "epoch": 2.183838383838384,
+ "grad_norm": 1.8687561750411987,
+ "learning_rate": 2.818181818181818e-06,
+ "loss": 0.7718763947486877,
+ "mean_token_accuracy": 0.7982413172721863,
+ "num_tokens": 17711104.0,
+ "step": 1081
+ },
+ {
+ "entropy": 1.0613173246383667,
+ "epoch": 2.1858585858585857,
+ "grad_norm": 2.072643995285034,
+ "learning_rate": 2.8161616161616163e-06,
+ "loss": 1.0554629564285278,
+ "mean_token_accuracy": 0.734306275844574,
+ "num_tokens": 17727488.0,
+ "step": 1082
+ },
+ {
+ "entropy": 1.0348066091537476,
+ "epoch": 2.187878787878788,
+ "grad_norm": 1.8455089330673218,
+ "learning_rate": 2.8141414141414142e-06,
+ "loss": 1.0231983661651611,
+ "mean_token_accuracy": 0.7382755279541016,
+ "num_tokens": 17743872.0,
+ "step": 1083
+ },
+ {
+ "entropy": 1.271000862121582,
+ "epoch": 2.18989898989899,
+ "grad_norm": 2.3292477130889893,
+ "learning_rate": 2.812121212121212e-06,
+ "loss": 1.2822595834732056,
+ "mean_token_accuracy": 0.6816682815551758,
+ "num_tokens": 17760256.0,
+ "step": 1084
+ },
+ {
+ "entropy": 1.62147855758667,
+ "epoch": 2.191919191919192,
+ "grad_norm": 2.3269946575164795,
+ "learning_rate": 2.81010101010101e-06,
+ "loss": 1.6017215251922607,
+ "mean_token_accuracy": 0.635808527469635,
+ "num_tokens": 17776640.0,
+ "step": 1085
+ },
+ {
+ "entropy": 1.4834544658660889,
+ "epoch": 2.193939393939394,
+ "grad_norm": 2.223590135574341,
+ "learning_rate": 2.808080808080808e-06,
+ "loss": 1.496992826461792,
+ "mean_token_accuracy": 0.6494870781898499,
+ "num_tokens": 17793024.0,
+ "step": 1086
+ },
+ {
+ "entropy": 1.4784536361694336,
+ "epoch": 2.195959595959596,
+ "grad_norm": 2.169416666030884,
+ "learning_rate": 2.806060606060606e-06,
+ "loss": 1.4766731262207031,
+ "mean_token_accuracy": 0.6543722748756409,
+ "num_tokens": 17809408.0,
+ "step": 1087
+ },
+ {
+ "entropy": 1.3650349378585815,
+ "epoch": 2.197979797979798,
+ "grad_norm": 2.1369845867156982,
+ "learning_rate": 2.804040404040404e-06,
+ "loss": 1.3475263118743896,
+ "mean_token_accuracy": 0.6712262034416199,
+ "num_tokens": 17825792.0,
+ "step": 1088
+ },
+ {
+ "entropy": 1.052316665649414,
+ "epoch": 2.2,
+ "grad_norm": 2.3176774978637695,
+ "learning_rate": 2.802020202020202e-06,
+ "loss": 1.056478500366211,
+ "mean_token_accuracy": 0.7374206185340881,
+ "num_tokens": 17842176.0,
+ "step": 1089
+ },
+ {
+ "entropy": 1.1928741931915283,
+ "epoch": 2.202020202020202,
+ "grad_norm": 2.2518808841705322,
+ "learning_rate": 2.8000000000000003e-06,
+ "loss": 1.2238969802856445,
+ "mean_token_accuracy": 0.7048119306564331,
+ "num_tokens": 17858560.0,
+ "step": 1090
+ },
+ {
+ "entropy": 1.135977864265442,
+ "epoch": 2.2040404040404042,
+ "grad_norm": 2.2688262462615967,
+ "learning_rate": 2.7979797979797986e-06,
+ "loss": 1.1330978870391846,
+ "mean_token_accuracy": 0.7053004503250122,
+ "num_tokens": 17874944.0,
+ "step": 1091
+ },
+ {
+ "entropy": 1.3542495965957642,
+ "epoch": 2.206060606060606,
+ "grad_norm": 2.1078014373779297,
+ "learning_rate": 2.7959595959595965e-06,
+ "loss": 1.3983874320983887,
+ "mean_token_accuracy": 0.6840498447418213,
+ "num_tokens": 17891328.0,
+ "step": 1092
+ },
+ {
+ "entropy": 1.1861188411712646,
+ "epoch": 2.2080808080808083,
+ "grad_norm": 2.348931312561035,
+ "learning_rate": 2.7939393939393944e-06,
+ "loss": 1.1982170343399048,
+ "mean_token_accuracy": 0.7014533281326294,
+ "num_tokens": 17907712.0,
+ "step": 1093
+ },
+ {
+ "entropy": 1.2858752012252808,
+ "epoch": 2.21010101010101,
+ "grad_norm": 2.1746175289154053,
+ "learning_rate": 2.7919191919191923e-06,
+ "loss": 1.302664041519165,
+ "mean_token_accuracy": 0.6818515062332153,
+ "num_tokens": 17924096.0,
+ "step": 1094
+ },
+ {
+ "entropy": 1.4817839860916138,
+ "epoch": 2.212121212121212,
+ "grad_norm": 1.9842815399169922,
+ "learning_rate": 2.78989898989899e-06,
+ "loss": 1.5031331777572632,
+ "mean_token_accuracy": 0.6546165347099304,
+ "num_tokens": 17940480.0,
+ "step": 1095
+ },
+ {
+ "entropy": 1.2977368831634521,
+ "epoch": 2.214141414141414,
+ "grad_norm": 2.0394232273101807,
+ "learning_rate": 2.7878787878787885e-06,
+ "loss": 1.3185319900512695,
+ "mean_token_accuracy": 0.6885075569152832,
+ "num_tokens": 17956864.0,
+ "step": 1096
+ },
+ {
+ "entropy": 1.010884404182434,
+ "epoch": 2.216161616161616,
+ "grad_norm": 2.1113393306732178,
+ "learning_rate": 2.7858585858585864e-06,
+ "loss": 1.0239849090576172,
+ "mean_token_accuracy": 0.7426111102104187,
+ "num_tokens": 17973248.0,
+ "step": 1097
+ },
+ {
+ "entropy": 1.6405178308486938,
+ "epoch": 2.2181818181818183,
+ "grad_norm": 2.220865249633789,
+ "learning_rate": 2.7838383838383842e-06,
+ "loss": 1.660733938217163,
+ "mean_token_accuracy": 0.6250610947608948,
+ "num_tokens": 17989632.0,
+ "step": 1098
+ },
+ {
+ "entropy": 1.2822861671447754,
+ "epoch": 2.22020202020202,
+ "grad_norm": 2.1409878730773926,
+ "learning_rate": 2.781818181818182e-06,
+ "loss": 1.2675114870071411,
+ "mean_token_accuracy": 0.7099413871765137,
+ "num_tokens": 18006016.0,
+ "step": 1099
+ },
+ {
+ "entropy": 1.5835071802139282,
+ "epoch": 2.2222222222222223,
+ "grad_norm": 2.32753586769104,
+ "learning_rate": 2.77979797979798e-06,
+ "loss": 1.585759162902832,
+ "mean_token_accuracy": 0.6321446299552917,
+ "num_tokens": 18022400.0,
+ "step": 1100
+ },
+ {
+ "epoch": 2.2222222222222223,
+ "eval_entropy": 1.3726552551792515,
+ "eval_loss": 1.5380089282989502,
+ "eval_mean_token_accuracy": 0.645188374384757,
+ "eval_num_tokens": 18022400.0,
+ "eval_runtime": 43.7591,
+ "eval_samples_per_second": 22.624,
+ "eval_steps_per_second": 2.834,
+ "step": 1100
+ },
+ {
+ "entropy": 1.0727043151855469,
+ "epoch": 2.224242424242424,
+ "grad_norm": 2.2024266719818115,
+ "learning_rate": 2.7777777777777783e-06,
+ "loss": 1.066201090812683,
+ "mean_token_accuracy": 0.7272838354110718,
+ "num_tokens": 18038784.0,
+ "step": 1101
+ },
+ {
+ "entropy": 1.1188048124313354,
+ "epoch": 2.2262626262626264,
+ "grad_norm": 2.032862663269043,
+ "learning_rate": 2.7757575757575762e-06,
+ "loss": 1.1292006969451904,
+ "mean_token_accuracy": 0.7198949456214905,
+ "num_tokens": 18055168.0,
+ "step": 1102
+ },
+ {
+ "entropy": 1.3658474683761597,
+ "epoch": 2.228282828282828,
+ "grad_norm": 2.081120491027832,
+ "learning_rate": 2.773737373737374e-06,
+ "loss": 1.3503609895706177,
+ "mean_token_accuracy": 0.6908280253410339,
+ "num_tokens": 18071552.0,
+ "step": 1103
+ },
+ {
+ "entropy": 1.5833252668380737,
+ "epoch": 2.2303030303030305,
+ "grad_norm": 2.174055576324463,
+ "learning_rate": 2.771717171717172e-06,
+ "loss": 1.5991246700286865,
+ "mean_token_accuracy": 0.6322056651115417,
+ "num_tokens": 18087936.0,
+ "step": 1104
+ },
+ {
+ "entropy": 1.2178122997283936,
+ "epoch": 2.2323232323232323,
+ "grad_norm": 2.328620195388794,
+ "learning_rate": 2.76969696969697e-06,
+ "loss": 1.2162861824035645,
+ "mean_token_accuracy": 0.6916218996047974,
+ "num_tokens": 18104320.0,
+ "step": 1105
+ },
+ {
+ "entropy": 1.2894606590270996,
+ "epoch": 2.2343434343434345,
+ "grad_norm": 2.236239433288574,
+ "learning_rate": 2.7676767676767678e-06,
+ "loss": 1.2695279121398926,
+ "mean_token_accuracy": 0.689728856086731,
+ "num_tokens": 18120704.0,
+ "step": 1106
+ },
+ {
+ "entropy": 1.0141552686691284,
+ "epoch": 2.2363636363636363,
+ "grad_norm": 2.000847339630127,
+ "learning_rate": 2.765656565656566e-06,
+ "loss": 1.0147396326065063,
+ "mean_token_accuracy": 0.7469467520713806,
+ "num_tokens": 18137088.0,
+ "step": 1107
+ },
+ {
+ "entropy": 1.150480031967163,
+ "epoch": 2.2383838383838386,
+ "grad_norm": 2.15156888961792,
+ "learning_rate": 2.763636363636364e-06,
+ "loss": 1.1593115329742432,
+ "mean_token_accuracy": 0.7129335403442383,
+ "num_tokens": 18153472.0,
+ "step": 1108
+ },
+ {
+ "entropy": 1.4387518167495728,
+ "epoch": 2.2404040404040404,
+ "grad_norm": 2.3262319564819336,
+ "learning_rate": 2.761616161616162e-06,
+ "loss": 1.4393596649169922,
+ "mean_token_accuracy": 0.6572422981262207,
+ "num_tokens": 18169856.0,
+ "step": 1109
+ },
+ {
+ "entropy": 1.205459475517273,
+ "epoch": 2.242424242424242,
+ "grad_norm": 2.223987579345703,
+ "learning_rate": 2.7595959595959597e-06,
+ "loss": 1.1871377229690552,
+ "mean_token_accuracy": 0.7018197178840637,
+ "num_tokens": 18186240.0,
+ "step": 1110
+ },
+ {
+ "entropy": 1.428817868232727,
+ "epoch": 2.2444444444444445,
+ "grad_norm": 2.091627836227417,
+ "learning_rate": 2.7575757575757576e-06,
+ "loss": 1.4262187480926514,
+ "mean_token_accuracy": 0.6708598136901855,
+ "num_tokens": 18202624.0,
+ "step": 1111
+ },
+ {
+ "entropy": 1.420682668685913,
+ "epoch": 2.2464646464646463,
+ "grad_norm": 2.30232310295105,
+ "learning_rate": 2.755555555555556e-06,
+ "loss": 1.439821720123291,
+ "mean_token_accuracy": 0.6508305072784424,
+ "num_tokens": 18219008.0,
+ "step": 1112
+ },
+ {
+ "entropy": 1.2365189790725708,
+ "epoch": 2.2484848484848485,
+ "grad_norm": 2.1747143268585205,
+ "learning_rate": 2.753535353535354e-06,
+ "loss": 1.2286595106124878,
+ "mean_token_accuracy": 0.7011480331420898,
+ "num_tokens": 18235392.0,
+ "step": 1113
+ },
+ {
+ "entropy": 1.0271798372268677,
+ "epoch": 2.2505050505050503,
+ "grad_norm": 2.1767842769622803,
+ "learning_rate": 2.7515151515151517e-06,
+ "loss": 1.018047571182251,
+ "mean_token_accuracy": 0.7297264337539673,
+ "num_tokens": 18251776.0,
+ "step": 1114
+ },
+ {
+ "entropy": 1.229053258895874,
+ "epoch": 2.2525252525252526,
+ "grad_norm": 2.390345811843872,
+ "learning_rate": 2.7494949494949496e-06,
+ "loss": 1.250126838684082,
+ "mean_token_accuracy": 0.6869809627532959,
+ "num_tokens": 18268160.0,
+ "step": 1115
+ },
+ {
+ "entropy": 1.4062843322753906,
+ "epoch": 2.2545454545454544,
+ "grad_norm": 2.220144271850586,
+ "learning_rate": 2.7474747474747475e-06,
+ "loss": 1.4262535572052002,
+ "mean_token_accuracy": 0.6592574715614319,
+ "num_tokens": 18284544.0,
+ "step": 1116
+ },
+ {
+ "entropy": 1.1991937160491943,
+ "epoch": 2.2565656565656567,
+ "grad_norm": 1.9910556077957153,
+ "learning_rate": 2.7454545454545454e-06,
+ "loss": 1.1869618892669678,
+ "mean_token_accuracy": 0.7049340605735779,
+ "num_tokens": 18300928.0,
+ "step": 1117
+ },
+ {
+ "entropy": 1.3147934675216675,
+ "epoch": 2.2585858585858585,
+ "grad_norm": 2.033942222595215,
+ "learning_rate": 2.7434343434343437e-06,
+ "loss": 1.3236720561981201,
+ "mean_token_accuracy": 0.6854543089866638,
+ "num_tokens": 18317312.0,
+ "step": 1118
+ },
+ {
+ "entropy": 1.2828413248062134,
+ "epoch": 2.2606060606060607,
+ "grad_norm": 2.2729172706604004,
+ "learning_rate": 2.7414141414141416e-06,
+ "loss": 1.2601265907287598,
+ "mean_token_accuracy": 0.6866145730018616,
+ "num_tokens": 18333696.0,
+ "step": 1119
+ },
+ {
+ "entropy": 1.9708930253982544,
+ "epoch": 2.2626262626262625,
+ "grad_norm": 2.4053614139556885,
+ "learning_rate": 2.7393939393939395e-06,
+ "loss": 1.9953765869140625,
+ "mean_token_accuracy": 0.5719345211982727,
+ "num_tokens": 18350080.0,
+ "step": 1120
+ },
+ {
+ "entropy": 1.071018099784851,
+ "epoch": 2.264646464646465,
+ "grad_norm": 2.0399091243743896,
+ "learning_rate": 2.7373737373737374e-06,
+ "loss": 1.0672862529754639,
+ "mean_token_accuracy": 0.72007817029953,
+ "num_tokens": 18366464.0,
+ "step": 1121
+ },
+ {
+ "entropy": 1.8741599321365356,
+ "epoch": 2.2666666666666666,
+ "grad_norm": 2.0918753147125244,
+ "learning_rate": 2.7353535353535353e-06,
+ "loss": 1.8604744672775269,
+ "mean_token_accuracy": 0.598436713218689,
+ "num_tokens": 18382848.0,
+ "step": 1122
+ },
+ {
+ "entropy": 1.0931968688964844,
+ "epoch": 2.268686868686869,
+ "grad_norm": 2.161102294921875,
+ "learning_rate": 2.7333333333333336e-06,
+ "loss": 1.1152517795562744,
+ "mean_token_accuracy": 0.7209330797195435,
+ "num_tokens": 18399232.0,
+ "step": 1123
+ },
+ {
+ "entropy": 1.3318616151809692,
+ "epoch": 2.2707070707070707,
+ "grad_norm": 2.200199842453003,
+ "learning_rate": 2.7313131313131315e-06,
+ "loss": 1.3456424474716187,
+ "mean_token_accuracy": 0.6789203882217407,
+ "num_tokens": 18415616.0,
+ "step": 1124
+ },
+ {
+ "entropy": 1.3777986764907837,
+ "epoch": 2.2727272727272725,
+ "grad_norm": 2.328887701034546,
+ "learning_rate": 2.7292929292929293e-06,
+ "loss": 1.3955169916152954,
+ "mean_token_accuracy": 0.6679897308349609,
+ "num_tokens": 18432000.0,
+ "step": 1125
+ },
+ {
+ "entropy": 1.0748711824417114,
+ "epoch": 2.2747474747474747,
+ "grad_norm": 2.151282787322998,
+ "learning_rate": 2.7272727272727272e-06,
+ "loss": 1.0380895137786865,
+ "mean_token_accuracy": 0.736932098865509,
+ "num_tokens": 18448384.0,
+ "step": 1126
+ },
+ {
+ "entropy": 1.4420548677444458,
+ "epoch": 2.276767676767677,
+ "grad_norm": 2.133190870285034,
+ "learning_rate": 2.725252525252525e-06,
+ "loss": 1.4465012550354004,
+ "mean_token_accuracy": 0.6784929037094116,
+ "num_tokens": 18464768.0,
+ "step": 1127
+ },
+ {
+ "entropy": 0.9478949904441833,
+ "epoch": 2.278787878787879,
+ "grad_norm": 2.0902628898620605,
+ "learning_rate": 2.7232323232323234e-06,
+ "loss": 0.9504396915435791,
+ "mean_token_accuracy": 0.7458475828170776,
+ "num_tokens": 18481152.0,
+ "step": 1128
+ },
+ {
+ "entropy": 1.367720127105713,
+ "epoch": 2.2808080808080806,
+ "grad_norm": 2.0413167476654053,
+ "learning_rate": 2.7212121212121213e-06,
+ "loss": 1.3884644508361816,
+ "mean_token_accuracy": 0.6762945652008057,
+ "num_tokens": 18497536.0,
+ "step": 1129
+ },
+ {
+ "entropy": 1.6931284666061401,
+ "epoch": 2.282828282828283,
+ "grad_norm": 2.188426971435547,
+ "learning_rate": 2.719191919191919e-06,
+ "loss": 1.7233154773712158,
+ "mean_token_accuracy": 0.6323277950286865,
+ "num_tokens": 18513920.0,
+ "step": 1130
+ },
+ {
+ "entropy": 1.563072681427002,
+ "epoch": 2.2848484848484847,
+ "grad_norm": 2.241102695465088,
+ "learning_rate": 2.717171717171717e-06,
+ "loss": 1.5677220821380615,
+ "mean_token_accuracy": 0.640693724155426,
+ "num_tokens": 18530304.0,
+ "step": 1131
+ },
+ {
+ "entropy": 1.3332083225250244,
+ "epoch": 2.286868686868687,
+ "grad_norm": 2.209522008895874,
+ "learning_rate": 2.715151515151516e-06,
+ "loss": 1.337693452835083,
+ "mean_token_accuracy": 0.6715925931930542,
+ "num_tokens": 18546688.0,
+ "step": 1132
+ },
+ {
+ "entropy": 1.5389035940170288,
+ "epoch": 2.2888888888888888,
+ "grad_norm": 2.212047815322876,
+ "learning_rate": 2.7131313131313137e-06,
+ "loss": 1.52158784866333,
+ "mean_token_accuracy": 0.6447850465774536,
+ "num_tokens": 18563072.0,
+ "step": 1133
+ },
+ {
+ "entropy": 1.2687056064605713,
+ "epoch": 2.290909090909091,
+ "grad_norm": 2.1673405170440674,
+ "learning_rate": 2.7111111111111116e-06,
+ "loss": 1.2689636945724487,
+ "mean_token_accuracy": 0.6911333799362183,
+ "num_tokens": 18579456.0,
+ "step": 1134
+ },
+ {
+ "entropy": 1.1464369297027588,
+ "epoch": 2.292929292929293,
+ "grad_norm": 1.9283273220062256,
+ "learning_rate": 2.7090909090909095e-06,
+ "loss": 1.1613389253616333,
+ "mean_token_accuracy": 0.7145212292671204,
+ "num_tokens": 18595840.0,
+ "step": 1135
+ },
+ {
+ "entropy": 1.327767252922058,
+ "epoch": 2.294949494949495,
+ "grad_norm": 2.2193424701690674,
+ "learning_rate": 2.7070707070707074e-06,
+ "loss": 1.3209152221679688,
+ "mean_token_accuracy": 0.6740962266921997,
+ "num_tokens": 18612224.0,
+ "step": 1136
+ },
+ {
+ "entropy": 1.5820066928863525,
+ "epoch": 2.296969696969697,
+ "grad_norm": 2.2020492553710938,
+ "learning_rate": 2.7050505050505057e-06,
+ "loss": 1.603272795677185,
+ "mean_token_accuracy": 0.6400830745697021,
+ "num_tokens": 18628608.0,
+ "step": 1137
+ },
+ {
+ "entropy": 1.279178500175476,
+ "epoch": 2.298989898989899,
+ "grad_norm": 2.1509714126586914,
+ "learning_rate": 2.7030303030303036e-06,
+ "loss": 1.2742427587509155,
+ "mean_token_accuracy": 0.689667820930481,
+ "num_tokens": 18644992.0,
+ "step": 1138
+ },
+ {
+ "entropy": 1.2199708223342896,
+ "epoch": 2.301010101010101,
+ "grad_norm": 2.0223684310913086,
+ "learning_rate": 2.7010101010101015e-06,
+ "loss": 1.2254796028137207,
+ "mean_token_accuracy": 0.6971787810325623,
+ "num_tokens": 18661376.0,
+ "step": 1139
+ },
+ {
+ "entropy": 1.1640081405639648,
+ "epoch": 2.303030303030303,
+ "grad_norm": 2.1907923221588135,
+ "learning_rate": 2.6989898989898994e-06,
+ "loss": 1.170613408088684,
+ "mean_token_accuracy": 0.6979726552963257,
+ "num_tokens": 18677760.0,
+ "step": 1140
+ },
+ {
+ "entropy": 1.3213385343551636,
+ "epoch": 2.305050505050505,
+ "grad_norm": 2.170276641845703,
+ "learning_rate": 2.6969696969696972e-06,
+ "loss": 1.3338474035263062,
+ "mean_token_accuracy": 0.672874927520752,
+ "num_tokens": 18694144.0,
+ "step": 1141
+ },
+ {
+ "entropy": 1.6036078929901123,
+ "epoch": 2.3070707070707073,
+ "grad_norm": 2.3499386310577393,
+ "learning_rate": 2.694949494949495e-06,
+ "loss": 1.5859179496765137,
+ "mean_token_accuracy": 0.6349536180496216,
+ "num_tokens": 18710528.0,
+ "step": 1142
+ },
+ {
+ "entropy": 1.2171586751937866,
+ "epoch": 2.309090909090909,
+ "grad_norm": 2.0380377769470215,
+ "learning_rate": 2.6929292929292934e-06,
+ "loss": 1.2191288471221924,
+ "mean_token_accuracy": 0.7063385248184204,
+ "num_tokens": 18726912.0,
+ "step": 1143
+ },
+ {
+ "entropy": 1.165902018547058,
+ "epoch": 2.311111111111111,
+ "grad_norm": 2.077310085296631,
+ "learning_rate": 2.6909090909090913e-06,
+ "loss": 1.1728768348693848,
+ "mean_token_accuracy": 0.7075598239898682,
+ "num_tokens": 18743296.0,
+ "step": 1144
+ },
+ {
+ "entropy": 1.5172539949417114,
+ "epoch": 2.313131313131313,
+ "grad_norm": 2.0132439136505127,
+ "learning_rate": 2.6888888888888892e-06,
+ "loss": 1.5073471069335938,
+ "mean_token_accuracy": 0.6546775698661804,
+ "num_tokens": 18759680.0,
+ "step": 1145
+ },
+ {
+ "entropy": 1.3588616847991943,
+ "epoch": 2.315151515151515,
+ "grad_norm": 2.1851465702056885,
+ "learning_rate": 2.686868686868687e-06,
+ "loss": 1.4111744165420532,
+ "mean_token_accuracy": 0.6750732660293579,
+ "num_tokens": 18776064.0,
+ "step": 1146
+ },
+ {
+ "entropy": 1.1948320865631104,
+ "epoch": 2.317171717171717,
+ "grad_norm": 2.1817259788513184,
+ "learning_rate": 2.684848484848485e-06,
+ "loss": 1.188377857208252,
+ "mean_token_accuracy": 0.7122007608413696,
+ "num_tokens": 18792448.0,
+ "step": 1147
+ },
+ {
+ "entropy": 1.7049702405929565,
+ "epoch": 2.319191919191919,
+ "grad_norm": 2.309352159500122,
+ "learning_rate": 2.6828282828282833e-06,
+ "loss": 1.6966536045074463,
+ "mean_token_accuracy": 0.6052759885787964,
+ "num_tokens": 18808832.0,
+ "step": 1148
+ },
+ {
+ "entropy": 1.0853440761566162,
+ "epoch": 2.3212121212121213,
+ "grad_norm": 2.1501059532165527,
+ "learning_rate": 2.680808080808081e-06,
+ "loss": 1.0747997760772705,
+ "mean_token_accuracy": 0.7195896506309509,
+ "num_tokens": 18825216.0,
+ "step": 1149
+ },
+ {
+ "entropy": 1.209170937538147,
+ "epoch": 2.323232323232323,
+ "grad_norm": 2.105555772781372,
+ "learning_rate": 2.678787878787879e-06,
+ "loss": 1.2055076360702515,
+ "mean_token_accuracy": 0.7062774896621704,
+ "num_tokens": 18841600.0,
+ "step": 1150
+ },
+ {
+ "epoch": 2.323232323232323,
+ "eval_entropy": 1.3799022854335847,
+ "eval_loss": 1.5365934371948242,
+ "eval_mean_token_accuracy": 0.6452573234996488,
+ "eval_num_tokens": 18841600.0,
+ "eval_runtime": 43.7362,
+ "eval_samples_per_second": 22.636,
+ "eval_steps_per_second": 2.835,
+ "step": 1150
+ },
+ {
+ "entropy": 0.9533048868179321,
+ "epoch": 2.3252525252525253,
+ "grad_norm": 1.9681413173675537,
+ "learning_rate": 2.676767676767677e-06,
+ "loss": 0.9284776449203491,
+ "mean_token_accuracy": 0.7609916925430298,
+ "num_tokens": 18857984.0,
+ "step": 1151
+ },
+ {
+ "entropy": 1.575872778892517,
+ "epoch": 2.327272727272727,
+ "grad_norm": 2.15085768699646,
+ "learning_rate": 2.674747474747475e-06,
+ "loss": 1.5716705322265625,
+ "mean_token_accuracy": 0.6415486335754395,
+ "num_tokens": 18874368.0,
+ "step": 1152
+ },
+ {
+ "entropy": 1.1680493354797363,
+ "epoch": 2.3292929292929294,
+ "grad_norm": 2.189589023590088,
+ "learning_rate": 2.6727272727272727e-06,
+ "loss": 1.1651779413223267,
+ "mean_token_accuracy": 0.7156203985214233,
+ "num_tokens": 18890752.0,
+ "step": 1153
+ },
+ {
+ "entropy": 1.1636664867401123,
+ "epoch": 2.3313131313131312,
+ "grad_norm": 1.9992915391921997,
+ "learning_rate": 2.670707070707071e-06,
+ "loss": 1.1650042533874512,
+ "mean_token_accuracy": 0.7116512060165405,
+ "num_tokens": 18907136.0,
+ "step": 1154
+ },
+ {
+ "entropy": 1.205094337463379,
+ "epoch": 2.3333333333333335,
+ "grad_norm": 2.070323944091797,
+ "learning_rate": 2.668686868686869e-06,
+ "loss": 1.2071596384048462,
+ "mean_token_accuracy": 0.7018197178840637,
+ "num_tokens": 18923520.0,
+ "step": 1155
+ },
+ {
+ "entropy": 1.2256643772125244,
+ "epoch": 2.3353535353535353,
+ "grad_norm": 2.144339084625244,
+ "learning_rate": 2.666666666666667e-06,
+ "loss": 1.2168457508087158,
+ "mean_token_accuracy": 0.7010869383811951,
+ "num_tokens": 18939904.0,
+ "step": 1156
+ },
+ {
+ "entropy": 1.631039023399353,
+ "epoch": 2.3373737373737375,
+ "grad_norm": 2.2886807918548584,
+ "learning_rate": 2.6646464646464647e-06,
+ "loss": 1.6534450054168701,
+ "mean_token_accuracy": 0.6248778700828552,
+ "num_tokens": 18956288.0,
+ "step": 1157
+ },
+ {
+ "entropy": 1.0360562801361084,
+ "epoch": 2.3393939393939394,
+ "grad_norm": 2.137491226196289,
+ "learning_rate": 2.6626262626262626e-06,
+ "loss": 1.0167723894119263,
+ "mean_token_accuracy": 0.7330239415168762,
+ "num_tokens": 18972672.0,
+ "step": 1158
+ },
+ {
+ "entropy": 1.2292720079421997,
+ "epoch": 2.341414141414141,
+ "grad_norm": 3.1518330574035645,
+ "learning_rate": 2.660606060606061e-06,
+ "loss": 1.2180495262145996,
+ "mean_token_accuracy": 0.6957743167877197,
+ "num_tokens": 18989056.0,
+ "step": 1159
+ },
+ {
+ "entropy": 1.2210739850997925,
+ "epoch": 2.3434343434343434,
+ "grad_norm": 2.1314170360565186,
+ "learning_rate": 2.658585858585859e-06,
+ "loss": 1.2399296760559082,
+ "mean_token_accuracy": 0.7047508358955383,
+ "num_tokens": 19005440.0,
+ "step": 1160
+ },
+ {
+ "entropy": 1.2762846946716309,
+ "epoch": 2.3454545454545457,
+ "grad_norm": 2.1767046451568604,
+ "learning_rate": 2.6565656565656567e-06,
+ "loss": 1.2640776634216309,
+ "mean_token_accuracy": 0.6954079270362854,
+ "num_tokens": 19021824.0,
+ "step": 1161
+ },
+ {
+ "entropy": 1.271180272102356,
+ "epoch": 2.3474747474747475,
+ "grad_norm": 2.121891975402832,
+ "learning_rate": 2.6545454545454546e-06,
+ "loss": 1.2666800022125244,
+ "mean_token_accuracy": 0.6880801320075989,
+ "num_tokens": 19038208.0,
+ "step": 1162
+ },
+ {
+ "entropy": 1.5604509115219116,
+ "epoch": 2.3494949494949493,
+ "grad_norm": 2.320765495300293,
+ "learning_rate": 2.6525252525252525e-06,
+ "loss": 1.5687410831451416,
+ "mean_token_accuracy": 0.627198338508606,
+ "num_tokens": 19054592.0,
+ "step": 1163
+ },
+ {
+ "entropy": 1.1606905460357666,
+ "epoch": 2.3515151515151516,
+ "grad_norm": 2.1167874336242676,
+ "learning_rate": 2.6505050505050508e-06,
+ "loss": 1.1879122257232666,
+ "mean_token_accuracy": 0.7130556702613831,
+ "num_tokens": 19070976.0,
+ "step": 1164
+ },
+ {
+ "entropy": 1.5421797037124634,
+ "epoch": 2.3535353535353534,
+ "grad_norm": 2.144657850265503,
+ "learning_rate": 2.6484848484848487e-06,
+ "loss": 1.5562596321105957,
+ "mean_token_accuracy": 0.6350146532058716,
+ "num_tokens": 19087360.0,
+ "step": 1165
+ },
+ {
+ "entropy": 1.2803337574005127,
+ "epoch": 2.3555555555555556,
+ "grad_norm": 2.0440914630889893,
+ "learning_rate": 2.6464646464646466e-06,
+ "loss": 1.3008983135223389,
+ "mean_token_accuracy": 0.6868588328361511,
+ "num_tokens": 19103744.0,
+ "step": 1166
+ },
+ {
+ "entropy": 1.1863176822662354,
+ "epoch": 2.3575757575757574,
+ "grad_norm": 2.231771469116211,
+ "learning_rate": 2.6444444444444444e-06,
+ "loss": 1.1863257884979248,
+ "mean_token_accuracy": 0.7069491744041443,
+ "num_tokens": 19120128.0,
+ "step": 1167
+ },
+ {
+ "entropy": 1.1186378002166748,
+ "epoch": 2.3595959595959597,
+ "grad_norm": 2.0704708099365234,
+ "learning_rate": 2.6424242424242423e-06,
+ "loss": 1.1196620464324951,
+ "mean_token_accuracy": 0.726062536239624,
+ "num_tokens": 19136512.0,
+ "step": 1168
+ },
+ {
+ "entropy": 1.2110928297042847,
+ "epoch": 2.3616161616161615,
+ "grad_norm": 2.251551866531372,
+ "learning_rate": 2.6404040404040402e-06,
+ "loss": 1.2020606994628906,
+ "mean_token_accuracy": 0.7058500051498413,
+ "num_tokens": 19152896.0,
+ "step": 1169
+ },
+ {
+ "entropy": 1.302309513092041,
+ "epoch": 2.3636363636363638,
+ "grad_norm": 2.171961545944214,
+ "learning_rate": 2.6383838383838385e-06,
+ "loss": 1.3029515743255615,
+ "mean_token_accuracy": 0.673363447189331,
+ "num_tokens": 19169280.0,
+ "step": 1170
+ },
+ {
+ "entropy": 1.6036607027053833,
+ "epoch": 2.3656565656565656,
+ "grad_norm": 2.336979627609253,
+ "learning_rate": 2.6363636363636364e-06,
+ "loss": 1.6076864004135132,
+ "mean_token_accuracy": 0.630984365940094,
+ "num_tokens": 19185664.0,
+ "step": 1171
+ },
+ {
+ "entropy": 1.431620478630066,
+ "epoch": 2.367676767676768,
+ "grad_norm": 2.115891218185425,
+ "learning_rate": 2.6343434343434343e-06,
+ "loss": 1.4436556100845337,
+ "mean_token_accuracy": 0.6623717546463013,
+ "num_tokens": 19202048.0,
+ "step": 1172
+ },
+ {
+ "entropy": 1.4537419080734253,
+ "epoch": 2.3696969696969696,
+ "grad_norm": 2.497891664505005,
+ "learning_rate": 2.632323232323232e-06,
+ "loss": 1.48027503490448,
+ "mean_token_accuracy": 0.6352589130401611,
+ "num_tokens": 19218432.0,
+ "step": 1173
+ },
+ {
+ "entropy": 1.292363166809082,
+ "epoch": 2.371717171717172,
+ "grad_norm": 2.0235633850097656,
+ "learning_rate": 2.63030303030303e-06,
+ "loss": 1.3111207485198975,
+ "mean_token_accuracy": 0.6873473525047302,
+ "num_tokens": 19234816.0,
+ "step": 1174
+ },
+ {
+ "entropy": 1.2336944341659546,
+ "epoch": 2.3737373737373737,
+ "grad_norm": 2.342643976211548,
+ "learning_rate": 2.628282828282829e-06,
+ "loss": 1.2427266836166382,
+ "mean_token_accuracy": 0.6844772696495056,
+ "num_tokens": 19251200.0,
+ "step": 1175
+ },
+ {
+ "entropy": 1.166039228439331,
+ "epoch": 2.375757575757576,
+ "grad_norm": 2.157937526702881,
+ "learning_rate": 2.6262626262626267e-06,
+ "loss": 1.167286992073059,
+ "mean_token_accuracy": 0.7115290760993958,
+ "num_tokens": 19267584.0,
+ "step": 1176
+ },
+ {
+ "entropy": 1.3824102878570557,
+ "epoch": 2.3777777777777778,
+ "grad_norm": 2.2874608039855957,
+ "learning_rate": 2.6242424242424246e-06,
+ "loss": 1.404619574546814,
+ "mean_token_accuracy": 0.6707376837730408,
+ "num_tokens": 19283968.0,
+ "step": 1177
+ },
+ {
+ "entropy": 0.8266427516937256,
+ "epoch": 2.3797979797979796,
+ "grad_norm": 2.098217010498047,
+ "learning_rate": 2.6222222222222225e-06,
+ "loss": 0.821276068687439,
+ "mean_token_accuracy": 0.7726551294326782,
+ "num_tokens": 19300352.0,
+ "step": 1178
+ },
+ {
+ "entropy": 1.4335079193115234,
+ "epoch": 2.381818181818182,
+ "grad_norm": 2.3051438331604004,
+ "learning_rate": 2.620202020202021e-06,
+ "loss": 1.437370777130127,
+ "mean_token_accuracy": 0.6520518064498901,
+ "num_tokens": 19316736.0,
+ "step": 1179
+ },
+ {
+ "entropy": 1.0816318988800049,
+ "epoch": 2.3838383838383836,
+ "grad_norm": 2.0723016262054443,
+ "learning_rate": 2.6181818181818187e-06,
+ "loss": 1.0697057247161865,
+ "mean_token_accuracy": 0.7304592132568359,
+ "num_tokens": 19333120.0,
+ "step": 1180
+ },
+ {
+ "entropy": 1.2515255212783813,
+ "epoch": 2.385858585858586,
+ "grad_norm": 2.32125186920166,
+ "learning_rate": 2.6161616161616166e-06,
+ "loss": 1.256905198097229,
+ "mean_token_accuracy": 0.697300910949707,
+ "num_tokens": 19349504.0,
+ "step": 1181
+ },
+ {
+ "entropy": 1.0271868705749512,
+ "epoch": 2.3878787878787877,
+ "grad_norm": 2.0145103931427,
+ "learning_rate": 2.6141414141414145e-06,
+ "loss": 1.0158754587173462,
+ "mean_token_accuracy": 0.7518930435180664,
+ "num_tokens": 19365888.0,
+ "step": 1182
+ },
+ {
+ "entropy": 1.2139984369277954,
+ "epoch": 2.38989898989899,
+ "grad_norm": 2.1393911838531494,
+ "learning_rate": 2.6121212121212123e-06,
+ "loss": 1.2150651216506958,
+ "mean_token_accuracy": 0.7135441899299622,
+ "num_tokens": 19382272.0,
+ "step": 1183
+ },
+ {
+ "entropy": 1.076101303100586,
+ "epoch": 2.391919191919192,
+ "grad_norm": 2.1664907932281494,
+ "learning_rate": 2.6101010101010107e-06,
+ "loss": 1.0744431018829346,
+ "mean_token_accuracy": 0.7267953157424927,
+ "num_tokens": 19398656.0,
+ "step": 1184
+ },
+ {
+ "entropy": 1.5176570415496826,
+ "epoch": 2.393939393939394,
+ "grad_norm": 2.0756311416625977,
+ "learning_rate": 2.6080808080808085e-06,
+ "loss": 1.5209624767303467,
+ "mean_token_accuracy": 0.647777259349823,
+ "num_tokens": 19415040.0,
+ "step": 1185
+ },
+ {
+ "entropy": 1.0946495532989502,
+ "epoch": 2.395959595959596,
+ "grad_norm": 2.0545496940612793,
+ "learning_rate": 2.6060606060606064e-06,
+ "loss": 1.0858100652694702,
+ "mean_token_accuracy": 0.7226428985595703,
+ "num_tokens": 19431424.0,
+ "step": 1186
+ },
+ {
+ "entropy": 0.7713431715965271,
+ "epoch": 2.397979797979798,
+ "grad_norm": 2.0606112480163574,
+ "learning_rate": 2.6040404040404043e-06,
+ "loss": 0.7782045006752014,
+ "mean_token_accuracy": 0.7804103493690491,
+ "num_tokens": 19447808.0,
+ "step": 1187
+ },
+ {
+ "entropy": 1.2803305387496948,
+ "epoch": 2.4,
+ "grad_norm": 2.2125155925750732,
+ "learning_rate": 2.602020202020202e-06,
+ "loss": 1.2971004247665405,
+ "mean_token_accuracy": 0.6787371635437012,
+ "num_tokens": 19464192.0,
+ "step": 1188
+ },
+ {
+ "entropy": 0.9156424403190613,
+ "epoch": 2.402020202020202,
+ "grad_norm": 2.032531499862671,
+ "learning_rate": 2.6e-06,
+ "loss": 0.9119267463684082,
+ "mean_token_accuracy": 0.7627015113830566,
+ "num_tokens": 19480576.0,
+ "step": 1189
+ },
+ {
+ "entropy": 1.2698813676834106,
+ "epoch": 2.404040404040404,
+ "grad_norm": 2.2204320430755615,
+ "learning_rate": 2.5979797979797984e-06,
+ "loss": 1.2934629917144775,
+ "mean_token_accuracy": 0.682584285736084,
+ "num_tokens": 19496960.0,
+ "step": 1190
+ },
+ {
+ "entropy": 1.5677871704101562,
+ "epoch": 2.4060606060606062,
+ "grad_norm": 2.229994773864746,
+ "learning_rate": 2.5959595959595963e-06,
+ "loss": 1.591191053390503,
+ "mean_token_accuracy": 0.6297020316123962,
+ "num_tokens": 19513344.0,
+ "step": 1191
+ },
+ {
+ "entropy": 1.3057011365890503,
+ "epoch": 2.408080808080808,
+ "grad_norm": 2.4158618450164795,
+ "learning_rate": 2.593939393939394e-06,
+ "loss": 1.2989580631256104,
+ "mean_token_accuracy": 0.689728856086731,
+ "num_tokens": 19529728.0,
+ "step": 1192
+ },
+ {
+ "entropy": 1.3805862665176392,
+ "epoch": 2.41010101010101,
+ "grad_norm": 2.2611825466156006,
+ "learning_rate": 2.591919191919192e-06,
+ "loss": 1.385383129119873,
+ "mean_token_accuracy": 0.6634709239006042,
+ "num_tokens": 19546112.0,
+ "step": 1193
+ },
+ {
+ "entropy": 1.8123539686203003,
+ "epoch": 2.412121212121212,
+ "grad_norm": 2.3038923740386963,
+ "learning_rate": 2.58989898989899e-06,
+ "loss": 1.80764901638031,
+ "mean_token_accuracy": 0.5925744771957397,
+ "num_tokens": 19562496.0,
+ "step": 1194
+ },
+ {
+ "entropy": 1.1476300954818726,
+ "epoch": 2.4141414141414144,
+ "grad_norm": 2.2417445182800293,
+ "learning_rate": 2.5878787878787883e-06,
+ "loss": 1.1802024841308594,
+ "mean_token_accuracy": 0.7130556702613831,
+ "num_tokens": 19578880.0,
+ "step": 1195
+ },
+ {
+ "entropy": 1.6678205728530884,
+ "epoch": 2.416161616161616,
+ "grad_norm": 2.399136781692505,
+ "learning_rate": 2.585858585858586e-06,
+ "loss": 1.6780188083648682,
+ "mean_token_accuracy": 0.6161455512046814,
+ "num_tokens": 19595264.0,
+ "step": 1196
+ },
+ {
+ "entropy": 1.053451418876648,
+ "epoch": 2.418181818181818,
+ "grad_norm": 1.9980698823928833,
+ "learning_rate": 2.583838383838384e-06,
+ "loss": 1.0241587162017822,
+ "mean_token_accuracy": 0.7372984886169434,
+ "num_tokens": 19611648.0,
+ "step": 1197
+ },
+ {
+ "entropy": 1.5389430522918701,
+ "epoch": 2.4202020202020202,
+ "grad_norm": 2.171046733856201,
+ "learning_rate": 2.581818181818182e-06,
+ "loss": 1.5499310493469238,
+ "mean_token_accuracy": 0.6422203183174133,
+ "num_tokens": 19628032.0,
+ "step": 1198
+ },
+ {
+ "entropy": 1.4151558876037598,
+ "epoch": 2.422222222222222,
+ "grad_norm": 2.185184955596924,
+ "learning_rate": 2.57979797979798e-06,
+ "loss": 1.4296989440917969,
+ "mean_token_accuracy": 0.663593053817749,
+ "num_tokens": 19644416.0,
+ "step": 1199
+ },
+ {
+ "entropy": 1.1455156803131104,
+ "epoch": 2.4242424242424243,
+ "grad_norm": 2.224477767944336,
+ "learning_rate": 2.577777777777778e-06,
+ "loss": 1.1682045459747314,
+ "mean_token_accuracy": 0.7084147334098816,
+ "num_tokens": 19660800.0,
+ "step": 1200
+ },
+ {
+ "epoch": 2.4242424242424243,
+ "eval_entropy": 1.3761802339746105,
+ "eval_loss": 1.53595769405365,
+ "eval_mean_token_accuracy": 0.6453439944213436,
+ "eval_num_tokens": 19660800.0,
+ "eval_runtime": 43.8176,
+ "eval_samples_per_second": 22.594,
+ "eval_steps_per_second": 2.83,
+ "step": 1200
+ },
+ {
+ "entropy": 1.6039199829101562,
+ "epoch": 2.426262626262626,
+ "grad_norm": 2.122072696685791,
+ "learning_rate": 2.575757575757576e-06,
+ "loss": 1.625109314918518,
+ "mean_token_accuracy": 0.6306179761886597,
+ "num_tokens": 19677184.0,
+ "step": 1201
+ },
+ {
+ "entropy": 1.2364906072616577,
+ "epoch": 2.4282828282828284,
+ "grad_norm": 2.175671339035034,
+ "learning_rate": 2.573737373737374e-06,
+ "loss": 1.2398111820220947,
+ "mean_token_accuracy": 0.6886907815933228,
+ "num_tokens": 19693568.0,
+ "step": 1202
+ },
+ {
+ "entropy": 1.2937804460525513,
+ "epoch": 2.43030303030303,
+ "grad_norm": 2.1206040382385254,
+ "learning_rate": 2.571717171717172e-06,
+ "loss": 1.3073123693466187,
+ "mean_token_accuracy": 0.6762945652008057,
+ "num_tokens": 19709952.0,
+ "step": 1203
+ },
+ {
+ "entropy": 1.0239282846450806,
+ "epoch": 2.4323232323232324,
+ "grad_norm": 1.863510012626648,
+ "learning_rate": 2.5696969696969697e-06,
+ "loss": 1.015241026878357,
+ "mean_token_accuracy": 0.7452369332313538,
+ "num_tokens": 19726336.0,
+ "step": 1204
+ },
+ {
+ "entropy": 1.2327015399932861,
+ "epoch": 2.4343434343434343,
+ "grad_norm": 1.968019962310791,
+ "learning_rate": 2.5676767676767676e-06,
+ "loss": 1.2275054454803467,
+ "mean_token_accuracy": 0.6974841356277466,
+ "num_tokens": 19742720.0,
+ "step": 1205
+ },
+ {
+ "entropy": 1.130920171737671,
+ "epoch": 2.4363636363636365,
+ "grad_norm": 2.096205949783325,
+ "learning_rate": 2.565656565656566e-06,
+ "loss": 1.1229937076568604,
+ "mean_token_accuracy": 0.7102466821670532,
+ "num_tokens": 19759104.0,
+ "step": 1206
+ },
+ {
+ "entropy": 0.9204118251800537,
+ "epoch": 2.4383838383838383,
+ "grad_norm": 1.7956496477127075,
+ "learning_rate": 2.5636363636363638e-06,
+ "loss": 0.9276518821716309,
+ "mean_token_accuracy": 0.7644113302230835,
+ "num_tokens": 19775488.0,
+ "step": 1207
+ },
+ {
+ "entropy": 0.9552480578422546,
+ "epoch": 2.4404040404040406,
+ "grad_norm": 2.0838167667388916,
+ "learning_rate": 2.5616161616161617e-06,
+ "loss": 0.9676632881164551,
+ "mean_token_accuracy": 0.7421225905418396,
+ "num_tokens": 19791872.0,
+ "step": 1208
+ },
+ {
+ "entropy": 1.0109010934829712,
+ "epoch": 2.4424242424242424,
+ "grad_norm": 2.1739392280578613,
+ "learning_rate": 2.5595959595959595e-06,
+ "loss": 1.0201005935668945,
+ "mean_token_accuracy": 0.7366267442703247,
+ "num_tokens": 19808256.0,
+ "step": 1209
+ },
+ {
+ "entropy": 1.4037907123565674,
+ "epoch": 2.4444444444444446,
+ "grad_norm": 2.151865005493164,
+ "learning_rate": 2.5575757575757574e-06,
+ "loss": 1.4110395908355713,
+ "mean_token_accuracy": 0.6682339906692505,
+ "num_tokens": 19824640.0,
+ "step": 1210
+ },
+ {
+ "entropy": 1.2449947595596313,
+ "epoch": 2.4464646464646465,
+ "grad_norm": 2.065934896469116,
+ "learning_rate": 2.5555555555555557e-06,
+ "loss": 1.2521324157714844,
+ "mean_token_accuracy": 0.6952247023582458,
+ "num_tokens": 19841024.0,
+ "step": 1211
+ },
+ {
+ "entropy": 1.2075072526931763,
+ "epoch": 2.4484848484848483,
+ "grad_norm": 2.197885274887085,
+ "learning_rate": 2.5535353535353536e-06,
+ "loss": 1.2041654586791992,
+ "mean_token_accuracy": 0.69840008020401,
+ "num_tokens": 19857408.0,
+ "step": 1212
+ },
+ {
+ "entropy": 1.0454095602035522,
+ "epoch": 2.4505050505050505,
+ "grad_norm": 2.0778188705444336,
+ "learning_rate": 2.5515151515151515e-06,
+ "loss": 1.0392906665802002,
+ "mean_token_accuracy": 0.74187833070755,
+ "num_tokens": 19873792.0,
+ "step": 1213
+ },
+ {
+ "entropy": 1.3138911724090576,
+ "epoch": 2.4525252525252528,
+ "grad_norm": 2.1871910095214844,
+ "learning_rate": 2.5494949494949494e-06,
+ "loss": 1.317048192024231,
+ "mean_token_accuracy": 0.6720811128616333,
+ "num_tokens": 19890176.0,
+ "step": 1214
+ },
+ {
+ "entropy": 1.263680338859558,
+ "epoch": 2.4545454545454546,
+ "grad_norm": 2.129873752593994,
+ "learning_rate": 2.5474747474747473e-06,
+ "loss": 1.277045726776123,
+ "mean_token_accuracy": 0.7011480331420898,
+ "num_tokens": 19906560.0,
+ "step": 1215
+ },
+ {
+ "entropy": 0.9653311967849731,
+ "epoch": 2.4565656565656564,
+ "grad_norm": 2.1161508560180664,
+ "learning_rate": 2.5454545454545456e-06,
+ "loss": 0.9699352979660034,
+ "mean_token_accuracy": 0.7437713742256165,
+ "num_tokens": 19922944.0,
+ "step": 1216
+ },
+ {
+ "entropy": 1.350392460823059,
+ "epoch": 2.4585858585858587,
+ "grad_norm": 2.0958914756774902,
+ "learning_rate": 2.5434343434343435e-06,
+ "loss": 1.3442802429199219,
+ "mean_token_accuracy": 0.6768441796302795,
+ "num_tokens": 19939328.0,
+ "step": 1217
+ },
+ {
+ "entropy": 1.2484683990478516,
+ "epoch": 2.4606060606060605,
+ "grad_norm": 2.191668748855591,
+ "learning_rate": 2.541414141414142e-06,
+ "loss": 1.2663497924804688,
+ "mean_token_accuracy": 0.6958964467048645,
+ "num_tokens": 19955712.0,
+ "step": 1218
+ },
+ {
+ "entropy": 0.9751343131065369,
+ "epoch": 2.4626262626262627,
+ "grad_norm": 2.0976815223693848,
+ "learning_rate": 2.5393939393939397e-06,
+ "loss": 0.9734562635421753,
+ "mean_token_accuracy": 0.7372373938560486,
+ "num_tokens": 19972096.0,
+ "step": 1219
+ },
+ {
+ "entropy": 1.1655267477035522,
+ "epoch": 2.4646464646464645,
+ "grad_norm": 2.3504767417907715,
+ "learning_rate": 2.537373737373738e-06,
+ "loss": 1.1775636672973633,
+ "mean_token_accuracy": 0.7023082375526428,
+ "num_tokens": 19988480.0,
+ "step": 1220
+ },
+ {
+ "entropy": 1.3543325662612915,
+ "epoch": 2.466666666666667,
+ "grad_norm": 2.038313627243042,
+ "learning_rate": 2.535353535353536e-06,
+ "loss": 1.3447773456573486,
+ "mean_token_accuracy": 0.6779433488845825,
+ "num_tokens": 20004864.0,
+ "step": 1221
+ },
+ {
+ "entropy": 1.1814380884170532,
+ "epoch": 2.4686868686868686,
+ "grad_norm": 2.186947822570801,
+ "learning_rate": 2.5333333333333338e-06,
+ "loss": 1.186347246170044,
+ "mean_token_accuracy": 0.7064606547355652,
+ "num_tokens": 20021248.0,
+ "step": 1222
+ },
+ {
+ "entropy": 0.9775183796882629,
+ "epoch": 2.470707070707071,
+ "grad_norm": 2.144923210144043,
+ "learning_rate": 2.5313131313131317e-06,
+ "loss": 1.0010652542114258,
+ "mean_token_accuracy": 0.7385808229446411,
+ "num_tokens": 20037632.0,
+ "step": 1223
+ },
+ {
+ "entropy": 1.6076546907424927,
+ "epoch": 2.4727272727272727,
+ "grad_norm": 2.1664817333221436,
+ "learning_rate": 2.5292929292929296e-06,
+ "loss": 1.618837594985962,
+ "mean_token_accuracy": 0.6297630667686462,
+ "num_tokens": 20054016.0,
+ "step": 1224
+ },
+ {
+ "entropy": 1.0457805395126343,
+ "epoch": 2.474747474747475,
+ "grad_norm": 2.154000997543335,
+ "learning_rate": 2.5272727272727274e-06,
+ "loss": 1.0406007766723633,
+ "mean_token_accuracy": 0.7322911620140076,
+ "num_tokens": 20070400.0,
+ "step": 1225
+ },
+ {
+ "entropy": 1.065863847732544,
+ "epoch": 2.4767676767676767,
+ "grad_norm": 2.062626361846924,
+ "learning_rate": 2.5252525252525258e-06,
+ "loss": 1.0432696342468262,
+ "mean_token_accuracy": 0.7371152639389038,
+ "num_tokens": 20086784.0,
+ "step": 1226
+ },
+ {
+ "entropy": 1.5166304111480713,
+ "epoch": 2.4787878787878785,
+ "grad_norm": 2.089587926864624,
+ "learning_rate": 2.5232323232323236e-06,
+ "loss": 1.5205962657928467,
+ "mean_token_accuracy": 0.653822660446167,
+ "num_tokens": 20103168.0,
+ "step": 1227
+ },
+ {
+ "entropy": 1.5969332456588745,
+ "epoch": 2.480808080808081,
+ "grad_norm": 2.297184705734253,
+ "learning_rate": 2.5212121212121215e-06,
+ "loss": 1.611903190612793,
+ "mean_token_accuracy": 0.6246336102485657,
+ "num_tokens": 20119552.0,
+ "step": 1228
+ },
+ {
+ "entropy": 1.8385323286056519,
+ "epoch": 2.482828282828283,
+ "grad_norm": 2.0568299293518066,
+ "learning_rate": 2.5191919191919194e-06,
+ "loss": 1.848689079284668,
+ "mean_token_accuracy": 0.6079018115997314,
+ "num_tokens": 20135936.0,
+ "step": 1229
+ },
+ {
+ "entropy": 1.1586394309997559,
+ "epoch": 2.484848484848485,
+ "grad_norm": 1.91037917137146,
+ "learning_rate": 2.5171717171717173e-06,
+ "loss": 1.1482139825820923,
+ "mean_token_accuracy": 0.7165364027023315,
+ "num_tokens": 20152320.0,
+ "step": 1230
+ },
+ {
+ "entropy": 1.6905426979064941,
+ "epoch": 2.4868686868686867,
+ "grad_norm": 2.2317121028900146,
+ "learning_rate": 2.5151515151515156e-06,
+ "loss": 1.7028450965881348,
+ "mean_token_accuracy": 0.6125427484512329,
+ "num_tokens": 20168704.0,
+ "step": 1231
+ },
+ {
+ "entropy": 1.1329548358917236,
+ "epoch": 2.488888888888889,
+ "grad_norm": 2.112380027770996,
+ "learning_rate": 2.5131313131313135e-06,
+ "loss": 1.120601773262024,
+ "mean_token_accuracy": 0.7181240916252136,
+ "num_tokens": 20185088.0,
+ "step": 1232
+ },
+ {
+ "entropy": 1.2013109922409058,
+ "epoch": 2.4909090909090907,
+ "grad_norm": 2.087054491043091,
+ "learning_rate": 2.5111111111111114e-06,
+ "loss": 1.2169899940490723,
+ "mean_token_accuracy": 0.7089032530784607,
+ "num_tokens": 20201472.0,
+ "step": 1233
+ },
+ {
+ "entropy": 1.187458872795105,
+ "epoch": 2.492929292929293,
+ "grad_norm": 2.431793689727783,
+ "learning_rate": 2.5090909090909093e-06,
+ "loss": 1.2183618545532227,
+ "mean_token_accuracy": 0.6925378441810608,
+ "num_tokens": 20217856.0,
+ "step": 1234
+ },
+ {
+ "entropy": 1.2123541831970215,
+ "epoch": 2.494949494949495,
+ "grad_norm": 2.1715121269226074,
+ "learning_rate": 2.507070707070707e-06,
+ "loss": 1.2254973649978638,
+ "mean_token_accuracy": 0.703285276889801,
+ "num_tokens": 20234240.0,
+ "step": 1235
+ },
+ {
+ "entropy": 1.0582001209259033,
+ "epoch": 2.496969696969697,
+ "grad_norm": 2.222578763961792,
+ "learning_rate": 2.5050505050505055e-06,
+ "loss": 1.061441421508789,
+ "mean_token_accuracy": 0.7247191071510315,
+ "num_tokens": 20250624.0,
+ "step": 1236
+ },
+ {
+ "entropy": 1.5102840662002563,
+ "epoch": 2.498989898989899,
+ "grad_norm": 2.27860951423645,
+ "learning_rate": 2.5030303030303034e-06,
+ "loss": 1.519880771636963,
+ "mean_token_accuracy": 0.6581583023071289,
+ "num_tokens": 20267008.0,
+ "step": 1237
+ },
+ {
+ "entropy": 1.222317099571228,
+ "epoch": 2.501010101010101,
+ "grad_norm": 2.182908773422241,
+ "learning_rate": 2.5010101010101013e-06,
+ "loss": 1.2382556200027466,
+ "mean_token_accuracy": 0.6908891201019287,
+ "num_tokens": 20283392.0,
+ "step": 1238
+ },
+ {
+ "entropy": 1.252193570137024,
+ "epoch": 2.503030303030303,
+ "grad_norm": 2.2065556049346924,
+ "learning_rate": 2.498989898989899e-06,
+ "loss": 1.2510521411895752,
+ "mean_token_accuracy": 0.6914386749267578,
+ "num_tokens": 20299776.0,
+ "step": 1239
+ },
+ {
+ "entropy": 1.2909793853759766,
+ "epoch": 2.505050505050505,
+ "grad_norm": 2.374338150024414,
+ "learning_rate": 2.496969696969697e-06,
+ "loss": 1.3144972324371338,
+ "mean_token_accuracy": 0.6847215294837952,
+ "num_tokens": 20316160.0,
+ "step": 1240
+ },
+ {
+ "entropy": 1.5218966007232666,
+ "epoch": 2.507070707070707,
+ "grad_norm": 2.3914759159088135,
+ "learning_rate": 2.494949494949495e-06,
+ "loss": 1.5194602012634277,
+ "mean_token_accuracy": 0.6398998498916626,
+ "num_tokens": 20332544.0,
+ "step": 1241
+ },
+ {
+ "entropy": 1.2130569219589233,
+ "epoch": 2.509090909090909,
+ "grad_norm": 2.2857706546783447,
+ "learning_rate": 2.4929292929292932e-06,
+ "loss": 1.2035828828811646,
+ "mean_token_accuracy": 0.6977283954620361,
+ "num_tokens": 20348928.0,
+ "step": 1242
+ },
+ {
+ "entropy": 1.623070478439331,
+ "epoch": 2.511111111111111,
+ "grad_norm": 2.3757028579711914,
+ "learning_rate": 2.490909090909091e-06,
+ "loss": 1.654674768447876,
+ "mean_token_accuracy": 0.6242061257362366,
+ "num_tokens": 20365312.0,
+ "step": 1243
+ },
+ {
+ "entropy": 1.6762864589691162,
+ "epoch": 2.5131313131313133,
+ "grad_norm": 2.239466667175293,
+ "learning_rate": 2.488888888888889e-06,
+ "loss": 1.6712524890899658,
+ "mean_token_accuracy": 0.6280532479286194,
+ "num_tokens": 20381696.0,
+ "step": 1244
+ },
+ {
+ "entropy": 1.6754344701766968,
+ "epoch": 2.515151515151515,
+ "grad_norm": 2.2599003314971924,
+ "learning_rate": 2.486868686868687e-06,
+ "loss": 1.6697289943695068,
+ "mean_token_accuracy": 0.6158402562141418,
+ "num_tokens": 20398080.0,
+ "step": 1245
+ },
+ {
+ "entropy": 1.6282587051391602,
+ "epoch": 2.517171717171717,
+ "grad_norm": 2.0555050373077393,
+ "learning_rate": 2.4848484848484848e-06,
+ "loss": 1.6177103519439697,
+ "mean_token_accuracy": 0.63312166929245,
+ "num_tokens": 20414464.0,
+ "step": 1246
+ },
+ {
+ "entropy": 1.3457770347595215,
+ "epoch": 2.519191919191919,
+ "grad_norm": 2.3396496772766113,
+ "learning_rate": 2.482828282828283e-06,
+ "loss": 1.3288640975952148,
+ "mean_token_accuracy": 0.6720811128616333,
+ "num_tokens": 20430848.0,
+ "step": 1247
+ },
+ {
+ "entropy": 1.5269931554794312,
+ "epoch": 2.5212121212121215,
+ "grad_norm": 2.303053617477417,
+ "learning_rate": 2.480808080808081e-06,
+ "loss": 1.4953217506408691,
+ "mean_token_accuracy": 0.639167070388794,
+ "num_tokens": 20447232.0,
+ "step": 1248
+ },
+ {
+ "entropy": 1.3399546146392822,
+ "epoch": 2.5232323232323233,
+ "grad_norm": 2.3238017559051514,
+ "learning_rate": 2.478787878787879e-06,
+ "loss": 1.337552547454834,
+ "mean_token_accuracy": 0.6790425181388855,
+ "num_tokens": 20463616.0,
+ "step": 1249
+ },
+ {
+ "entropy": 1.0954687595367432,
+ "epoch": 2.525252525252525,
+ "grad_norm": 2.2266361713409424,
+ "learning_rate": 2.476767676767677e-06,
+ "loss": 1.074552297592163,
+ "mean_token_accuracy": 0.7192232608795166,
+ "num_tokens": 20480000.0,
+ "step": 1250
+ },
+ {
+ "epoch": 2.525252525252525,
+ "eval_entropy": 1.375552624464035,
+ "eval_loss": 1.5356626510620117,
+ "eval_mean_token_accuracy": 0.6455749580937047,
+ "eval_num_tokens": 20480000.0,
+ "eval_runtime": 43.8024,
+ "eval_samples_per_second": 22.602,
+ "eval_steps_per_second": 2.831,
+ "step": 1250
+ },
+ {
+ "entropy": 1.329379916191101,
+ "epoch": 2.5272727272727273,
+ "grad_norm": 2.288421392440796,
+ "learning_rate": 2.474747474747475e-06,
+ "loss": 1.3166120052337646,
+ "mean_token_accuracy": 0.6743404865264893,
+ "num_tokens": 20496384.0,
+ "step": 1251
+ },
+ {
+ "entropy": 1.3155947923660278,
+ "epoch": 2.529292929292929,
+ "grad_norm": 2.2791004180908203,
+ "learning_rate": 2.472727272727273e-06,
+ "loss": 1.3281152248382568,
+ "mean_token_accuracy": 0.6623717546463013,
+ "num_tokens": 20512768.0,
+ "step": 1252
+ },
+ {
+ "entropy": 1.2875827550888062,
+ "epoch": 2.5313131313131314,
+ "grad_norm": 2.193981409072876,
+ "learning_rate": 2.470707070707071e-06,
+ "loss": 1.280174732208252,
+ "mean_token_accuracy": 0.6996824741363525,
+ "num_tokens": 20529152.0,
+ "step": 1253
+ },
+ {
+ "entropy": 1.1551588773727417,
+ "epoch": 2.533333333333333,
+ "grad_norm": 2.297534942626953,
+ "learning_rate": 2.468686868686869e-06,
+ "loss": 1.1548980474472046,
+ "mean_token_accuracy": 0.7089643478393555,
+ "num_tokens": 20545536.0,
+ "step": 1254
+ },
+ {
+ "entropy": 1.1836578845977783,
+ "epoch": 2.5353535353535355,
+ "grad_norm": 2.1724929809570312,
+ "learning_rate": 2.466666666666667e-06,
+ "loss": 1.1787464618682861,
+ "mean_token_accuracy": 0.705422580242157,
+ "num_tokens": 20561920.0,
+ "step": 1255
+ },
+ {
+ "entropy": 1.0938197374343872,
+ "epoch": 2.5373737373737373,
+ "grad_norm": 2.247507095336914,
+ "learning_rate": 2.464646464646465e-06,
+ "loss": 1.0977723598480225,
+ "mean_token_accuracy": 0.7205055952072144,
+ "num_tokens": 20578304.0,
+ "step": 1256
+ },
+ {
+ "entropy": 1.790277361869812,
+ "epoch": 2.5393939393939395,
+ "grad_norm": 2.314159631729126,
+ "learning_rate": 2.462626262626263e-06,
+ "loss": 1.792272925376892,
+ "mean_token_accuracy": 0.6231069564819336,
+ "num_tokens": 20594688.0,
+ "step": 1257
+ },
+ {
+ "entropy": 1.385352611541748,
+ "epoch": 2.5414141414141413,
+ "grad_norm": 2.2065250873565674,
+ "learning_rate": 2.4606060606060607e-06,
+ "loss": 1.4161381721496582,
+ "mean_token_accuracy": 0.673363447189331,
+ "num_tokens": 20611072.0,
+ "step": 1258
+ },
+ {
+ "entropy": 1.5186140537261963,
+ "epoch": 2.5434343434343436,
+ "grad_norm": 2.277163028717041,
+ "learning_rate": 2.4585858585858586e-06,
+ "loss": 1.5157082080841064,
+ "mean_token_accuracy": 0.6420371532440186,
+ "num_tokens": 20627456.0,
+ "step": 1259
+ },
+ {
+ "entropy": 1.6179429292678833,
+ "epoch": 2.5454545454545454,
+ "grad_norm": 2.1880221366882324,
+ "learning_rate": 2.456565656565657e-06,
+ "loss": 1.6365107297897339,
+ "mean_token_accuracy": 0.6164509057998657,
+ "num_tokens": 20643840.0,
+ "step": 1260
+ },
+ {
+ "entropy": 1.5839602947235107,
+ "epoch": 2.5474747474747472,
+ "grad_norm": 2.3658924102783203,
+ "learning_rate": 2.454545454545455e-06,
+ "loss": 1.593346118927002,
+ "mean_token_accuracy": 0.6339765787124634,
+ "num_tokens": 20660224.0,
+ "step": 1261
+ },
+ {
+ "entropy": 1.3825792074203491,
+ "epoch": 2.5494949494949495,
+ "grad_norm": 2.3293564319610596,
+ "learning_rate": 2.4525252525252527e-06,
+ "loss": 1.44181489944458,
+ "mean_token_accuracy": 0.6660356521606445,
+ "num_tokens": 20676608.0,
+ "step": 1262
+ },
+ {
+ "entropy": 1.0796198844909668,
+ "epoch": 2.5515151515151517,
+ "grad_norm": 2.1055870056152344,
+ "learning_rate": 2.4505050505050506e-06,
+ "loss": 1.0861892700195312,
+ "mean_token_accuracy": 0.7263067960739136,
+ "num_tokens": 20692992.0,
+ "step": 1263
+ },
+ {
+ "entropy": 1.4059629440307617,
+ "epoch": 2.5535353535353535,
+ "grad_norm": 2.137063980102539,
+ "learning_rate": 2.4484848484848485e-06,
+ "loss": 1.4127235412597656,
+ "mean_token_accuracy": 0.6654250025749207,
+ "num_tokens": 20709376.0,
+ "step": 1264
+ },
+ {
+ "entropy": 1.2902467250823975,
+ "epoch": 2.5555555555555554,
+ "grad_norm": 2.175401210784912,
+ "learning_rate": 2.4464646464646468e-06,
+ "loss": 1.2854642868041992,
+ "mean_token_accuracy": 0.6966292262077332,
+ "num_tokens": 20725760.0,
+ "step": 1265
+ },
+ {
+ "entropy": 1.7958943843841553,
+ "epoch": 2.5575757575757576,
+ "grad_norm": 2.3500523567199707,
+ "learning_rate": 2.4444444444444447e-06,
+ "loss": 1.811906337738037,
+ "mean_token_accuracy": 0.5961162447929382,
+ "num_tokens": 20742144.0,
+ "step": 1266
+ },
+ {
+ "entropy": 1.7406131029129028,
+ "epoch": 2.55959595959596,
+ "grad_norm": 2.3911163806915283,
+ "learning_rate": 2.4424242424242426e-06,
+ "loss": 1.7608356475830078,
+ "mean_token_accuracy": 0.6002076268196106,
+ "num_tokens": 20758528.0,
+ "step": 1267
+ },
+ {
+ "entropy": 1.440137267112732,
+ "epoch": 2.5616161616161617,
+ "grad_norm": 2.057145833969116,
+ "learning_rate": 2.4404040404040404e-06,
+ "loss": 1.4450485706329346,
+ "mean_token_accuracy": 0.6570591330528259,
+ "num_tokens": 20774912.0,
+ "step": 1268
+ },
+ {
+ "entropy": 1.510685682296753,
+ "epoch": 2.5636363636363635,
+ "grad_norm": 2.379920244216919,
+ "learning_rate": 2.4383838383838383e-06,
+ "loss": 1.5113954544067383,
+ "mean_token_accuracy": 0.6450293064117432,
+ "num_tokens": 20791296.0,
+ "step": 1269
+ },
+ {
+ "entropy": 1.3595963716506958,
+ "epoch": 2.5656565656565657,
+ "grad_norm": 2.226229190826416,
+ "learning_rate": 2.4363636363636366e-06,
+ "loss": 1.3598928451538086,
+ "mean_token_accuracy": 0.6723864078521729,
+ "num_tokens": 20807680.0,
+ "step": 1270
+ },
+ {
+ "entropy": 1.3053373098373413,
+ "epoch": 2.5676767676767676,
+ "grad_norm": 2.181692361831665,
+ "learning_rate": 2.4343434343434345e-06,
+ "loss": 1.283501148223877,
+ "mean_token_accuracy": 0.672874927520752,
+ "num_tokens": 20824064.0,
+ "step": 1271
+ },
+ {
+ "entropy": 0.9631388187408447,
+ "epoch": 2.56969696969697,
+ "grad_norm": 2.083442449569702,
+ "learning_rate": 2.432323232323233e-06,
+ "loss": 0.9463216066360474,
+ "mean_token_accuracy": 0.7476184368133545,
+ "num_tokens": 20840448.0,
+ "step": 1272
+ },
+ {
+ "entropy": 1.0792899131774902,
+ "epoch": 2.5717171717171716,
+ "grad_norm": 2.204257011413574,
+ "learning_rate": 2.4303030303030307e-06,
+ "loss": 1.0787214040756226,
+ "mean_token_accuracy": 0.7241694927215576,
+ "num_tokens": 20856832.0,
+ "step": 1273
+ },
+ {
+ "entropy": 1.0843226909637451,
+ "epoch": 2.573737373737374,
+ "grad_norm": 2.0145976543426514,
+ "learning_rate": 2.4282828282828286e-06,
+ "loss": 1.0722554922103882,
+ "mean_token_accuracy": 0.7263678312301636,
+ "num_tokens": 20873216.0,
+ "step": 1274
+ },
+ {
+ "entropy": 1.5040221214294434,
+ "epoch": 2.5757575757575757,
+ "grad_norm": 2.3076000213623047,
+ "learning_rate": 2.4262626262626265e-06,
+ "loss": 1.5152506828308105,
+ "mean_token_accuracy": 0.6420371532440186,
+ "num_tokens": 20889600.0,
+ "step": 1275
+ },
+ {
+ "entropy": 1.5882561206817627,
+ "epoch": 2.5777777777777775,
+ "grad_norm": 2.093400716781616,
+ "learning_rate": 2.4242424242424244e-06,
+ "loss": 1.5827500820159912,
+ "mean_token_accuracy": 0.6293356418609619,
+ "num_tokens": 20905984.0,
+ "step": 1276
+ },
+ {
+ "entropy": 1.1761770248413086,
+ "epoch": 2.5797979797979798,
+ "grad_norm": 1.9723676443099976,
+ "learning_rate": 2.4222222222222223e-06,
+ "loss": 1.1729084253311157,
+ "mean_token_accuracy": 0.7160478830337524,
+ "num_tokens": 20922368.0,
+ "step": 1277
+ },
+ {
+ "entropy": 1.5020796060562134,
+ "epoch": 2.581818181818182,
+ "grad_norm": 2.198739767074585,
+ "learning_rate": 2.4202020202020206e-06,
+ "loss": 1.5012025833129883,
+ "mean_token_accuracy": 0.6408768892288208,
+ "num_tokens": 20938752.0,
+ "step": 1278
+ },
+ {
+ "entropy": 1.0234061479568481,
+ "epoch": 2.583838383838384,
+ "grad_norm": 2.0784571170806885,
+ "learning_rate": 2.4181818181818185e-06,
+ "loss": 1.0343588590621948,
+ "mean_token_accuracy": 0.7372984886169434,
+ "num_tokens": 20955136.0,
+ "step": 1279
+ },
+ {
+ "entropy": 1.6372766494750977,
+ "epoch": 2.5858585858585856,
+ "grad_norm": 2.339104652404785,
+ "learning_rate": 2.4161616161616164e-06,
+ "loss": 1.6200227737426758,
+ "mean_token_accuracy": 0.6249389052391052,
+ "num_tokens": 20971520.0,
+ "step": 1280
+ },
+ {
+ "entropy": 1.3991668224334717,
+ "epoch": 2.587878787878788,
+ "grad_norm": 2.33329439163208,
+ "learning_rate": 2.4141414141414143e-06,
+ "loss": 1.434525489807129,
+ "mean_token_accuracy": 0.6656692624092102,
+ "num_tokens": 20987904.0,
+ "step": 1281
+ },
+ {
+ "entropy": 0.9680843949317932,
+ "epoch": 2.58989898989899,
+ "grad_norm": 2.0454914569854736,
+ "learning_rate": 2.412121212121212e-06,
+ "loss": 0.9875590801239014,
+ "mean_token_accuracy": 0.7512823939323425,
+ "num_tokens": 21004288.0,
+ "step": 1282
+ },
+ {
+ "entropy": 1.1564005613327026,
+ "epoch": 2.591919191919192,
+ "grad_norm": 2.0994997024536133,
+ "learning_rate": 2.4101010101010105e-06,
+ "loss": 1.1473734378814697,
+ "mean_token_accuracy": 0.709208607673645,
+ "num_tokens": 21020672.0,
+ "step": 1283
+ },
+ {
+ "entropy": 1.435438871383667,
+ "epoch": 2.5939393939393938,
+ "grad_norm": 2.369999408721924,
+ "learning_rate": 2.4080808080808083e-06,
+ "loss": 1.4502406120300293,
+ "mean_token_accuracy": 0.6505251526832581,
+ "num_tokens": 21037056.0,
+ "step": 1284
+ },
+ {
+ "entropy": 1.358853816986084,
+ "epoch": 2.595959595959596,
+ "grad_norm": 4.034506320953369,
+ "learning_rate": 2.4060606060606062e-06,
+ "loss": 1.3811532258987427,
+ "mean_token_accuracy": 0.6720200181007385,
+ "num_tokens": 21053440.0,
+ "step": 1285
+ },
+ {
+ "entropy": 1.2741261720657349,
+ "epoch": 2.597979797979798,
+ "grad_norm": 2.231471061706543,
+ "learning_rate": 2.404040404040404e-06,
+ "loss": 1.2811863422393799,
+ "mean_token_accuracy": 0.6806302070617676,
+ "num_tokens": 21069824.0,
+ "step": 1286
+ },
+ {
+ "entropy": 1.3789101839065552,
+ "epoch": 2.6,
+ "grad_norm": 2.287457227706909,
+ "learning_rate": 2.402020202020202e-06,
+ "loss": 1.389050006866455,
+ "mean_token_accuracy": 0.6687836050987244,
+ "num_tokens": 21086208.0,
+ "step": 1287
+ },
+ {
+ "entropy": 1.0021989345550537,
+ "epoch": 2.602020202020202,
+ "grad_norm": 1.9996925592422485,
+ "learning_rate": 2.4000000000000003e-06,
+ "loss": 1.028086543083191,
+ "mean_token_accuracy": 0.7479237914085388,
+ "num_tokens": 21102592.0,
+ "step": 1288
+ },
+ {
+ "entropy": 1.4363199472427368,
+ "epoch": 2.604040404040404,
+ "grad_norm": 2.203239917755127,
+ "learning_rate": 2.397979797979798e-06,
+ "loss": 1.4645861387252808,
+ "mean_token_accuracy": 0.6590132117271423,
+ "num_tokens": 21118976.0,
+ "step": 1289
+ },
+ {
+ "entropy": 1.307484745979309,
+ "epoch": 2.606060606060606,
+ "grad_norm": 1.9925199747085571,
+ "learning_rate": 2.395959595959596e-06,
+ "loss": 1.3217129707336426,
+ "mean_token_accuracy": 0.6787371635437012,
+ "num_tokens": 21135360.0,
+ "step": 1290
+ },
+ {
+ "entropy": 1.2430635690689087,
+ "epoch": 2.608080808080808,
+ "grad_norm": 2.1385529041290283,
+ "learning_rate": 2.393939393939394e-06,
+ "loss": 1.238807201385498,
+ "mean_token_accuracy": 0.6949804425239563,
+ "num_tokens": 21151744.0,
+ "step": 1291
+ },
+ {
+ "entropy": 1.0351316928863525,
+ "epoch": 2.61010101010101,
+ "grad_norm": 2.2897744178771973,
+ "learning_rate": 2.3919191919191923e-06,
+ "loss": 1.032408595085144,
+ "mean_token_accuracy": 0.7267953157424927,
+ "num_tokens": 21168128.0,
+ "step": 1292
+ },
+ {
+ "entropy": 1.2167868614196777,
+ "epoch": 2.6121212121212123,
+ "grad_norm": 2.021771192550659,
+ "learning_rate": 2.38989898989899e-06,
+ "loss": 1.223731279373169,
+ "mean_token_accuracy": 0.7067049145698547,
+ "num_tokens": 21184512.0,
+ "step": 1293
+ },
+ {
+ "entropy": 1.0386477708816528,
+ "epoch": 2.614141414141414,
+ "grad_norm": 2.048285722732544,
+ "learning_rate": 2.387878787878788e-06,
+ "loss": 1.0437978506088257,
+ "mean_token_accuracy": 0.7332682013511658,
+ "num_tokens": 21200896.0,
+ "step": 1294
+ },
+ {
+ "entropy": 1.5832154750823975,
+ "epoch": 2.616161616161616,
+ "grad_norm": 2.214010715484619,
+ "learning_rate": 2.385858585858586e-06,
+ "loss": 1.5869958400726318,
+ "mean_token_accuracy": 0.6240839958190918,
+ "num_tokens": 21217280.0,
+ "step": 1295
+ },
+ {
+ "entropy": 1.4032398462295532,
+ "epoch": 2.618181818181818,
+ "grad_norm": 2.069399833679199,
+ "learning_rate": 2.3838383838383843e-06,
+ "loss": 1.40470290184021,
+ "mean_token_accuracy": 0.6671348214149475,
+ "num_tokens": 21233664.0,
+ "step": 1296
+ },
+ {
+ "entropy": 1.1057566404342651,
+ "epoch": 2.6202020202020204,
+ "grad_norm": 2.090847969055176,
+ "learning_rate": 2.381818181818182e-06,
+ "loss": 1.099064826965332,
+ "mean_token_accuracy": 0.7186736464500427,
+ "num_tokens": 21250048.0,
+ "step": 1297
+ },
+ {
+ "entropy": 1.1217924356460571,
+ "epoch": 2.6222222222222222,
+ "grad_norm": 2.2802088260650635,
+ "learning_rate": 2.37979797979798e-06,
+ "loss": 1.1425195932388306,
+ "mean_token_accuracy": 0.717452347278595,
+ "num_tokens": 21266432.0,
+ "step": 1298
+ },
+ {
+ "entropy": 1.3103245496749878,
+ "epoch": 2.624242424242424,
+ "grad_norm": 2.2961838245391846,
+ "learning_rate": 2.377777777777778e-06,
+ "loss": 1.325084924697876,
+ "mean_token_accuracy": 0.6704323291778564,
+ "num_tokens": 21282816.0,
+ "step": 1299
+ },
+ {
+ "entropy": 1.397556185722351,
+ "epoch": 2.6262626262626263,
+ "grad_norm": 2.14385986328125,
+ "learning_rate": 2.375757575757576e-06,
+ "loss": 1.3891851902008057,
+ "mean_token_accuracy": 0.6722642779350281,
+ "num_tokens": 21299200.0,
+ "step": 1300
+ },
+ {
+ "epoch": 2.6262626262626263,
+ "eval_entropy": 1.377457697064646,
+ "eval_loss": 1.5343767404556274,
+ "eval_mean_token_accuracy": 0.6456591679203895,
+ "eval_num_tokens": 21299200.0,
+ "eval_runtime": 43.8661,
+ "eval_samples_per_second": 22.569,
+ "eval_steps_per_second": 2.827,
+ "step": 1300
+ },
+ {
+ "entropy": 1.4542138576507568,
+ "epoch": 2.6282828282828286,
+ "grad_norm": 2.2326884269714355,
+ "learning_rate": 2.373737373737374e-06,
+ "loss": 1.459869384765625,
+ "mean_token_accuracy": 0.6545554399490356,
+ "num_tokens": 21315584.0,
+ "step": 1301
+ },
+ {
+ "entropy": 1.144904613494873,
+ "epoch": 2.6303030303030304,
+ "grad_norm": 2.0662593841552734,
+ "learning_rate": 2.371717171717172e-06,
+ "loss": 1.151012897491455,
+ "mean_token_accuracy": 0.7232535481452942,
+ "num_tokens": 21331968.0,
+ "step": 1302
+ },
+ {
+ "entropy": 1.2142722606658936,
+ "epoch": 2.632323232323232,
+ "grad_norm": 2.206763982772827,
+ "learning_rate": 2.36969696969697e-06,
+ "loss": 1.2040069103240967,
+ "mean_token_accuracy": 0.7002320289611816,
+ "num_tokens": 21348352.0,
+ "step": 1303
+ },
+ {
+ "entropy": 1.7695715427398682,
+ "epoch": 2.6343434343434344,
+ "grad_norm": 2.1346793174743652,
+ "learning_rate": 2.367676767676768e-06,
+ "loss": 1.7356187105178833,
+ "mean_token_accuracy": 0.6051538586616516,
+ "num_tokens": 21364736.0,
+ "step": 1304
+ },
+ {
+ "entropy": 1.351786494255066,
+ "epoch": 2.6363636363636362,
+ "grad_norm": 2.291290283203125,
+ "learning_rate": 2.3656565656565657e-06,
+ "loss": 1.3471777439117432,
+ "mean_token_accuracy": 0.6797142028808594,
+ "num_tokens": 21381120.0,
+ "step": 1305
+ },
+ {
+ "entropy": 1.202124834060669,
+ "epoch": 2.6383838383838385,
+ "grad_norm": 2.1541965007781982,
+ "learning_rate": 2.363636363636364e-06,
+ "loss": 1.2085479497909546,
+ "mean_token_accuracy": 0.7031631469726562,
+ "num_tokens": 21397504.0,
+ "step": 1306
+ },
+ {
+ "entropy": 0.9269154667854309,
+ "epoch": 2.6404040404040403,
+ "grad_norm": 2.1681666374206543,
+ "learning_rate": 2.361616161616162e-06,
+ "loss": 0.9319549798965454,
+ "mean_token_accuracy": 0.7553126811981201,
+ "num_tokens": 21413888.0,
+ "step": 1307
+ },
+ {
+ "entropy": 1.0244951248168945,
+ "epoch": 2.6424242424242426,
+ "grad_norm": 1.9392226934432983,
+ "learning_rate": 2.3595959595959598e-06,
+ "loss": 0.9993175268173218,
+ "mean_token_accuracy": 0.7588544487953186,
+ "num_tokens": 21430272.0,
+ "step": 1308
+ },
+ {
+ "entropy": 1.256042718887329,
+ "epoch": 2.6444444444444444,
+ "grad_norm": 2.346327304840088,
+ "learning_rate": 2.3575757575757577e-06,
+ "loss": 1.2512779235839844,
+ "mean_token_accuracy": 0.6858206987380981,
+ "num_tokens": 21446656.0,
+ "step": 1309
+ },
+ {
+ "entropy": 1.0318644046783447,
+ "epoch": 2.6464646464646466,
+ "grad_norm": 1.988356113433838,
+ "learning_rate": 2.3555555555555555e-06,
+ "loss": 1.0428593158721924,
+ "mean_token_accuracy": 0.7383365631103516,
+ "num_tokens": 21463040.0,
+ "step": 1310
+ },
+ {
+ "entropy": 1.3145314455032349,
+ "epoch": 2.6484848484848484,
+ "grad_norm": 2.1777143478393555,
+ "learning_rate": 2.3535353535353534e-06,
+ "loss": 1.3207441568374634,
+ "mean_token_accuracy": 0.6881411671638489,
+ "num_tokens": 21479424.0,
+ "step": 1311
+ },
+ {
+ "entropy": 1.2858450412750244,
+ "epoch": 2.6505050505050507,
+ "grad_norm": 2.2654671669006348,
+ "learning_rate": 2.3515151515151517e-06,
+ "loss": 1.2940181493759155,
+ "mean_token_accuracy": 0.6831338405609131,
+ "num_tokens": 21495808.0,
+ "step": 1312
+ },
+ {
+ "entropy": 1.4315612316131592,
+ "epoch": 2.6525252525252525,
+ "grad_norm": 2.140456438064575,
+ "learning_rate": 2.3494949494949496e-06,
+ "loss": 1.4298607110977173,
+ "mean_token_accuracy": 0.6585246920585632,
+ "num_tokens": 21512192.0,
+ "step": 1313
+ },
+ {
+ "entropy": 1.2628042697906494,
+ "epoch": 2.6545454545454543,
+ "grad_norm": 2.387079954147339,
+ "learning_rate": 2.347474747474748e-06,
+ "loss": 1.2526830434799194,
+ "mean_token_accuracy": 0.6882632970809937,
+ "num_tokens": 21528576.0,
+ "step": 1314
+ },
+ {
+ "entropy": 1.3501923084259033,
+ "epoch": 2.6565656565656566,
+ "grad_norm": 2.17873215675354,
+ "learning_rate": 2.345454545454546e-06,
+ "loss": 1.3459683656692505,
+ "mean_token_accuracy": 0.6783707737922668,
+ "num_tokens": 21544960.0,
+ "step": 1315
+ },
+ {
+ "entropy": 1.4369540214538574,
+ "epoch": 2.658585858585859,
+ "grad_norm": 2.335825204849243,
+ "learning_rate": 2.3434343434343437e-06,
+ "loss": 1.4296600818634033,
+ "mean_token_accuracy": 0.6534562706947327,
+ "num_tokens": 21561344.0,
+ "step": 1316
+ },
+ {
+ "entropy": 1.1612765789031982,
+ "epoch": 2.6606060606060606,
+ "grad_norm": 2.194164276123047,
+ "learning_rate": 2.3414141414141416e-06,
+ "loss": 1.1527385711669922,
+ "mean_token_accuracy": 0.7140327095985413,
+ "num_tokens": 21577728.0,
+ "step": 1317
+ },
+ {
+ "entropy": 1.0844523906707764,
+ "epoch": 2.6626262626262625,
+ "grad_norm": 2.1018459796905518,
+ "learning_rate": 2.3393939393939395e-06,
+ "loss": 1.0732126235961914,
+ "mean_token_accuracy": 0.7245969772338867,
+ "num_tokens": 21594112.0,
+ "step": 1318
+ },
+ {
+ "entropy": 1.1760655641555786,
+ "epoch": 2.6646464646464647,
+ "grad_norm": 2.1674582958221436,
+ "learning_rate": 2.337373737373738e-06,
+ "loss": 1.1928012371063232,
+ "mean_token_accuracy": 0.693453848361969,
+ "num_tokens": 21610496.0,
+ "step": 1319
+ },
+ {
+ "entropy": 1.3627854585647583,
+ "epoch": 2.6666666666666665,
+ "grad_norm": 2.2469325065612793,
+ "learning_rate": 2.3353535353535357e-06,
+ "loss": 1.375096321105957,
+ "mean_token_accuracy": 0.6670737862586975,
+ "num_tokens": 21626880.0,
+ "step": 1320
+ },
+ {
+ "entropy": 1.0562385320663452,
+ "epoch": 2.6686868686868688,
+ "grad_norm": 2.3479180335998535,
+ "learning_rate": 2.3333333333333336e-06,
+ "loss": 1.0513578653335571,
+ "mean_token_accuracy": 0.7401685118675232,
+ "num_tokens": 21643264.0,
+ "step": 1321
+ },
+ {
+ "entropy": 1.1622484922409058,
+ "epoch": 2.6707070707070706,
+ "grad_norm": 2.218021869659424,
+ "learning_rate": 2.3313131313131315e-06,
+ "loss": 1.172579288482666,
+ "mean_token_accuracy": 0.7042623162269592,
+ "num_tokens": 21659648.0,
+ "step": 1322
+ },
+ {
+ "entropy": 1.2139835357666016,
+ "epoch": 2.672727272727273,
+ "grad_norm": 2.2875492572784424,
+ "learning_rate": 2.3292929292929294e-06,
+ "loss": 1.2015337944030762,
+ "mean_token_accuracy": 0.7027357220649719,
+ "num_tokens": 21676032.0,
+ "step": 1323
+ },
+ {
+ "entropy": 1.7094130516052246,
+ "epoch": 2.6747474747474747,
+ "grad_norm": 2.2344024181365967,
+ "learning_rate": 2.3272727272727277e-06,
+ "loss": 1.7565287351608276,
+ "mean_token_accuracy": 0.6025891304016113,
+ "num_tokens": 21692416.0,
+ "step": 1324
+ },
+ {
+ "entropy": 1.1033966541290283,
+ "epoch": 2.676767676767677,
+ "grad_norm": 2.2697556018829346,
+ "learning_rate": 2.3252525252525256e-06,
+ "loss": 1.1173789501190186,
+ "mean_token_accuracy": 0.7180629968643188,
+ "num_tokens": 21708800.0,
+ "step": 1325
+ },
+ {
+ "entropy": 0.9456706643104553,
+ "epoch": 2.6787878787878787,
+ "grad_norm": 2.1194941997528076,
+ "learning_rate": 2.3232323232323234e-06,
+ "loss": 0.9613161087036133,
+ "mean_token_accuracy": 0.7573277950286865,
+ "num_tokens": 21725184.0,
+ "step": 1326
+ },
+ {
+ "entropy": 1.098917841911316,
+ "epoch": 2.680808080808081,
+ "grad_norm": 2.2719058990478516,
+ "learning_rate": 2.3212121212121213e-06,
+ "loss": 1.116957664489746,
+ "mean_token_accuracy": 0.709269642829895,
+ "num_tokens": 21741568.0,
+ "step": 1327
+ },
+ {
+ "entropy": 1.5637823343276978,
+ "epoch": 2.682828282828283,
+ "grad_norm": 2.257596731185913,
+ "learning_rate": 2.3191919191919192e-06,
+ "loss": 1.5726745128631592,
+ "mean_token_accuracy": 0.6359306573867798,
+ "num_tokens": 21757952.0,
+ "step": 1328
+ },
+ {
+ "entropy": 0.9321528673171997,
+ "epoch": 2.6848484848484846,
+ "grad_norm": 2.0641798973083496,
+ "learning_rate": 2.317171717171717e-06,
+ "loss": 0.9297301769256592,
+ "mean_token_accuracy": 0.7602589130401611,
+ "num_tokens": 21774336.0,
+ "step": 1329
+ },
+ {
+ "entropy": 1.2314496040344238,
+ "epoch": 2.686868686868687,
+ "grad_norm": 2.143329620361328,
+ "learning_rate": 2.3151515151515154e-06,
+ "loss": 1.2292556762695312,
+ "mean_token_accuracy": 0.6940034031867981,
+ "num_tokens": 21790720.0,
+ "step": 1330
+ },
+ {
+ "entropy": 1.195270299911499,
+ "epoch": 2.688888888888889,
+ "grad_norm": 2.291335105895996,
+ "learning_rate": 2.3131313131313133e-06,
+ "loss": 1.1904417276382446,
+ "mean_token_accuracy": 0.7023082375526428,
+ "num_tokens": 21807104.0,
+ "step": 1331
+ },
+ {
+ "entropy": 0.8783636093139648,
+ "epoch": 2.690909090909091,
+ "grad_norm": 2.069147825241089,
+ "learning_rate": 2.311111111111111e-06,
+ "loss": 0.8936513066291809,
+ "mean_token_accuracy": 0.7703346610069275,
+ "num_tokens": 21823488.0,
+ "step": 1332
+ },
+ {
+ "entropy": 1.3100625276565552,
+ "epoch": 2.6929292929292927,
+ "grad_norm": 2.396043300628662,
+ "learning_rate": 2.309090909090909e-06,
+ "loss": 1.3054237365722656,
+ "mean_token_accuracy": 0.6758671402931213,
+ "num_tokens": 21839872.0,
+ "step": 1333
+ },
+ {
+ "entropy": 0.9413285255432129,
+ "epoch": 2.694949494949495,
+ "grad_norm": 2.0738699436187744,
+ "learning_rate": 2.307070707070707e-06,
+ "loss": 0.9171029329299927,
+ "mean_token_accuracy": 0.7571446299552917,
+ "num_tokens": 21856256.0,
+ "step": 1334
+ },
+ {
+ "entropy": 1.268250584602356,
+ "epoch": 2.6969696969696972,
+ "grad_norm": 2.0988986492156982,
+ "learning_rate": 2.3050505050505053e-06,
+ "loss": 1.257792353630066,
+ "mean_token_accuracy": 0.6866145730018616,
+ "num_tokens": 21872640.0,
+ "step": 1335
+ },
+ {
+ "entropy": 1.3402502536773682,
+ "epoch": 2.698989898989899,
+ "grad_norm": 2.08109712600708,
+ "learning_rate": 2.303030303030303e-06,
+ "loss": 1.3316476345062256,
+ "mean_token_accuracy": 0.6869198679924011,
+ "num_tokens": 21889024.0,
+ "step": 1336
+ },
+ {
+ "entropy": 1.5612709522247314,
+ "epoch": 2.701010101010101,
+ "grad_norm": 2.2410786151885986,
+ "learning_rate": 2.3010101010101015e-06,
+ "loss": 1.5604108572006226,
+ "mean_token_accuracy": 0.6339765787124634,
+ "num_tokens": 21905408.0,
+ "step": 1337
+ },
+ {
+ "entropy": 1.0962555408477783,
+ "epoch": 2.703030303030303,
+ "grad_norm": 2.289069890975952,
+ "learning_rate": 2.2989898989898994e-06,
+ "loss": 1.1016948223114014,
+ "mean_token_accuracy": 0.714093804359436,
+ "num_tokens": 21921792.0,
+ "step": 1338
+ },
+ {
+ "entropy": 1.600198745727539,
+ "epoch": 2.705050505050505,
+ "grad_norm": 2.152244806289673,
+ "learning_rate": 2.2969696969696973e-06,
+ "loss": 1.608877420425415,
+ "mean_token_accuracy": 0.6394724249839783,
+ "num_tokens": 21938176.0,
+ "step": 1339
+ },
+ {
+ "entropy": 1.4823815822601318,
+ "epoch": 2.707070707070707,
+ "grad_norm": 2.142822742462158,
+ "learning_rate": 2.294949494949495e-06,
+ "loss": 1.4775569438934326,
+ "mean_token_accuracy": 0.6656082272529602,
+ "num_tokens": 21954560.0,
+ "step": 1340
+ },
+ {
+ "entropy": 1.1100481748580933,
+ "epoch": 2.709090909090909,
+ "grad_norm": 2.271402597427368,
+ "learning_rate": 2.292929292929293e-06,
+ "loss": 1.1097328662872314,
+ "mean_token_accuracy": 0.7238031029701233,
+ "num_tokens": 21970944.0,
+ "step": 1341
+ },
+ {
+ "entropy": 1.3482670783996582,
+ "epoch": 2.7111111111111112,
+ "grad_norm": 2.136296510696411,
+ "learning_rate": 2.2909090909090913e-06,
+ "loss": 1.3715920448303223,
+ "mean_token_accuracy": 0.670615553855896,
+ "num_tokens": 21987328.0,
+ "step": 1342
+ },
+ {
+ "entropy": 1.3454313278198242,
+ "epoch": 2.713131313131313,
+ "grad_norm": 1.8919764757156372,
+ "learning_rate": 2.2888888888888892e-06,
+ "loss": 1.3545129299163818,
+ "mean_token_accuracy": 0.6950415372848511,
+ "num_tokens": 22003712.0,
+ "step": 1343
+ },
+ {
+ "entropy": 1.314072847366333,
+ "epoch": 2.7151515151515153,
+ "grad_norm": 2.0495684146881104,
+ "learning_rate": 2.286868686868687e-06,
+ "loss": 1.3117344379425049,
+ "mean_token_accuracy": 0.6800195574760437,
+ "num_tokens": 22020096.0,
+ "step": 1344
+ },
+ {
+ "entropy": 0.7977169156074524,
+ "epoch": 2.717171717171717,
+ "grad_norm": 2.050006866455078,
+ "learning_rate": 2.284848484848485e-06,
+ "loss": 0.7982609868049622,
+ "mean_token_accuracy": 0.7774181962013245,
+ "num_tokens": 22036480.0,
+ "step": 1345
+ },
+ {
+ "entropy": 1.6121824979782104,
+ "epoch": 2.7191919191919194,
+ "grad_norm": 2.1058602333068848,
+ "learning_rate": 2.282828282828283e-06,
+ "loss": 1.6340572834014893,
+ "mean_token_accuracy": 0.6278700828552246,
+ "num_tokens": 22052864.0,
+ "step": 1346
+ },
+ {
+ "entropy": 1.2698943614959717,
+ "epoch": 2.721212121212121,
+ "grad_norm": 1.9856007099151611,
+ "learning_rate": 2.2808080808080808e-06,
+ "loss": 1.2857555150985718,
+ "mean_token_accuracy": 0.7088422179222107,
+ "num_tokens": 22069248.0,
+ "step": 1347
+ },
+ {
+ "entropy": 1.6614608764648438,
+ "epoch": 2.723232323232323,
+ "grad_norm": 2.1818666458129883,
+ "learning_rate": 2.278787878787879e-06,
+ "loss": 1.6615056991577148,
+ "mean_token_accuracy": 0.6217635273933411,
+ "num_tokens": 22085632.0,
+ "step": 1348
+ },
+ {
+ "entropy": 1.542864203453064,
+ "epoch": 2.7252525252525253,
+ "grad_norm": 2.1022093296051025,
+ "learning_rate": 2.276767676767677e-06,
+ "loss": 1.5696978569030762,
+ "mean_token_accuracy": 0.6462506055831909,
+ "num_tokens": 22102016.0,
+ "step": 1349
+ },
+ {
+ "entropy": 1.0613912343978882,
+ "epoch": 2.7272727272727275,
+ "grad_norm": 2.213463068008423,
+ "learning_rate": 2.274747474747475e-06,
+ "loss": 1.0789921283721924,
+ "mean_token_accuracy": 0.7288104295730591,
+ "num_tokens": 22118400.0,
+ "step": 1350
+ },
+ {
+ "epoch": 2.7272727272727275,
+ "eval_entropy": 1.362602738603469,
+ "eval_loss": 1.5345665216445923,
+ "eval_mean_token_accuracy": 0.6458812678052533,
+ "eval_num_tokens": 22118400.0,
+ "eval_runtime": 43.734,
+ "eval_samples_per_second": 22.637,
+ "eval_steps_per_second": 2.835,
+ "step": 1350
+ },
+ {
+ "entropy": 1.175439715385437,
+ "epoch": 2.7292929292929293,
+ "grad_norm": 2.3636603355407715,
+ "learning_rate": 2.2727272727272728e-06,
+ "loss": 1.2032190561294556,
+ "mean_token_accuracy": 0.689667820930481,
+ "num_tokens": 22134784.0,
+ "step": 1351
+ },
+ {
+ "entropy": 1.056649923324585,
+ "epoch": 2.731313131313131,
+ "grad_norm": 2.111971139907837,
+ "learning_rate": 2.2707070707070706e-06,
+ "loss": 1.0646127462387085,
+ "mean_token_accuracy": 0.7132999300956726,
+ "num_tokens": 22151168.0,
+ "step": 1352
+ },
+ {
+ "entropy": 1.401559591293335,
+ "epoch": 2.7333333333333334,
+ "grad_norm": 2.269573211669922,
+ "learning_rate": 2.268686868686869e-06,
+ "loss": 1.405207633972168,
+ "mean_token_accuracy": 0.6612115502357483,
+ "num_tokens": 22167552.0,
+ "step": 1353
+ },
+ {
+ "entropy": 0.9708455204963684,
+ "epoch": 2.735353535353535,
+ "grad_norm": 2.3020989894866943,
+ "learning_rate": 2.266666666666667e-06,
+ "loss": 0.9864540100097656,
+ "mean_token_accuracy": 0.7361993193626404,
+ "num_tokens": 22183936.0,
+ "step": 1354
+ },
+ {
+ "entropy": 1.2840858697891235,
+ "epoch": 2.7373737373737375,
+ "grad_norm": 2.4489457607269287,
+ "learning_rate": 2.2646464646464647e-06,
+ "loss": 1.2992041110992432,
+ "mean_token_accuracy": 0.6783097386360168,
+ "num_tokens": 22200320.0,
+ "step": 1355
+ },
+ {
+ "entropy": 1.6704803705215454,
+ "epoch": 2.7393939393939393,
+ "grad_norm": 2.213287591934204,
+ "learning_rate": 2.262626262626263e-06,
+ "loss": 1.7022172212600708,
+ "mean_token_accuracy": 0.6433194875717163,
+ "num_tokens": 22216704.0,
+ "step": 1356
+ },
+ {
+ "entropy": 1.2570815086364746,
+ "epoch": 2.7414141414141415,
+ "grad_norm": 2.250062942504883,
+ "learning_rate": 2.260606060606061e-06,
+ "loss": 1.2799533605575562,
+ "mean_token_accuracy": 0.6823400259017944,
+ "num_tokens": 22233088.0,
+ "step": 1357
+ },
+ {
+ "entropy": 1.0966694355010986,
+ "epoch": 2.7434343434343433,
+ "grad_norm": 2.24507212638855,
+ "learning_rate": 2.258585858585859e-06,
+ "loss": 1.0954806804656982,
+ "mean_token_accuracy": 0.7264289259910583,
+ "num_tokens": 22249472.0,
+ "step": 1358
+ },
+ {
+ "entropy": 1.6675825119018555,
+ "epoch": 2.7454545454545456,
+ "grad_norm": 2.1572930812835693,
+ "learning_rate": 2.2565656565656567e-06,
+ "loss": 1.6607719659805298,
+ "mean_token_accuracy": 0.6113214492797852,
+ "num_tokens": 22265856.0,
+ "step": 1359
+ },
+ {
+ "entropy": 1.247696042060852,
+ "epoch": 2.7474747474747474,
+ "grad_norm": 2.105797290802002,
+ "learning_rate": 2.254545454545455e-06,
+ "loss": 1.2314106225967407,
+ "mean_token_accuracy": 0.6982169151306152,
+ "num_tokens": 22282240.0,
+ "step": 1360
+ },
+ {
+ "entropy": 1.6081417798995972,
+ "epoch": 2.7494949494949497,
+ "grad_norm": 2.1624395847320557,
+ "learning_rate": 2.252525252525253e-06,
+ "loss": 1.6011556386947632,
+ "mean_token_accuracy": 0.6282364726066589,
+ "num_tokens": 22298624.0,
+ "step": 1361
+ },
+ {
+ "entropy": 1.1124155521392822,
+ "epoch": 2.7515151515151515,
+ "grad_norm": 2.141758918762207,
+ "learning_rate": 2.250505050505051e-06,
+ "loss": 1.1052089929580688,
+ "mean_token_accuracy": 0.7202003002166748,
+ "num_tokens": 22315008.0,
+ "step": 1362
+ },
+ {
+ "entropy": 1.093716025352478,
+ "epoch": 2.7535353535353533,
+ "grad_norm": 2.2610108852386475,
+ "learning_rate": 2.2484848484848487e-06,
+ "loss": 1.095299482345581,
+ "mean_token_accuracy": 0.714154839515686,
+ "num_tokens": 22331392.0,
+ "step": 1363
+ },
+ {
+ "entropy": 1.353977918624878,
+ "epoch": 2.7555555555555555,
+ "grad_norm": 2.171783447265625,
+ "learning_rate": 2.2464646464646466e-06,
+ "loss": 1.3497436046600342,
+ "mean_token_accuracy": 0.6914386749267578,
+ "num_tokens": 22347776.0,
+ "step": 1364
+ },
+ {
+ "entropy": 1.4595965147018433,
+ "epoch": 2.757575757575758,
+ "grad_norm": 2.2337677478790283,
+ "learning_rate": 2.2444444444444445e-06,
+ "loss": 1.4730737209320068,
+ "mean_token_accuracy": 0.6578529477119446,
+ "num_tokens": 22364160.0,
+ "step": 1365
+ },
+ {
+ "entropy": 1.5668566226959229,
+ "epoch": 2.7595959595959596,
+ "grad_norm": 2.2195982933044434,
+ "learning_rate": 2.2424242424242428e-06,
+ "loss": 1.5661592483520508,
+ "mean_token_accuracy": 0.6397166848182678,
+ "num_tokens": 22380544.0,
+ "step": 1366
+ },
+ {
+ "entropy": 1.4419786930084229,
+ "epoch": 2.7616161616161614,
+ "grad_norm": 2.202221393585205,
+ "learning_rate": 2.2404040404040407e-06,
+ "loss": 1.4466135501861572,
+ "mean_token_accuracy": 0.6698827743530273,
+ "num_tokens": 22396928.0,
+ "step": 1367
+ },
+ {
+ "entropy": 1.43394136428833,
+ "epoch": 2.7636363636363637,
+ "grad_norm": 2.1969001293182373,
+ "learning_rate": 2.2383838383838385e-06,
+ "loss": 1.433483362197876,
+ "mean_token_accuracy": 0.6655471324920654,
+ "num_tokens": 22413312.0,
+ "step": 1368
+ },
+ {
+ "entropy": 1.048527717590332,
+ "epoch": 2.765656565656566,
+ "grad_norm": 2.142869472503662,
+ "learning_rate": 2.2363636363636364e-06,
+ "loss": 1.0373283624649048,
+ "mean_token_accuracy": 0.7396189570426941,
+ "num_tokens": 22429696.0,
+ "step": 1369
+ },
+ {
+ "entropy": 1.4282819032669067,
+ "epoch": 2.7676767676767677,
+ "grad_norm": 2.1550097465515137,
+ "learning_rate": 2.2343434343434343e-06,
+ "loss": 1.419884443283081,
+ "mean_token_accuracy": 0.6712262034416199,
+ "num_tokens": 22446080.0,
+ "step": 1370
+ },
+ {
+ "entropy": 1.5361018180847168,
+ "epoch": 2.7696969696969695,
+ "grad_norm": 2.1175451278686523,
+ "learning_rate": 2.2323232323232326e-06,
+ "loss": 1.5433859825134277,
+ "mean_token_accuracy": 0.636907696723938,
+ "num_tokens": 22462464.0,
+ "step": 1371
+ },
+ {
+ "entropy": 1.326242208480835,
+ "epoch": 2.771717171717172,
+ "grad_norm": 2.084632635116577,
+ "learning_rate": 2.2303030303030305e-06,
+ "loss": 1.3364207744598389,
+ "mean_token_accuracy": 0.6809965968132019,
+ "num_tokens": 22478848.0,
+ "step": 1372
+ },
+ {
+ "entropy": 1.255878210067749,
+ "epoch": 2.7737373737373736,
+ "grad_norm": 2.142777919769287,
+ "learning_rate": 2.2282828282828284e-06,
+ "loss": 1.264795184135437,
+ "mean_token_accuracy": 0.6875916123390198,
+ "num_tokens": 22495232.0,
+ "step": 1373
+ },
+ {
+ "entropy": 1.2916159629821777,
+ "epoch": 2.775757575757576,
+ "grad_norm": 2.115644693374634,
+ "learning_rate": 2.2262626262626263e-06,
+ "loss": 1.2945363521575928,
+ "mean_token_accuracy": 0.6895456910133362,
+ "num_tokens": 22511616.0,
+ "step": 1374
+ },
+ {
+ "entropy": 1.4490385055541992,
+ "epoch": 2.7777777777777777,
+ "grad_norm": 2.2078094482421875,
+ "learning_rate": 2.224242424242424e-06,
+ "loss": 1.4401804208755493,
+ "mean_token_accuracy": 0.6637151837348938,
+ "num_tokens": 22528000.0,
+ "step": 1375
+ },
+ {
+ "entropy": 1.2710940837860107,
+ "epoch": 2.77979797979798,
+ "grad_norm": 2.1877286434173584,
+ "learning_rate": 2.222222222222222e-06,
+ "loss": 1.2672982215881348,
+ "mean_token_accuracy": 0.6958353519439697,
+ "num_tokens": 22544384.0,
+ "step": 1376
+ },
+ {
+ "entropy": 1.3981144428253174,
+ "epoch": 2.7818181818181817,
+ "grad_norm": 2.2816240787506104,
+ "learning_rate": 2.2202020202020204e-06,
+ "loss": 1.4073718786239624,
+ "mean_token_accuracy": 0.6583414673805237,
+ "num_tokens": 22560768.0,
+ "step": 1377
+ },
+ {
+ "entropy": 1.541896104812622,
+ "epoch": 2.783838383838384,
+ "grad_norm": 2.3664891719818115,
+ "learning_rate": 2.2181818181818187e-06,
+ "loss": 1.5782896280288696,
+ "mean_token_accuracy": 0.6264045238494873,
+ "num_tokens": 22577152.0,
+ "step": 1378
+ },
+ {
+ "entropy": 1.1470069885253906,
+ "epoch": 2.785858585858586,
+ "grad_norm": 2.334867000579834,
+ "learning_rate": 2.2161616161616166e-06,
+ "loss": 1.1734097003936768,
+ "mean_token_accuracy": 0.6981558203697205,
+ "num_tokens": 22593536.0,
+ "step": 1379
+ },
+ {
+ "entropy": 1.3938566446304321,
+ "epoch": 2.787878787878788,
+ "grad_norm": 2.493093967437744,
+ "learning_rate": 2.2141414141414145e-06,
+ "loss": 1.4161372184753418,
+ "mean_token_accuracy": 0.6610283255577087,
+ "num_tokens": 22609920.0,
+ "step": 1380
+ },
+ {
+ "entropy": 1.2948062419891357,
+ "epoch": 2.78989898989899,
+ "grad_norm": 2.227708578109741,
+ "learning_rate": 2.2121212121212124e-06,
+ "loss": 1.3157434463500977,
+ "mean_token_accuracy": 0.6835002303123474,
+ "num_tokens": 22626304.0,
+ "step": 1381
+ },
+ {
+ "entropy": 0.9506068229675293,
+ "epoch": 2.7919191919191917,
+ "grad_norm": 2.1482889652252197,
+ "learning_rate": 2.2101010101010102e-06,
+ "loss": 0.950823187828064,
+ "mean_token_accuracy": 0.7500610947608948,
+ "num_tokens": 22642688.0,
+ "step": 1382
+ },
+ {
+ "entropy": 1.4429880380630493,
+ "epoch": 2.793939393939394,
+ "grad_norm": 2.8189656734466553,
+ "learning_rate": 2.208080808080808e-06,
+ "loss": 1.4801222085952759,
+ "mean_token_accuracy": 0.6469223499298096,
+ "num_tokens": 22659072.0,
+ "step": 1383
+ },
+ {
+ "entropy": 1.3517380952835083,
+ "epoch": 2.795959595959596,
+ "grad_norm": 2.047236442565918,
+ "learning_rate": 2.2060606060606064e-06,
+ "loss": 1.3715345859527588,
+ "mean_token_accuracy": 0.6921714544296265,
+ "num_tokens": 22675456.0,
+ "step": 1384
+ },
+ {
+ "entropy": 1.3647958040237427,
+ "epoch": 2.797979797979798,
+ "grad_norm": 2.296548366546631,
+ "learning_rate": 2.2040404040404043e-06,
+ "loss": 1.3669397830963135,
+ "mean_token_accuracy": 0.6769663095474243,
+ "num_tokens": 22691840.0,
+ "step": 1385
+ },
+ {
+ "entropy": 1.2830337285995483,
+ "epoch": 2.8,
+ "grad_norm": 2.222038745880127,
+ "learning_rate": 2.2020202020202022e-06,
+ "loss": 1.2938398122787476,
+ "mean_token_accuracy": 0.6786150336265564,
+ "num_tokens": 22708224.0,
+ "step": 1386
+ },
+ {
+ "entropy": 1.1581721305847168,
+ "epoch": 2.802020202020202,
+ "grad_norm": 1.9979426860809326,
+ "learning_rate": 2.2e-06,
+ "loss": 1.1363141536712646,
+ "mean_token_accuracy": 0.7106130719184875,
+ "num_tokens": 22724608.0,
+ "step": 1387
+ },
+ {
+ "entropy": 1.4855602979660034,
+ "epoch": 2.804040404040404,
+ "grad_norm": 2.2763257026672363,
+ "learning_rate": 2.197979797979798e-06,
+ "loss": 1.5020172595977783,
+ "mean_token_accuracy": 0.6478993892669678,
+ "num_tokens": 22740992.0,
+ "step": 1388
+ },
+ {
+ "entropy": 1.1942899227142334,
+ "epoch": 2.806060606060606,
+ "grad_norm": 2.1685962677001953,
+ "learning_rate": 2.1959595959595963e-06,
+ "loss": 1.188063621520996,
+ "mean_token_accuracy": 0.705483615398407,
+ "num_tokens": 22757376.0,
+ "step": 1389
+ },
+ {
+ "entropy": 1.1369596719741821,
+ "epoch": 2.808080808080808,
+ "grad_norm": 2.1820614337921143,
+ "learning_rate": 2.193939393939394e-06,
+ "loss": 1.1348206996917725,
+ "mean_token_accuracy": 0.7253297567367554,
+ "num_tokens": 22773760.0,
+ "step": 1390
+ },
+ {
+ "entropy": 1.2539693117141724,
+ "epoch": 2.81010101010101,
+ "grad_norm": 2.0384390354156494,
+ "learning_rate": 2.191919191919192e-06,
+ "loss": 1.250205636024475,
+ "mean_token_accuracy": 0.6993771195411682,
+ "num_tokens": 22790144.0,
+ "step": 1391
+ },
+ {
+ "entropy": 1.7445363998413086,
+ "epoch": 2.812121212121212,
+ "grad_norm": 2.55062198638916,
+ "learning_rate": 2.18989898989899e-06,
+ "loss": 1.7788689136505127,
+ "mean_token_accuracy": 0.607107937335968,
+ "num_tokens": 22806528.0,
+ "step": 1392
+ },
+ {
+ "entropy": 1.2282429933547974,
+ "epoch": 2.8141414141414143,
+ "grad_norm": 2.1791000366210938,
+ "learning_rate": 2.187878787878788e-06,
+ "loss": 1.2460708618164062,
+ "mean_token_accuracy": 0.697239875793457,
+ "num_tokens": 22822912.0,
+ "step": 1393
+ },
+ {
+ "entropy": 1.2007265090942383,
+ "epoch": 2.816161616161616,
+ "grad_norm": 2.4067599773406982,
+ "learning_rate": 2.1858585858585858e-06,
+ "loss": 1.2013460397720337,
+ "mean_token_accuracy": 0.6963238716125488,
+ "num_tokens": 22839296.0,
+ "step": 1394
+ },
+ {
+ "entropy": 1.7315549850463867,
+ "epoch": 2.8181818181818183,
+ "grad_norm": 2.15682053565979,
+ "learning_rate": 2.183838383838384e-06,
+ "loss": 1.6833488941192627,
+ "mean_token_accuracy": 0.6170004606246948,
+ "num_tokens": 22855680.0,
+ "step": 1395
+ },
+ {
+ "entropy": 1.325459599494934,
+ "epoch": 2.82020202020202,
+ "grad_norm": 2.0464699268341064,
+ "learning_rate": 2.181818181818182e-06,
+ "loss": 1.3265891075134277,
+ "mean_token_accuracy": 0.6802638173103333,
+ "num_tokens": 22872064.0,
+ "step": 1396
+ },
+ {
+ "entropy": 1.0391966104507446,
+ "epoch": 2.822222222222222,
+ "grad_norm": 2.3595499992370605,
+ "learning_rate": 2.17979797979798e-06,
+ "loss": 1.0515706539154053,
+ "mean_token_accuracy": 0.7299706935882568,
+ "num_tokens": 22888448.0,
+ "step": 1397
+ },
+ {
+ "entropy": 1.5484130382537842,
+ "epoch": 2.824242424242424,
+ "grad_norm": 2.24381422996521,
+ "learning_rate": 2.1777777777777777e-06,
+ "loss": 1.5656720399856567,
+ "mean_token_accuracy": 0.6390449404716492,
+ "num_tokens": 22904832.0,
+ "step": 1398
+ },
+ {
+ "entropy": 1.2196807861328125,
+ "epoch": 2.8262626262626265,
+ "grad_norm": 2.3043432235717773,
+ "learning_rate": 2.175757575757576e-06,
+ "loss": 1.2172417640686035,
+ "mean_token_accuracy": 0.6985833048820496,
+ "num_tokens": 22921216.0,
+ "step": 1399
+ },
+ {
+ "entropy": 1.171323537826538,
+ "epoch": 2.8282828282828283,
+ "grad_norm": 2.34149169921875,
+ "learning_rate": 2.173737373737374e-06,
+ "loss": 1.1752846240997314,
+ "mean_token_accuracy": 0.6979726552963257,
+ "num_tokens": 22937600.0,
+ "step": 1400
+ },
+ {
+ "epoch": 2.8282828282828283,
+ "eval_entropy": 1.3776377598124165,
+ "eval_loss": 1.533128261566162,
+ "eval_mean_token_accuracy": 0.645915245336871,
+ "eval_num_tokens": 22937600.0,
+ "eval_runtime": 43.7952,
+ "eval_samples_per_second": 22.605,
+ "eval_steps_per_second": 2.831,
+ "step": 1400
+ },
+ {
+ "entropy": 1.2173237800598145,
+ "epoch": 2.83030303030303,
+ "grad_norm": 2.245490789413452,
+ "learning_rate": 2.171717171717172e-06,
+ "loss": 1.1928956508636475,
+ "mean_token_accuracy": 0.6988885998725891,
+ "num_tokens": 22953984.0,
+ "step": 1401
+ },
+ {
+ "entropy": 1.4213385581970215,
+ "epoch": 2.8323232323232324,
+ "grad_norm": 2.192051649093628,
+ "learning_rate": 2.16969696969697e-06,
+ "loss": 1.4340262413024902,
+ "mean_token_accuracy": 0.6622496247291565,
+ "num_tokens": 22970368.0,
+ "step": 1402
+ },
+ {
+ "entropy": 1.922193169593811,
+ "epoch": 2.8343434343434346,
+ "grad_norm": 2.0441250801086426,
+ "learning_rate": 2.167676767676768e-06,
+ "loss": 1.9164612293243408,
+ "mean_token_accuracy": 0.5942232608795166,
+ "num_tokens": 22986752.0,
+ "step": 1403
+ },
+ {
+ "entropy": 1.3124938011169434,
+ "epoch": 2.8363636363636364,
+ "grad_norm": 2.245614528656006,
+ "learning_rate": 2.165656565656566e-06,
+ "loss": 1.3055529594421387,
+ "mean_token_accuracy": 0.680385947227478,
+ "num_tokens": 23003136.0,
+ "step": 1404
+ },
+ {
+ "entropy": 0.9893879294395447,
+ "epoch": 2.8383838383838382,
+ "grad_norm": 2.1041340827941895,
+ "learning_rate": 2.163636363636364e-06,
+ "loss": 0.9646933078765869,
+ "mean_token_accuracy": 0.7441377639770508,
+ "num_tokens": 23019520.0,
+ "step": 1405
+ },
+ {
+ "entropy": 1.4011882543563843,
+ "epoch": 2.8404040404040405,
+ "grad_norm": 2.5371105670928955,
+ "learning_rate": 2.1616161616161617e-06,
+ "loss": 1.381878137588501,
+ "mean_token_accuracy": 0.6522960662841797,
+ "num_tokens": 23035904.0,
+ "step": 1406
+ },
+ {
+ "entropy": 1.4563852548599243,
+ "epoch": 2.8424242424242423,
+ "grad_norm": 2.1308720111846924,
+ "learning_rate": 2.15959595959596e-06,
+ "loss": 1.4763004779815674,
+ "mean_token_accuracy": 0.6792256832122803,
+ "num_tokens": 23052288.0,
+ "step": 1407
+ },
+ {
+ "entropy": 0.8676514625549316,
+ "epoch": 2.8444444444444446,
+ "grad_norm": 2.016085147857666,
+ "learning_rate": 2.157575757575758e-06,
+ "loss": 0.8672611713409424,
+ "mean_token_accuracy": 0.7660601139068604,
+ "num_tokens": 23068672.0,
+ "step": 1408
+ },
+ {
+ "entropy": 1.2011700868606567,
+ "epoch": 2.8464646464646464,
+ "grad_norm": 2.1641454696655273,
+ "learning_rate": 2.1555555555555558e-06,
+ "loss": 1.2113897800445557,
+ "mean_token_accuracy": 0.7180019617080688,
+ "num_tokens": 23085056.0,
+ "step": 1409
+ },
+ {
+ "entropy": 1.7813633680343628,
+ "epoch": 2.8484848484848486,
+ "grad_norm": 2.1768839359283447,
+ "learning_rate": 2.1535353535353537e-06,
+ "loss": 1.7897224426269531,
+ "mean_token_accuracy": 0.6129701733589172,
+ "num_tokens": 23101440.0,
+ "step": 1410
+ },
+ {
+ "entropy": 1.0219851732254028,
+ "epoch": 2.8505050505050504,
+ "grad_norm": 2.299229145050049,
+ "learning_rate": 2.1515151515151515e-06,
+ "loss": 1.0073617696762085,
+ "mean_token_accuracy": 0.7240473628044128,
+ "num_tokens": 23117824.0,
+ "step": 1411
+ },
+ {
+ "entropy": 1.3342534303665161,
+ "epoch": 2.8525252525252527,
+ "grad_norm": 2.165161371231079,
+ "learning_rate": 2.1494949494949494e-06,
+ "loss": 1.3219788074493408,
+ "mean_token_accuracy": 0.6764777898788452,
+ "num_tokens": 23134208.0,
+ "step": 1412
+ },
+ {
+ "entropy": 1.1618846654891968,
+ "epoch": 2.8545454545454545,
+ "grad_norm": 2.1276957988739014,
+ "learning_rate": 2.1474747474747477e-06,
+ "loss": 1.1667187213897705,
+ "mean_token_accuracy": 0.7214215993881226,
+ "num_tokens": 23150592.0,
+ "step": 1413
+ },
+ {
+ "entropy": 1.3673025369644165,
+ "epoch": 2.8565656565656568,
+ "grad_norm": 2.249702215194702,
+ "learning_rate": 2.1454545454545456e-06,
+ "loss": 1.380744218826294,
+ "mean_token_accuracy": 0.6656082272529602,
+ "num_tokens": 23166976.0,
+ "step": 1414
+ },
+ {
+ "entropy": 1.4118674993515015,
+ "epoch": 2.8585858585858586,
+ "grad_norm": 2.1488327980041504,
+ "learning_rate": 2.1434343434343435e-06,
+ "loss": 1.4007214307785034,
+ "mean_token_accuracy": 0.6663410067558289,
+ "num_tokens": 23183360.0,
+ "step": 1415
+ },
+ {
+ "entropy": 1.4398424625396729,
+ "epoch": 2.8606060606060604,
+ "grad_norm": 2.2609732151031494,
+ "learning_rate": 2.1414141414141414e-06,
+ "loss": 1.4657073020935059,
+ "mean_token_accuracy": 0.6483268141746521,
+ "num_tokens": 23199744.0,
+ "step": 1416
+ },
+ {
+ "entropy": 1.2922199964523315,
+ "epoch": 2.8626262626262626,
+ "grad_norm": 2.1911425590515137,
+ "learning_rate": 2.1393939393939393e-06,
+ "loss": 1.3118690252304077,
+ "mean_token_accuracy": 0.6951025724411011,
+ "num_tokens": 23216128.0,
+ "step": 1417
+ },
+ {
+ "entropy": 1.6888933181762695,
+ "epoch": 2.864646464646465,
+ "grad_norm": 2.2805919647216797,
+ "learning_rate": 2.1373737373737376e-06,
+ "loss": 1.6726739406585693,
+ "mean_token_accuracy": 0.6136419177055359,
+ "num_tokens": 23232512.0,
+ "step": 1418
+ },
+ {
+ "entropy": 1.289890170097351,
+ "epoch": 2.8666666666666667,
+ "grad_norm": 2.092263698577881,
+ "learning_rate": 2.1353535353535355e-06,
+ "loss": 1.2876317501068115,
+ "mean_token_accuracy": 0.6835613250732422,
+ "num_tokens": 23248896.0,
+ "step": 1419
+ },
+ {
+ "entropy": 1.361393690109253,
+ "epoch": 2.8686868686868685,
+ "grad_norm": 2.1863481998443604,
+ "learning_rate": 2.133333333333334e-06,
+ "loss": 1.3719209432601929,
+ "mean_token_accuracy": 0.663532018661499,
+ "num_tokens": 23265280.0,
+ "step": 1420
+ },
+ {
+ "entropy": 1.1187894344329834,
+ "epoch": 2.8707070707070708,
+ "grad_norm": 2.256434679031372,
+ "learning_rate": 2.1313131313131317e-06,
+ "loss": 1.1177198886871338,
+ "mean_token_accuracy": 0.7082926034927368,
+ "num_tokens": 23281664.0,
+ "step": 1421
+ },
+ {
+ "entropy": 0.8779795169830322,
+ "epoch": 2.8727272727272726,
+ "grad_norm": 2.095226526260376,
+ "learning_rate": 2.1292929292929296e-06,
+ "loss": 0.8675153851509094,
+ "mean_token_accuracy": 0.7647777199745178,
+ "num_tokens": 23298048.0,
+ "step": 1422
+ },
+ {
+ "entropy": 1.458146095275879,
+ "epoch": 2.874747474747475,
+ "grad_norm": 2.225179433822632,
+ "learning_rate": 2.1272727272727275e-06,
+ "loss": 1.4488263130187988,
+ "mean_token_accuracy": 0.6640205383300781,
+ "num_tokens": 23314432.0,
+ "step": 1423
+ },
+ {
+ "entropy": 1.187269926071167,
+ "epoch": 2.8767676767676766,
+ "grad_norm": 2.023710250854492,
+ "learning_rate": 2.1252525252525254e-06,
+ "loss": 1.182060718536377,
+ "mean_token_accuracy": 0.7017586827278137,
+ "num_tokens": 23330816.0,
+ "step": 1424
+ },
+ {
+ "entropy": 1.1446877717971802,
+ "epoch": 2.878787878787879,
+ "grad_norm": 2.301314115524292,
+ "learning_rate": 2.1232323232323237e-06,
+ "loss": 1.1507803201675415,
+ "mean_token_accuracy": 0.7048119306564331,
+ "num_tokens": 23347200.0,
+ "step": 1425
+ },
+ {
+ "entropy": 1.3453346490859985,
+ "epoch": 2.8808080808080807,
+ "grad_norm": 2.0338571071624756,
+ "learning_rate": 2.1212121212121216e-06,
+ "loss": 1.345799207687378,
+ "mean_token_accuracy": 0.6966292262077332,
+ "num_tokens": 23363584.0,
+ "step": 1426
+ },
+ {
+ "entropy": 1.2638238668441772,
+ "epoch": 2.882828282828283,
+ "grad_norm": 2.1575632095336914,
+ "learning_rate": 2.1191919191919194e-06,
+ "loss": 1.2882521152496338,
+ "mean_token_accuracy": 0.6901563405990601,
+ "num_tokens": 23379968.0,
+ "step": 1427
+ },
+ {
+ "entropy": 1.4490493535995483,
+ "epoch": 2.8848484848484848,
+ "grad_norm": 2.1735994815826416,
+ "learning_rate": 2.1171717171717173e-06,
+ "loss": 1.455702781677246,
+ "mean_token_accuracy": 0.6538837552070618,
+ "num_tokens": 23396352.0,
+ "step": 1428
+ },
+ {
+ "entropy": 1.3873618841171265,
+ "epoch": 2.886868686868687,
+ "grad_norm": 2.3870506286621094,
+ "learning_rate": 2.1151515151515152e-06,
+ "loss": 1.3657323122024536,
+ "mean_token_accuracy": 0.6692110300064087,
+ "num_tokens": 23412736.0,
+ "step": 1429
+ },
+ {
+ "entropy": 1.298308253288269,
+ "epoch": 2.888888888888889,
+ "grad_norm": 2.079481363296509,
+ "learning_rate": 2.113131313131313e-06,
+ "loss": 1.30775785446167,
+ "mean_token_accuracy": 0.6834391951560974,
+ "num_tokens": 23429120.0,
+ "step": 1430
+ },
+ {
+ "entropy": 1.1544564962387085,
+ "epoch": 2.8909090909090907,
+ "grad_norm": 2.2400357723236084,
+ "learning_rate": 2.1111111111111114e-06,
+ "loss": 1.17466139793396,
+ "mean_token_accuracy": 0.7093307375907898,
+ "num_tokens": 23445504.0,
+ "step": 1431
+ },
+ {
+ "entropy": 1.4711883068084717,
+ "epoch": 2.892929292929293,
+ "grad_norm": 2.1779074668884277,
+ "learning_rate": 2.1090909090909093e-06,
+ "loss": 1.4626137018203735,
+ "mean_token_accuracy": 0.6602955460548401,
+ "num_tokens": 23461888.0,
+ "step": 1432
+ },
+ {
+ "entropy": 1.4876407384872437,
+ "epoch": 2.894949494949495,
+ "grad_norm": 2.278554677963257,
+ "learning_rate": 2.107070707070707e-06,
+ "loss": 1.5031921863555908,
+ "mean_token_accuracy": 0.6570591330528259,
+ "num_tokens": 23478272.0,
+ "step": 1433
+ },
+ {
+ "entropy": 1.328355073928833,
+ "epoch": 2.896969696969697,
+ "grad_norm": 2.034842014312744,
+ "learning_rate": 2.105050505050505e-06,
+ "loss": 1.3163859844207764,
+ "mean_token_accuracy": 0.6878358721733093,
+ "num_tokens": 23494656.0,
+ "step": 1434
+ },
+ {
+ "entropy": 1.487100601196289,
+ "epoch": 2.898989898989899,
+ "grad_norm": 2.1980724334716797,
+ "learning_rate": 2.103030303030303e-06,
+ "loss": 1.4886071681976318,
+ "mean_token_accuracy": 0.6521739363670349,
+ "num_tokens": 23511040.0,
+ "step": 1435
+ },
+ {
+ "entropy": 1.4826854467391968,
+ "epoch": 2.901010101010101,
+ "grad_norm": 2.5214874744415283,
+ "learning_rate": 2.1010101010101013e-06,
+ "loss": 1.4610626697540283,
+ "mean_token_accuracy": 0.6503419876098633,
+ "num_tokens": 23527424.0,
+ "step": 1436
+ },
+ {
+ "entropy": 1.00849187374115,
+ "epoch": 2.9030303030303033,
+ "grad_norm": 2.1452033519744873,
+ "learning_rate": 2.098989898989899e-06,
+ "loss": 0.9949107766151428,
+ "mean_token_accuracy": 0.7457864880561829,
+ "num_tokens": 23543808.0,
+ "step": 1437
+ },
+ {
+ "entropy": 1.0621757507324219,
+ "epoch": 2.905050505050505,
+ "grad_norm": 2.1871836185455322,
+ "learning_rate": 2.096969696969697e-06,
+ "loss": 1.0466980934143066,
+ "mean_token_accuracy": 0.7275280952453613,
+ "num_tokens": 23560192.0,
+ "step": 1438
+ },
+ {
+ "entropy": 1.3413732051849365,
+ "epoch": 2.907070707070707,
+ "grad_norm": 2.2954704761505127,
+ "learning_rate": 2.094949494949495e-06,
+ "loss": 1.3602681159973145,
+ "mean_token_accuracy": 0.6686003804206848,
+ "num_tokens": 23576576.0,
+ "step": 1439
+ },
+ {
+ "entropy": 0.8138323426246643,
+ "epoch": 2.909090909090909,
+ "grad_norm": 1.9476791620254517,
+ "learning_rate": 2.092929292929293e-06,
+ "loss": 0.8208089470863342,
+ "mean_token_accuracy": 0.7796775698661804,
+ "num_tokens": 23592960.0,
+ "step": 1440
+ },
+ {
+ "entropy": 0.9035854339599609,
+ "epoch": 2.911111111111111,
+ "grad_norm": 1.9753291606903076,
+ "learning_rate": 2.090909090909091e-06,
+ "loss": 0.8889603614807129,
+ "mean_token_accuracy": 0.7610527873039246,
+ "num_tokens": 23609344.0,
+ "step": 1441
+ },
+ {
+ "entropy": 0.9301351308822632,
+ "epoch": 2.9131313131313132,
+ "grad_norm": 2.244597911834717,
+ "learning_rate": 2.088888888888889e-06,
+ "loss": 0.9369313716888428,
+ "mean_token_accuracy": 0.7603810429573059,
+ "num_tokens": 23625728.0,
+ "step": 1442
+ },
+ {
+ "entropy": 0.8740416765213013,
+ "epoch": 2.915151515151515,
+ "grad_norm": 2.0207667350769043,
+ "learning_rate": 2.0868686868686873e-06,
+ "loss": 0.8751406073570251,
+ "mean_token_accuracy": 0.7634342908859253,
+ "num_tokens": 23642112.0,
+ "step": 1443
+ },
+ {
+ "entropy": 1.044225811958313,
+ "epoch": 2.9171717171717173,
+ "grad_norm": 2.8100409507751465,
+ "learning_rate": 2.0848484848484852e-06,
+ "loss": 1.0568115711212158,
+ "mean_token_accuracy": 0.7291157841682434,
+ "num_tokens": 23658496.0,
+ "step": 1444
+ },
+ {
+ "entropy": 1.1093257665634155,
+ "epoch": 2.919191919191919,
+ "grad_norm": 2.2957236766815186,
+ "learning_rate": 2.082828282828283e-06,
+ "loss": 1.1235377788543701,
+ "mean_token_accuracy": 0.7108573317527771,
+ "num_tokens": 23674880.0,
+ "step": 1445
+ },
+ {
+ "entropy": 1.2211518287658691,
+ "epoch": 2.9212121212121214,
+ "grad_norm": 2.022287130355835,
+ "learning_rate": 2.080808080808081e-06,
+ "loss": 1.241473913192749,
+ "mean_token_accuracy": 0.7020029425621033,
+ "num_tokens": 23691264.0,
+ "step": 1446
+ },
+ {
+ "entropy": 1.3636398315429688,
+ "epoch": 2.923232323232323,
+ "grad_norm": 2.1484291553497314,
+ "learning_rate": 2.078787878787879e-06,
+ "loss": 1.3876409530639648,
+ "mean_token_accuracy": 0.6714093685150146,
+ "num_tokens": 23707648.0,
+ "step": 1447
+ },
+ {
+ "entropy": 1.1615006923675537,
+ "epoch": 2.9252525252525254,
+ "grad_norm": 2.775181293487549,
+ "learning_rate": 2.0767676767676768e-06,
+ "loss": 1.1792278289794922,
+ "mean_token_accuracy": 0.7070102691650391,
+ "num_tokens": 23724032.0,
+ "step": 1448
+ },
+ {
+ "entropy": 1.6308530569076538,
+ "epoch": 2.9272727272727272,
+ "grad_norm": 2.3813729286193848,
+ "learning_rate": 2.074747474747475e-06,
+ "loss": 1.6642422676086426,
+ "mean_token_accuracy": 0.6171225905418396,
+ "num_tokens": 23740416.0,
+ "step": 1449
+ },
+ {
+ "entropy": 1.2764222621917725,
+ "epoch": 2.929292929292929,
+ "grad_norm": 2.234309196472168,
+ "learning_rate": 2.072727272727273e-06,
+ "loss": 1.2756493091583252,
+ "mean_token_accuracy": 0.6824010610580444,
+ "num_tokens": 23756800.0,
+ "step": 1450
+ },
+ {
+ "epoch": 2.929292929292929,
+ "eval_entropy": 1.366065975639128,
+ "eval_loss": 1.5328131914138794,
+ "eval_mean_token_accuracy": 0.6461491642459747,
+ "eval_num_tokens": 23756800.0,
+ "eval_runtime": 43.8548,
+ "eval_samples_per_second": 22.574,
+ "eval_steps_per_second": 2.828,
+ "step": 1450
+ },
+ {
+ "entropy": 1.178619623184204,
+ "epoch": 2.9313131313131313,
+ "grad_norm": 2.0946547985076904,
+ "learning_rate": 2.070707070707071e-06,
+ "loss": 1.1751608848571777,
+ "mean_token_accuracy": 0.704384446144104,
+ "num_tokens": 23773184.0,
+ "step": 1451
+ },
+ {
+ "entropy": 1.319455862045288,
+ "epoch": 2.9333333333333336,
+ "grad_norm": 2.1925816535949707,
+ "learning_rate": 2.0686868686868688e-06,
+ "loss": 1.3229246139526367,
+ "mean_token_accuracy": 0.6867977380752563,
+ "num_tokens": 23789568.0,
+ "step": 1452
+ },
+ {
+ "entropy": 1.787271499633789,
+ "epoch": 2.9353535353535354,
+ "grad_norm": 2.374494791030884,
+ "learning_rate": 2.0666666666666666e-06,
+ "loss": 1.7788429260253906,
+ "mean_token_accuracy": 0.589154839515686,
+ "num_tokens": 23805952.0,
+ "step": 1453
+ },
+ {
+ "entropy": 1.25193452835083,
+ "epoch": 2.937373737373737,
+ "grad_norm": 2.3499720096588135,
+ "learning_rate": 2.064646464646465e-06,
+ "loss": 1.2651758193969727,
+ "mean_token_accuracy": 0.6863092184066772,
+ "num_tokens": 23822336.0,
+ "step": 1454
+ },
+ {
+ "entropy": 1.355589747428894,
+ "epoch": 2.9393939393939394,
+ "grad_norm": 2.228123426437378,
+ "learning_rate": 2.062626262626263e-06,
+ "loss": 1.3623197078704834,
+ "mean_token_accuracy": 0.6623717546463013,
+ "num_tokens": 23838720.0,
+ "step": 1455
+ },
+ {
+ "entropy": 1.2293426990509033,
+ "epoch": 2.9414141414141413,
+ "grad_norm": 2.4237186908721924,
+ "learning_rate": 2.0606060606060607e-06,
+ "loss": 1.2226693630218506,
+ "mean_token_accuracy": 0.6903395056724548,
+ "num_tokens": 23855104.0,
+ "step": 1456
+ },
+ {
+ "entropy": 1.2312723398208618,
+ "epoch": 2.9434343434343435,
+ "grad_norm": 2.2350525856018066,
+ "learning_rate": 2.0585858585858586e-06,
+ "loss": 1.235978364944458,
+ "mean_token_accuracy": 0.696201741695404,
+ "num_tokens": 23871488.0,
+ "step": 1457
+ },
+ {
+ "entropy": 1.2731300592422485,
+ "epoch": 2.9454545454545453,
+ "grad_norm": 2.488929033279419,
+ "learning_rate": 2.0565656565656565e-06,
+ "loss": 1.2763257026672363,
+ "mean_token_accuracy": 0.6765388250350952,
+ "num_tokens": 23887872.0,
+ "step": 1458
+ },
+ {
+ "entropy": 1.1283669471740723,
+ "epoch": 2.9474747474747476,
+ "grad_norm": 2.116241693496704,
+ "learning_rate": 2.054545454545455e-06,
+ "loss": 1.1279038190841675,
+ "mean_token_accuracy": 0.7123228907585144,
+ "num_tokens": 23904256.0,
+ "step": 1459
+ },
+ {
+ "entropy": 1.4969894886016846,
+ "epoch": 2.9494949494949494,
+ "grad_norm": 2.17519474029541,
+ "learning_rate": 2.0525252525252527e-06,
+ "loss": 1.5177810192108154,
+ "mean_token_accuracy": 0.6544333100318909,
+ "num_tokens": 23920640.0,
+ "step": 1460
+ },
+ {
+ "entropy": 1.3133295774459839,
+ "epoch": 2.9515151515151516,
+ "grad_norm": 2.149486780166626,
+ "learning_rate": 2.0505050505050506e-06,
+ "loss": 1.2985190153121948,
+ "mean_token_accuracy": 0.6745237112045288,
+ "num_tokens": 23937024.0,
+ "step": 1461
+ },
+ {
+ "entropy": 1.2777348756790161,
+ "epoch": 2.9535353535353535,
+ "grad_norm": 2.124922752380371,
+ "learning_rate": 2.0484848484848485e-06,
+ "loss": 1.2731128931045532,
+ "mean_token_accuracy": 0.692415714263916,
+ "num_tokens": 23953408.0,
+ "step": 1462
+ },
+ {
+ "entropy": 1.0544995069503784,
+ "epoch": 2.9555555555555557,
+ "grad_norm": 2.177363872528076,
+ "learning_rate": 2.046464646464647e-06,
+ "loss": 1.0588994026184082,
+ "mean_token_accuracy": 0.7378480434417725,
+ "num_tokens": 23969792.0,
+ "step": 1463
+ },
+ {
+ "entropy": 1.441129446029663,
+ "epoch": 2.9575757575757575,
+ "grad_norm": 2.5355494022369385,
+ "learning_rate": 2.0444444444444447e-06,
+ "loss": 1.4684169292449951,
+ "mean_token_accuracy": 0.6495481133460999,
+ "num_tokens": 23986176.0,
+ "step": 1464
+ },
+ {
+ "entropy": 1.3359375,
+ "epoch": 2.9595959595959593,
+ "grad_norm": 2.309727191925049,
+ "learning_rate": 2.0424242424242426e-06,
+ "loss": 1.339503288269043,
+ "mean_token_accuracy": 0.6707376837730408,
+ "num_tokens": 24002560.0,
+ "step": 1465
+ },
+ {
+ "entropy": 1.5922675132751465,
+ "epoch": 2.9616161616161616,
+ "grad_norm": 2.191904067993164,
+ "learning_rate": 2.0404040404040405e-06,
+ "loss": 1.613649845123291,
+ "mean_token_accuracy": 0.635869562625885,
+ "num_tokens": 24018944.0,
+ "step": 1466
+ },
+ {
+ "entropy": 1.152869701385498,
+ "epoch": 2.963636363636364,
+ "grad_norm": 2.1656837463378906,
+ "learning_rate": 2.0383838383838388e-06,
+ "loss": 1.1791510581970215,
+ "mean_token_accuracy": 0.7013311982154846,
+ "num_tokens": 24035328.0,
+ "step": 1467
+ },
+ {
+ "entropy": 1.491494059562683,
+ "epoch": 2.9656565656565657,
+ "grad_norm": 2.397916555404663,
+ "learning_rate": 2.0363636363636367e-06,
+ "loss": 1.4803133010864258,
+ "mean_token_accuracy": 0.6395334601402283,
+ "num_tokens": 24051712.0,
+ "step": 1468
+ },
+ {
+ "entropy": 1.165432095527649,
+ "epoch": 2.9676767676767675,
+ "grad_norm": 2.0728659629821777,
+ "learning_rate": 2.0343434343434345e-06,
+ "loss": 1.1676859855651855,
+ "mean_token_accuracy": 0.7170249223709106,
+ "num_tokens": 24068096.0,
+ "step": 1469
+ },
+ {
+ "entropy": 0.7055315971374512,
+ "epoch": 2.9696969696969697,
+ "grad_norm": 1.949097752571106,
+ "learning_rate": 2.0323232323232324e-06,
+ "loss": 0.7094426155090332,
+ "mean_token_accuracy": 0.8047752976417542,
+ "num_tokens": 24084480.0,
+ "step": 1470
+ },
+ {
+ "entropy": 1.5079377889633179,
+ "epoch": 2.971717171717172,
+ "grad_norm": 2.282525062561035,
+ "learning_rate": 2.0303030303030303e-06,
+ "loss": 1.5309293270111084,
+ "mean_token_accuracy": 0.65486079454422,
+ "num_tokens": 24100864.0,
+ "step": 1471
+ },
+ {
+ "entropy": 1.6969871520996094,
+ "epoch": 2.973737373737374,
+ "grad_norm": 2.2211408615112305,
+ "learning_rate": 2.0282828282828286e-06,
+ "loss": 1.6919132471084595,
+ "mean_token_accuracy": 0.6259770393371582,
+ "num_tokens": 24117248.0,
+ "step": 1472
+ },
+ {
+ "entropy": 1.3981496095657349,
+ "epoch": 2.9757575757575756,
+ "grad_norm": 2.27559232711792,
+ "learning_rate": 2.0262626262626265e-06,
+ "loss": 1.4188807010650635,
+ "mean_token_accuracy": 0.6502198576927185,
+ "num_tokens": 24133632.0,
+ "step": 1473
+ },
+ {
+ "entropy": 1.2966978549957275,
+ "epoch": 2.977777777777778,
+ "grad_norm": 2.2315874099731445,
+ "learning_rate": 2.0242424242424244e-06,
+ "loss": 1.3034720420837402,
+ "mean_token_accuracy": 0.6849047541618347,
+ "num_tokens": 24150016.0,
+ "step": 1474
+ },
+ {
+ "entropy": 1.4312105178833008,
+ "epoch": 2.9797979797979797,
+ "grad_norm": 2.3630731105804443,
+ "learning_rate": 2.0222222222222223e-06,
+ "loss": 1.412591814994812,
+ "mean_token_accuracy": 0.6673180460929871,
+ "num_tokens": 24166400.0,
+ "step": 1475
+ },
+ {
+ "entropy": 1.246411681175232,
+ "epoch": 2.981818181818182,
+ "grad_norm": 2.101142644882202,
+ "learning_rate": 2.02020202020202e-06,
+ "loss": 1.2674810886383057,
+ "mean_token_accuracy": 0.691255509853363,
+ "num_tokens": 24182784.0,
+ "step": 1476
+ },
+ {
+ "entropy": 1.2367678880691528,
+ "epoch": 2.9838383838383837,
+ "grad_norm": 2.3414289951324463,
+ "learning_rate": 2.0181818181818185e-06,
+ "loss": 1.254493236541748,
+ "mean_token_accuracy": 0.6811797618865967,
+ "num_tokens": 24199168.0,
+ "step": 1477
+ },
+ {
+ "entropy": 1.3566304445266724,
+ "epoch": 2.985858585858586,
+ "grad_norm": 2.3050434589385986,
+ "learning_rate": 2.0161616161616164e-06,
+ "loss": 1.3336361646652222,
+ "mean_token_accuracy": 0.6761724352836609,
+ "num_tokens": 24215552.0,
+ "step": 1478
+ },
+ {
+ "entropy": 1.3928223848342896,
+ "epoch": 2.987878787878788,
+ "grad_norm": 2.2100086212158203,
+ "learning_rate": 2.0141414141414143e-06,
+ "loss": 1.4050222635269165,
+ "mean_token_accuracy": 0.6573033928871155,
+ "num_tokens": 24231936.0,
+ "step": 1479
+ },
+ {
+ "entropy": 1.067913293838501,
+ "epoch": 2.98989898989899,
+ "grad_norm": 2.0196874141693115,
+ "learning_rate": 2.012121212121212e-06,
+ "loss": 1.0836842060089111,
+ "mean_token_accuracy": 0.7427943348884583,
+ "num_tokens": 24248320.0,
+ "step": 1480
+ },
+ {
+ "entropy": 1.077475905418396,
+ "epoch": 2.991919191919192,
+ "grad_norm": 2.117917776107788,
+ "learning_rate": 2.01010101010101e-06,
+ "loss": 1.0825674533843994,
+ "mean_token_accuracy": 0.7288104295730591,
+ "num_tokens": 24264704.0,
+ "step": 1481
+ },
+ {
+ "entropy": 1.092706561088562,
+ "epoch": 2.993939393939394,
+ "grad_norm": 2.143916130065918,
+ "learning_rate": 2.008080808080808e-06,
+ "loss": 1.1073365211486816,
+ "mean_token_accuracy": 0.7271006107330322,
+ "num_tokens": 24281088.0,
+ "step": 1482
+ },
+ {
+ "entropy": 1.3734595775604248,
+ "epoch": 2.995959595959596,
+ "grad_norm": 2.3651299476623535,
+ "learning_rate": 2.0060606060606062e-06,
+ "loss": 1.3319274187088013,
+ "mean_token_accuracy": 0.6729360222816467,
+ "num_tokens": 24297472.0,
+ "step": 1483
+ },
+ {
+ "entropy": 1.5602202415466309,
+ "epoch": 2.9979797979797977,
+ "grad_norm": 2.4872968196868896,
+ "learning_rate": 2.004040404040404e-06,
+ "loss": 1.5969634056091309,
+ "mean_token_accuracy": 0.6352589130401611,
+ "num_tokens": 24313856.0,
+ "step": 1484
+ },
+ {
+ "entropy": 1.4174549579620361,
+ "epoch": 3.0,
+ "grad_norm": 2.1410610675811768,
+ "learning_rate": 2.0020202020202024e-06,
+ "loss": 1.4155133962631226,
+ "mean_token_accuracy": 0.6683561205863953,
+ "num_tokens": 24330240.0,
+ "step": 1485
+ },
+ {
+ "entropy": 1.5710192918777466,
+ "epoch": 3.002020202020202,
+ "grad_norm": 2.1068179607391357,
+ "learning_rate": 2.0000000000000003e-06,
+ "loss": 1.5049738883972168,
+ "mean_token_accuracy": 0.6563873887062073,
+ "num_tokens": 24346624.0,
+ "step": 1486
+ },
+ {
+ "entropy": 1.2311826944351196,
+ "epoch": 3.004040404040404,
+ "grad_norm": 2.238032579421997,
+ "learning_rate": 1.9979797979797982e-06,
+ "loss": 1.1740195751190186,
+ "mean_token_accuracy": 0.7018197178840637,
+ "num_tokens": 24363008.0,
+ "step": 1487
+ },
+ {
+ "entropy": 1.0681109428405762,
+ "epoch": 3.006060606060606,
+ "grad_norm": 2.0808138847351074,
+ "learning_rate": 1.995959595959596e-06,
+ "loss": 1.0269291400909424,
+ "mean_token_accuracy": 0.739130437374115,
+ "num_tokens": 24379392.0,
+ "step": 1488
+ },
+ {
+ "entropy": 1.2553848028182983,
+ "epoch": 3.008080808080808,
+ "grad_norm": 2.155320644378662,
+ "learning_rate": 1.993939393939394e-06,
+ "loss": 1.209822654724121,
+ "mean_token_accuracy": 0.7072545289993286,
+ "num_tokens": 24395776.0,
+ "step": 1489
+ },
+ {
+ "entropy": 1.5376615524291992,
+ "epoch": 3.01010101010101,
+ "grad_norm": 2.376460075378418,
+ "learning_rate": 1.9919191919191923e-06,
+ "loss": 1.485018014907837,
+ "mean_token_accuracy": 0.6547996997833252,
+ "num_tokens": 24412160.0,
+ "step": 1490
+ },
+ {
+ "entropy": 1.0649843215942383,
+ "epoch": 3.012121212121212,
+ "grad_norm": 2.1056692600250244,
+ "learning_rate": 1.98989898989899e-06,
+ "loss": 1.037630558013916,
+ "mean_token_accuracy": 0.7341841459274292,
+ "num_tokens": 24428544.0,
+ "step": 1491
+ },
+ {
+ "entropy": 1.3711966276168823,
+ "epoch": 3.014141414141414,
+ "grad_norm": 2.238981008529663,
+ "learning_rate": 1.987878787878788e-06,
+ "loss": 1.3336507081985474,
+ "mean_token_accuracy": 0.6914386749267578,
+ "num_tokens": 24444928.0,
+ "step": 1492
+ },
+ {
+ "entropy": 1.3955618143081665,
+ "epoch": 3.0161616161616163,
+ "grad_norm": 2.183464765548706,
+ "learning_rate": 1.985858585858586e-06,
+ "loss": 1.3517842292785645,
+ "mean_token_accuracy": 0.6735466718673706,
+ "num_tokens": 24461312.0,
+ "step": 1493
+ },
+ {
+ "entropy": 1.0063103437423706,
+ "epoch": 3.018181818181818,
+ "grad_norm": 2.1911609172821045,
+ "learning_rate": 1.983838383838384e-06,
+ "loss": 0.9973118305206299,
+ "mean_token_accuracy": 0.7280166149139404,
+ "num_tokens": 24477696.0,
+ "step": 1494
+ },
+ {
+ "entropy": 1.3080904483795166,
+ "epoch": 3.0202020202020203,
+ "grad_norm": 2.1129884719848633,
+ "learning_rate": 1.981818181818182e-06,
+ "loss": 1.2758756875991821,
+ "mean_token_accuracy": 0.7027357220649719,
+ "num_tokens": 24494080.0,
+ "step": 1495
+ },
+ {
+ "entropy": 1.5414087772369385,
+ "epoch": 3.022222222222222,
+ "grad_norm": 2.1628341674804688,
+ "learning_rate": 1.97979797979798e-06,
+ "loss": 1.527054786682129,
+ "mean_token_accuracy": 0.6569370031356812,
+ "num_tokens": 24510464.0,
+ "step": 1496
+ },
+ {
+ "entropy": 1.44732666015625,
+ "epoch": 3.0242424242424244,
+ "grad_norm": 2.2525594234466553,
+ "learning_rate": 1.977777777777778e-06,
+ "loss": 1.3968576192855835,
+ "mean_token_accuracy": 0.6604787707328796,
+ "num_tokens": 24526848.0,
+ "step": 1497
+ },
+ {
+ "entropy": 1.522241234779358,
+ "epoch": 3.026262626262626,
+ "grad_norm": 2.1826529502868652,
+ "learning_rate": 1.975757575757576e-06,
+ "loss": 1.529256820678711,
+ "mean_token_accuracy": 0.6493649482727051,
+ "num_tokens": 24543232.0,
+ "step": 1498
+ },
+ {
+ "entropy": 0.8186452984809875,
+ "epoch": 3.0282828282828285,
+ "grad_norm": 2.1004087924957275,
+ "learning_rate": 1.9737373737373737e-06,
+ "loss": 0.8136417865753174,
+ "mean_token_accuracy": 0.7819370031356812,
+ "num_tokens": 24559616.0,
+ "step": 1499
+ },
+ {
+ "entropy": 1.2122654914855957,
+ "epoch": 3.0303030303030303,
+ "grad_norm": 2.4567720890045166,
+ "learning_rate": 1.9717171717171716e-06,
+ "loss": 1.236933946609497,
+ "mean_token_accuracy": 0.6828895807266235,
+ "num_tokens": 24576000.0,
+ "step": 1500
+ },
+ {
+ "epoch": 3.0303030303030303,
+ "eval_entropy": 1.3221501962792488,
+ "eval_loss": 1.5418624877929688,
+ "eval_mean_token_accuracy": 0.6455981039231823,
+ "eval_num_tokens": 24576000.0,
+ "eval_runtime": 43.8085,
+ "eval_samples_per_second": 22.598,
+ "eval_steps_per_second": 2.831,
+ "step": 1500
+ },
+ {
+ "entropy": 0.9493642449378967,
+ "epoch": 3.0323232323232325,
+ "grad_norm": 2.278731346130371,
+ "learning_rate": 1.96969696969697e-06,
+ "loss": 0.9456083178520203,
+ "mean_token_accuracy": 0.7484733462333679,
+ "num_tokens": 24592384.0,
+ "step": 1501
+ },
+ {
+ "entropy": 1.4509791135787964,
+ "epoch": 3.0343434343434343,
+ "grad_norm": 2.112172842025757,
+ "learning_rate": 1.967676767676768e-06,
+ "loss": 1.451171875,
+ "mean_token_accuracy": 0.6633487939834595,
+ "num_tokens": 24608768.0,
+ "step": 1502
+ },
+ {
+ "entropy": 1.6493220329284668,
+ "epoch": 3.036363636363636,
+ "grad_norm": 2.0969111919403076,
+ "learning_rate": 1.9656565656565657e-06,
+ "loss": 1.6576147079467773,
+ "mean_token_accuracy": 0.628724992275238,
+ "num_tokens": 24625152.0,
+ "step": 1503
+ },
+ {
+ "entropy": 1.5533742904663086,
+ "epoch": 3.0383838383838384,
+ "grad_norm": 2.1630685329437256,
+ "learning_rate": 1.9636363636363636e-06,
+ "loss": 1.556575894355774,
+ "mean_token_accuracy": 0.6745237112045288,
+ "num_tokens": 24641536.0,
+ "step": 1504
+ },
+ {
+ "entropy": 1.382328987121582,
+ "epoch": 3.04040404040404,
+ "grad_norm": 2.1493992805480957,
+ "learning_rate": 1.9616161616161615e-06,
+ "loss": 1.3725603818893433,
+ "mean_token_accuracy": 0.6811797618865967,
+ "num_tokens": 24657920.0,
+ "step": 1505
+ },
+ {
+ "entropy": 1.276682734489441,
+ "epoch": 3.0424242424242425,
+ "grad_norm": 2.342043161392212,
+ "learning_rate": 1.9595959595959598e-06,
+ "loss": 1.2861356735229492,
+ "mean_token_accuracy": 0.6843551397323608,
+ "num_tokens": 24674304.0,
+ "step": 1506
+ },
+ {
+ "entropy": 1.3813475370407104,
+ "epoch": 3.0444444444444443,
+ "grad_norm": 2.246891736984253,
+ "learning_rate": 1.9575757575757577e-06,
+ "loss": 1.3686306476593018,
+ "mean_token_accuracy": 0.6649364829063416,
+ "num_tokens": 24690688.0,
+ "step": 1507
+ },
+ {
+ "entropy": 1.2101274728775024,
+ "epoch": 3.0464646464646465,
+ "grad_norm": 2.2917656898498535,
+ "learning_rate": 1.955555555555556e-06,
+ "loss": 1.2131311893463135,
+ "mean_token_accuracy": 0.6958353519439697,
+ "num_tokens": 24707072.0,
+ "step": 1508
+ },
+ {
+ "entropy": 0.7796710133552551,
+ "epoch": 3.0484848484848484,
+ "grad_norm": 2.1023900508880615,
+ "learning_rate": 1.953535353535354e-06,
+ "loss": 0.7828338146209717,
+ "mean_token_accuracy": 0.7879824042320251,
+ "num_tokens": 24723456.0,
+ "step": 1509
+ },
+ {
+ "entropy": 0.8980224132537842,
+ "epoch": 3.0505050505050506,
+ "grad_norm": 2.259763240814209,
+ "learning_rate": 1.9515151515151518e-06,
+ "loss": 0.8986867070198059,
+ "mean_token_accuracy": 0.751038134098053,
+ "num_tokens": 24739840.0,
+ "step": 1510
+ },
+ {
+ "entropy": 1.1927038431167603,
+ "epoch": 3.0525252525252524,
+ "grad_norm": 2.2032012939453125,
+ "learning_rate": 1.9494949494949496e-06,
+ "loss": 1.1801973581314087,
+ "mean_token_accuracy": 0.7048729658126831,
+ "num_tokens": 24756224.0,
+ "step": 1511
+ },
+ {
+ "entropy": 0.8241261839866638,
+ "epoch": 3.0545454545454547,
+ "grad_norm": 1.9027519226074219,
+ "learning_rate": 1.9474747474747475e-06,
+ "loss": 0.8152725100517273,
+ "mean_token_accuracy": 0.7793111801147461,
+ "num_tokens": 24772608.0,
+ "step": 1512
+ },
+ {
+ "entropy": 1.1086668968200684,
+ "epoch": 3.0565656565656565,
+ "grad_norm": 2.4891581535339355,
+ "learning_rate": 1.945454545454546e-06,
+ "loss": 1.1167728900909424,
+ "mean_token_accuracy": 0.7162310481071472,
+ "num_tokens": 24788992.0,
+ "step": 1513
+ },
+ {
+ "entropy": 1.0804646015167236,
+ "epoch": 3.0585858585858587,
+ "grad_norm": 2.3115813732147217,
+ "learning_rate": 1.9434343434343437e-06,
+ "loss": 1.1059165000915527,
+ "mean_token_accuracy": 0.7208109498023987,
+ "num_tokens": 24805376.0,
+ "step": 1514
+ },
+ {
+ "entropy": 1.2628663778305054,
+ "epoch": 3.0606060606060606,
+ "grad_norm": 2.077954053878784,
+ "learning_rate": 1.9414141414141416e-06,
+ "loss": 1.2748162746429443,
+ "mean_token_accuracy": 0.697300910949707,
+ "num_tokens": 24821760.0,
+ "step": 1515
+ },
+ {
+ "entropy": 1.1143958568572998,
+ "epoch": 3.062626262626263,
+ "grad_norm": 2.3562631607055664,
+ "learning_rate": 1.9393939393939395e-06,
+ "loss": 1.127720832824707,
+ "mean_token_accuracy": 0.7080483436584473,
+ "num_tokens": 24838144.0,
+ "step": 1516
+ },
+ {
+ "entropy": 1.3625105619430542,
+ "epoch": 3.0646464646464646,
+ "grad_norm": 2.592428684234619,
+ "learning_rate": 1.9373737373737374e-06,
+ "loss": 1.3909248113632202,
+ "mean_token_accuracy": 0.6637151837348938,
+ "num_tokens": 24854528.0,
+ "step": 1517
+ },
+ {
+ "entropy": 1.032320261001587,
+ "epoch": 3.066666666666667,
+ "grad_norm": 2.400594711303711,
+ "learning_rate": 1.9353535353535353e-06,
+ "loss": 1.0697423219680786,
+ "mean_token_accuracy": 0.7302759885787964,
+ "num_tokens": 24870912.0,
+ "step": 1518
+ },
+ {
+ "entropy": 1.019578456878662,
+ "epoch": 3.0686868686868687,
+ "grad_norm": 2.137122869491577,
+ "learning_rate": 1.9333333333333336e-06,
+ "loss": 0.9904043674468994,
+ "mean_token_accuracy": 0.7461528778076172,
+ "num_tokens": 24887296.0,
+ "step": 1519
+ },
+ {
+ "entropy": 1.4399018287658691,
+ "epoch": 3.0707070707070705,
+ "grad_norm": 2.343580484390259,
+ "learning_rate": 1.9313131313131315e-06,
+ "loss": 1.429957628250122,
+ "mean_token_accuracy": 0.6615168452262878,
+ "num_tokens": 24903680.0,
+ "step": 1520
+ },
+ {
+ "entropy": 0.946161150932312,
+ "epoch": 3.0727272727272728,
+ "grad_norm": 1.9037182331085205,
+ "learning_rate": 1.9292929292929294e-06,
+ "loss": 0.943518877029419,
+ "mean_token_accuracy": 0.7626404762268066,
+ "num_tokens": 24920064.0,
+ "step": 1521
+ },
+ {
+ "entropy": 1.2314492464065552,
+ "epoch": 3.0747474747474746,
+ "grad_norm": 2.0896897315979004,
+ "learning_rate": 1.9272727272727273e-06,
+ "loss": 1.2434325218200684,
+ "mean_token_accuracy": 0.7122007608413696,
+ "num_tokens": 24936448.0,
+ "step": 1522
+ },
+ {
+ "entropy": 1.4557143449783325,
+ "epoch": 3.076767676767677,
+ "grad_norm": 2.162768602371216,
+ "learning_rate": 1.925252525252525e-06,
+ "loss": 1.4258036613464355,
+ "mean_token_accuracy": 0.6709819436073303,
+ "num_tokens": 24952832.0,
+ "step": 1523
+ },
+ {
+ "entropy": 1.1379952430725098,
+ "epoch": 3.0787878787878786,
+ "grad_norm": 2.322523832321167,
+ "learning_rate": 1.9232323232323235e-06,
+ "loss": 1.1321945190429688,
+ "mean_token_accuracy": 0.7206888198852539,
+ "num_tokens": 24969216.0,
+ "step": 1524
+ },
+ {
+ "entropy": 1.4143598079681396,
+ "epoch": 3.080808080808081,
+ "grad_norm": 2.2130038738250732,
+ "learning_rate": 1.9212121212121213e-06,
+ "loss": 1.4286584854125977,
+ "mean_token_accuracy": 0.6595627665519714,
+ "num_tokens": 24985600.0,
+ "step": 1525
+ },
+ {
+ "entropy": 0.9206982254981995,
+ "epoch": 3.0828282828282827,
+ "grad_norm": 2.150385618209839,
+ "learning_rate": 1.9191919191919192e-06,
+ "loss": 0.9122598171234131,
+ "mean_token_accuracy": 0.7564728856086731,
+ "num_tokens": 25001984.0,
+ "step": 1526
+ },
+ {
+ "entropy": 1.3168023824691772,
+ "epoch": 3.084848484848485,
+ "grad_norm": 2.2106544971466064,
+ "learning_rate": 1.9171717171717175e-06,
+ "loss": 1.3181350231170654,
+ "mean_token_accuracy": 0.6904616355895996,
+ "num_tokens": 25018368.0,
+ "step": 1527
+ },
+ {
+ "entropy": 1.2383761405944824,
+ "epoch": 3.0868686868686868,
+ "grad_norm": 2.1377766132354736,
+ "learning_rate": 1.9151515151515154e-06,
+ "loss": 1.2224464416503906,
+ "mean_token_accuracy": 0.7129335403442383,
+ "num_tokens": 25034752.0,
+ "step": 1528
+ },
+ {
+ "entropy": 1.6698395013809204,
+ "epoch": 3.088888888888889,
+ "grad_norm": 2.354219913482666,
+ "learning_rate": 1.9131313131313133e-06,
+ "loss": 1.67041015625,
+ "mean_token_accuracy": 0.6486932039260864,
+ "num_tokens": 25051136.0,
+ "step": 1529
+ },
+ {
+ "entropy": 1.1346160173416138,
+ "epoch": 3.090909090909091,
+ "grad_norm": 2.0619447231292725,
+ "learning_rate": 1.9111111111111112e-06,
+ "loss": 1.1436500549316406,
+ "mean_token_accuracy": 0.721177339553833,
+ "num_tokens": 25067520.0,
+ "step": 1530
+ },
+ {
+ "entropy": 1.4769412279129028,
+ "epoch": 3.092929292929293,
+ "grad_norm": 2.272189140319824,
+ "learning_rate": 1.9090909090909095e-06,
+ "loss": 1.4578258991241455,
+ "mean_token_accuracy": 0.6602955460548401,
+ "num_tokens": 25083904.0,
+ "step": 1531
+ },
+ {
+ "entropy": 1.3661279678344727,
+ "epoch": 3.094949494949495,
+ "grad_norm": 2.381265878677368,
+ "learning_rate": 1.9070707070707072e-06,
+ "loss": 1.3745051622390747,
+ "mean_token_accuracy": 0.6734245419502258,
+ "num_tokens": 25100288.0,
+ "step": 1532
+ },
+ {
+ "entropy": 1.0367696285247803,
+ "epoch": 3.096969696969697,
+ "grad_norm": 2.337775707244873,
+ "learning_rate": 1.9050505050505053e-06,
+ "loss": 1.0290615558624268,
+ "mean_token_accuracy": 0.7355275750160217,
+ "num_tokens": 25116672.0,
+ "step": 1533
+ },
+ {
+ "entropy": 0.9273203015327454,
+ "epoch": 3.098989898989899,
+ "grad_norm": 2.0960872173309326,
+ "learning_rate": 1.9030303030303032e-06,
+ "loss": 0.9162085056304932,
+ "mean_token_accuracy": 0.7639839053153992,
+ "num_tokens": 25133056.0,
+ "step": 1534
+ },
+ {
+ "entropy": 1.018664002418518,
+ "epoch": 3.101010101010101,
+ "grad_norm": 2.243028163909912,
+ "learning_rate": 1.9010101010101013e-06,
+ "loss": 1.019313097000122,
+ "mean_token_accuracy": 0.7399242520332336,
+ "num_tokens": 25149440.0,
+ "step": 1535
+ },
+ {
+ "entropy": 1.3178181648254395,
+ "epoch": 3.103030303030303,
+ "grad_norm": 2.305586338043213,
+ "learning_rate": 1.8989898989898992e-06,
+ "loss": 1.3278884887695312,
+ "mean_token_accuracy": 0.6838055849075317,
+ "num_tokens": 25165824.0,
+ "step": 1536
+ },
+ {
+ "entropy": 1.0808820724487305,
+ "epoch": 3.105050505050505,
+ "grad_norm": 2.2185559272766113,
+ "learning_rate": 1.896969696969697e-06,
+ "loss": 1.0852123498916626,
+ "mean_token_accuracy": 0.7187347412109375,
+ "num_tokens": 25182208.0,
+ "step": 1537
+ },
+ {
+ "entropy": 1.1828639507293701,
+ "epoch": 3.107070707070707,
+ "grad_norm": 2.5469112396240234,
+ "learning_rate": 1.8949494949494952e-06,
+ "loss": 1.1897902488708496,
+ "mean_token_accuracy": 0.697239875793457,
+ "num_tokens": 25198592.0,
+ "step": 1538
+ },
+ {
+ "entropy": 1.3447293043136597,
+ "epoch": 3.109090909090909,
+ "grad_norm": 2.2092692852020264,
+ "learning_rate": 1.892929292929293e-06,
+ "loss": 1.3312615156173706,
+ "mean_token_accuracy": 0.6797142028808594,
+ "num_tokens": 25214976.0,
+ "step": 1539
+ },
+ {
+ "entropy": 1.392472743988037,
+ "epoch": 3.111111111111111,
+ "grad_norm": 2.3445911407470703,
+ "learning_rate": 1.890909090909091e-06,
+ "loss": 1.3892791271209717,
+ "mean_token_accuracy": 0.6742794513702393,
+ "num_tokens": 25231360.0,
+ "step": 1540
+ },
+ {
+ "entropy": 1.422629475593567,
+ "epoch": 3.113131313131313,
+ "grad_norm": 2.4753165245056152,
+ "learning_rate": 1.888888888888889e-06,
+ "loss": 1.432596206665039,
+ "mean_token_accuracy": 0.6670126914978027,
+ "num_tokens": 25247744.0,
+ "step": 1541
+ },
+ {
+ "entropy": 1.5371577739715576,
+ "epoch": 3.1151515151515152,
+ "grad_norm": 2.3083088397979736,
+ "learning_rate": 1.886868686868687e-06,
+ "loss": 1.5318076610565186,
+ "mean_token_accuracy": 0.6301905512809753,
+ "num_tokens": 25264128.0,
+ "step": 1542
+ },
+ {
+ "entropy": 1.1179860830307007,
+ "epoch": 3.117171717171717,
+ "grad_norm": 2.1037089824676514,
+ "learning_rate": 1.884848484848485e-06,
+ "loss": 1.123746395111084,
+ "mean_token_accuracy": 0.7327185869216919,
+ "num_tokens": 25280512.0,
+ "step": 1543
+ },
+ {
+ "entropy": 1.046632170677185,
+ "epoch": 3.1191919191919193,
+ "grad_norm": 2.267493963241577,
+ "learning_rate": 1.882828282828283e-06,
+ "loss": 1.027912974357605,
+ "mean_token_accuracy": 0.7275280952453613,
+ "num_tokens": 25296896.0,
+ "step": 1544
+ },
+ {
+ "entropy": 1.2848094701766968,
+ "epoch": 3.121212121212121,
+ "grad_norm": 2.1797454357147217,
+ "learning_rate": 1.8808080808080808e-06,
+ "loss": 1.3054044246673584,
+ "mean_token_accuracy": 0.7058500051498413,
+ "num_tokens": 25313280.0,
+ "step": 1545
+ },
+ {
+ "entropy": 1.132463812828064,
+ "epoch": 3.1232323232323234,
+ "grad_norm": 2.2171123027801514,
+ "learning_rate": 1.878787878787879e-06,
+ "loss": 1.1395090818405151,
+ "mean_token_accuracy": 0.7204445600509644,
+ "num_tokens": 25329664.0,
+ "step": 1546
+ },
+ {
+ "entropy": 1.084518551826477,
+ "epoch": 3.125252525252525,
+ "grad_norm": 2.2163641452789307,
+ "learning_rate": 1.8767676767676768e-06,
+ "loss": 1.093268871307373,
+ "mean_token_accuracy": 0.7297264337539673,
+ "num_tokens": 25346048.0,
+ "step": 1547
+ },
+ {
+ "entropy": 1.1031688451766968,
+ "epoch": 3.1272727272727274,
+ "grad_norm": 2.3715343475341797,
+ "learning_rate": 1.874747474747475e-06,
+ "loss": 1.1064910888671875,
+ "mean_token_accuracy": 0.7130556702613831,
+ "num_tokens": 25362432.0,
+ "step": 1548
+ },
+ {
+ "entropy": 1.515089988708496,
+ "epoch": 3.1292929292929292,
+ "grad_norm": 2.0995519161224365,
+ "learning_rate": 1.872727272727273e-06,
+ "loss": 1.5236327648162842,
+ "mean_token_accuracy": 0.6491206884384155,
+ "num_tokens": 25378816.0,
+ "step": 1549
+ },
+ {
+ "entropy": 1.26030695438385,
+ "epoch": 3.1313131313131315,
+ "grad_norm": 2.2167913913726807,
+ "learning_rate": 1.8707070707070709e-06,
+ "loss": 1.2750146389007568,
+ "mean_token_accuracy": 0.6991939544677734,
+ "num_tokens": 25395200.0,
+ "step": 1550
+ },
+ {
+ "epoch": 3.1313131313131315,
+ "eval_entropy": 1.3187109231948853,
+ "eval_loss": 1.5490814447402954,
+ "eval_mean_token_accuracy": 0.6446880385760339,
+ "eval_num_tokens": 25395200.0,
+ "eval_runtime": 43.7612,
+ "eval_samples_per_second": 22.623,
+ "eval_steps_per_second": 2.834,
+ "step": 1550
+ },
+ {
+ "entropy": 1.044468641281128,
+ "epoch": 3.1333333333333333,
+ "grad_norm": 2.189656972885132,
+ "learning_rate": 1.868686868686869e-06,
+ "loss": 1.035596251487732,
+ "mean_token_accuracy": 0.7405959963798523,
+ "num_tokens": 25411584.0,
+ "step": 1551
+ },
+ {
+ "entropy": 1.286029577255249,
+ "epoch": 3.1353535353535356,
+ "grad_norm": 2.3911969661712646,
+ "learning_rate": 1.8666666666666669e-06,
+ "loss": 1.287235975265503,
+ "mean_token_accuracy": 0.6918661594390869,
+ "num_tokens": 25427968.0,
+ "step": 1552
+ },
+ {
+ "entropy": 1.4401317834854126,
+ "epoch": 3.1373737373737374,
+ "grad_norm": 2.452929973602295,
+ "learning_rate": 1.864646464646465e-06,
+ "loss": 1.4605298042297363,
+ "mean_token_accuracy": 0.6466169953346252,
+ "num_tokens": 25444352.0,
+ "step": 1553
+ },
+ {
+ "entropy": 1.316934585571289,
+ "epoch": 3.1393939393939396,
+ "grad_norm": 2.4121034145355225,
+ "learning_rate": 1.8626262626262629e-06,
+ "loss": 1.3266665935516357,
+ "mean_token_accuracy": 0.6759892702102661,
+ "num_tokens": 25460736.0,
+ "step": 1554
+ },
+ {
+ "entropy": 1.2644363641738892,
+ "epoch": 3.1414141414141414,
+ "grad_norm": 2.2631289958953857,
+ "learning_rate": 1.8606060606060607e-06,
+ "loss": 1.2905628681182861,
+ "mean_token_accuracy": 0.6833170652389526,
+ "num_tokens": 25477120.0,
+ "step": 1555
+ },
+ {
+ "entropy": 1.3172475099563599,
+ "epoch": 3.1434343434343432,
+ "grad_norm": 2.357954502105713,
+ "learning_rate": 1.8585858585858588e-06,
+ "loss": 1.316823124885559,
+ "mean_token_accuracy": 0.6763556599617004,
+ "num_tokens": 25493504.0,
+ "step": 1556
+ },
+ {
+ "entropy": 1.392463207244873,
+ "epoch": 3.1454545454545455,
+ "grad_norm": 2.491037130355835,
+ "learning_rate": 1.8565656565656567e-06,
+ "loss": 1.3718677759170532,
+ "mean_token_accuracy": 0.6508915424346924,
+ "num_tokens": 25509888.0,
+ "step": 1557
+ },
+ {
+ "entropy": 1.3607147932052612,
+ "epoch": 3.1474747474747473,
+ "grad_norm": 2.306100845336914,
+ "learning_rate": 1.8545454545454546e-06,
+ "loss": 1.3523163795471191,
+ "mean_token_accuracy": 0.6557157039642334,
+ "num_tokens": 25526272.0,
+ "step": 1558
+ },
+ {
+ "entropy": 0.9888757467269897,
+ "epoch": 3.1494949494949496,
+ "grad_norm": 2.125560998916626,
+ "learning_rate": 1.8525252525252527e-06,
+ "loss": 0.9713444709777832,
+ "mean_token_accuracy": 0.7584269642829895,
+ "num_tokens": 25542656.0,
+ "step": 1559
+ },
+ {
+ "entropy": 1.286476492881775,
+ "epoch": 3.1515151515151514,
+ "grad_norm": 2.149113416671753,
+ "learning_rate": 1.8505050505050506e-06,
+ "loss": 1.2688651084899902,
+ "mean_token_accuracy": 0.6882022619247437,
+ "num_tokens": 25559040.0,
+ "step": 1560
+ },
+ {
+ "entropy": 0.9255673289299011,
+ "epoch": 3.1535353535353536,
+ "grad_norm": 2.1615419387817383,
+ "learning_rate": 1.8484848484848487e-06,
+ "loss": 0.9265275597572327,
+ "mean_token_accuracy": 0.7542135119438171,
+ "num_tokens": 25575424.0,
+ "step": 1561
+ },
+ {
+ "entropy": 1.4960277080535889,
+ "epoch": 3.1555555555555554,
+ "grad_norm": 2.183372974395752,
+ "learning_rate": 1.8464646464646466e-06,
+ "loss": 1.4737393856048584,
+ "mean_token_accuracy": 0.6516854166984558,
+ "num_tokens": 25591808.0,
+ "step": 1562
+ },
+ {
+ "entropy": 1.2294487953186035,
+ "epoch": 3.1575757575757577,
+ "grad_norm": 2.389885425567627,
+ "learning_rate": 1.8444444444444445e-06,
+ "loss": 1.210254192352295,
+ "mean_token_accuracy": 0.6988885998725891,
+ "num_tokens": 25608192.0,
+ "step": 1563
+ },
+ {
+ "entropy": 1.0153220891952515,
+ "epoch": 3.1595959595959595,
+ "grad_norm": 2.131989002227783,
+ "learning_rate": 1.8424242424242426e-06,
+ "loss": 1.0005543231964111,
+ "mean_token_accuracy": 0.7426722049713135,
+ "num_tokens": 25624576.0,
+ "step": 1564
+ },
+ {
+ "entropy": 1.2591265439987183,
+ "epoch": 3.1616161616161618,
+ "grad_norm": 2.150913715362549,
+ "learning_rate": 1.8404040404040405e-06,
+ "loss": 1.2284668684005737,
+ "mean_token_accuracy": 0.6993771195411682,
+ "num_tokens": 25640960.0,
+ "step": 1565
+ },
+ {
+ "entropy": 1.009209394454956,
+ "epoch": 3.1636363636363636,
+ "grad_norm": 2.090205192565918,
+ "learning_rate": 1.8383838383838384e-06,
+ "loss": 0.9917510747909546,
+ "mean_token_accuracy": 0.7412066459655762,
+ "num_tokens": 25657344.0,
+ "step": 1566
+ },
+ {
+ "entropy": 1.716966986656189,
+ "epoch": 3.165656565656566,
+ "grad_norm": 2.2880618572235107,
+ "learning_rate": 1.8363636363636365e-06,
+ "loss": 1.7331494092941284,
+ "mean_token_accuracy": 0.6106497049331665,
+ "num_tokens": 25673728.0,
+ "step": 1567
+ },
+ {
+ "entropy": 1.5511447191238403,
+ "epoch": 3.1676767676767676,
+ "grad_norm": 2.371710777282715,
+ "learning_rate": 1.8343434343434343e-06,
+ "loss": 1.5570940971374512,
+ "mean_token_accuracy": 0.6477161645889282,
+ "num_tokens": 25690112.0,
+ "step": 1568
+ },
+ {
+ "entropy": 1.325405240058899,
+ "epoch": 3.16969696969697,
+ "grad_norm": 2.3747830390930176,
+ "learning_rate": 1.8323232323232324e-06,
+ "loss": 1.3260624408721924,
+ "mean_token_accuracy": 0.6791035532951355,
+ "num_tokens": 25706496.0,
+ "step": 1569
+ },
+ {
+ "entropy": 1.2529335021972656,
+ "epoch": 3.1717171717171717,
+ "grad_norm": 2.162775754928589,
+ "learning_rate": 1.8303030303030305e-06,
+ "loss": 1.257356882095337,
+ "mean_token_accuracy": 0.7066438794136047,
+ "num_tokens": 25722880.0,
+ "step": 1570
+ },
+ {
+ "entropy": 1.116769790649414,
+ "epoch": 3.1737373737373735,
+ "grad_norm": 2.217358350753784,
+ "learning_rate": 1.8282828282828286e-06,
+ "loss": 1.1458317041397095,
+ "mean_token_accuracy": 0.7087811231613159,
+ "num_tokens": 25739264.0,
+ "step": 1571
+ },
+ {
+ "entropy": 1.3465793132781982,
+ "epoch": 3.175757575757576,
+ "grad_norm": 2.225062370300293,
+ "learning_rate": 1.8262626262626265e-06,
+ "loss": 1.380183219909668,
+ "mean_token_accuracy": 0.6872862577438354,
+ "num_tokens": 25755648.0,
+ "step": 1572
+ },
+ {
+ "entropy": 1.237407922744751,
+ "epoch": 3.1777777777777776,
+ "grad_norm": 2.31829833984375,
+ "learning_rate": 1.8242424242424244e-06,
+ "loss": 1.247010588645935,
+ "mean_token_accuracy": 0.7034074068069458,
+ "num_tokens": 25772032.0,
+ "step": 1573
+ },
+ {
+ "entropy": 1.3967171907424927,
+ "epoch": 3.17979797979798,
+ "grad_norm": 2.4075193405151367,
+ "learning_rate": 1.8222222222222225e-06,
+ "loss": 1.4189220666885376,
+ "mean_token_accuracy": 0.6609672904014587,
+ "num_tokens": 25788416.0,
+ "step": 1574
+ },
+ {
+ "entropy": 1.4262871742248535,
+ "epoch": 3.1818181818181817,
+ "grad_norm": 2.507885217666626,
+ "learning_rate": 1.8202020202020204e-06,
+ "loss": 1.4657065868377686,
+ "mean_token_accuracy": 0.6502198576927185,
+ "num_tokens": 25804800.0,
+ "step": 1575
+ },
+ {
+ "entropy": 1.1763924360275269,
+ "epoch": 3.183838383838384,
+ "grad_norm": 2.179891586303711,
+ "learning_rate": 1.8181818181818183e-06,
+ "loss": 1.1837804317474365,
+ "mean_token_accuracy": 0.7209941148757935,
+ "num_tokens": 25821184.0,
+ "step": 1576
+ },
+ {
+ "entropy": 1.119240403175354,
+ "epoch": 3.1858585858585857,
+ "grad_norm": 2.300708532333374,
+ "learning_rate": 1.8161616161616164e-06,
+ "loss": 1.1251145601272583,
+ "mean_token_accuracy": 0.7134220600128174,
+ "num_tokens": 25837568.0,
+ "step": 1577
+ },
+ {
+ "entropy": 1.3045564889907837,
+ "epoch": 3.187878787878788,
+ "grad_norm": 2.2904021739959717,
+ "learning_rate": 1.8141414141414143e-06,
+ "loss": 1.3114243745803833,
+ "mean_token_accuracy": 0.691316545009613,
+ "num_tokens": 25853952.0,
+ "step": 1578
+ },
+ {
+ "entropy": 1.0507739782333374,
+ "epoch": 3.18989898989899,
+ "grad_norm": 2.0942912101745605,
+ "learning_rate": 1.8121212121212124e-06,
+ "loss": 1.043527364730835,
+ "mean_token_accuracy": 0.7244748473167419,
+ "num_tokens": 25870336.0,
+ "step": 1579
+ },
+ {
+ "entropy": 1.2050808668136597,
+ "epoch": 3.191919191919192,
+ "grad_norm": 2.131643295288086,
+ "learning_rate": 1.8101010101010103e-06,
+ "loss": 1.1917791366577148,
+ "mean_token_accuracy": 0.7046898007392883,
+ "num_tokens": 25886720.0,
+ "step": 1580
+ },
+ {
+ "entropy": 1.3607940673828125,
+ "epoch": 3.193939393939394,
+ "grad_norm": 2.2744040489196777,
+ "learning_rate": 1.8080808080808082e-06,
+ "loss": 1.347360610961914,
+ "mean_token_accuracy": 0.6780043840408325,
+ "num_tokens": 25903104.0,
+ "step": 1581
+ },
+ {
+ "entropy": 1.4996237754821777,
+ "epoch": 3.195959595959596,
+ "grad_norm": 2.3050780296325684,
+ "learning_rate": 1.8060606060606063e-06,
+ "loss": 1.5023889541625977,
+ "mean_token_accuracy": 0.6423424482345581,
+ "num_tokens": 25919488.0,
+ "step": 1582
+ },
+ {
+ "entropy": 1.1950902938842773,
+ "epoch": 3.197979797979798,
+ "grad_norm": 2.408799648284912,
+ "learning_rate": 1.8040404040404041e-06,
+ "loss": 1.1868454217910767,
+ "mean_token_accuracy": 0.6998046040534973,
+ "num_tokens": 25935872.0,
+ "step": 1583
+ },
+ {
+ "entropy": 1.5395914316177368,
+ "epoch": 3.2,
+ "grad_norm": 2.4104502201080322,
+ "learning_rate": 1.802020202020202e-06,
+ "loss": 1.5414860248565674,
+ "mean_token_accuracy": 0.6382511258125305,
+ "num_tokens": 25952256.0,
+ "step": 1584
+ },
+ {
+ "entropy": 1.0701062679290771,
+ "epoch": 3.202020202020202,
+ "grad_norm": 2.2250685691833496,
+ "learning_rate": 1.8000000000000001e-06,
+ "loss": 1.0787923336029053,
+ "mean_token_accuracy": 0.7307645082473755,
+ "num_tokens": 25968640.0,
+ "step": 1585
+ },
+ {
+ "entropy": 1.0968927145004272,
+ "epoch": 3.2040404040404042,
+ "grad_norm": 2.3169026374816895,
+ "learning_rate": 1.797979797979798e-06,
+ "loss": 1.1023577451705933,
+ "mean_token_accuracy": 0.7217879891395569,
+ "num_tokens": 25985024.0,
+ "step": 1586
+ },
+ {
+ "entropy": 1.0547802448272705,
+ "epoch": 3.206060606060606,
+ "grad_norm": 2.1187734603881836,
+ "learning_rate": 1.7959595959595961e-06,
+ "loss": 1.0536352396011353,
+ "mean_token_accuracy": 0.733146071434021,
+ "num_tokens": 26001408.0,
+ "step": 1587
+ },
+ {
+ "entropy": 1.821456789970398,
+ "epoch": 3.2080808080808083,
+ "grad_norm": 2.2364695072174072,
+ "learning_rate": 1.793939393939394e-06,
+ "loss": 1.8386787176132202,
+ "mean_token_accuracy": 0.5963605046272278,
+ "num_tokens": 26017792.0,
+ "step": 1588
+ },
+ {
+ "entropy": 1.5173720121383667,
+ "epoch": 3.21010101010101,
+ "grad_norm": 2.2785775661468506,
+ "learning_rate": 1.7919191919191919e-06,
+ "loss": 1.5099977254867554,
+ "mean_token_accuracy": 0.6417317986488342,
+ "num_tokens": 26034176.0,
+ "step": 1589
+ },
+ {
+ "entropy": 1.5751056671142578,
+ "epoch": 3.212121212121212,
+ "grad_norm": 2.2048606872558594,
+ "learning_rate": 1.78989898989899e-06,
+ "loss": 1.5845966339111328,
+ "mean_token_accuracy": 0.633671224117279,
+ "num_tokens": 26050560.0,
+ "step": 1590
+ },
+ {
+ "entropy": 1.3307859897613525,
+ "epoch": 3.214141414141414,
+ "grad_norm": 2.310291051864624,
+ "learning_rate": 1.787878787878788e-06,
+ "loss": 1.3058912754058838,
+ "mean_token_accuracy": 0.6883854269981384,
+ "num_tokens": 26066944.0,
+ "step": 1591
+ },
+ {
+ "entropy": 0.9954612255096436,
+ "epoch": 3.216161616161616,
+ "grad_norm": 2.1447651386260986,
+ "learning_rate": 1.7858585858585862e-06,
+ "loss": 0.975112795829773,
+ "mean_token_accuracy": 0.7406570315361023,
+ "num_tokens": 26083328.0,
+ "step": 1592
+ },
+ {
+ "entropy": 1.1513571739196777,
+ "epoch": 3.2181818181818183,
+ "grad_norm": 2.3595945835113525,
+ "learning_rate": 1.783838383838384e-06,
+ "loss": 1.1416239738464355,
+ "mean_token_accuracy": 0.7137274146080017,
+ "num_tokens": 26099712.0,
+ "step": 1593
+ },
+ {
+ "entropy": 0.9992931485176086,
+ "epoch": 3.22020202020202,
+ "grad_norm": 2.1605300903320312,
+ "learning_rate": 1.781818181818182e-06,
+ "loss": 1.0023880004882812,
+ "mean_token_accuracy": 0.738092303276062,
+ "num_tokens": 26116096.0,
+ "step": 1594
+ },
+ {
+ "entropy": 1.6680811643600464,
+ "epoch": 3.2222222222222223,
+ "grad_norm": 2.3036210536956787,
+ "learning_rate": 1.77979797979798e-06,
+ "loss": 1.7154381275177002,
+ "mean_token_accuracy": 0.6182217597961426,
+ "num_tokens": 26132480.0,
+ "step": 1595
+ },
+ {
+ "entropy": 1.113036036491394,
+ "epoch": 3.224242424242424,
+ "grad_norm": 2.3098108768463135,
+ "learning_rate": 1.777777777777778e-06,
+ "loss": 1.1006368398666382,
+ "mean_token_accuracy": 0.7120175957679749,
+ "num_tokens": 26148864.0,
+ "step": 1596
+ },
+ {
+ "entropy": 1.0055886507034302,
+ "epoch": 3.2262626262626264,
+ "grad_norm": 2.3227128982543945,
+ "learning_rate": 1.775757575757576e-06,
+ "loss": 1.0274525880813599,
+ "mean_token_accuracy": 0.7253297567367554,
+ "num_tokens": 26165248.0,
+ "step": 1597
+ },
+ {
+ "entropy": 1.139039397239685,
+ "epoch": 3.228282828282828,
+ "grad_norm": 2.415011405944824,
+ "learning_rate": 1.773737373737374e-06,
+ "loss": 1.1494395732879639,
+ "mean_token_accuracy": 0.7061553597450256,
+ "num_tokens": 26181632.0,
+ "step": 1598
+ },
+ {
+ "entropy": 1.1573477983474731,
+ "epoch": 3.2303030303030305,
+ "grad_norm": 2.2666749954223633,
+ "learning_rate": 1.7717171717171718e-06,
+ "loss": 1.1615169048309326,
+ "mean_token_accuracy": 0.7168416976928711,
+ "num_tokens": 26198016.0,
+ "step": 1599
+ },
+ {
+ "entropy": 1.1301828622817993,
+ "epoch": 3.2323232323232323,
+ "grad_norm": 2.345222234725952,
+ "learning_rate": 1.76969696969697e-06,
+ "loss": 1.1062344312667847,
+ "mean_token_accuracy": 0.7164753079414368,
+ "num_tokens": 26214400.0,
+ "step": 1600
+ },
+ {
+ "epoch": 3.2323232323232323,
+ "eval_entropy": 1.3245953331070561,
+ "eval_loss": 1.5489962100982666,
+ "eval_mean_token_accuracy": 0.644533898080549,
+ "eval_num_tokens": 26214400.0,
+ "eval_runtime": 43.7619,
+ "eval_samples_per_second": 22.622,
+ "eval_steps_per_second": 2.834,
+ "step": 1600
+ },
+ {
+ "entropy": 1.4168665409088135,
+ "epoch": 3.2343434343434345,
+ "grad_norm": 2.3160741329193115,
+ "learning_rate": 1.7676767676767678e-06,
+ "loss": 1.403707504272461,
+ "mean_token_accuracy": 0.6696385145187378,
+ "num_tokens": 26230784.0,
+ "step": 1601
+ },
+ {
+ "entropy": 1.4689940214157104,
+ "epoch": 3.2363636363636363,
+ "grad_norm": 2.1691131591796875,
+ "learning_rate": 1.7656565656565657e-06,
+ "loss": 1.463477611541748,
+ "mean_token_accuracy": 0.6482046842575073,
+ "num_tokens": 26247168.0,
+ "step": 1602
+ },
+ {
+ "entropy": 1.2312567234039307,
+ "epoch": 3.2383838383838386,
+ "grad_norm": 2.4240810871124268,
+ "learning_rate": 1.7636363636363638e-06,
+ "loss": 1.2253142595291138,
+ "mean_token_accuracy": 0.6933317184448242,
+ "num_tokens": 26263552.0,
+ "step": 1603
+ },
+ {
+ "entropy": 1.5346348285675049,
+ "epoch": 3.2404040404040404,
+ "grad_norm": 2.417025566101074,
+ "learning_rate": 1.7616161616161617e-06,
+ "loss": 1.5418915748596191,
+ "mean_token_accuracy": 0.6389228105545044,
+ "num_tokens": 26279936.0,
+ "step": 1604
+ },
+ {
+ "entropy": 1.4432504177093506,
+ "epoch": 3.242424242424242,
+ "grad_norm": 2.2379117012023926,
+ "learning_rate": 1.7595959595959598e-06,
+ "loss": 1.4543676376342773,
+ "mean_token_accuracy": 0.6502198576927185,
+ "num_tokens": 26296320.0,
+ "step": 1605
+ },
+ {
+ "entropy": 1.54794442653656,
+ "epoch": 3.2444444444444445,
+ "grad_norm": 2.293386936187744,
+ "learning_rate": 1.7575757575757577e-06,
+ "loss": 1.525251865386963,
+ "mean_token_accuracy": 0.652723491191864,
+ "num_tokens": 26312704.0,
+ "step": 1606
+ },
+ {
+ "entropy": 1.3418866395950317,
+ "epoch": 3.2464646464646463,
+ "grad_norm": 2.143143892288208,
+ "learning_rate": 1.7555555555555556e-06,
+ "loss": 1.3268392086029053,
+ "mean_token_accuracy": 0.6758060455322266,
+ "num_tokens": 26329088.0,
+ "step": 1607
+ },
+ {
+ "entropy": 1.3032151460647583,
+ "epoch": 3.2484848484848485,
+ "grad_norm": 2.0813350677490234,
+ "learning_rate": 1.7535353535353537e-06,
+ "loss": 1.3105076551437378,
+ "mean_token_accuracy": 0.6873473525047302,
+ "num_tokens": 26345472.0,
+ "step": 1608
+ },
+ {
+ "entropy": 1.258314847946167,
+ "epoch": 3.2505050505050503,
+ "grad_norm": 2.346703052520752,
+ "learning_rate": 1.7515151515151516e-06,
+ "loss": 1.2453927993774414,
+ "mean_token_accuracy": 0.6931484937667847,
+ "num_tokens": 26361856.0,
+ "step": 1609
+ },
+ {
+ "entropy": 1.1887613534927368,
+ "epoch": 3.2525252525252526,
+ "grad_norm": 2.2838189601898193,
+ "learning_rate": 1.7494949494949494e-06,
+ "loss": 1.2008028030395508,
+ "mean_token_accuracy": 0.7044455409049988,
+ "num_tokens": 26378240.0,
+ "step": 1610
+ },
+ {
+ "entropy": 1.2221968173980713,
+ "epoch": 3.2545454545454544,
+ "grad_norm": 2.201186418533325,
+ "learning_rate": 1.7474747474747475e-06,
+ "loss": 1.2126820087432861,
+ "mean_token_accuracy": 0.6948583126068115,
+ "num_tokens": 26394624.0,
+ "step": 1611
+ },
+ {
+ "entropy": 1.3306292295455933,
+ "epoch": 3.2565656565656567,
+ "grad_norm": 2.1501541137695312,
+ "learning_rate": 1.7454545454545456e-06,
+ "loss": 1.3301104307174683,
+ "mean_token_accuracy": 0.6844772696495056,
+ "num_tokens": 26411008.0,
+ "step": 1612
+ },
+ {
+ "entropy": 1.0881599187850952,
+ "epoch": 3.2585858585858585,
+ "grad_norm": 2.092081069946289,
+ "learning_rate": 1.7434343434343437e-06,
+ "loss": 1.0868645906448364,
+ "mean_token_accuracy": 0.7172691822052002,
+ "num_tokens": 26427392.0,
+ "step": 1613
+ },
+ {
+ "entropy": 1.1937274932861328,
+ "epoch": 3.2606060606060607,
+ "grad_norm": 2.563627243041992,
+ "learning_rate": 1.7414141414141416e-06,
+ "loss": 1.1793221235275269,
+ "mean_token_accuracy": 0.7015144228935242,
+ "num_tokens": 26443776.0,
+ "step": 1614
+ },
+ {
+ "entropy": 1.1501706838607788,
+ "epoch": 3.2626262626262625,
+ "grad_norm": 2.2964398860931396,
+ "learning_rate": 1.7393939393939397e-06,
+ "loss": 1.1438164710998535,
+ "mean_token_accuracy": 0.7105520367622375,
+ "num_tokens": 26460160.0,
+ "step": 1615
+ },
+ {
+ "entropy": 1.276473045349121,
+ "epoch": 3.264646464646465,
+ "grad_norm": 2.4301693439483643,
+ "learning_rate": 1.7373737373737376e-06,
+ "loss": 1.2897876501083374,
+ "mean_token_accuracy": 0.6933927536010742,
+ "num_tokens": 26476544.0,
+ "step": 1616
+ },
+ {
+ "entropy": 1.1926265954971313,
+ "epoch": 3.2666666666666666,
+ "grad_norm": 2.1503496170043945,
+ "learning_rate": 1.7353535353535355e-06,
+ "loss": 1.1882628202438354,
+ "mean_token_accuracy": 0.7013922929763794,
+ "num_tokens": 26492928.0,
+ "step": 1617
+ },
+ {
+ "entropy": 1.2843140363693237,
+ "epoch": 3.268686868686869,
+ "grad_norm": 2.312652587890625,
+ "learning_rate": 1.7333333333333336e-06,
+ "loss": 1.3045732975006104,
+ "mean_token_accuracy": 0.6849047541618347,
+ "num_tokens": 26509312.0,
+ "step": 1618
+ },
+ {
+ "entropy": 1.0693352222442627,
+ "epoch": 3.2707070707070707,
+ "grad_norm": 2.2860965728759766,
+ "learning_rate": 1.7313131313131315e-06,
+ "loss": 1.044942855834961,
+ "mean_token_accuracy": 0.72832190990448,
+ "num_tokens": 26525696.0,
+ "step": 1619
+ },
+ {
+ "entropy": 1.5462334156036377,
+ "epoch": 3.2727272727272725,
+ "grad_norm": 2.395378589630127,
+ "learning_rate": 1.7292929292929294e-06,
+ "loss": 1.5529801845550537,
+ "mean_token_accuracy": 0.6301294565200806,
+ "num_tokens": 26542080.0,
+ "step": 1620
+ },
+ {
+ "entropy": 1.0582746267318726,
+ "epoch": 3.2747474747474747,
+ "grad_norm": 2.198370933532715,
+ "learning_rate": 1.7272727272727275e-06,
+ "loss": 1.0674792528152466,
+ "mean_token_accuracy": 0.7289936542510986,
+ "num_tokens": 26558464.0,
+ "step": 1621
+ },
+ {
+ "entropy": 1.133894920349121,
+ "epoch": 3.276767676767677,
+ "grad_norm": 2.2087979316711426,
+ "learning_rate": 1.7252525252525254e-06,
+ "loss": 1.1322801113128662,
+ "mean_token_accuracy": 0.7148876190185547,
+ "num_tokens": 26574848.0,
+ "step": 1622
+ },
+ {
+ "entropy": 1.0465091466903687,
+ "epoch": 3.278787878787879,
+ "grad_norm": 2.4500746726989746,
+ "learning_rate": 1.7232323232323235e-06,
+ "loss": 1.042628526687622,
+ "mean_token_accuracy": 0.72832190990448,
+ "num_tokens": 26591232.0,
+ "step": 1623
+ },
+ {
+ "entropy": 1.3930310010910034,
+ "epoch": 3.2808080808080806,
+ "grad_norm": 2.2968733310699463,
+ "learning_rate": 1.7212121212121214e-06,
+ "loss": 1.3903234004974365,
+ "mean_token_accuracy": 0.6658524870872498,
+ "num_tokens": 26607616.0,
+ "step": 1624
+ },
+ {
+ "entropy": 1.4873614311218262,
+ "epoch": 3.282828282828283,
+ "grad_norm": 2.404604434967041,
+ "learning_rate": 1.7191919191919192e-06,
+ "loss": 1.4828845262527466,
+ "mean_token_accuracy": 0.6601123809814453,
+ "num_tokens": 26624000.0,
+ "step": 1625
+ },
+ {
+ "entropy": 1.4326503276824951,
+ "epoch": 3.2848484848484847,
+ "grad_norm": 2.3051655292510986,
+ "learning_rate": 1.7171717171717173e-06,
+ "loss": 1.4187182188034058,
+ "mean_token_accuracy": 0.6783097386360168,
+ "num_tokens": 26640384.0,
+ "step": 1626
+ },
+ {
+ "entropy": 1.067203402519226,
+ "epoch": 3.286868686868687,
+ "grad_norm": 2.1361143589019775,
+ "learning_rate": 1.7151515151515152e-06,
+ "loss": 1.0571379661560059,
+ "mean_token_accuracy": 0.7386419177055359,
+ "num_tokens": 26656768.0,
+ "step": 1627
+ },
+ {
+ "entropy": 0.9872021675109863,
+ "epoch": 3.2888888888888888,
+ "grad_norm": 2.084920883178711,
+ "learning_rate": 1.7131313131313131e-06,
+ "loss": 0.9779852628707886,
+ "mean_token_accuracy": 0.7591597437858582,
+ "num_tokens": 26673152.0,
+ "step": 1628
+ },
+ {
+ "entropy": 1.261454701423645,
+ "epoch": 3.290909090909091,
+ "grad_norm": 2.2008469104766846,
+ "learning_rate": 1.7111111111111112e-06,
+ "loss": 1.2701423168182373,
+ "mean_token_accuracy": 0.6892403364181519,
+ "num_tokens": 26689536.0,
+ "step": 1629
+ },
+ {
+ "entropy": 1.6490918397903442,
+ "epoch": 3.292929292929293,
+ "grad_norm": 2.2967329025268555,
+ "learning_rate": 1.7090909090909091e-06,
+ "loss": 1.6521049737930298,
+ "mean_token_accuracy": 0.6223741769790649,
+ "num_tokens": 26705920.0,
+ "step": 1630
+ },
+ {
+ "entropy": 1.241626262664795,
+ "epoch": 3.294949494949495,
+ "grad_norm": 2.3993594646453857,
+ "learning_rate": 1.707070707070707e-06,
+ "loss": 1.2557122707366943,
+ "mean_token_accuracy": 0.6982169151306152,
+ "num_tokens": 26722304.0,
+ "step": 1631
+ },
+ {
+ "entropy": 1.2038613557815552,
+ "epoch": 3.296969696969697,
+ "grad_norm": 2.6470985412597656,
+ "learning_rate": 1.705050505050505e-06,
+ "loss": 1.220262885093689,
+ "mean_token_accuracy": 0.702186107635498,
+ "num_tokens": 26738688.0,
+ "step": 1632
+ },
+ {
+ "entropy": 1.0352909564971924,
+ "epoch": 3.298989898989899,
+ "grad_norm": 2.30973219871521,
+ "learning_rate": 1.703030303030303e-06,
+ "loss": 1.0416737794876099,
+ "mean_token_accuracy": 0.7299095988273621,
+ "num_tokens": 26755072.0,
+ "step": 1633
+ },
+ {
+ "entropy": 1.0178903341293335,
+ "epoch": 3.301010101010101,
+ "grad_norm": 2.3068888187408447,
+ "learning_rate": 1.7010101010101013e-06,
+ "loss": 1.021026849746704,
+ "mean_token_accuracy": 0.7370542287826538,
+ "num_tokens": 26771456.0,
+ "step": 1634
+ },
+ {
+ "entropy": 1.1840031147003174,
+ "epoch": 3.303030303030303,
+ "grad_norm": 2.1070966720581055,
+ "learning_rate": 1.6989898989898992e-06,
+ "loss": 1.191172480583191,
+ "mean_token_accuracy": 0.7166585326194763,
+ "num_tokens": 26787840.0,
+ "step": 1635
+ },
+ {
+ "entropy": 1.3296533823013306,
+ "epoch": 3.305050505050505,
+ "grad_norm": 2.4398109912872314,
+ "learning_rate": 1.6969696969696973e-06,
+ "loss": 1.338945984840393,
+ "mean_token_accuracy": 0.67666095495224,
+ "num_tokens": 26804224.0,
+ "step": 1636
+ },
+ {
+ "entropy": 1.1331745386123657,
+ "epoch": 3.3070707070707073,
+ "grad_norm": 2.103158473968506,
+ "learning_rate": 1.6949494949494952e-06,
+ "loss": 1.139038324356079,
+ "mean_token_accuracy": 0.7154372334480286,
+ "num_tokens": 26820608.0,
+ "step": 1637
+ },
+ {
+ "entropy": 1.3300442695617676,
+ "epoch": 3.309090909090909,
+ "grad_norm": 2.4576940536499023,
+ "learning_rate": 1.692929292929293e-06,
+ "loss": 1.3464535474777222,
+ "mean_token_accuracy": 0.6715925931930542,
+ "num_tokens": 26836992.0,
+ "step": 1638
+ },
+ {
+ "entropy": 1.5538347959518433,
+ "epoch": 3.311111111111111,
+ "grad_norm": 2.4972009658813477,
+ "learning_rate": 1.6909090909090912e-06,
+ "loss": 1.544262409210205,
+ "mean_token_accuracy": 0.6377015113830566,
+ "num_tokens": 26853376.0,
+ "step": 1639
+ },
+ {
+ "entropy": 1.2610400915145874,
+ "epoch": 3.313131313131313,
+ "grad_norm": 2.354304790496826,
+ "learning_rate": 1.688888888888889e-06,
+ "loss": 1.2615927457809448,
+ "mean_token_accuracy": 0.6805691123008728,
+ "num_tokens": 26869760.0,
+ "step": 1640
+ },
+ {
+ "entropy": 0.9232268333435059,
+ "epoch": 3.315151515151515,
+ "grad_norm": 2.2459871768951416,
+ "learning_rate": 1.6868686868686871e-06,
+ "loss": 0.9295459985733032,
+ "mean_token_accuracy": 0.7508549094200134,
+ "num_tokens": 26886144.0,
+ "step": 1641
+ },
+ {
+ "entropy": 1.3030803203582764,
+ "epoch": 3.317171717171717,
+ "grad_norm": 2.086177110671997,
+ "learning_rate": 1.684848484848485e-06,
+ "loss": 1.3260836601257324,
+ "mean_token_accuracy": 0.6850268840789795,
+ "num_tokens": 26902528.0,
+ "step": 1642
+ },
+ {
+ "entropy": 1.0115952491760254,
+ "epoch": 3.319191919191919,
+ "grad_norm": 2.3342647552490234,
+ "learning_rate": 1.682828282828283e-06,
+ "loss": 1.0125452280044556,
+ "mean_token_accuracy": 0.7252076268196106,
+ "num_tokens": 26918912.0,
+ "step": 1643
+ },
+ {
+ "entropy": 0.9997871518135071,
+ "epoch": 3.3212121212121213,
+ "grad_norm": 2.0844249725341797,
+ "learning_rate": 1.680808080808081e-06,
+ "loss": 1.0233726501464844,
+ "mean_token_accuracy": 0.7490839958190918,
+ "num_tokens": 26935296.0,
+ "step": 1644
+ },
+ {
+ "entropy": 1.3131269216537476,
+ "epoch": 3.323232323232323,
+ "grad_norm": 2.279741048812866,
+ "learning_rate": 1.678787878787879e-06,
+ "loss": 1.3082493543624878,
+ "mean_token_accuracy": 0.6856985688209534,
+ "num_tokens": 26951680.0,
+ "step": 1645
+ },
+ {
+ "entropy": 1.5594273805618286,
+ "epoch": 3.3252525252525253,
+ "grad_norm": 2.423882246017456,
+ "learning_rate": 1.6767676767676768e-06,
+ "loss": 1.5775840282440186,
+ "mean_token_accuracy": 0.662493884563446,
+ "num_tokens": 26968064.0,
+ "step": 1646
+ },
+ {
+ "entropy": 1.2298411130905151,
+ "epoch": 3.327272727272727,
+ "grad_norm": 2.237107515335083,
+ "learning_rate": 1.674747474747475e-06,
+ "loss": 1.1941357851028442,
+ "mean_token_accuracy": 0.7106130719184875,
+ "num_tokens": 26984448.0,
+ "step": 1647
+ },
+ {
+ "entropy": 0.986072301864624,
+ "epoch": 3.3292929292929294,
+ "grad_norm": 2.2843995094299316,
+ "learning_rate": 1.6727272727272728e-06,
+ "loss": 0.9964872598648071,
+ "mean_token_accuracy": 0.7433438897132874,
+ "num_tokens": 27000832.0,
+ "step": 1648
+ },
+ {
+ "entropy": 1.2612992525100708,
+ "epoch": 3.3313131313131312,
+ "grad_norm": 2.2401392459869385,
+ "learning_rate": 1.6707070707070707e-06,
+ "loss": 1.2401535511016846,
+ "mean_token_accuracy": 0.6914997696876526,
+ "num_tokens": 27017216.0,
+ "step": 1649
+ },
+ {
+ "entropy": 0.8876240849494934,
+ "epoch": 3.3333333333333335,
+ "grad_norm": 2.055438995361328,
+ "learning_rate": 1.6686868686868688e-06,
+ "loss": 0.8796688914299011,
+ "mean_token_accuracy": 0.7739985585212708,
+ "num_tokens": 27033600.0,
+ "step": 1650
+ },
+ {
+ "epoch": 3.3333333333333335,
+ "eval_entropy": 1.3213856729768938,
+ "eval_loss": 1.5484445095062256,
+ "eval_mean_token_accuracy": 0.6446653873689713,
+ "eval_num_tokens": 27033600.0,
+ "eval_runtime": 43.7594,
+ "eval_samples_per_second": 22.624,
+ "eval_steps_per_second": 2.834,
+ "step": 1650
+ },
+ {
+ "entropy": 1.8024967908859253,
+ "epoch": 3.3353535353535353,
+ "grad_norm": 2.316970109939575,
+ "learning_rate": 1.6666666666666667e-06,
+ "loss": 1.8035298585891724,
+ "mean_token_accuracy": 0.6057034730911255,
+ "num_tokens": 27049984.0,
+ "step": 1651
+ },
+ {
+ "entropy": 0.8543419241905212,
+ "epoch": 3.3373737373737375,
+ "grad_norm": 2.2566277980804443,
+ "learning_rate": 1.6646464646464648e-06,
+ "loss": 0.8558261394500732,
+ "mean_token_accuracy": 0.7656326293945312,
+ "num_tokens": 27066368.0,
+ "step": 1652
+ },
+ {
+ "entropy": 1.3101993799209595,
+ "epoch": 3.3393939393939394,
+ "grad_norm": 2.490753650665283,
+ "learning_rate": 1.6626262626262626e-06,
+ "loss": 1.326491355895996,
+ "mean_token_accuracy": 0.6656082272529602,
+ "num_tokens": 27082752.0,
+ "step": 1653
+ },
+ {
+ "entropy": 1.226317048072815,
+ "epoch": 3.341414141414141,
+ "grad_norm": 2.438649892807007,
+ "learning_rate": 1.6606060606060605e-06,
+ "loss": 1.2096238136291504,
+ "mean_token_accuracy": 0.6954079270362854,
+ "num_tokens": 27099136.0,
+ "step": 1654
+ },
+ {
+ "entropy": 1.1575508117675781,
+ "epoch": 3.3434343434343434,
+ "grad_norm": 2.130772113800049,
+ "learning_rate": 1.6585858585858588e-06,
+ "loss": 1.1718693971633911,
+ "mean_token_accuracy": 0.7256350517272949,
+ "num_tokens": 27115520.0,
+ "step": 1655
+ },
+ {
+ "entropy": 1.4135949611663818,
+ "epoch": 3.3454545454545457,
+ "grad_norm": 2.3037710189819336,
+ "learning_rate": 1.6565656565656567e-06,
+ "loss": 1.408097505569458,
+ "mean_token_accuracy": 0.6669516563415527,
+ "num_tokens": 27131904.0,
+ "step": 1656
+ },
+ {
+ "entropy": 0.977189838886261,
+ "epoch": 3.3474747474747475,
+ "grad_norm": 2.3096542358398438,
+ "learning_rate": 1.6545454545454548e-06,
+ "loss": 0.9848534464836121,
+ "mean_token_accuracy": 0.7306423783302307,
+ "num_tokens": 27148288.0,
+ "step": 1657
+ },
+ {
+ "entropy": 1.6641992330551147,
+ "epoch": 3.3494949494949493,
+ "grad_norm": 2.2562549114227295,
+ "learning_rate": 1.6525252525252527e-06,
+ "loss": 1.6742463111877441,
+ "mean_token_accuracy": 0.6157181262969971,
+ "num_tokens": 27164672.0,
+ "step": 1658
+ },
+ {
+ "entropy": 1.2698633670806885,
+ "epoch": 3.3515151515151516,
+ "grad_norm": 2.426100492477417,
+ "learning_rate": 1.6505050505050508e-06,
+ "loss": 1.2856690883636475,
+ "mean_token_accuracy": 0.6808133721351624,
+ "num_tokens": 27181056.0,
+ "step": 1659
+ },
+ {
+ "entropy": 1.4621158838272095,
+ "epoch": 3.3535353535353534,
+ "grad_norm": 2.3147833347320557,
+ "learning_rate": 1.6484848484848487e-06,
+ "loss": 1.468501091003418,
+ "mean_token_accuracy": 0.6610283255577087,
+ "num_tokens": 27197440.0,
+ "step": 1660
+ },
+ {
+ "entropy": 1.249279499053955,
+ "epoch": 3.3555555555555556,
+ "grad_norm": 2.237351894378662,
+ "learning_rate": 1.6464646464646466e-06,
+ "loss": 1.2397043704986572,
+ "mean_token_accuracy": 0.6951636672019958,
+ "num_tokens": 27213824.0,
+ "step": 1661
+ },
+ {
+ "entropy": 0.9608144164085388,
+ "epoch": 3.3575757575757574,
+ "grad_norm": 2.2147974967956543,
+ "learning_rate": 1.6444444444444447e-06,
+ "loss": 0.9720965623855591,
+ "mean_token_accuracy": 0.7529922127723694,
+ "num_tokens": 27230208.0,
+ "step": 1662
+ },
+ {
+ "entropy": 1.5832558870315552,
+ "epoch": 3.3595959595959597,
+ "grad_norm": 2.325833559036255,
+ "learning_rate": 1.6424242424242426e-06,
+ "loss": 1.5746217966079712,
+ "mean_token_accuracy": 0.6457010507583618,
+ "num_tokens": 27246592.0,
+ "step": 1663
+ },
+ {
+ "entropy": 1.35304856300354,
+ "epoch": 3.3616161616161615,
+ "grad_norm": 2.509624481201172,
+ "learning_rate": 1.6404040404040405e-06,
+ "loss": 1.3489930629730225,
+ "mean_token_accuracy": 0.6699438095092773,
+ "num_tokens": 27262976.0,
+ "step": 1664
+ },
+ {
+ "entropy": 1.3161576986312866,
+ "epoch": 3.3636363636363638,
+ "grad_norm": 2.393994092941284,
+ "learning_rate": 1.6383838383838386e-06,
+ "loss": 1.303264856338501,
+ "mean_token_accuracy": 0.6855764389038086,
+ "num_tokens": 27279360.0,
+ "step": 1665
+ },
+ {
+ "entropy": 1.1878900527954102,
+ "epoch": 3.3656565656565656,
+ "grad_norm": 2.180389165878296,
+ "learning_rate": 1.6363636363636365e-06,
+ "loss": 1.182593584060669,
+ "mean_token_accuracy": 0.7091475129127502,
+ "num_tokens": 27295744.0,
+ "step": 1666
+ },
+ {
+ "entropy": 1.3966656923294067,
+ "epoch": 3.367676767676768,
+ "grad_norm": 2.097033739089966,
+ "learning_rate": 1.6343434343434344e-06,
+ "loss": 1.3977530002593994,
+ "mean_token_accuracy": 0.6893014311790466,
+ "num_tokens": 27312128.0,
+ "step": 1667
+ },
+ {
+ "entropy": 1.298252820968628,
+ "epoch": 3.3696969696969696,
+ "grad_norm": 2.301867723464966,
+ "learning_rate": 1.6323232323232325e-06,
+ "loss": 1.2884492874145508,
+ "mean_token_accuracy": 0.6819125413894653,
+ "num_tokens": 27328512.0,
+ "step": 1668
+ },
+ {
+ "entropy": 1.0709348917007446,
+ "epoch": 3.371717171717172,
+ "grad_norm": 2.323345422744751,
+ "learning_rate": 1.6303030303030303e-06,
+ "loss": 1.0587565898895264,
+ "mean_token_accuracy": 0.7345505356788635,
+ "num_tokens": 27344896.0,
+ "step": 1669
+ },
+ {
+ "entropy": 1.4100518226623535,
+ "epoch": 3.3737373737373737,
+ "grad_norm": 2.3652331829071045,
+ "learning_rate": 1.6282828282828284e-06,
+ "loss": 1.4043725728988647,
+ "mean_token_accuracy": 0.670615553855896,
+ "num_tokens": 27361280.0,
+ "step": 1670
+ },
+ {
+ "entropy": 1.3682993650436401,
+ "epoch": 3.375757575757576,
+ "grad_norm": 2.4122917652130127,
+ "learning_rate": 1.6262626262626263e-06,
+ "loss": 1.3637261390686035,
+ "mean_token_accuracy": 0.665730357170105,
+ "num_tokens": 27377664.0,
+ "step": 1671
+ },
+ {
+ "entropy": 1.1528232097625732,
+ "epoch": 3.3777777777777778,
+ "grad_norm": 2.1029880046844482,
+ "learning_rate": 1.6242424242424242e-06,
+ "loss": 1.1693168878555298,
+ "mean_token_accuracy": 0.7155593633651733,
+ "num_tokens": 27394048.0,
+ "step": 1672
+ },
+ {
+ "entropy": 1.0737035274505615,
+ "epoch": 3.3797979797979796,
+ "grad_norm": 2.0809051990509033,
+ "learning_rate": 1.6222222222222223e-06,
+ "loss": 1.0787606239318848,
+ "mean_token_accuracy": 0.7236199378967285,
+ "num_tokens": 27410432.0,
+ "step": 1673
+ },
+ {
+ "entropy": 0.992008626461029,
+ "epoch": 3.381818181818182,
+ "grad_norm": 2.216033935546875,
+ "learning_rate": 1.6202020202020202e-06,
+ "loss": 0.9949617385864258,
+ "mean_token_accuracy": 0.7416340708732605,
+ "num_tokens": 27426816.0,
+ "step": 1674
+ },
+ {
+ "entropy": 1.3799123764038086,
+ "epoch": 3.3838383838383836,
+ "grad_norm": 2.218331813812256,
+ "learning_rate": 1.618181818181818e-06,
+ "loss": 1.400660514831543,
+ "mean_token_accuracy": 0.6707376837730408,
+ "num_tokens": 27443200.0,
+ "step": 1675
+ },
+ {
+ "entropy": 1.125901222229004,
+ "epoch": 3.385858585858586,
+ "grad_norm": 2.1194028854370117,
+ "learning_rate": 1.6161616161616164e-06,
+ "loss": 1.128243088722229,
+ "mean_token_accuracy": 0.7427332401275635,
+ "num_tokens": 27459584.0,
+ "step": 1676
+ },
+ {
+ "entropy": 1.347123384475708,
+ "epoch": 3.3878787878787877,
+ "grad_norm": 2.3231191635131836,
+ "learning_rate": 1.6141414141414145e-06,
+ "loss": 1.3301351070404053,
+ "mean_token_accuracy": 0.6792256832122803,
+ "num_tokens": 27475968.0,
+ "step": 1677
+ },
+ {
+ "entropy": 1.3429654836654663,
+ "epoch": 3.38989898989899,
+ "grad_norm": 2.3116424083709717,
+ "learning_rate": 1.6121212121212124e-06,
+ "loss": 1.3509732484817505,
+ "mean_token_accuracy": 0.6730581521987915,
+ "num_tokens": 27492352.0,
+ "step": 1678
+ },
+ {
+ "entropy": 1.0976272821426392,
+ "epoch": 3.391919191919192,
+ "grad_norm": 2.2770488262176514,
+ "learning_rate": 1.6101010101010103e-06,
+ "loss": 1.0915334224700928,
+ "mean_token_accuracy": 0.72826087474823,
+ "num_tokens": 27508736.0,
+ "step": 1679
+ },
+ {
+ "entropy": 1.6415719985961914,
+ "epoch": 3.393939393939394,
+ "grad_norm": 2.224179267883301,
+ "learning_rate": 1.6080808080808084e-06,
+ "loss": 1.6517691612243652,
+ "mean_token_accuracy": 0.6277479529380798,
+ "num_tokens": 27525120.0,
+ "step": 1680
+ },
+ {
+ "entropy": 1.2124820947647095,
+ "epoch": 3.395959595959596,
+ "grad_norm": 2.1096651554107666,
+ "learning_rate": 1.6060606060606063e-06,
+ "loss": 1.230252981185913,
+ "mean_token_accuracy": 0.7074987888336182,
+ "num_tokens": 27541504.0,
+ "step": 1681
+ },
+ {
+ "entropy": 1.4807898998260498,
+ "epoch": 3.397979797979798,
+ "grad_norm": 2.1753411293029785,
+ "learning_rate": 1.6040404040404042e-06,
+ "loss": 1.4894174337387085,
+ "mean_token_accuracy": 0.6585246920585632,
+ "num_tokens": 27557888.0,
+ "step": 1682
+ },
+ {
+ "entropy": 1.31308913230896,
+ "epoch": 3.4,
+ "grad_norm": 2.251997947692871,
+ "learning_rate": 1.6020202020202023e-06,
+ "loss": 1.3479615449905396,
+ "mean_token_accuracy": 0.6800805926322937,
+ "num_tokens": 27574272.0,
+ "step": 1683
+ },
+ {
+ "entropy": 1.1820056438446045,
+ "epoch": 3.402020202020202,
+ "grad_norm": 2.074507474899292,
+ "learning_rate": 1.6000000000000001e-06,
+ "loss": 1.1995149850845337,
+ "mean_token_accuracy": 0.7117733359336853,
+ "num_tokens": 27590656.0,
+ "step": 1684
+ },
+ {
+ "entropy": 1.2405987977981567,
+ "epoch": 3.404040404040404,
+ "grad_norm": 1.9572449922561646,
+ "learning_rate": 1.597979797979798e-06,
+ "loss": 1.228973627090454,
+ "mean_token_accuracy": 0.7143990993499756,
+ "num_tokens": 27607040.0,
+ "step": 1685
+ },
+ {
+ "entropy": 1.3054970502853394,
+ "epoch": 3.4060606060606062,
+ "grad_norm": 2.174598217010498,
+ "learning_rate": 1.5959595959595961e-06,
+ "loss": 1.2918202877044678,
+ "mean_token_accuracy": 0.7015754580497742,
+ "num_tokens": 27623424.0,
+ "step": 1686
+ },
+ {
+ "entropy": 1.2568256855010986,
+ "epoch": 3.408080808080808,
+ "grad_norm": 2.1647098064422607,
+ "learning_rate": 1.593939393939394e-06,
+ "loss": 1.2464665174484253,
+ "mean_token_accuracy": 0.7046898007392883,
+ "num_tokens": 27639808.0,
+ "step": 1687
+ },
+ {
+ "entropy": 0.9985859990119934,
+ "epoch": 3.41010101010101,
+ "grad_norm": 2.230548143386841,
+ "learning_rate": 1.5919191919191921e-06,
+ "loss": 0.9941741824150085,
+ "mean_token_accuracy": 0.7424889802932739,
+ "num_tokens": 27656192.0,
+ "step": 1688
+ },
+ {
+ "entropy": 1.2002249956130981,
+ "epoch": 3.412121212121212,
+ "grad_norm": 2.211632490158081,
+ "learning_rate": 1.58989898989899e-06,
+ "loss": 1.1901870965957642,
+ "mean_token_accuracy": 0.7168416976928711,
+ "num_tokens": 27672576.0,
+ "step": 1689
+ },
+ {
+ "entropy": 1.5074162483215332,
+ "epoch": 3.4141414141414144,
+ "grad_norm": 2.3280093669891357,
+ "learning_rate": 1.5878787878787879e-06,
+ "loss": 1.5252172946929932,
+ "mean_token_accuracy": 0.6465559601783752,
+ "num_tokens": 27688960.0,
+ "step": 1690
+ },
+ {
+ "entropy": 1.1828655004501343,
+ "epoch": 3.416161616161616,
+ "grad_norm": 2.3439552783966064,
+ "learning_rate": 1.585858585858586e-06,
+ "loss": 1.196863055229187,
+ "mean_token_accuracy": 0.7009037733078003,
+ "num_tokens": 27705344.0,
+ "step": 1691
+ },
+ {
+ "entropy": 1.176308035850525,
+ "epoch": 3.418181818181818,
+ "grad_norm": 2.460371255874634,
+ "learning_rate": 1.5838383838383839e-06,
+ "loss": 1.1818437576293945,
+ "mean_token_accuracy": 0.6998656392097473,
+ "num_tokens": 27721728.0,
+ "step": 1692
+ },
+ {
+ "entropy": 1.1989773511886597,
+ "epoch": 3.4202020202020202,
+ "grad_norm": 2.406575918197632,
+ "learning_rate": 1.5818181818181818e-06,
+ "loss": 1.2158204317092896,
+ "mean_token_accuracy": 0.6949194073677063,
+ "num_tokens": 27738112.0,
+ "step": 1693
+ },
+ {
+ "entropy": 1.537143588066101,
+ "epoch": 3.422222222222222,
+ "grad_norm": 2.298275947570801,
+ "learning_rate": 1.5797979797979799e-06,
+ "loss": 1.5335557460784912,
+ "mean_token_accuracy": 0.6571812629699707,
+ "num_tokens": 27754496.0,
+ "step": 1694
+ },
+ {
+ "entropy": 1.3271749019622803,
+ "epoch": 3.4242424242424243,
+ "grad_norm": 2.309699058532715,
+ "learning_rate": 1.5777777777777778e-06,
+ "loss": 1.3499958515167236,
+ "mean_token_accuracy": 0.6943087577819824,
+ "num_tokens": 27770880.0,
+ "step": 1695
+ },
+ {
+ "entropy": 1.4381791353225708,
+ "epoch": 3.426262626262626,
+ "grad_norm": 2.2183644771575928,
+ "learning_rate": 1.5757575757575759e-06,
+ "loss": 1.4350523948669434,
+ "mean_token_accuracy": 0.6568148732185364,
+ "num_tokens": 27787264.0,
+ "step": 1696
+ },
+ {
+ "entropy": 0.9867562055587769,
+ "epoch": 3.4282828282828284,
+ "grad_norm": 2.13663387298584,
+ "learning_rate": 1.5737373737373737e-06,
+ "loss": 0.9871019124984741,
+ "mean_token_accuracy": 0.7538471221923828,
+ "num_tokens": 27803648.0,
+ "step": 1697
+ },
+ {
+ "entropy": 1.0085591077804565,
+ "epoch": 3.43030303030303,
+ "grad_norm": 2.3447134494781494,
+ "learning_rate": 1.571717171717172e-06,
+ "loss": 1.0313187837600708,
+ "mean_token_accuracy": 0.7325354218482971,
+ "num_tokens": 27820032.0,
+ "step": 1698
+ },
+ {
+ "entropy": 0.9242914915084839,
+ "epoch": 3.4323232323232324,
+ "grad_norm": 2.16290545463562,
+ "learning_rate": 1.56969696969697e-06,
+ "loss": 0.9127025604248047,
+ "mean_token_accuracy": 0.7653273344039917,
+ "num_tokens": 27836416.0,
+ "step": 1699
+ },
+ {
+ "entropy": 1.4173065423965454,
+ "epoch": 3.4343434343434343,
+ "grad_norm": 2.192754030227661,
+ "learning_rate": 1.5676767676767678e-06,
+ "loss": 1.4122503995895386,
+ "mean_token_accuracy": 0.6770884394645691,
+ "num_tokens": 27852800.0,
+ "step": 1700
+ },
+ {
+ "epoch": 3.4343434343434343,
+ "eval_entropy": 1.3203289614569755,
+ "eval_loss": 1.5483064651489258,
+ "eval_mean_token_accuracy": 0.6447958845284677,
+ "eval_num_tokens": 27852800.0,
+ "eval_runtime": 43.7896,
+ "eval_samples_per_second": 22.608,
+ "eval_steps_per_second": 2.832,
+ "step": 1700
+ },
+ {
+ "entropy": 1.4677248001098633,
+ "epoch": 3.4363636363636365,
+ "grad_norm": 2.07183575630188,
+ "learning_rate": 1.565656565656566e-06,
+ "loss": 1.4772744178771973,
+ "mean_token_accuracy": 0.6927821040153503,
+ "num_tokens": 27869184.0,
+ "step": 1701
+ },
+ {
+ "entropy": 1.845194935798645,
+ "epoch": 3.4383838383838383,
+ "grad_norm": 1.9974539279937744,
+ "learning_rate": 1.5636363636363638e-06,
+ "loss": 1.8598425388336182,
+ "mean_token_accuracy": 0.5933683514595032,
+ "num_tokens": 27885568.0,
+ "step": 1702
+ },
+ {
+ "entropy": 1.1044740676879883,
+ "epoch": 3.4404040404040406,
+ "grad_norm": 2.199014186859131,
+ "learning_rate": 1.5616161616161617e-06,
+ "loss": 1.119746446609497,
+ "mean_token_accuracy": 0.7175744771957397,
+ "num_tokens": 27901952.0,
+ "step": 1703
+ },
+ {
+ "entropy": 0.920816957950592,
+ "epoch": 3.4424242424242424,
+ "grad_norm": 2.3172857761383057,
+ "learning_rate": 1.5595959595959598e-06,
+ "loss": 0.908664882183075,
+ "mean_token_accuracy": 0.7598314881324768,
+ "num_tokens": 27918336.0,
+ "step": 1704
+ },
+ {
+ "entropy": 1.4610254764556885,
+ "epoch": 3.4444444444444446,
+ "grad_norm": 2.5196661949157715,
+ "learning_rate": 1.5575757575757577e-06,
+ "loss": 1.4767801761627197,
+ "mean_token_accuracy": 0.652662456035614,
+ "num_tokens": 27934720.0,
+ "step": 1705
+ },
+ {
+ "entropy": 1.0131516456604004,
+ "epoch": 3.4464646464646465,
+ "grad_norm": 2.2522189617156982,
+ "learning_rate": 1.5555555555555558e-06,
+ "loss": 1.0143696069717407,
+ "mean_token_accuracy": 0.7348558902740479,
+ "num_tokens": 27951104.0,
+ "step": 1706
+ },
+ {
+ "entropy": 1.2054252624511719,
+ "epoch": 3.4484848484848483,
+ "grad_norm": 2.1769580841064453,
+ "learning_rate": 1.5535353535353537e-06,
+ "loss": 1.2020167112350464,
+ "mean_token_accuracy": 0.7076209187507629,
+ "num_tokens": 27967488.0,
+ "step": 1707
+ },
+ {
+ "entropy": 1.1004403829574585,
+ "epoch": 3.4505050505050505,
+ "grad_norm": 2.3803768157958984,
+ "learning_rate": 1.5515151515151516e-06,
+ "loss": 1.1220088005065918,
+ "mean_token_accuracy": 0.712506115436554,
+ "num_tokens": 27983872.0,
+ "step": 1708
+ },
+ {
+ "entropy": 1.423991322517395,
+ "epoch": 3.4525252525252528,
+ "grad_norm": 2.2648093700408936,
+ "learning_rate": 1.5494949494949497e-06,
+ "loss": 1.432944893836975,
+ "mean_token_accuracy": 0.6545554399490356,
+ "num_tokens": 28000256.0,
+ "step": 1709
+ },
+ {
+ "entropy": 1.5308680534362793,
+ "epoch": 3.4545454545454546,
+ "grad_norm": 2.1772451400756836,
+ "learning_rate": 1.5474747474747476e-06,
+ "loss": 1.5395504236221313,
+ "mean_token_accuracy": 0.6433805823326111,
+ "num_tokens": 28016640.0,
+ "step": 1710
+ },
+ {
+ "entropy": 1.3868193626403809,
+ "epoch": 3.4565656565656564,
+ "grad_norm": 2.1782312393188477,
+ "learning_rate": 1.5454545454545454e-06,
+ "loss": 1.3730117082595825,
+ "mean_token_accuracy": 0.6857596635818481,
+ "num_tokens": 28033024.0,
+ "step": 1711
+ },
+ {
+ "entropy": 0.6769964694976807,
+ "epoch": 3.4585858585858587,
+ "grad_norm": 1.941037654876709,
+ "learning_rate": 1.5434343434343435e-06,
+ "loss": 0.6863309144973755,
+ "mean_token_accuracy": 0.8126526474952698,
+ "num_tokens": 28049408.0,
+ "step": 1712
+ },
+ {
+ "entropy": 1.1113249063491821,
+ "epoch": 3.4606060606060605,
+ "grad_norm": 2.0039165019989014,
+ "learning_rate": 1.5414141414141414e-06,
+ "loss": 1.1056404113769531,
+ "mean_token_accuracy": 0.7314362525939941,
+ "num_tokens": 28065792.0,
+ "step": 1713
+ },
+ {
+ "entropy": 0.8827037811279297,
+ "epoch": 3.4626262626262627,
+ "grad_norm": 2.2235255241394043,
+ "learning_rate": 1.5393939393939395e-06,
+ "loss": 0.8903871178627014,
+ "mean_token_accuracy": 0.7547020316123962,
+ "num_tokens": 28082176.0,
+ "step": 1714
+ },
+ {
+ "entropy": 1.2060964107513428,
+ "epoch": 3.4646464646464645,
+ "grad_norm": 2.2997398376464844,
+ "learning_rate": 1.5373737373737374e-06,
+ "loss": 1.2247741222381592,
+ "mean_token_accuracy": 0.6979726552963257,
+ "num_tokens": 28098560.0,
+ "step": 1715
+ },
+ {
+ "entropy": 1.2543354034423828,
+ "epoch": 3.466666666666667,
+ "grad_norm": 2.2603039741516113,
+ "learning_rate": 1.5353535353535353e-06,
+ "loss": 1.2638390064239502,
+ "mean_token_accuracy": 0.6919271945953369,
+ "num_tokens": 28114944.0,
+ "step": 1716
+ },
+ {
+ "entropy": 1.4569451808929443,
+ "epoch": 3.4686868686868686,
+ "grad_norm": 2.3623344898223877,
+ "learning_rate": 1.5333333333333334e-06,
+ "loss": 1.4669182300567627,
+ "mean_token_accuracy": 0.6596238613128662,
+ "num_tokens": 28131328.0,
+ "step": 1717
+ },
+ {
+ "entropy": 1.0979474782943726,
+ "epoch": 3.470707070707071,
+ "grad_norm": 2.282487154006958,
+ "learning_rate": 1.5313131313131313e-06,
+ "loss": 1.123124361038208,
+ "mean_token_accuracy": 0.7218490242958069,
+ "num_tokens": 28147712.0,
+ "step": 1718
+ },
+ {
+ "entropy": 1.3850172758102417,
+ "epoch": 3.4727272727272727,
+ "grad_norm": 2.406338930130005,
+ "learning_rate": 1.5292929292929296e-06,
+ "loss": 1.361925482749939,
+ "mean_token_accuracy": 0.672874927520752,
+ "num_tokens": 28164096.0,
+ "step": 1719
+ },
+ {
+ "entropy": 1.3522284030914307,
+ "epoch": 3.474747474747475,
+ "grad_norm": 2.171316385269165,
+ "learning_rate": 1.5272727272727275e-06,
+ "loss": 1.3782949447631836,
+ "mean_token_accuracy": 0.6841108798980713,
+ "num_tokens": 28180480.0,
+ "step": 1720
+ },
+ {
+ "entropy": 1.4793171882629395,
+ "epoch": 3.4767676767676767,
+ "grad_norm": 2.29586124420166,
+ "learning_rate": 1.5252525252525254e-06,
+ "loss": 1.4899311065673828,
+ "mean_token_accuracy": 0.6422203183174133,
+ "num_tokens": 28196864.0,
+ "step": 1721
+ },
+ {
+ "entropy": 1.0572689771652222,
+ "epoch": 3.4787878787878785,
+ "grad_norm": 2.0798633098602295,
+ "learning_rate": 1.5232323232323235e-06,
+ "loss": 1.0417135953903198,
+ "mean_token_accuracy": 0.7372373938560486,
+ "num_tokens": 28213248.0,
+ "step": 1722
+ },
+ {
+ "entropy": 1.0050398111343384,
+ "epoch": 3.480808080808081,
+ "grad_norm": 2.205551862716675,
+ "learning_rate": 1.5212121212121214e-06,
+ "loss": 1.0072510242462158,
+ "mean_token_accuracy": 0.7414509057998657,
+ "num_tokens": 28229632.0,
+ "step": 1723
+ },
+ {
+ "entropy": 0.9438449144363403,
+ "epoch": 3.482828282828283,
+ "grad_norm": 2.177335262298584,
+ "learning_rate": 1.5191919191919195e-06,
+ "loss": 0.9125795364379883,
+ "mean_token_accuracy": 0.7605642676353455,
+ "num_tokens": 28246016.0,
+ "step": 1724
+ },
+ {
+ "entropy": 0.8600634336471558,
+ "epoch": 3.484848484848485,
+ "grad_norm": 2.286794662475586,
+ "learning_rate": 1.5171717171717174e-06,
+ "loss": 0.8473736047744751,
+ "mean_token_accuracy": 0.7839521169662476,
+ "num_tokens": 28262400.0,
+ "step": 1725
+ },
+ {
+ "entropy": 1.0827349424362183,
+ "epoch": 3.4868686868686867,
+ "grad_norm": 2.176316738128662,
+ "learning_rate": 1.5151515151515152e-06,
+ "loss": 1.0760138034820557,
+ "mean_token_accuracy": 0.7195896506309509,
+ "num_tokens": 28278784.0,
+ "step": 1726
+ },
+ {
+ "entropy": 1.3604737520217896,
+ "epoch": 3.488888888888889,
+ "grad_norm": 2.3393819332122803,
+ "learning_rate": 1.5131313131313133e-06,
+ "loss": 1.3466029167175293,
+ "mean_token_accuracy": 0.6731802821159363,
+ "num_tokens": 28295168.0,
+ "step": 1727
+ },
+ {
+ "entropy": 1.4611977338790894,
+ "epoch": 3.4909090909090907,
+ "grad_norm": 2.3396732807159424,
+ "learning_rate": 1.5111111111111112e-06,
+ "loss": 1.474691390991211,
+ "mean_token_accuracy": 0.6497313380241394,
+ "num_tokens": 28311552.0,
+ "step": 1728
+ },
+ {
+ "entropy": 1.1734099388122559,
+ "epoch": 3.492929292929293,
+ "grad_norm": 2.154437780380249,
+ "learning_rate": 1.5090909090909091e-06,
+ "loss": 1.1631931066513062,
+ "mean_token_accuracy": 0.7048119306564331,
+ "num_tokens": 28327936.0,
+ "step": 1729
+ },
+ {
+ "entropy": 1.0015299320220947,
+ "epoch": 3.494949494949495,
+ "grad_norm": 2.273815393447876,
+ "learning_rate": 1.5070707070707072e-06,
+ "loss": 0.9818742871284485,
+ "mean_token_accuracy": 0.7361382246017456,
+ "num_tokens": 28344320.0,
+ "step": 1730
+ },
+ {
+ "entropy": 1.0342366695404053,
+ "epoch": 3.496969696969697,
+ "grad_norm": 2.1011879444122314,
+ "learning_rate": 1.5050505050505051e-06,
+ "loss": 1.02262282371521,
+ "mean_token_accuracy": 0.7448094487190247,
+ "num_tokens": 28360704.0,
+ "step": 1731
+ },
+ {
+ "entropy": 1.6318060159683228,
+ "epoch": 3.498989898989899,
+ "grad_norm": 2.338040590286255,
+ "learning_rate": 1.5030303030303032e-06,
+ "loss": 1.6215972900390625,
+ "mean_token_accuracy": 0.6225574016571045,
+ "num_tokens": 28377088.0,
+ "step": 1732
+ },
+ {
+ "entropy": 1.2391737699508667,
+ "epoch": 3.501010101010101,
+ "grad_norm": 2.2958035469055176,
+ "learning_rate": 1.501010101010101e-06,
+ "loss": 1.2481398582458496,
+ "mean_token_accuracy": 0.6922325491905212,
+ "num_tokens": 28393472.0,
+ "step": 1733
+ },
+ {
+ "entropy": 1.09321129322052,
+ "epoch": 3.503030303030303,
+ "grad_norm": 2.117506980895996,
+ "learning_rate": 1.498989898989899e-06,
+ "loss": 1.100616455078125,
+ "mean_token_accuracy": 0.7240473628044128,
+ "num_tokens": 28409856.0,
+ "step": 1734
+ },
+ {
+ "entropy": 1.2746680974960327,
+ "epoch": 3.505050505050505,
+ "grad_norm": 2.2428109645843506,
+ "learning_rate": 1.496969696969697e-06,
+ "loss": 1.278875470161438,
+ "mean_token_accuracy": 0.688629686832428,
+ "num_tokens": 28426240.0,
+ "step": 1735
+ },
+ {
+ "entropy": 0.9300931096076965,
+ "epoch": 3.507070707070707,
+ "grad_norm": 2.0250492095947266,
+ "learning_rate": 1.494949494949495e-06,
+ "loss": 0.9304848909378052,
+ "mean_token_accuracy": 0.7613580822944641,
+ "num_tokens": 28442624.0,
+ "step": 1736
+ },
+ {
+ "entropy": 1.5743480920791626,
+ "epoch": 3.509090909090909,
+ "grad_norm": 2.251974582672119,
+ "learning_rate": 1.4929292929292929e-06,
+ "loss": 1.570695400238037,
+ "mean_token_accuracy": 0.6520518064498901,
+ "num_tokens": 28459008.0,
+ "step": 1737
+ },
+ {
+ "entropy": 1.0316240787506104,
+ "epoch": 3.511111111111111,
+ "grad_norm": 2.187208652496338,
+ "learning_rate": 1.490909090909091e-06,
+ "loss": 1.022985816001892,
+ "mean_token_accuracy": 0.7444430589675903,
+ "num_tokens": 28475392.0,
+ "step": 1738
+ },
+ {
+ "entropy": 1.0719387531280518,
+ "epoch": 3.5131313131313133,
+ "grad_norm": 2.1497936248779297,
+ "learning_rate": 1.4888888888888888e-06,
+ "loss": 1.087444543838501,
+ "mean_token_accuracy": 0.7261846661567688,
+ "num_tokens": 28491776.0,
+ "step": 1739
+ },
+ {
+ "entropy": 1.679359793663025,
+ "epoch": 3.515151515151515,
+ "grad_norm": 2.2430360317230225,
+ "learning_rate": 1.4868686868686872e-06,
+ "loss": 1.6981315612792969,
+ "mean_token_accuracy": 0.6258549094200134,
+ "num_tokens": 28508160.0,
+ "step": 1740
+ },
+ {
+ "entropy": 1.0614526271820068,
+ "epoch": 3.517171717171717,
+ "grad_norm": 2.220466375350952,
+ "learning_rate": 1.484848484848485e-06,
+ "loss": 1.0519301891326904,
+ "mean_token_accuracy": 0.7313751578330994,
+ "num_tokens": 28524544.0,
+ "step": 1741
+ },
+ {
+ "entropy": 1.267195224761963,
+ "epoch": 3.519191919191919,
+ "grad_norm": 1.9448764324188232,
+ "learning_rate": 1.4828282828282831e-06,
+ "loss": 1.2594044208526611,
+ "mean_token_accuracy": 0.7093917727470398,
+ "num_tokens": 28540928.0,
+ "step": 1742
+ },
+ {
+ "entropy": 1.0900983810424805,
+ "epoch": 3.5212121212121215,
+ "grad_norm": 2.292336940765381,
+ "learning_rate": 1.480808080808081e-06,
+ "loss": 1.0952692031860352,
+ "mean_token_accuracy": 0.7146433591842651,
+ "num_tokens": 28557312.0,
+ "step": 1743
+ },
+ {
+ "entropy": 1.4188793897628784,
+ "epoch": 3.5232323232323233,
+ "grad_norm": 2.355440139770508,
+ "learning_rate": 1.478787878787879e-06,
+ "loss": 1.4508819580078125,
+ "mean_token_accuracy": 0.6642037034034729,
+ "num_tokens": 28573696.0,
+ "step": 1744
+ },
+ {
+ "entropy": 1.52566397190094,
+ "epoch": 3.525252525252525,
+ "grad_norm": 2.2767279148101807,
+ "learning_rate": 1.476767676767677e-06,
+ "loss": 1.5568758249282837,
+ "mean_token_accuracy": 0.6472276449203491,
+ "num_tokens": 28590080.0,
+ "step": 1745
+ },
+ {
+ "entropy": 1.4695135354995728,
+ "epoch": 3.5272727272727273,
+ "grad_norm": 2.3080222606658936,
+ "learning_rate": 1.474747474747475e-06,
+ "loss": 1.4671947956085205,
+ "mean_token_accuracy": 0.6530898809432983,
+ "num_tokens": 28606464.0,
+ "step": 1746
+ },
+ {
+ "entropy": 1.438908338546753,
+ "epoch": 3.529292929292929,
+ "grad_norm": 2.485170364379883,
+ "learning_rate": 1.4727272727272728e-06,
+ "loss": 1.4531956911087036,
+ "mean_token_accuracy": 0.6664020419120789,
+ "num_tokens": 28622848.0,
+ "step": 1747
+ },
+ {
+ "entropy": 1.5307536125183105,
+ "epoch": 3.5313131313131314,
+ "grad_norm": 2.3280844688415527,
+ "learning_rate": 1.470707070707071e-06,
+ "loss": 1.5298030376434326,
+ "mean_token_accuracy": 0.642892062664032,
+ "num_tokens": 28639232.0,
+ "step": 1748
+ },
+ {
+ "entropy": 1.429180383682251,
+ "epoch": 3.533333333333333,
+ "grad_norm": 2.36387038230896,
+ "learning_rate": 1.4686868686868688e-06,
+ "loss": 1.4191480875015259,
+ "mean_token_accuracy": 0.6469833850860596,
+ "num_tokens": 28655616.0,
+ "step": 1749
+ },
+ {
+ "entropy": 0.9336722493171692,
+ "epoch": 3.5353535353535355,
+ "grad_norm": 2.1873295307159424,
+ "learning_rate": 1.4666666666666669e-06,
+ "loss": 0.9257663488388062,
+ "mean_token_accuracy": 0.7629457712173462,
+ "num_tokens": 28672000.0,
+ "step": 1750
+ },
+ {
+ "epoch": 3.5353535353535355,
+ "eval_entropy": 1.3185442157330052,
+ "eval_loss": 1.548416018486023,
+ "eval_mean_token_accuracy": 0.6448259276728476,
+ "eval_num_tokens": 28672000.0,
+ "eval_runtime": 43.8376,
+ "eval_samples_per_second": 22.583,
+ "eval_steps_per_second": 2.829,
+ "step": 1750
+ },
+ {
+ "entropy": 1.2647572755813599,
+ "epoch": 3.5373737373737373,
+ "grad_norm": 2.1273581981658936,
+ "learning_rate": 1.4646464646464648e-06,
+ "loss": 1.2557384967803955,
+ "mean_token_accuracy": 0.6935759782791138,
+ "num_tokens": 28688384.0,
+ "step": 1751
+ },
+ {
+ "entropy": 1.4252692461013794,
+ "epoch": 3.5393939393939395,
+ "grad_norm": 2.052034616470337,
+ "learning_rate": 1.4626262626262627e-06,
+ "loss": 1.43434739112854,
+ "mean_token_accuracy": 0.6618832349777222,
+ "num_tokens": 28704768.0,
+ "step": 1752
+ },
+ {
+ "entropy": 1.6918041706085205,
+ "epoch": 3.5414141414141413,
+ "grad_norm": 2.20965313911438,
+ "learning_rate": 1.4606060606060608e-06,
+ "loss": 1.707175850868225,
+ "mean_token_accuracy": 0.6127870082855225,
+ "num_tokens": 28721152.0,
+ "step": 1753
+ },
+ {
+ "entropy": 1.5122125148773193,
+ "epoch": 3.5434343434343436,
+ "grad_norm": 2.2061564922332764,
+ "learning_rate": 1.4585858585858586e-06,
+ "loss": 1.5119690895080566,
+ "mean_token_accuracy": 0.6488153338432312,
+ "num_tokens": 28737536.0,
+ "step": 1754
+ },
+ {
+ "entropy": 0.9536482095718384,
+ "epoch": 3.5454545454545454,
+ "grad_norm": 2.310809373855591,
+ "learning_rate": 1.4565656565656565e-06,
+ "loss": 0.9411349296569824,
+ "mean_token_accuracy": 0.7589765787124634,
+ "num_tokens": 28753920.0,
+ "step": 1755
+ },
+ {
+ "entropy": 1.5147753953933716,
+ "epoch": 3.5474747474747472,
+ "grad_norm": 2.520862340927124,
+ "learning_rate": 1.4545454545454546e-06,
+ "loss": 1.5047588348388672,
+ "mean_token_accuracy": 0.639167070388794,
+ "num_tokens": 28770304.0,
+ "step": 1756
+ },
+ {
+ "entropy": 1.3881752490997314,
+ "epoch": 3.5494949494949495,
+ "grad_norm": 2.3421478271484375,
+ "learning_rate": 1.4525252525252525e-06,
+ "loss": 1.3775359392166138,
+ "mean_token_accuracy": 0.6719589829444885,
+ "num_tokens": 28786688.0,
+ "step": 1757
+ },
+ {
+ "entropy": 1.0432544946670532,
+ "epoch": 3.5515151515151517,
+ "grad_norm": 2.235539674758911,
+ "learning_rate": 1.4505050505050506e-06,
+ "loss": 1.055377721786499,
+ "mean_token_accuracy": 0.7360160946846008,
+ "num_tokens": 28803072.0,
+ "step": 1758
+ },
+ {
+ "entropy": 1.2739083766937256,
+ "epoch": 3.5535353535353535,
+ "grad_norm": 2.295605421066284,
+ "learning_rate": 1.4484848484848485e-06,
+ "loss": 1.289074420928955,
+ "mean_token_accuracy": 0.6969345211982727,
+ "num_tokens": 28819456.0,
+ "step": 1759
+ },
+ {
+ "entropy": 1.192219853401184,
+ "epoch": 3.5555555555555554,
+ "grad_norm": 2.1039023399353027,
+ "learning_rate": 1.4464646464646464e-06,
+ "loss": 1.2043861150741577,
+ "mean_token_accuracy": 0.7080483436584473,
+ "num_tokens": 28835840.0,
+ "step": 1760
+ },
+ {
+ "entropy": 1.002448320388794,
+ "epoch": 3.5575757575757576,
+ "grad_norm": 2.476088523864746,
+ "learning_rate": 1.4444444444444445e-06,
+ "loss": 1.0271326303482056,
+ "mean_token_accuracy": 0.7325354218482971,
+ "num_tokens": 28852224.0,
+ "step": 1761
+ },
+ {
+ "entropy": 1.4694463014602661,
+ "epoch": 3.55959595959596,
+ "grad_norm": 2.298769235610962,
+ "learning_rate": 1.4424242424242426e-06,
+ "loss": 1.4576467275619507,
+ "mean_token_accuracy": 0.6591353416442871,
+ "num_tokens": 28868608.0,
+ "step": 1762
+ },
+ {
+ "entropy": 1.4025219678878784,
+ "epoch": 3.5616161616161617,
+ "grad_norm": 2.148528575897217,
+ "learning_rate": 1.4404040404040407e-06,
+ "loss": 1.3884464502334595,
+ "mean_token_accuracy": 0.6655471324920654,
+ "num_tokens": 28884992.0,
+ "step": 1763
+ },
+ {
+ "entropy": 1.2829358577728271,
+ "epoch": 3.5636363636363635,
+ "grad_norm": 2.1344001293182373,
+ "learning_rate": 1.4383838383838386e-06,
+ "loss": 1.3100465536117554,
+ "mean_token_accuracy": 0.6872862577438354,
+ "num_tokens": 28901376.0,
+ "step": 1764
+ },
+ {
+ "entropy": 1.2500309944152832,
+ "epoch": 3.5656565656565657,
+ "grad_norm": 2.1933422088623047,
+ "learning_rate": 1.4363636363636365e-06,
+ "loss": 1.2566546201705933,
+ "mean_token_accuracy": 0.7057889699935913,
+ "num_tokens": 28917760.0,
+ "step": 1765
+ },
+ {
+ "entropy": 1.4471250772476196,
+ "epoch": 3.5676767676767676,
+ "grad_norm": 2.2238707542419434,
+ "learning_rate": 1.4343434343434346e-06,
+ "loss": 1.4434895515441895,
+ "mean_token_accuracy": 0.6715925931930542,
+ "num_tokens": 28934144.0,
+ "step": 1766
+ },
+ {
+ "entropy": 1.2484838962554932,
+ "epoch": 3.56969696969697,
+ "grad_norm": 2.272259473800659,
+ "learning_rate": 1.4323232323232325e-06,
+ "loss": 1.2546260356903076,
+ "mean_token_accuracy": 0.6996824741363525,
+ "num_tokens": 28950528.0,
+ "step": 1767
+ },
+ {
+ "entropy": 1.326412558555603,
+ "epoch": 3.5717171717171716,
+ "grad_norm": 2.157593250274658,
+ "learning_rate": 1.4303030303030306e-06,
+ "loss": 1.3175407648086548,
+ "mean_token_accuracy": 0.7037127614021301,
+ "num_tokens": 28966912.0,
+ "step": 1768
+ },
+ {
+ "entropy": 1.314650297164917,
+ "epoch": 3.573737373737374,
+ "grad_norm": 2.146393299102783,
+ "learning_rate": 1.4282828282828284e-06,
+ "loss": 1.3232550621032715,
+ "mean_token_accuracy": 0.6864924430847168,
+ "num_tokens": 28983296.0,
+ "step": 1769
+ },
+ {
+ "entropy": 1.2547006607055664,
+ "epoch": 3.5757575757575757,
+ "grad_norm": 2.2809762954711914,
+ "learning_rate": 1.4262626262626263e-06,
+ "loss": 1.2578558921813965,
+ "mean_token_accuracy": 0.689667820930481,
+ "num_tokens": 28999680.0,
+ "step": 1770
+ },
+ {
+ "entropy": 1.1902955770492554,
+ "epoch": 3.5777777777777775,
+ "grad_norm": 2.5193467140197754,
+ "learning_rate": 1.4242424242424244e-06,
+ "loss": 1.1852655410766602,
+ "mean_token_accuracy": 0.699499249458313,
+ "num_tokens": 29016064.0,
+ "step": 1771
+ },
+ {
+ "entropy": 1.075490951538086,
+ "epoch": 3.5797979797979798,
+ "grad_norm": 2.055223226547241,
+ "learning_rate": 1.4222222222222223e-06,
+ "loss": 1.09010648727417,
+ "mean_token_accuracy": 0.7302759885787964,
+ "num_tokens": 29032448.0,
+ "step": 1772
+ },
+ {
+ "entropy": 1.1549168825149536,
+ "epoch": 3.581818181818182,
+ "grad_norm": 2.1802778244018555,
+ "learning_rate": 1.4202020202020202e-06,
+ "loss": 1.1465229988098145,
+ "mean_token_accuracy": 0.7187957763671875,
+ "num_tokens": 29048832.0,
+ "step": 1773
+ },
+ {
+ "entropy": 1.0966418981552124,
+ "epoch": 3.583838383838384,
+ "grad_norm": 2.354775905609131,
+ "learning_rate": 1.4181818181818183e-06,
+ "loss": 1.0915825366973877,
+ "mean_token_accuracy": 0.7263067960739136,
+ "num_tokens": 29065216.0,
+ "step": 1774
+ },
+ {
+ "entropy": 1.3264915943145752,
+ "epoch": 3.5858585858585856,
+ "grad_norm": 2.291811943054199,
+ "learning_rate": 1.4161616161616162e-06,
+ "loss": 1.3168433904647827,
+ "mean_token_accuracy": 0.6811797618865967,
+ "num_tokens": 29081600.0,
+ "step": 1775
+ },
+ {
+ "entropy": 1.4051839113235474,
+ "epoch": 3.587878787878788,
+ "grad_norm": 2.4314849376678467,
+ "learning_rate": 1.4141414141414143e-06,
+ "loss": 1.4150385856628418,
+ "mean_token_accuracy": 0.6616389751434326,
+ "num_tokens": 29097984.0,
+ "step": 1776
+ },
+ {
+ "entropy": 1.045767903327942,
+ "epoch": 3.58989898989899,
+ "grad_norm": 2.361785650253296,
+ "learning_rate": 1.4121212121212122e-06,
+ "loss": 1.0414667129516602,
+ "mean_token_accuracy": 0.726062536239624,
+ "num_tokens": 29114368.0,
+ "step": 1777
+ },
+ {
+ "entropy": 1.0930231809616089,
+ "epoch": 3.591919191919192,
+ "grad_norm": 2.2943761348724365,
+ "learning_rate": 1.41010101010101e-06,
+ "loss": 1.116988182067871,
+ "mean_token_accuracy": 0.7142159342765808,
+ "num_tokens": 29130752.0,
+ "step": 1778
+ },
+ {
+ "entropy": 2.053565502166748,
+ "epoch": 3.5939393939393938,
+ "grad_norm": 2.274879217147827,
+ "learning_rate": 1.4080808080808082e-06,
+ "loss": 2.012974262237549,
+ "mean_token_accuracy": 0.5475085377693176,
+ "num_tokens": 29147136.0,
+ "step": 1779
+ },
+ {
+ "entropy": 1.666791319847107,
+ "epoch": 3.595959595959596,
+ "grad_norm": 2.3387227058410645,
+ "learning_rate": 1.406060606060606e-06,
+ "loss": 1.6848305463790894,
+ "mean_token_accuracy": 0.6218246221542358,
+ "num_tokens": 29163520.0,
+ "step": 1780
+ },
+ {
+ "entropy": 1.2312790155410767,
+ "epoch": 3.597979797979798,
+ "grad_norm": 2.1445071697235107,
+ "learning_rate": 1.404040404040404e-06,
+ "loss": 1.2197356224060059,
+ "mean_token_accuracy": 0.7146433591842651,
+ "num_tokens": 29179904.0,
+ "step": 1781
+ },
+ {
+ "entropy": 1.2540696859359741,
+ "epoch": 3.6,
+ "grad_norm": 2.3021488189697266,
+ "learning_rate": 1.402020202020202e-06,
+ "loss": 1.2441315650939941,
+ "mean_token_accuracy": 0.697239875793457,
+ "num_tokens": 29196288.0,
+ "step": 1782
+ },
+ {
+ "entropy": 1.3095312118530273,
+ "epoch": 3.602020202020202,
+ "grad_norm": 2.3379111289978027,
+ "learning_rate": 1.4000000000000001e-06,
+ "loss": 1.299248218536377,
+ "mean_token_accuracy": 0.6789814233779907,
+ "num_tokens": 29212672.0,
+ "step": 1783
+ },
+ {
+ "entropy": 1.0785331726074219,
+ "epoch": 3.604040404040404,
+ "grad_norm": 2.1612391471862793,
+ "learning_rate": 1.3979797979797982e-06,
+ "loss": 1.0741904973983765,
+ "mean_token_accuracy": 0.7294821739196777,
+ "num_tokens": 29229056.0,
+ "step": 1784
+ },
+ {
+ "entropy": 1.5178064107894897,
+ "epoch": 3.606060606060606,
+ "grad_norm": 2.7717669010162354,
+ "learning_rate": 1.3959595959595961e-06,
+ "loss": 1.4960123300552368,
+ "mean_token_accuracy": 0.6555935740470886,
+ "num_tokens": 29245440.0,
+ "step": 1785
+ },
+ {
+ "entropy": 0.9794894456863403,
+ "epoch": 3.608080808080808,
+ "grad_norm": 2.128119707107544,
+ "learning_rate": 1.3939393939393942e-06,
+ "loss": 0.964859664440155,
+ "mean_token_accuracy": 0.7516487836837769,
+ "num_tokens": 29261824.0,
+ "step": 1786
+ },
+ {
+ "entropy": 1.1770693063735962,
+ "epoch": 3.61010101010101,
+ "grad_norm": 2.2583603858947754,
+ "learning_rate": 1.3919191919191921e-06,
+ "loss": 1.163985252380371,
+ "mean_token_accuracy": 0.7167806625366211,
+ "num_tokens": 29278208.0,
+ "step": 1787
+ },
+ {
+ "entropy": 1.2855029106140137,
+ "epoch": 3.6121212121212123,
+ "grad_norm": 2.1924304962158203,
+ "learning_rate": 1.38989898989899e-06,
+ "loss": 1.3204423189163208,
+ "mean_token_accuracy": 0.6933927536010742,
+ "num_tokens": 29294592.0,
+ "step": 1788
+ },
+ {
+ "entropy": 1.1483845710754395,
+ "epoch": 3.614141414141414,
+ "grad_norm": 2.414332151412964,
+ "learning_rate": 1.3878787878787881e-06,
+ "loss": 1.154718041419983,
+ "mean_token_accuracy": 0.6999267339706421,
+ "num_tokens": 29310976.0,
+ "step": 1789
+ },
+ {
+ "entropy": 1.3682032823562622,
+ "epoch": 3.616161616161616,
+ "grad_norm": 2.2198476791381836,
+ "learning_rate": 1.385858585858586e-06,
+ "loss": 1.3729476928710938,
+ "mean_token_accuracy": 0.6736077070236206,
+ "num_tokens": 29327360.0,
+ "step": 1790
+ },
+ {
+ "entropy": 1.0998579263687134,
+ "epoch": 3.618181818181818,
+ "grad_norm": 2.328645706176758,
+ "learning_rate": 1.3838383838383839e-06,
+ "loss": 1.0919283628463745,
+ "mean_token_accuracy": 0.7213605046272278,
+ "num_tokens": 29343744.0,
+ "step": 1791
+ },
+ {
+ "entropy": 1.4821339845657349,
+ "epoch": 3.6202020202020204,
+ "grad_norm": 2.346853256225586,
+ "learning_rate": 1.381818181818182e-06,
+ "loss": 1.4857574701309204,
+ "mean_token_accuracy": 0.6529677510261536,
+ "num_tokens": 29360128.0,
+ "step": 1792
+ },
+ {
+ "entropy": 1.2080271244049072,
+ "epoch": 3.6222222222222222,
+ "grad_norm": 2.4182422161102295,
+ "learning_rate": 1.3797979797979799e-06,
+ "loss": 1.2055883407592773,
+ "mean_token_accuracy": 0.7100635170936584,
+ "num_tokens": 29376512.0,
+ "step": 1793
+ },
+ {
+ "entropy": 1.2058414220809937,
+ "epoch": 3.624242424242424,
+ "grad_norm": 2.216625690460205,
+ "learning_rate": 1.377777777777778e-06,
+ "loss": 1.2196989059448242,
+ "mean_token_accuracy": 0.7110405564308167,
+ "num_tokens": 29392896.0,
+ "step": 1794
+ },
+ {
+ "entropy": 1.2888880968093872,
+ "epoch": 3.6262626262626263,
+ "grad_norm": 2.339319944381714,
+ "learning_rate": 1.3757575757575759e-06,
+ "loss": 1.277637004852295,
+ "mean_token_accuracy": 0.6922935843467712,
+ "num_tokens": 29409280.0,
+ "step": 1795
+ },
+ {
+ "entropy": 1.3091189861297607,
+ "epoch": 3.6282828282828286,
+ "grad_norm": 2.2296347618103027,
+ "learning_rate": 1.3737373737373738e-06,
+ "loss": 1.3214037418365479,
+ "mean_token_accuracy": 0.6828285455703735,
+ "num_tokens": 29425664.0,
+ "step": 1796
+ },
+ {
+ "entropy": 1.3359358310699463,
+ "epoch": 3.6303030303030304,
+ "grad_norm": 2.3027234077453613,
+ "learning_rate": 1.3717171717171718e-06,
+ "loss": 1.3537523746490479,
+ "mean_token_accuracy": 0.6944919228553772,
+ "num_tokens": 29442048.0,
+ "step": 1797
+ },
+ {
+ "entropy": 1.4622148275375366,
+ "epoch": 3.632323232323232,
+ "grad_norm": 2.2217557430267334,
+ "learning_rate": 1.3696969696969697e-06,
+ "loss": 1.4778163433074951,
+ "mean_token_accuracy": 0.6425867080688477,
+ "num_tokens": 29458432.0,
+ "step": 1798
+ },
+ {
+ "entropy": 1.1139435768127441,
+ "epoch": 3.6343434343434344,
+ "grad_norm": 2.1497981548309326,
+ "learning_rate": 1.3676767676767676e-06,
+ "loss": 1.1079224348068237,
+ "mean_token_accuracy": 0.7202613353729248,
+ "num_tokens": 29474816.0,
+ "step": 1799
+ },
+ {
+ "entropy": 1.4551507234573364,
+ "epoch": 3.6363636363636362,
+ "grad_norm": 2.138988971710205,
+ "learning_rate": 1.3656565656565657e-06,
+ "loss": 1.469781517982483,
+ "mean_token_accuracy": 0.6572422981262207,
+ "num_tokens": 29491200.0,
+ "step": 1800
+ },
+ {
+ "epoch": 3.6363636363636362,
+ "eval_entropy": 1.3177791665638647,
+ "eval_loss": 1.5487139225006104,
+ "eval_mean_token_accuracy": 0.6448490720602774,
+ "eval_num_tokens": 29491200.0,
+ "eval_runtime": 43.82,
+ "eval_samples_per_second": 22.592,
+ "eval_steps_per_second": 2.83,
+ "step": 1800
+ },
+ {
+ "entropy": 1.3909705877304077,
+ "epoch": 3.6383838383838385,
+ "grad_norm": 2.2257578372955322,
+ "learning_rate": 1.3636363636363636e-06,
+ "loss": 1.4063682556152344,
+ "mean_token_accuracy": 0.6611504554748535,
+ "num_tokens": 29507584.0,
+ "step": 1801
+ },
+ {
+ "entropy": 1.2972772121429443,
+ "epoch": 3.6404040404040403,
+ "grad_norm": 2.3228883743286133,
+ "learning_rate": 1.3616161616161617e-06,
+ "loss": 1.2978641986846924,
+ "mean_token_accuracy": 0.6862481832504272,
+ "num_tokens": 29523968.0,
+ "step": 1802
+ },
+ {
+ "entropy": 0.8828009366989136,
+ "epoch": 3.6424242424242426,
+ "grad_norm": 2.294520616531372,
+ "learning_rate": 1.3595959595959596e-06,
+ "loss": 0.8818602561950684,
+ "mean_token_accuracy": 0.7627626061439514,
+ "num_tokens": 29540352.0,
+ "step": 1803
+ },
+ {
+ "entropy": 1.3855260610580444,
+ "epoch": 3.6444444444444444,
+ "grad_norm": 2.160240411758423,
+ "learning_rate": 1.357575757575758e-06,
+ "loss": 1.3808461427688599,
+ "mean_token_accuracy": 0.6831949353218079,
+ "num_tokens": 29556736.0,
+ "step": 1804
+ },
+ {
+ "entropy": 1.1665101051330566,
+ "epoch": 3.6464646464646466,
+ "grad_norm": 2.3000075817108154,
+ "learning_rate": 1.3555555555555558e-06,
+ "loss": 1.1676980257034302,
+ "mean_token_accuracy": 0.7067049145698547,
+ "num_tokens": 29573120.0,
+ "step": 1805
+ },
+ {
+ "entropy": 1.05939519405365,
+ "epoch": 3.6484848484848484,
+ "grad_norm": 2.204202651977539,
+ "learning_rate": 1.3535353535353537e-06,
+ "loss": 1.0654125213623047,
+ "mean_token_accuracy": 0.7197728157043457,
+ "num_tokens": 29589504.0,
+ "step": 1806
+ },
+ {
+ "entropy": 1.355467677116394,
+ "epoch": 3.6505050505050507,
+ "grad_norm": 2.264705181121826,
+ "learning_rate": 1.3515151515151518e-06,
+ "loss": 1.3571414947509766,
+ "mean_token_accuracy": 0.6802638173103333,
+ "num_tokens": 29605888.0,
+ "step": 1807
+ },
+ {
+ "entropy": 1.1876951456069946,
+ "epoch": 3.6525252525252525,
+ "grad_norm": 2.2944509983062744,
+ "learning_rate": 1.3494949494949497e-06,
+ "loss": 1.1937611103057861,
+ "mean_token_accuracy": 0.7105520367622375,
+ "num_tokens": 29622272.0,
+ "step": 1808
+ },
+ {
+ "entropy": 1.410723090171814,
+ "epoch": 3.6545454545454543,
+ "grad_norm": 2.329960584640503,
+ "learning_rate": 1.3474747474747476e-06,
+ "loss": 1.4132624864578247,
+ "mean_token_accuracy": 0.656448483467102,
+ "num_tokens": 29638656.0,
+ "step": 1809
+ },
+ {
+ "entropy": 1.018915057182312,
+ "epoch": 3.6565656565656566,
+ "grad_norm": 2.1073148250579834,
+ "learning_rate": 1.3454545454545457e-06,
+ "loss": 1.017066478729248,
+ "mean_token_accuracy": 0.7398021221160889,
+ "num_tokens": 29655040.0,
+ "step": 1810
+ },
+ {
+ "entropy": 1.385291337966919,
+ "epoch": 3.658585858585859,
+ "grad_norm": 2.4099128246307373,
+ "learning_rate": 1.3434343434343436e-06,
+ "loss": 1.3809027671813965,
+ "mean_token_accuracy": 0.6530288457870483,
+ "num_tokens": 29671424.0,
+ "step": 1811
+ },
+ {
+ "entropy": 1.0589433908462524,
+ "epoch": 3.6606060606060606,
+ "grad_norm": 2.3987228870391846,
+ "learning_rate": 1.3414141414141417e-06,
+ "loss": 1.051498293876648,
+ "mean_token_accuracy": 0.7296043038368225,
+ "num_tokens": 29687808.0,
+ "step": 1812
+ },
+ {
+ "entropy": 1.0399458408355713,
+ "epoch": 3.6626262626262625,
+ "grad_norm": 1.9951106309890747,
+ "learning_rate": 1.3393939393939395e-06,
+ "loss": 1.0482317209243774,
+ "mean_token_accuracy": 0.745175838470459,
+ "num_tokens": 29704192.0,
+ "step": 1813
+ },
+ {
+ "entropy": 1.3024410009384155,
+ "epoch": 3.6646464646464647,
+ "grad_norm": 2.1363778114318848,
+ "learning_rate": 1.3373737373737374e-06,
+ "loss": 1.2995185852050781,
+ "mean_token_accuracy": 0.6889960765838623,
+ "num_tokens": 29720576.0,
+ "step": 1814
+ },
+ {
+ "entropy": 1.604906678199768,
+ "epoch": 3.6666666666666665,
+ "grad_norm": 2.2837791442871094,
+ "learning_rate": 1.3353535353535355e-06,
+ "loss": 1.6025669574737549,
+ "mean_token_accuracy": 0.6281143426895142,
+ "num_tokens": 29736960.0,
+ "step": 1815
+ },
+ {
+ "entropy": 1.0479960441589355,
+ "epoch": 3.6686868686868688,
+ "grad_norm": 2.200199842453003,
+ "learning_rate": 1.3333333333333334e-06,
+ "loss": 1.0443310737609863,
+ "mean_token_accuracy": 0.7302149534225464,
+ "num_tokens": 29753344.0,
+ "step": 1816
+ },
+ {
+ "entropy": 1.1220474243164062,
+ "epoch": 3.6707070707070706,
+ "grad_norm": 2.278193712234497,
+ "learning_rate": 1.3313131313131313e-06,
+ "loss": 1.1253700256347656,
+ "mean_token_accuracy": 0.717391312122345,
+ "num_tokens": 29769728.0,
+ "step": 1817
+ },
+ {
+ "entropy": 0.7910162210464478,
+ "epoch": 3.672727272727273,
+ "grad_norm": 2.171074151992798,
+ "learning_rate": 1.3292929292929294e-06,
+ "loss": 0.7818440198898315,
+ "mean_token_accuracy": 0.7832193374633789,
+ "num_tokens": 29786112.0,
+ "step": 1818
+ },
+ {
+ "entropy": 1.2416874170303345,
+ "epoch": 3.6747474747474747,
+ "grad_norm": 2.41068959236145,
+ "learning_rate": 1.3272727272727273e-06,
+ "loss": 1.2441900968551636,
+ "mean_token_accuracy": 0.6918050646781921,
+ "num_tokens": 29802496.0,
+ "step": 1819
+ },
+ {
+ "entropy": 1.1947519779205322,
+ "epoch": 3.676767676767677,
+ "grad_norm": 2.3515799045562744,
+ "learning_rate": 1.3252525252525254e-06,
+ "loss": 1.1756401062011719,
+ "mean_token_accuracy": 0.7001098990440369,
+ "num_tokens": 29818880.0,
+ "step": 1820
+ },
+ {
+ "entropy": 1.0655851364135742,
+ "epoch": 3.6787878787878787,
+ "grad_norm": 2.3352251052856445,
+ "learning_rate": 1.3232323232323233e-06,
+ "loss": 1.0540491342544556,
+ "mean_token_accuracy": 0.7285051345825195,
+ "num_tokens": 29835264.0,
+ "step": 1821
+ },
+ {
+ "entropy": 1.1881550550460815,
+ "epoch": 3.680808080808081,
+ "grad_norm": 2.483917474746704,
+ "learning_rate": 1.3212121212121212e-06,
+ "loss": 1.199643611907959,
+ "mean_token_accuracy": 0.6998046040534973,
+ "num_tokens": 29851648.0,
+ "step": 1822
+ },
+ {
+ "entropy": 1.2031608819961548,
+ "epoch": 3.682828282828283,
+ "grad_norm": 2.1765546798706055,
+ "learning_rate": 1.3191919191919193e-06,
+ "loss": 1.2054451704025269,
+ "mean_token_accuracy": 0.7191011309623718,
+ "num_tokens": 29868032.0,
+ "step": 1823
+ },
+ {
+ "entropy": 0.8889259099960327,
+ "epoch": 3.6848484848484846,
+ "grad_norm": 2.3284575939178467,
+ "learning_rate": 1.3171717171717172e-06,
+ "loss": 0.8793114423751831,
+ "mean_token_accuracy": 0.7661211490631104,
+ "num_tokens": 29884416.0,
+ "step": 1824
+ },
+ {
+ "entropy": 1.1559280157089233,
+ "epoch": 3.686868686868687,
+ "grad_norm": 2.2942442893981934,
+ "learning_rate": 1.315151515151515e-06,
+ "loss": 1.1748852729797363,
+ "mean_token_accuracy": 0.7139105796813965,
+ "num_tokens": 29900800.0,
+ "step": 1825
+ },
+ {
+ "entropy": 1.1315678358078003,
+ "epoch": 3.688888888888889,
+ "grad_norm": 2.0435519218444824,
+ "learning_rate": 1.3131313131313134e-06,
+ "loss": 1.123443603515625,
+ "mean_token_accuracy": 0.7311308979988098,
+ "num_tokens": 29917184.0,
+ "step": 1826
+ },
+ {
+ "entropy": 1.063825249671936,
+ "epoch": 3.690909090909091,
+ "grad_norm": 2.3425393104553223,
+ "learning_rate": 1.3111111111111112e-06,
+ "loss": 1.0752053260803223,
+ "mean_token_accuracy": 0.7281997799873352,
+ "num_tokens": 29933568.0,
+ "step": 1827
+ },
+ {
+ "entropy": 0.9652242660522461,
+ "epoch": 3.6929292929292927,
+ "grad_norm": 2.2079033851623535,
+ "learning_rate": 1.3090909090909093e-06,
+ "loss": 0.969928503036499,
+ "mean_token_accuracy": 0.7507938742637634,
+ "num_tokens": 29949952.0,
+ "step": 1828
+ },
+ {
+ "entropy": 1.4131009578704834,
+ "epoch": 3.694949494949495,
+ "grad_norm": 2.368713617324829,
+ "learning_rate": 1.3070707070707072e-06,
+ "loss": 1.4113342761993408,
+ "mean_token_accuracy": 0.6670737862586975,
+ "num_tokens": 29966336.0,
+ "step": 1829
+ },
+ {
+ "entropy": 1.4447262287139893,
+ "epoch": 3.6969696969696972,
+ "grad_norm": 2.142131805419922,
+ "learning_rate": 1.3050505050505053e-06,
+ "loss": 1.4544841051101685,
+ "mean_token_accuracy": 0.669516384601593,
+ "num_tokens": 29982720.0,
+ "step": 1830
+ },
+ {
+ "entropy": 1.2524710893630981,
+ "epoch": 3.698989898989899,
+ "grad_norm": 2.096489429473877,
+ "learning_rate": 1.3030303030303032e-06,
+ "loss": 1.2251232862472534,
+ "mean_token_accuracy": 0.7231924533843994,
+ "num_tokens": 29999104.0,
+ "step": 1831
+ },
+ {
+ "entropy": 1.2173584699630737,
+ "epoch": 3.701010101010101,
+ "grad_norm": 2.265334367752075,
+ "learning_rate": 1.301010101010101e-06,
+ "loss": 1.2278194427490234,
+ "mean_token_accuracy": 0.7136052846908569,
+ "num_tokens": 30015488.0,
+ "step": 1832
+ },
+ {
+ "entropy": 1.1699647903442383,
+ "epoch": 3.703030303030303,
+ "grad_norm": 2.274156093597412,
+ "learning_rate": 1.2989898989898992e-06,
+ "loss": 1.1550461053848267,
+ "mean_token_accuracy": 0.7139716744422913,
+ "num_tokens": 30031872.0,
+ "step": 1833
+ },
+ {
+ "entropy": 1.113942265510559,
+ "epoch": 3.705050505050505,
+ "grad_norm": 2.386925220489502,
+ "learning_rate": 1.296969696969697e-06,
+ "loss": 1.1161434650421143,
+ "mean_token_accuracy": 0.7212994694709778,
+ "num_tokens": 30048256.0,
+ "step": 1834
+ },
+ {
+ "entropy": 1.4958065748214722,
+ "epoch": 3.707070707070707,
+ "grad_norm": 2.2243921756744385,
+ "learning_rate": 1.294949494949495e-06,
+ "loss": 1.4940879344940186,
+ "mean_token_accuracy": 0.6489985585212708,
+ "num_tokens": 30064640.0,
+ "step": 1835
+ },
+ {
+ "entropy": 1.047438144683838,
+ "epoch": 3.709090909090909,
+ "grad_norm": 2.237632989883423,
+ "learning_rate": 1.292929292929293e-06,
+ "loss": 1.0492076873779297,
+ "mean_token_accuracy": 0.7387640476226807,
+ "num_tokens": 30081024.0,
+ "step": 1836
+ },
+ {
+ "entropy": 1.4816311597824097,
+ "epoch": 3.7111111111111112,
+ "grad_norm": 2.2184505462646484,
+ "learning_rate": 1.290909090909091e-06,
+ "loss": 1.483510971069336,
+ "mean_token_accuracy": 0.660906195640564,
+ "num_tokens": 30097408.0,
+ "step": 1837
+ },
+ {
+ "entropy": 1.1734580993652344,
+ "epoch": 3.713131313131313,
+ "grad_norm": 2.284940242767334,
+ "learning_rate": 1.288888888888889e-06,
+ "loss": 1.1763725280761719,
+ "mean_token_accuracy": 0.708109438419342,
+ "num_tokens": 30113792.0,
+ "step": 1838
+ },
+ {
+ "entropy": 1.1073024272918701,
+ "epoch": 3.7151515151515153,
+ "grad_norm": 2.381448268890381,
+ "learning_rate": 1.286868686868687e-06,
+ "loss": 1.1168363094329834,
+ "mean_token_accuracy": 0.722337543964386,
+ "num_tokens": 30130176.0,
+ "step": 1839
+ },
+ {
+ "entropy": 1.2699110507965088,
+ "epoch": 3.717171717171717,
+ "grad_norm": 2.23447585105896,
+ "learning_rate": 1.2848484848484848e-06,
+ "loss": 1.2582776546478271,
+ "mean_token_accuracy": 0.6878358721733093,
+ "num_tokens": 30146560.0,
+ "step": 1840
+ },
+ {
+ "entropy": 1.421061635017395,
+ "epoch": 3.7191919191919194,
+ "grad_norm": 2.1020593643188477,
+ "learning_rate": 1.282828282828283e-06,
+ "loss": 1.415670394897461,
+ "mean_token_accuracy": 0.6722032427787781,
+ "num_tokens": 30162944.0,
+ "step": 1841
+ },
+ {
+ "entropy": 0.8924360871315002,
+ "epoch": 3.721212121212121,
+ "grad_norm": 2.1613609790802,
+ "learning_rate": 1.2808080808080808e-06,
+ "loss": 0.8864036202430725,
+ "mean_token_accuracy": 0.7616023421287537,
+ "num_tokens": 30179328.0,
+ "step": 1842
+ },
+ {
+ "entropy": 1.2242110967636108,
+ "epoch": 3.723232323232323,
+ "grad_norm": 2.472944498062134,
+ "learning_rate": 1.2787878787878787e-06,
+ "loss": 1.227367877960205,
+ "mean_token_accuracy": 0.6939423680305481,
+ "num_tokens": 30195712.0,
+ "step": 1843
+ },
+ {
+ "entropy": 1.2816494703292847,
+ "epoch": 3.7252525252525253,
+ "grad_norm": 2.123680353164673,
+ "learning_rate": 1.2767676767676768e-06,
+ "loss": 1.2535889148712158,
+ "mean_token_accuracy": 0.6894845962524414,
+ "num_tokens": 30212096.0,
+ "step": 1844
+ },
+ {
+ "entropy": 1.0055866241455078,
+ "epoch": 3.7272727272727275,
+ "grad_norm": 2.2397797107696533,
+ "learning_rate": 1.2747474747474747e-06,
+ "loss": 1.0046356916427612,
+ "mean_token_accuracy": 0.7375427484512329,
+ "num_tokens": 30228480.0,
+ "step": 1845
+ },
+ {
+ "entropy": 0.9162007570266724,
+ "epoch": 3.7292929292929293,
+ "grad_norm": 2.2464940547943115,
+ "learning_rate": 1.2727272727272728e-06,
+ "loss": 0.9192468523979187,
+ "mean_token_accuracy": 0.7609306573867798,
+ "num_tokens": 30244864.0,
+ "step": 1846
+ },
+ {
+ "entropy": 1.3691668510437012,
+ "epoch": 3.731313131313131,
+ "grad_norm": 2.766839027404785,
+ "learning_rate": 1.270707070707071e-06,
+ "loss": 1.3715851306915283,
+ "mean_token_accuracy": 0.6667684316635132,
+ "num_tokens": 30261248.0,
+ "step": 1847
+ },
+ {
+ "entropy": 1.140328049659729,
+ "epoch": 3.7333333333333334,
+ "grad_norm": 2.1093955039978027,
+ "learning_rate": 1.268686868686869e-06,
+ "loss": 1.1491321325302124,
+ "mean_token_accuracy": 0.7158646583557129,
+ "num_tokens": 30277632.0,
+ "step": 1848
+ },
+ {
+ "entropy": 1.3393927812576294,
+ "epoch": 3.735353535353535,
+ "grad_norm": 2.216357469558716,
+ "learning_rate": 1.2666666666666669e-06,
+ "loss": 1.3406658172607422,
+ "mean_token_accuracy": 0.6829506754875183,
+ "num_tokens": 30294016.0,
+ "step": 1849
+ },
+ {
+ "entropy": 1.0507596731185913,
+ "epoch": 3.7373737373737375,
+ "grad_norm": 2.287649154663086,
+ "learning_rate": 1.2646464646464648e-06,
+ "loss": 1.0469536781311035,
+ "mean_token_accuracy": 0.7306423783302307,
+ "num_tokens": 30310400.0,
+ "step": 1850
+ },
+ {
+ "epoch": 3.7373737373737375,
+ "eval_entropy": 1.3188492747083786,
+ "eval_loss": 1.5484505891799927,
+ "eval_mean_token_accuracy": 0.6448682820604693,
+ "eval_num_tokens": 30310400.0,
+ "eval_runtime": 43.7649,
+ "eval_samples_per_second": 22.621,
+ "eval_steps_per_second": 2.833,
+ "step": 1850
+ },
+ {
+ "entropy": 1.0057004690170288,
+ "epoch": 3.7393939393939393,
+ "grad_norm": 2.1135971546173096,
+ "learning_rate": 1.2626262626262629e-06,
+ "loss": 1.0220353603363037,
+ "mean_token_accuracy": 0.7348558902740479,
+ "num_tokens": 30326784.0,
+ "step": 1851
+ },
+ {
+ "entropy": 1.185084581375122,
+ "epoch": 3.7414141414141415,
+ "grad_norm": 2.388561725616455,
+ "learning_rate": 1.2606060606060608e-06,
+ "loss": 1.205190658569336,
+ "mean_token_accuracy": 0.7070102691650391,
+ "num_tokens": 30343168.0,
+ "step": 1852
+ },
+ {
+ "entropy": 1.2007622718811035,
+ "epoch": 3.7434343434343433,
+ "grad_norm": 2.3002219200134277,
+ "learning_rate": 1.2585858585858587e-06,
+ "loss": 1.196336269378662,
+ "mean_token_accuracy": 0.7073155641555786,
+ "num_tokens": 30359552.0,
+ "step": 1853
+ },
+ {
+ "entropy": 0.8792212605476379,
+ "epoch": 3.7454545454545456,
+ "grad_norm": 2.1559953689575195,
+ "learning_rate": 1.2565656565656568e-06,
+ "loss": 0.8786029815673828,
+ "mean_token_accuracy": 0.7723497748374939,
+ "num_tokens": 30375936.0,
+ "step": 1854
+ },
+ {
+ "entropy": 1.0509400367736816,
+ "epoch": 3.7474747474747474,
+ "grad_norm": 2.3014485836029053,
+ "learning_rate": 1.2545454545454546e-06,
+ "loss": 1.0421940088272095,
+ "mean_token_accuracy": 0.741389811038971,
+ "num_tokens": 30392320.0,
+ "step": 1855
+ },
+ {
+ "entropy": 1.4965882301330566,
+ "epoch": 3.7494949494949497,
+ "grad_norm": 2.255364179611206,
+ "learning_rate": 1.2525252525252527e-06,
+ "loss": 1.5137383937835693,
+ "mean_token_accuracy": 0.6535173654556274,
+ "num_tokens": 30408704.0,
+ "step": 1856
+ },
+ {
+ "entropy": 1.5112299919128418,
+ "epoch": 3.7515151515151515,
+ "grad_norm": 2.356682062149048,
+ "learning_rate": 1.2505050505050506e-06,
+ "loss": 1.4965360164642334,
+ "mean_token_accuracy": 0.639167070388794,
+ "num_tokens": 30425088.0,
+ "step": 1857
+ },
+ {
+ "entropy": 0.7364286780357361,
+ "epoch": 3.7535353535353533,
+ "grad_norm": 2.0973432064056396,
+ "learning_rate": 1.2484848484848485e-06,
+ "loss": 0.7423363327980042,
+ "mean_token_accuracy": 0.7923790812492371,
+ "num_tokens": 30441472.0,
+ "step": 1858
+ },
+ {
+ "entropy": 1.094409704208374,
+ "epoch": 3.7555555555555555,
+ "grad_norm": 2.174180746078491,
+ "learning_rate": 1.2464646464646466e-06,
+ "loss": 1.0954921245574951,
+ "mean_token_accuracy": 0.7165974378585815,
+ "num_tokens": 30457856.0,
+ "step": 1859
+ },
+ {
+ "entropy": 1.1821874380111694,
+ "epoch": 3.757575757575758,
+ "grad_norm": 2.253767728805542,
+ "learning_rate": 1.2444444444444445e-06,
+ "loss": 1.1888916492462158,
+ "mean_token_accuracy": 0.7046898007392883,
+ "num_tokens": 30474240.0,
+ "step": 1860
+ },
+ {
+ "entropy": 1.1281737089157104,
+ "epoch": 3.7595959595959596,
+ "grad_norm": 2.378880023956299,
+ "learning_rate": 1.2424242424242424e-06,
+ "loss": 1.1276686191558838,
+ "mean_token_accuracy": 0.712506115436554,
+ "num_tokens": 30490624.0,
+ "step": 1861
+ },
+ {
+ "entropy": 1.1477409601211548,
+ "epoch": 3.7616161616161614,
+ "grad_norm": 2.2524328231811523,
+ "learning_rate": 1.2404040404040405e-06,
+ "loss": 1.169572114944458,
+ "mean_token_accuracy": 0.7145823240280151,
+ "num_tokens": 30507008.0,
+ "step": 1862
+ },
+ {
+ "entropy": 1.1877309083938599,
+ "epoch": 3.7636363636363637,
+ "grad_norm": 2.5215635299682617,
+ "learning_rate": 1.2383838383838386e-06,
+ "loss": 1.1812067031860352,
+ "mean_token_accuracy": 0.7001709938049316,
+ "num_tokens": 30523392.0,
+ "step": 1863
+ },
+ {
+ "entropy": 1.2082164287567139,
+ "epoch": 3.765656565656566,
+ "grad_norm": 2.248847484588623,
+ "learning_rate": 1.2363636363636365e-06,
+ "loss": 1.2150661945343018,
+ "mean_token_accuracy": 0.7056668400764465,
+ "num_tokens": 30539776.0,
+ "step": 1864
+ },
+ {
+ "entropy": 1.0533868074417114,
+ "epoch": 3.7676767676767677,
+ "grad_norm": 2.4208297729492188,
+ "learning_rate": 1.2343434343434346e-06,
+ "loss": 1.0599069595336914,
+ "mean_token_accuracy": 0.7183683514595032,
+ "num_tokens": 30556160.0,
+ "step": 1865
+ },
+ {
+ "entropy": 1.3612878322601318,
+ "epoch": 3.7696969696969695,
+ "grad_norm": 2.2831575870513916,
+ "learning_rate": 1.2323232323232325e-06,
+ "loss": 1.359129548072815,
+ "mean_token_accuracy": 0.681485116481781,
+ "num_tokens": 30572544.0,
+ "step": 1866
+ },
+ {
+ "entropy": 1.3154997825622559,
+ "epoch": 3.771717171717172,
+ "grad_norm": 2.2212936878204346,
+ "learning_rate": 1.2303030303030304e-06,
+ "loss": 1.3217754364013672,
+ "mean_token_accuracy": 0.6781265139579773,
+ "num_tokens": 30588928.0,
+ "step": 1867
+ },
+ {
+ "entropy": 1.3559744358062744,
+ "epoch": 3.7737373737373736,
+ "grad_norm": 2.317963123321533,
+ "learning_rate": 1.2282828282828285e-06,
+ "loss": 1.3600891828536987,
+ "mean_token_accuracy": 0.6749511361122131,
+ "num_tokens": 30605312.0,
+ "step": 1868
+ },
+ {
+ "entropy": 1.0591025352478027,
+ "epoch": 3.775757575757576,
+ "grad_norm": 2.2434401512145996,
+ "learning_rate": 1.2262626262626263e-06,
+ "loss": 1.069009780883789,
+ "mean_token_accuracy": 0.7206888198852539,
+ "num_tokens": 30621696.0,
+ "step": 1869
+ },
+ {
+ "entropy": 1.4170074462890625,
+ "epoch": 3.7777777777777777,
+ "grad_norm": 2.395284652709961,
+ "learning_rate": 1.2242424242424242e-06,
+ "loss": 1.4480173587799072,
+ "mean_token_accuracy": 0.6687225103378296,
+ "num_tokens": 30638080.0,
+ "step": 1870
+ },
+ {
+ "entropy": 1.2455501556396484,
+ "epoch": 3.77979797979798,
+ "grad_norm": 2.308814525604248,
+ "learning_rate": 1.2222222222222223e-06,
+ "loss": 1.238303542137146,
+ "mean_token_accuracy": 0.693514883518219,
+ "num_tokens": 30654464.0,
+ "step": 1871
+ },
+ {
+ "entropy": 1.167949914932251,
+ "epoch": 3.7818181818181817,
+ "grad_norm": 2.292120933532715,
+ "learning_rate": 1.2202020202020202e-06,
+ "loss": 1.16925847530365,
+ "mean_token_accuracy": 0.7136052846908569,
+ "num_tokens": 30670848.0,
+ "step": 1872
+ },
+ {
+ "entropy": 1.1319386959075928,
+ "epoch": 3.783838383838384,
+ "grad_norm": 2.2560858726501465,
+ "learning_rate": 1.2181818181818183e-06,
+ "loss": 1.1299757957458496,
+ "mean_token_accuracy": 0.7148265838623047,
+ "num_tokens": 30687232.0,
+ "step": 1873
+ },
+ {
+ "entropy": 1.1161472797393799,
+ "epoch": 3.785858585858586,
+ "grad_norm": 2.375944137573242,
+ "learning_rate": 1.2161616161616164e-06,
+ "loss": 1.1136269569396973,
+ "mean_token_accuracy": 0.7096971273422241,
+ "num_tokens": 30703616.0,
+ "step": 1874
+ },
+ {
+ "entropy": 1.223272442817688,
+ "epoch": 3.787878787878788,
+ "grad_norm": 2.5566787719726562,
+ "learning_rate": 1.2141414141414143e-06,
+ "loss": 1.2140392065048218,
+ "mean_token_accuracy": 0.6939423680305481,
+ "num_tokens": 30720000.0,
+ "step": 1875
+ },
+ {
+ "entropy": 1.5127382278442383,
+ "epoch": 3.78989898989899,
+ "grad_norm": 2.2809665203094482,
+ "learning_rate": 1.2121212121212122e-06,
+ "loss": 1.508399248123169,
+ "mean_token_accuracy": 0.6579140424728394,
+ "num_tokens": 30736384.0,
+ "step": 1876
+ },
+ {
+ "entropy": 1.344763994216919,
+ "epoch": 3.7919191919191917,
+ "grad_norm": 2.3228111267089844,
+ "learning_rate": 1.2101010101010103e-06,
+ "loss": 1.3470673561096191,
+ "mean_token_accuracy": 0.6806302070617676,
+ "num_tokens": 30752768.0,
+ "step": 1877
+ },
+ {
+ "entropy": 1.2828526496887207,
+ "epoch": 3.793939393939394,
+ "grad_norm": 2.51246976852417,
+ "learning_rate": 1.2080808080808082e-06,
+ "loss": 1.2934420108795166,
+ "mean_token_accuracy": 0.6673790812492371,
+ "num_tokens": 30769152.0,
+ "step": 1878
+ },
+ {
+ "entropy": 0.9060937762260437,
+ "epoch": 3.795959595959596,
+ "grad_norm": 2.1415040493011475,
+ "learning_rate": 1.206060606060606e-06,
+ "loss": 0.8956457376480103,
+ "mean_token_accuracy": 0.7670371532440186,
+ "num_tokens": 30785536.0,
+ "step": 1879
+ },
+ {
+ "entropy": 1.1543211936950684,
+ "epoch": 3.797979797979798,
+ "grad_norm": 2.426781415939331,
+ "learning_rate": 1.2040404040404042e-06,
+ "loss": 1.1588597297668457,
+ "mean_token_accuracy": 0.7090253829956055,
+ "num_tokens": 30801920.0,
+ "step": 1880
+ },
+ {
+ "entropy": 1.3253799676895142,
+ "epoch": 3.8,
+ "grad_norm": 2.4828150272369385,
+ "learning_rate": 1.202020202020202e-06,
+ "loss": 1.3082318305969238,
+ "mean_token_accuracy": 0.6839887499809265,
+ "num_tokens": 30818304.0,
+ "step": 1881
+ },
+ {
+ "entropy": 1.1644330024719238,
+ "epoch": 3.802020202020202,
+ "grad_norm": 2.3429148197174072,
+ "learning_rate": 1.2000000000000002e-06,
+ "loss": 1.1645771265029907,
+ "mean_token_accuracy": 0.7025524973869324,
+ "num_tokens": 30834688.0,
+ "step": 1882
+ },
+ {
+ "entropy": 1.1643813848495483,
+ "epoch": 3.804040404040404,
+ "grad_norm": 2.2955002784729004,
+ "learning_rate": 1.197979797979798e-06,
+ "loss": 1.1721268892288208,
+ "mean_token_accuracy": 0.7041401863098145,
+ "num_tokens": 30851072.0,
+ "step": 1883
+ },
+ {
+ "entropy": 1.0957778692245483,
+ "epoch": 3.806060606060606,
+ "grad_norm": 2.244861125946045,
+ "learning_rate": 1.1959595959595961e-06,
+ "loss": 1.1006194353103638,
+ "mean_token_accuracy": 0.7057889699935913,
+ "num_tokens": 30867456.0,
+ "step": 1884
+ },
+ {
+ "entropy": 1.420525074005127,
+ "epoch": 3.808080808080808,
+ "grad_norm": 2.376477003097534,
+ "learning_rate": 1.193939393939394e-06,
+ "loss": 1.4241583347320557,
+ "mean_token_accuracy": 0.662493884563446,
+ "num_tokens": 30883840.0,
+ "step": 1885
+ },
+ {
+ "entropy": 1.4050343036651611,
+ "epoch": 3.81010101010101,
+ "grad_norm": 2.310051202774048,
+ "learning_rate": 1.1919191919191921e-06,
+ "loss": 1.4288474321365356,
+ "mean_token_accuracy": 0.6623107194900513,
+ "num_tokens": 30900224.0,
+ "step": 1886
+ },
+ {
+ "entropy": 1.1658165454864502,
+ "epoch": 3.812121212121212,
+ "grad_norm": 2.1743791103363037,
+ "learning_rate": 1.18989898989899e-06,
+ "loss": 1.1644586324691772,
+ "mean_token_accuracy": 0.7122007608413696,
+ "num_tokens": 30916608.0,
+ "step": 1887
+ },
+ {
+ "entropy": 1.1667358875274658,
+ "epoch": 3.8141414141414143,
+ "grad_norm": 2.092580556869507,
+ "learning_rate": 1.187878787878788e-06,
+ "loss": 1.1762518882751465,
+ "mean_token_accuracy": 0.718551516532898,
+ "num_tokens": 30932992.0,
+ "step": 1888
+ },
+ {
+ "entropy": 1.2293628454208374,
+ "epoch": 3.816161616161616,
+ "grad_norm": 2.2432591915130615,
+ "learning_rate": 1.185858585858586e-06,
+ "loss": 1.249151587486267,
+ "mean_token_accuracy": 0.7038959264755249,
+ "num_tokens": 30949376.0,
+ "step": 1889
+ },
+ {
+ "entropy": 1.104671597480774,
+ "epoch": 3.8181818181818183,
+ "grad_norm": 2.7913901805877686,
+ "learning_rate": 1.183838383838384e-06,
+ "loss": 1.107041597366333,
+ "mean_token_accuracy": 0.7236809730529785,
+ "num_tokens": 30965760.0,
+ "step": 1890
+ },
+ {
+ "entropy": 0.9601510763168335,
+ "epoch": 3.82020202020202,
+ "grad_norm": 2.0736911296844482,
+ "learning_rate": 1.181818181818182e-06,
+ "loss": 0.9680359363555908,
+ "mean_token_accuracy": 0.7562286257743835,
+ "num_tokens": 30982144.0,
+ "step": 1891
+ },
+ {
+ "entropy": 1.2994087934494019,
+ "epoch": 3.822222222222222,
+ "grad_norm": 2.2076780796051025,
+ "learning_rate": 1.1797979797979799e-06,
+ "loss": 1.3263574838638306,
+ "mean_token_accuracy": 0.6968123912811279,
+ "num_tokens": 30998528.0,
+ "step": 1892
+ },
+ {
+ "entropy": 1.3631376028060913,
+ "epoch": 3.824242424242424,
+ "grad_norm": 2.340932846069336,
+ "learning_rate": 1.1777777777777778e-06,
+ "loss": 1.3866732120513916,
+ "mean_token_accuracy": 0.6775158643722534,
+ "num_tokens": 31014912.0,
+ "step": 1893
+ },
+ {
+ "entropy": 0.8883916735649109,
+ "epoch": 3.8262626262626265,
+ "grad_norm": 2.2482478618621826,
+ "learning_rate": 1.1757575757575759e-06,
+ "loss": 0.8972976207733154,
+ "mean_token_accuracy": 0.7602589130401611,
+ "num_tokens": 31031296.0,
+ "step": 1894
+ },
+ {
+ "entropy": 1.2881001234054565,
+ "epoch": 3.8282828282828283,
+ "grad_norm": 2.2452962398529053,
+ "learning_rate": 1.173737373737374e-06,
+ "loss": 1.298166036605835,
+ "mean_token_accuracy": 0.6886907815933228,
+ "num_tokens": 31047680.0,
+ "step": 1895
+ },
+ {
+ "entropy": 1.2942661046981812,
+ "epoch": 3.83030303030303,
+ "grad_norm": 2.5083208084106445,
+ "learning_rate": 1.1717171717171719e-06,
+ "loss": 1.3013086318969727,
+ "mean_token_accuracy": 0.6803248524665833,
+ "num_tokens": 31064064.0,
+ "step": 1896
+ },
+ {
+ "entropy": 1.1769400835037231,
+ "epoch": 3.8323232323232324,
+ "grad_norm": 2.2185699939727783,
+ "learning_rate": 1.1696969696969697e-06,
+ "loss": 1.1732335090637207,
+ "mean_token_accuracy": 0.7107352018356323,
+ "num_tokens": 31080448.0,
+ "step": 1897
+ },
+ {
+ "entropy": 1.3242368698120117,
+ "epoch": 3.8343434343434346,
+ "grad_norm": 2.153888702392578,
+ "learning_rate": 1.1676767676767678e-06,
+ "loss": 1.3406283855438232,
+ "mean_token_accuracy": 0.6897899508476257,
+ "num_tokens": 31096832.0,
+ "step": 1898
+ },
+ {
+ "entropy": 0.9292052984237671,
+ "epoch": 3.8363636363636364,
+ "grad_norm": 2.1435623168945312,
+ "learning_rate": 1.1656565656565657e-06,
+ "loss": 0.9380660057067871,
+ "mean_token_accuracy": 0.7523204684257507,
+ "num_tokens": 31113216.0,
+ "step": 1899
+ },
+ {
+ "entropy": 1.2560123205184937,
+ "epoch": 3.8383838383838382,
+ "grad_norm": 2.292471408843994,
+ "learning_rate": 1.1636363636363638e-06,
+ "loss": 1.2524765729904175,
+ "mean_token_accuracy": 0.6988885998725891,
+ "num_tokens": 31129600.0,
+ "step": 1900
+ },
+ {
+ "epoch": 3.8383838383838382,
+ "eval_entropy": 1.323313660679325,
+ "eval_loss": 1.5475536584854126,
+ "eval_mean_token_accuracy": 0.6447510724106142,
+ "eval_num_tokens": 31129600.0,
+ "eval_runtime": 43.7651,
+ "eval_samples_per_second": 22.621,
+ "eval_steps_per_second": 2.833,
+ "step": 1900
+ },
+ {
+ "entropy": 1.0595868825912476,
+ "epoch": 3.8404040404040405,
+ "grad_norm": 2.2940235137939453,
+ "learning_rate": 1.1616161616161617e-06,
+ "loss": 1.0685131549835205,
+ "mean_token_accuracy": 0.7346116304397583,
+ "num_tokens": 31145984.0,
+ "step": 1901
+ },
+ {
+ "entropy": 1.1705970764160156,
+ "epoch": 3.8424242424242423,
+ "grad_norm": 2.200145959854126,
+ "learning_rate": 1.1595959595959596e-06,
+ "loss": 1.1744719743728638,
+ "mean_token_accuracy": 0.7096360325813293,
+ "num_tokens": 31162368.0,
+ "step": 1902
+ },
+ {
+ "entropy": 1.1404083967208862,
+ "epoch": 3.8444444444444446,
+ "grad_norm": 2.3402581214904785,
+ "learning_rate": 1.1575757575757577e-06,
+ "loss": 1.154109239578247,
+ "mean_token_accuracy": 0.7090253829956055,
+ "num_tokens": 31178752.0,
+ "step": 1903
+ },
+ {
+ "entropy": 1.0651684999465942,
+ "epoch": 3.8464646464646464,
+ "grad_norm": 2.1707308292388916,
+ "learning_rate": 1.1555555555555556e-06,
+ "loss": 1.0661014318466187,
+ "mean_token_accuracy": 0.7322300672531128,
+ "num_tokens": 31195136.0,
+ "step": 1904
+ },
+ {
+ "entropy": 1.4612106084823608,
+ "epoch": 3.8484848484848486,
+ "grad_norm": 2.3446035385131836,
+ "learning_rate": 1.1535353535353535e-06,
+ "loss": 1.4408762454986572,
+ "mean_token_accuracy": 0.6595017313957214,
+ "num_tokens": 31211520.0,
+ "step": 1905
+ },
+ {
+ "entropy": 1.2062592506408691,
+ "epoch": 3.8505050505050504,
+ "grad_norm": 2.244398593902588,
+ "learning_rate": 1.1515151515151516e-06,
+ "loss": 1.2011905908584595,
+ "mean_token_accuracy": 0.7075598239898682,
+ "num_tokens": 31227904.0,
+ "step": 1906
+ },
+ {
+ "entropy": 1.0715900659561157,
+ "epoch": 3.8525252525252527,
+ "grad_norm": 2.2543070316314697,
+ "learning_rate": 1.1494949494949497e-06,
+ "loss": 1.0634649991989136,
+ "mean_token_accuracy": 0.7305202484130859,
+ "num_tokens": 31244288.0,
+ "step": 1907
+ },
+ {
+ "entropy": 1.6116880178451538,
+ "epoch": 3.8545454545454545,
+ "grad_norm": 2.3540494441986084,
+ "learning_rate": 1.1474747474747476e-06,
+ "loss": 1.6299524307250977,
+ "mean_token_accuracy": 0.6351978778839111,
+ "num_tokens": 31260672.0,
+ "step": 1908
+ },
+ {
+ "entropy": 1.470803141593933,
+ "epoch": 3.8565656565656568,
+ "grad_norm": 2.2863407135009766,
+ "learning_rate": 1.1454545454545457e-06,
+ "loss": 1.4692970514297485,
+ "mean_token_accuracy": 0.6576697826385498,
+ "num_tokens": 31277056.0,
+ "step": 1909
+ },
+ {
+ "entropy": 1.0824670791625977,
+ "epoch": 3.8585858585858586,
+ "grad_norm": 2.3184683322906494,
+ "learning_rate": 1.1434343434343436e-06,
+ "loss": 1.0677670240402222,
+ "mean_token_accuracy": 0.7219101190567017,
+ "num_tokens": 31293440.0,
+ "step": 1910
+ },
+ {
+ "entropy": 1.027206540107727,
+ "epoch": 3.8606060606060604,
+ "grad_norm": 2.4096450805664062,
+ "learning_rate": 1.1414141414141414e-06,
+ "loss": 1.0224218368530273,
+ "mean_token_accuracy": 0.7347947955131531,
+ "num_tokens": 31309824.0,
+ "step": 1911
+ },
+ {
+ "entropy": 1.1143090724945068,
+ "epoch": 3.8626262626262626,
+ "grad_norm": 2.0057880878448486,
+ "learning_rate": 1.1393939393939395e-06,
+ "loss": 1.1075199842453003,
+ "mean_token_accuracy": 0.7245969772338867,
+ "num_tokens": 31326208.0,
+ "step": 1912
+ },
+ {
+ "entropy": 1.299835205078125,
+ "epoch": 3.864646464646465,
+ "grad_norm": 2.554501533508301,
+ "learning_rate": 1.1373737373737374e-06,
+ "loss": 1.2906074523925781,
+ "mean_token_accuracy": 0.681485116481781,
+ "num_tokens": 31342592.0,
+ "step": 1913
+ },
+ {
+ "entropy": 1.289318561553955,
+ "epoch": 3.8666666666666667,
+ "grad_norm": 2.4257376194000244,
+ "learning_rate": 1.1353535353535353e-06,
+ "loss": 1.281335711479187,
+ "mean_token_accuracy": 0.678798258304596,
+ "num_tokens": 31358976.0,
+ "step": 1914
+ },
+ {
+ "entropy": 1.0814216136932373,
+ "epoch": 3.8686868686868685,
+ "grad_norm": 2.5448758602142334,
+ "learning_rate": 1.1333333333333334e-06,
+ "loss": 1.0837101936340332,
+ "mean_token_accuracy": 0.7264899611473083,
+ "num_tokens": 31375360.0,
+ "step": 1915
+ },
+ {
+ "entropy": 0.8491103649139404,
+ "epoch": 3.8707070707070708,
+ "grad_norm": 2.0053293704986572,
+ "learning_rate": 1.1313131313131315e-06,
+ "loss": 0.8459927439689636,
+ "mean_token_accuracy": 0.775036633014679,
+ "num_tokens": 31391744.0,
+ "step": 1916
+ },
+ {
+ "entropy": 1.2331805229187012,
+ "epoch": 3.8727272727272726,
+ "grad_norm": 2.1101794242858887,
+ "learning_rate": 1.1292929292929294e-06,
+ "loss": 1.241546869277954,
+ "mean_token_accuracy": 0.7051783204078674,
+ "num_tokens": 31408128.0,
+ "step": 1917
+ },
+ {
+ "entropy": 0.8649861812591553,
+ "epoch": 3.874747474747475,
+ "grad_norm": 2.098615884780884,
+ "learning_rate": 1.1272727272727275e-06,
+ "loss": 0.8645999431610107,
+ "mean_token_accuracy": 0.7702125310897827,
+ "num_tokens": 31424512.0,
+ "step": 1918
+ },
+ {
+ "entropy": 1.1396632194519043,
+ "epoch": 3.8767676767676766,
+ "grad_norm": 2.2155094146728516,
+ "learning_rate": 1.1252525252525254e-06,
+ "loss": 1.1333739757537842,
+ "mean_token_accuracy": 0.7166585326194763,
+ "num_tokens": 31440896.0,
+ "step": 1919
+ },
+ {
+ "entropy": 1.375836968421936,
+ "epoch": 3.878787878787879,
+ "grad_norm": 2.3003385066986084,
+ "learning_rate": 1.1232323232323233e-06,
+ "loss": 1.379508376121521,
+ "mean_token_accuracy": 0.6699438095092773,
+ "num_tokens": 31457280.0,
+ "step": 1920
+ },
+ {
+ "entropy": 1.2705941200256348,
+ "epoch": 3.8808080808080807,
+ "grad_norm": 2.156363010406494,
+ "learning_rate": 1.1212121212121214e-06,
+ "loss": 1.266032099723816,
+ "mean_token_accuracy": 0.6869198679924011,
+ "num_tokens": 31473664.0,
+ "step": 1921
+ },
+ {
+ "entropy": 1.6494578123092651,
+ "epoch": 3.882828282828283,
+ "grad_norm": 2.420069932937622,
+ "learning_rate": 1.1191919191919193e-06,
+ "loss": 1.6762641668319702,
+ "mean_token_accuracy": 0.6144357323646545,
+ "num_tokens": 31490048.0,
+ "step": 1922
+ },
+ {
+ "entropy": 0.9549956917762756,
+ "epoch": 3.8848484848484848,
+ "grad_norm": 2.199296236038208,
+ "learning_rate": 1.1171717171717172e-06,
+ "loss": 0.9642987251281738,
+ "mean_token_accuracy": 0.7506717443466187,
+ "num_tokens": 31506432.0,
+ "step": 1923
+ },
+ {
+ "entropy": 1.5379301309585571,
+ "epoch": 3.886868686868687,
+ "grad_norm": 2.2627663612365723,
+ "learning_rate": 1.1151515151515153e-06,
+ "loss": 1.5375440120697021,
+ "mean_token_accuracy": 0.6497313380241394,
+ "num_tokens": 31522816.0,
+ "step": 1924
+ },
+ {
+ "entropy": 1.009248971939087,
+ "epoch": 3.888888888888889,
+ "grad_norm": 2.142307758331299,
+ "learning_rate": 1.1131313131313131e-06,
+ "loss": 1.014758586883545,
+ "mean_token_accuracy": 0.7470077872276306,
+ "num_tokens": 31539200.0,
+ "step": 1925
+ },
+ {
+ "entropy": 1.1543548107147217,
+ "epoch": 3.8909090909090907,
+ "grad_norm": 2.2896368503570557,
+ "learning_rate": 1.111111111111111e-06,
+ "loss": 1.169635534286499,
+ "mean_token_accuracy": 0.7079873085021973,
+ "num_tokens": 31555584.0,
+ "step": 1926
+ },
+ {
+ "entropy": 1.1072580814361572,
+ "epoch": 3.892929292929293,
+ "grad_norm": 2.4316368103027344,
+ "learning_rate": 1.1090909090909093e-06,
+ "loss": 1.0753037929534912,
+ "mean_token_accuracy": 0.7245358824729919,
+ "num_tokens": 31571968.0,
+ "step": 1927
+ },
+ {
+ "entropy": 1.0660669803619385,
+ "epoch": 3.894949494949495,
+ "grad_norm": 2.2569425106048584,
+ "learning_rate": 1.1070707070707072e-06,
+ "loss": 1.0584138631820679,
+ "mean_token_accuracy": 0.7323521971702576,
+ "num_tokens": 31588352.0,
+ "step": 1928
+ },
+ {
+ "entropy": 1.4635326862335205,
+ "epoch": 3.896969696969697,
+ "grad_norm": 2.4725160598754883,
+ "learning_rate": 1.1050505050505051e-06,
+ "loss": 1.4772144556045532,
+ "mean_token_accuracy": 0.6534562706947327,
+ "num_tokens": 31604736.0,
+ "step": 1929
+ },
+ {
+ "entropy": 1.2802455425262451,
+ "epoch": 3.898989898989899,
+ "grad_norm": 2.188870906829834,
+ "learning_rate": 1.1030303030303032e-06,
+ "loss": 1.2900553941726685,
+ "mean_token_accuracy": 0.6868588328361511,
+ "num_tokens": 31621120.0,
+ "step": 1930
+ },
+ {
+ "entropy": 1.0175695419311523,
+ "epoch": 3.901010101010101,
+ "grad_norm": 2.177603244781494,
+ "learning_rate": 1.1010101010101011e-06,
+ "loss": 1.0199849605560303,
+ "mean_token_accuracy": 0.7407791614532471,
+ "num_tokens": 31637504.0,
+ "step": 1931
+ },
+ {
+ "entropy": 1.0328937768936157,
+ "epoch": 3.9030303030303033,
+ "grad_norm": 2.3460423946380615,
+ "learning_rate": 1.098989898989899e-06,
+ "loss": 1.0519663095474243,
+ "mean_token_accuracy": 0.7301538586616516,
+ "num_tokens": 31653888.0,
+ "step": 1932
+ },
+ {
+ "entropy": 1.3149690628051758,
+ "epoch": 3.905050505050505,
+ "grad_norm": 2.4549903869628906,
+ "learning_rate": 1.096969696969697e-06,
+ "loss": 1.311675786972046,
+ "mean_token_accuracy": 0.6631045341491699,
+ "num_tokens": 31670272.0,
+ "step": 1933
+ },
+ {
+ "entropy": 1.0760291814804077,
+ "epoch": 3.907070707070707,
+ "grad_norm": 2.1368870735168457,
+ "learning_rate": 1.094949494949495e-06,
+ "loss": 1.0474696159362793,
+ "mean_token_accuracy": 0.740290641784668,
+ "num_tokens": 31686656.0,
+ "step": 1934
+ },
+ {
+ "entropy": 1.4127235412597656,
+ "epoch": 3.909090909090909,
+ "grad_norm": 2.329832077026367,
+ "learning_rate": 1.0929292929292929e-06,
+ "loss": 1.4108169078826904,
+ "mean_token_accuracy": 0.6663410067558289,
+ "num_tokens": 31703040.0,
+ "step": 1935
+ },
+ {
+ "entropy": 1.5287094116210938,
+ "epoch": 3.911111111111111,
+ "grad_norm": 2.2737598419189453,
+ "learning_rate": 1.090909090909091e-06,
+ "loss": 1.5544226169586182,
+ "mean_token_accuracy": 0.6364802122116089,
+ "num_tokens": 31719424.0,
+ "step": 1936
+ },
+ {
+ "entropy": 1.1318615674972534,
+ "epoch": 3.9131313131313132,
+ "grad_norm": 2.3448705673217773,
+ "learning_rate": 1.0888888888888889e-06,
+ "loss": 1.1228995323181152,
+ "mean_token_accuracy": 0.7198339104652405,
+ "num_tokens": 31735808.0,
+ "step": 1937
+ },
+ {
+ "entropy": 1.0484049320220947,
+ "epoch": 3.915151515151515,
+ "grad_norm": 2.5756354331970215,
+ "learning_rate": 1.086868686868687e-06,
+ "loss": 1.0507314205169678,
+ "mean_token_accuracy": 0.7214215993881226,
+ "num_tokens": 31752192.0,
+ "step": 1938
+ },
+ {
+ "entropy": 0.9962558150291443,
+ "epoch": 3.9171717171717173,
+ "grad_norm": 2.379638910293579,
+ "learning_rate": 1.084848484848485e-06,
+ "loss": 0.9964162707328796,
+ "mean_token_accuracy": 0.7351001501083374,
+ "num_tokens": 31768576.0,
+ "step": 1939
+ },
+ {
+ "entropy": 1.0813647508621216,
+ "epoch": 3.919191919191919,
+ "grad_norm": 2.202664852142334,
+ "learning_rate": 1.082828282828283e-06,
+ "loss": 1.0924289226531982,
+ "mean_token_accuracy": 0.7243527173995972,
+ "num_tokens": 31784960.0,
+ "step": 1940
+ },
+ {
+ "entropy": 1.3451746702194214,
+ "epoch": 3.9212121212121214,
+ "grad_norm": 2.2476084232330322,
+ "learning_rate": 1.0808080808080808e-06,
+ "loss": 1.3437637090682983,
+ "mean_token_accuracy": 0.681485116481781,
+ "num_tokens": 31801344.0,
+ "step": 1941
+ },
+ {
+ "entropy": 1.1332415342330933,
+ "epoch": 3.923232323232323,
+ "grad_norm": 2.414275884628296,
+ "learning_rate": 1.078787878787879e-06,
+ "loss": 1.1400260925292969,
+ "mean_token_accuracy": 0.7132388949394226,
+ "num_tokens": 31817728.0,
+ "step": 1942
+ },
+ {
+ "entropy": 1.087522029876709,
+ "epoch": 3.9252525252525254,
+ "grad_norm": 2.1204850673675537,
+ "learning_rate": 1.0767676767676768e-06,
+ "loss": 1.0787461996078491,
+ "mean_token_accuracy": 0.7150708436965942,
+ "num_tokens": 31834112.0,
+ "step": 1943
+ },
+ {
+ "entropy": 1.1856560707092285,
+ "epoch": 3.9272727272727272,
+ "grad_norm": 2.270273447036743,
+ "learning_rate": 1.0747474747474747e-06,
+ "loss": 1.1918301582336426,
+ "mean_token_accuracy": 0.7099413871765137,
+ "num_tokens": 31850496.0,
+ "step": 1944
+ },
+ {
+ "entropy": 1.1141650676727295,
+ "epoch": 3.929292929292929,
+ "grad_norm": 2.357293128967285,
+ "learning_rate": 1.0727272727272728e-06,
+ "loss": 1.0945638418197632,
+ "mean_token_accuracy": 0.7214826345443726,
+ "num_tokens": 31866880.0,
+ "step": 1945
+ },
+ {
+ "entropy": 1.4306713342666626,
+ "epoch": 3.9313131313131313,
+ "grad_norm": 2.1920864582061768,
+ "learning_rate": 1.0707070707070707e-06,
+ "loss": 1.4291378259658813,
+ "mean_token_accuracy": 0.6696995496749878,
+ "num_tokens": 31883264.0,
+ "step": 1946
+ },
+ {
+ "entropy": 1.0448427200317383,
+ "epoch": 3.9333333333333336,
+ "grad_norm": 2.2893853187561035,
+ "learning_rate": 1.0686868686868688e-06,
+ "loss": 1.0350053310394287,
+ "mean_token_accuracy": 0.7333292365074158,
+ "num_tokens": 31899648.0,
+ "step": 1947
+ },
+ {
+ "entropy": 0.9403035044670105,
+ "epoch": 3.9353535353535354,
+ "grad_norm": 2.036454200744629,
+ "learning_rate": 1.066666666666667e-06,
+ "loss": 0.9342324137687683,
+ "mean_token_accuracy": 0.7620298266410828,
+ "num_tokens": 31916032.0,
+ "step": 1948
+ },
+ {
+ "entropy": 1.1731034517288208,
+ "epoch": 3.937373737373737,
+ "grad_norm": 2.388814687728882,
+ "learning_rate": 1.0646464646464648e-06,
+ "loss": 1.1642717123031616,
+ "mean_token_accuracy": 0.705422580242157,
+ "num_tokens": 31932416.0,
+ "step": 1949
+ },
+ {
+ "entropy": 1.4009122848510742,
+ "epoch": 3.9393939393939394,
+ "grad_norm": 2.3011586666107178,
+ "learning_rate": 1.0626262626262627e-06,
+ "loss": 1.3879787921905518,
+ "mean_token_accuracy": 0.6805080771446228,
+ "num_tokens": 31948800.0,
+ "step": 1950
+ },
+ {
+ "epoch": 3.9393939393939394,
+ "eval_entropy": 1.319143979299453,
+ "eval_loss": 1.5476959943771362,
+ "eval_mean_token_accuracy": 0.6448943807232764,
+ "eval_num_tokens": 31948800.0,
+ "eval_runtime": 43.7596,
+ "eval_samples_per_second": 22.624,
+ "eval_steps_per_second": 2.834,
+ "step": 1950
+ },
+ {
+ "entropy": 1.2538156509399414,
+ "epoch": 3.9414141414141413,
+ "grad_norm": 2.5176167488098145,
+ "learning_rate": 1.0606060606060608e-06,
+ "loss": 1.2646058797836304,
+ "mean_token_accuracy": 0.6989496946334839,
+ "num_tokens": 31965184.0,
+ "step": 1951
+ },
+ {
+ "entropy": 1.4409568309783936,
+ "epoch": 3.9434343434343435,
+ "grad_norm": 2.3068838119506836,
+ "learning_rate": 1.0585858585858587e-06,
+ "loss": 1.4423179626464844,
+ "mean_token_accuracy": 0.6614558100700378,
+ "num_tokens": 31981568.0,
+ "step": 1952
+ },
+ {
+ "entropy": 1.4900083541870117,
+ "epoch": 3.9454545454545453,
+ "grad_norm": 2.3615455627441406,
+ "learning_rate": 1.0565656565656566e-06,
+ "loss": 1.4933650493621826,
+ "mean_token_accuracy": 0.6474108695983887,
+ "num_tokens": 31997952.0,
+ "step": 1953
+ },
+ {
+ "entropy": 1.2034417390823364,
+ "epoch": 3.9474747474747476,
+ "grad_norm": 2.4732308387756348,
+ "learning_rate": 1.0545454545454547e-06,
+ "loss": 1.209761619567871,
+ "mean_token_accuracy": 0.6927210688591003,
+ "num_tokens": 32014336.0,
+ "step": 1954
+ },
+ {
+ "entropy": 1.1165226697921753,
+ "epoch": 3.9494949494949494,
+ "grad_norm": 2.4827890396118164,
+ "learning_rate": 1.0525252525252525e-06,
+ "loss": 1.1289414167404175,
+ "mean_token_accuracy": 0.7070713043212891,
+ "num_tokens": 32030720.0,
+ "step": 1955
+ },
+ {
+ "entropy": 1.2843526601791382,
+ "epoch": 3.9515151515151516,
+ "grad_norm": 2.32797908782959,
+ "learning_rate": 1.0505050505050506e-06,
+ "loss": 1.2694849967956543,
+ "mean_token_accuracy": 0.6875916123390198,
+ "num_tokens": 32047104.0,
+ "step": 1956
+ },
+ {
+ "entropy": 1.3727971315383911,
+ "epoch": 3.9535353535353535,
+ "grad_norm": 2.320406198501587,
+ "learning_rate": 1.0484848484848485e-06,
+ "loss": 1.3948733806610107,
+ "mean_token_accuracy": 0.6679286956787109,
+ "num_tokens": 32063488.0,
+ "step": 1957
+ },
+ {
+ "entropy": 1.4869349002838135,
+ "epoch": 3.9555555555555557,
+ "grad_norm": 2.2239322662353516,
+ "learning_rate": 1.0464646464646464e-06,
+ "loss": 1.5088552236557007,
+ "mean_token_accuracy": 0.6551660895347595,
+ "num_tokens": 32079872.0,
+ "step": 1958
+ },
+ {
+ "entropy": 1.2804311513900757,
+ "epoch": 3.9575757575757575,
+ "grad_norm": 2.355632781982422,
+ "learning_rate": 1.0444444444444445e-06,
+ "loss": 1.2881189584732056,
+ "mean_token_accuracy": 0.6883854269981384,
+ "num_tokens": 32096256.0,
+ "step": 1959
+ },
+ {
+ "entropy": 1.469780683517456,
+ "epoch": 3.9595959595959593,
+ "grad_norm": 2.475818395614624,
+ "learning_rate": 1.0424242424242426e-06,
+ "loss": 1.4946942329406738,
+ "mean_token_accuracy": 0.6443576216697693,
+ "num_tokens": 32112640.0,
+ "step": 1960
+ },
+ {
+ "entropy": 1.0433298349380493,
+ "epoch": 3.9616161616161616,
+ "grad_norm": 2.335348606109619,
+ "learning_rate": 1.0404040404040405e-06,
+ "loss": 1.051847219467163,
+ "mean_token_accuracy": 0.7260014414787292,
+ "num_tokens": 32129024.0,
+ "step": 1961
+ },
+ {
+ "entropy": 1.060207486152649,
+ "epoch": 3.963636363636364,
+ "grad_norm": 2.465827465057373,
+ "learning_rate": 1.0383838383838384e-06,
+ "loss": 1.0738749504089355,
+ "mean_token_accuracy": 0.7250854969024658,
+ "num_tokens": 32145408.0,
+ "step": 1962
+ },
+ {
+ "entropy": 1.3710187673568726,
+ "epoch": 3.9656565656565657,
+ "grad_norm": 2.1859865188598633,
+ "learning_rate": 1.0363636363636365e-06,
+ "loss": 1.3790408372879028,
+ "mean_token_accuracy": 0.6800195574760437,
+ "num_tokens": 32161792.0,
+ "step": 1963
+ },
+ {
+ "entropy": 1.4380125999450684,
+ "epoch": 3.9676767676767675,
+ "grad_norm": 2.3186328411102295,
+ "learning_rate": 1.0343434343434344e-06,
+ "loss": 1.4353742599487305,
+ "mean_token_accuracy": 0.6627992391586304,
+ "num_tokens": 32178176.0,
+ "step": 1964
+ },
+ {
+ "entropy": 1.346064567565918,
+ "epoch": 3.9696969696969697,
+ "grad_norm": 2.455488920211792,
+ "learning_rate": 1.0323232323232325e-06,
+ "loss": 1.3504667282104492,
+ "mean_token_accuracy": 0.6808744668960571,
+ "num_tokens": 32194560.0,
+ "step": 1965
+ },
+ {
+ "entropy": 1.111870527267456,
+ "epoch": 3.971717171717172,
+ "grad_norm": 2.216954231262207,
+ "learning_rate": 1.0303030303030304e-06,
+ "loss": 1.1173250675201416,
+ "mean_token_accuracy": 0.7161700129508972,
+ "num_tokens": 32210944.0,
+ "step": 1966
+ },
+ {
+ "entropy": 1.570885181427002,
+ "epoch": 3.973737373737374,
+ "grad_norm": 2.5180883407592773,
+ "learning_rate": 1.0282828282828283e-06,
+ "loss": 1.550491213798523,
+ "mean_token_accuracy": 0.6203590631484985,
+ "num_tokens": 32227328.0,
+ "step": 1967
+ },
+ {
+ "entropy": 0.887984037399292,
+ "epoch": 3.9757575757575756,
+ "grad_norm": 2.193727970123291,
+ "learning_rate": 1.0262626262626264e-06,
+ "loss": 0.8939201831817627,
+ "mean_token_accuracy": 0.7674645781517029,
+ "num_tokens": 32243712.0,
+ "step": 1968
+ },
+ {
+ "entropy": 1.3493150472640991,
+ "epoch": 3.977777777777778,
+ "grad_norm": 2.320697784423828,
+ "learning_rate": 1.0242424242424242e-06,
+ "loss": 1.3643516302108765,
+ "mean_token_accuracy": 0.67659991979599,
+ "num_tokens": 32260096.0,
+ "step": 1969
+ },
+ {
+ "entropy": 1.2187929153442383,
+ "epoch": 3.9797979797979797,
+ "grad_norm": 2.2966790199279785,
+ "learning_rate": 1.0222222222222223e-06,
+ "loss": 1.2266913652420044,
+ "mean_token_accuracy": 0.701636552810669,
+ "num_tokens": 32276480.0,
+ "step": 1970
+ },
+ {
+ "entropy": 1.3222001791000366,
+ "epoch": 3.981818181818182,
+ "grad_norm": 2.484215497970581,
+ "learning_rate": 1.0202020202020202e-06,
+ "loss": 1.3209818601608276,
+ "mean_token_accuracy": 0.6782486438751221,
+ "num_tokens": 32292864.0,
+ "step": 1971
+ },
+ {
+ "entropy": 1.2982250452041626,
+ "epoch": 3.9838383838383837,
+ "grad_norm": 2.320852518081665,
+ "learning_rate": 1.0181818181818183e-06,
+ "loss": 1.3085607290267944,
+ "mean_token_accuracy": 0.6881411671638489,
+ "num_tokens": 32309248.0,
+ "step": 1972
+ },
+ {
+ "entropy": 1.3396943807601929,
+ "epoch": 3.985858585858586,
+ "grad_norm": 2.224574565887451,
+ "learning_rate": 1.0161616161616162e-06,
+ "loss": 1.3511459827423096,
+ "mean_token_accuracy": 0.6810576319694519,
+ "num_tokens": 32325632.0,
+ "step": 1973
+ },
+ {
+ "entropy": 1.2554755210876465,
+ "epoch": 3.987878787878788,
+ "grad_norm": 2.3757498264312744,
+ "learning_rate": 1.0141414141414143e-06,
+ "loss": 1.247807502746582,
+ "mean_token_accuracy": 0.687530517578125,
+ "num_tokens": 32342016.0,
+ "step": 1974
+ },
+ {
+ "entropy": 0.9622905850410461,
+ "epoch": 3.98989898989899,
+ "grad_norm": 2.2012438774108887,
+ "learning_rate": 1.0121212121212122e-06,
+ "loss": 0.9560307860374451,
+ "mean_token_accuracy": 0.7637396454811096,
+ "num_tokens": 32358400.0,
+ "step": 1975
+ },
+ {
+ "entropy": 1.1193549633026123,
+ "epoch": 3.991919191919192,
+ "grad_norm": 2.3705174922943115,
+ "learning_rate": 1.01010101010101e-06,
+ "loss": 1.11350679397583,
+ "mean_token_accuracy": 0.7154372334480286,
+ "num_tokens": 32374784.0,
+ "step": 1976
+ },
+ {
+ "entropy": 0.9361278414726257,
+ "epoch": 3.993939393939394,
+ "grad_norm": 2.174281358718872,
+ "learning_rate": 1.0080808080808082e-06,
+ "loss": 0.9388691186904907,
+ "mean_token_accuracy": 0.7485344409942627,
+ "num_tokens": 32391168.0,
+ "step": 1977
+ },
+ {
+ "entropy": 1.4282293319702148,
+ "epoch": 3.995959595959596,
+ "grad_norm": 2.1972262859344482,
+ "learning_rate": 1.006060606060606e-06,
+ "loss": 1.4244005680084229,
+ "mean_token_accuracy": 0.685332179069519,
+ "num_tokens": 32407552.0,
+ "step": 1978
+ },
+ {
+ "entropy": 1.221082091331482,
+ "epoch": 3.9979797979797977,
+ "grad_norm": 2.3935673236846924,
+ "learning_rate": 1.004040404040404e-06,
+ "loss": 1.1999341249465942,
+ "mean_token_accuracy": 0.6952857971191406,
+ "num_tokens": 32423936.0,
+ "step": 1979
+ },
+ {
+ "entropy": 1.1347371339797974,
+ "epoch": 4.0,
+ "grad_norm": 2.228233575820923,
+ "learning_rate": 1.002020202020202e-06,
+ "loss": 1.1378669738769531,
+ "mean_token_accuracy": 0.700537383556366,
+ "num_tokens": 32440320.0,
+ "step": 1980
+ },
+ {
+ "entropy": 1.1821788549423218,
+ "epoch": 4.002020202020202,
+ "grad_norm": 2.356572389602661,
+ "learning_rate": 1.0000000000000002e-06,
+ "loss": 1.1377968788146973,
+ "mean_token_accuracy": 0.708170473575592,
+ "num_tokens": 32456704.0,
+ "step": 1981
+ },
+ {
+ "entropy": 1.444516658782959,
+ "epoch": 4.004040404040404,
+ "grad_norm": 2.34165358543396,
+ "learning_rate": 9.97979797979798e-07,
+ "loss": 1.4065756797790527,
+ "mean_token_accuracy": 0.6689057350158691,
+ "num_tokens": 32473088.0,
+ "step": 1982
+ },
+ {
+ "entropy": 1.5600178241729736,
+ "epoch": 4.006060606060606,
+ "grad_norm": 2.147475004196167,
+ "learning_rate": 9.959595959595962e-07,
+ "loss": 1.5237784385681152,
+ "mean_token_accuracy": 0.6602345108985901,
+ "num_tokens": 32489472.0,
+ "step": 1983
+ },
+ {
+ "entropy": 1.3860907554626465,
+ "epoch": 4.008080808080808,
+ "grad_norm": 2.5050148963928223,
+ "learning_rate": 9.93939393939394e-07,
+ "loss": 1.3332314491271973,
+ "mean_token_accuracy": 0.6679897308349609,
+ "num_tokens": 32505856.0,
+ "step": 1984
+ },
+ {
+ "entropy": 1.2262948751449585,
+ "epoch": 4.01010101010101,
+ "grad_norm": 2.3707785606384277,
+ "learning_rate": 9.91919191919192e-07,
+ "loss": 1.1803091764450073,
+ "mean_token_accuracy": 0.6957743167877197,
+ "num_tokens": 32522240.0,
+ "step": 1985
+ },
+ {
+ "entropy": 1.2782310247421265,
+ "epoch": 4.012121212121212,
+ "grad_norm": 2.1337549686431885,
+ "learning_rate": 9.8989898989899e-07,
+ "loss": 1.2508152723312378,
+ "mean_token_accuracy": 0.6898509860038757,
+ "num_tokens": 32538624.0,
+ "step": 1986
+ },
+ {
+ "entropy": 1.2618422508239746,
+ "epoch": 4.014141414141414,
+ "grad_norm": 2.1014909744262695,
+ "learning_rate": 9.87878787878788e-07,
+ "loss": 1.2586122751235962,
+ "mean_token_accuracy": 0.7060942649841309,
+ "num_tokens": 32555008.0,
+ "step": 1987
+ },
+ {
+ "entropy": 1.4927153587341309,
+ "epoch": 4.016161616161616,
+ "grad_norm": 2.3997669219970703,
+ "learning_rate": 9.858585858585858e-07,
+ "loss": 1.458584189414978,
+ "mean_token_accuracy": 0.6642037034034729,
+ "num_tokens": 32571392.0,
+ "step": 1988
+ },
+ {
+ "entropy": 0.9929779767990112,
+ "epoch": 4.0181818181818185,
+ "grad_norm": 2.1199800968170166,
+ "learning_rate": 9.83838383838384e-07,
+ "loss": 0.9945173263549805,
+ "mean_token_accuracy": 0.7606863975524902,
+ "num_tokens": 32587776.0,
+ "step": 1989
+ },
+ {
+ "entropy": 1.2815021276474,
+ "epoch": 4.02020202020202,
+ "grad_norm": 2.2822794914245605,
+ "learning_rate": 9.818181818181818e-07,
+ "loss": 1.263106107711792,
+ "mean_token_accuracy": 0.7038959264755249,
+ "num_tokens": 32604160.0,
+ "step": 1990
+ },
+ {
+ "entropy": 1.2504572868347168,
+ "epoch": 4.022222222222222,
+ "grad_norm": 2.0446224212646484,
+ "learning_rate": 9.797979797979799e-07,
+ "loss": 1.2274131774902344,
+ "mean_token_accuracy": 0.7008426785469055,
+ "num_tokens": 32620544.0,
+ "step": 1991
+ },
+ {
+ "entropy": 1.1070255041122437,
+ "epoch": 4.024242424242424,
+ "grad_norm": 2.0727381706237793,
+ "learning_rate": 9.77777777777778e-07,
+ "loss": 1.082715392112732,
+ "mean_token_accuracy": 0.7302759885787964,
+ "num_tokens": 32636928.0,
+ "step": 1992
+ },
+ {
+ "entropy": 1.0761677026748657,
+ "epoch": 4.026262626262627,
+ "grad_norm": 2.0985822677612305,
+ "learning_rate": 9.757575757575759e-07,
+ "loss": 1.0548713207244873,
+ "mean_token_accuracy": 0.7333292365074158,
+ "num_tokens": 32653312.0,
+ "step": 1993
+ },
+ {
+ "entropy": 1.5109375715255737,
+ "epoch": 4.028282828282828,
+ "grad_norm": 2.426588535308838,
+ "learning_rate": 9.737373737373738e-07,
+ "loss": 1.4922651052474976,
+ "mean_token_accuracy": 0.6560820937156677,
+ "num_tokens": 32669696.0,
+ "step": 1994
+ },
+ {
+ "entropy": 1.2393261194229126,
+ "epoch": 4.03030303030303,
+ "grad_norm": 2.1131463050842285,
+ "learning_rate": 9.717171717171719e-07,
+ "loss": 1.2221450805664062,
+ "mean_token_accuracy": 0.7113458514213562,
+ "num_tokens": 32686080.0,
+ "step": 1995
+ },
+ {
+ "entropy": 1.2733023166656494,
+ "epoch": 4.0323232323232325,
+ "grad_norm": 2.42077898979187,
+ "learning_rate": 9.696969696969698e-07,
+ "loss": 1.266562581062317,
+ "mean_token_accuracy": 0.6916218996047974,
+ "num_tokens": 32702464.0,
+ "step": 1996
+ },
+ {
+ "entropy": 1.1015594005584717,
+ "epoch": 4.034343434343434,
+ "grad_norm": 2.3131461143493652,
+ "learning_rate": 9.676767676767676e-07,
+ "loss": 1.0699245929718018,
+ "mean_token_accuracy": 0.7295432090759277,
+ "num_tokens": 32718848.0,
+ "step": 1997
+ },
+ {
+ "entropy": 1.1488311290740967,
+ "epoch": 4.036363636363636,
+ "grad_norm": 2.460294723510742,
+ "learning_rate": 9.656565656565657e-07,
+ "loss": 1.1216062307357788,
+ "mean_token_accuracy": 0.7164753079414368,
+ "num_tokens": 32735232.0,
+ "step": 1998
+ },
+ {
+ "entropy": 1.1301594972610474,
+ "epoch": 4.038383838383838,
+ "grad_norm": 2.248311996459961,
+ "learning_rate": 9.636363636363636e-07,
+ "loss": 1.1271123886108398,
+ "mean_token_accuracy": 0.7147044539451599,
+ "num_tokens": 32751616.0,
+ "step": 1999
+ },
+ {
+ "entropy": 1.2287960052490234,
+ "epoch": 4.040404040404041,
+ "grad_norm": 2.191375732421875,
+ "learning_rate": 9.616161616161617e-07,
+ "loss": 1.2168409824371338,
+ "mean_token_accuracy": 0.7156814932823181,
+ "num_tokens": 32768000.0,
+ "step": 2000
+ },
+ {
+ "epoch": 4.040404040404041,
+ "eval_entropy": 1.294078712021151,
+ "eval_loss": 1.5553377866744995,
+ "eval_mean_token_accuracy": 0.6444841599272143,
+ "eval_num_tokens": 32768000.0,
+ "eval_runtime": 43.7303,
+ "eval_samples_per_second": 22.639,
+ "eval_steps_per_second": 2.836,
+ "step": 2000
+ },
+ {
+ "entropy": 1.0581393241882324,
+ "epoch": 4.042424242424242,
+ "grad_norm": 2.360215187072754,
+ "learning_rate": 9.595959595959596e-07,
+ "loss": 1.042952060699463,
+ "mean_token_accuracy": 0.7269784808158875,
+ "num_tokens": 32784384.0,
+ "step": 2001
+ },
+ {
+ "entropy": 1.0822330713272095,
+ "epoch": 4.044444444444444,
+ "grad_norm": 2.3221960067749023,
+ "learning_rate": 9.575757575757577e-07,
+ "loss": 1.0822763442993164,
+ "mean_token_accuracy": 0.7313751578330994,
+ "num_tokens": 32800768.0,
+ "step": 2002
+ },
+ {
+ "entropy": 1.1480356454849243,
+ "epoch": 4.0464646464646465,
+ "grad_norm": 2.063100576400757,
+ "learning_rate": 9.555555555555556e-07,
+ "loss": 1.123166799545288,
+ "mean_token_accuracy": 0.7303370833396912,
+ "num_tokens": 32817152.0,
+ "step": 2003
+ },
+ {
+ "entropy": 1.3967088460922241,
+ "epoch": 4.048484848484849,
+ "grad_norm": 2.17568039894104,
+ "learning_rate": 9.535353535353536e-07,
+ "loss": 1.4080045223236084,
+ "mean_token_accuracy": 0.6711040735244751,
+ "num_tokens": 32833536.0,
+ "step": 2004
+ },
+ {
+ "entropy": 1.2580901384353638,
+ "epoch": 4.05050505050505,
+ "grad_norm": 2.141166925430298,
+ "learning_rate": 9.515151515151516e-07,
+ "loss": 1.2547274827957153,
+ "mean_token_accuracy": 0.7031631469726562,
+ "num_tokens": 32849920.0,
+ "step": 2005
+ },
+ {
+ "entropy": 0.9946291446685791,
+ "epoch": 4.052525252525252,
+ "grad_norm": 2.490000009536743,
+ "learning_rate": 9.494949494949496e-07,
+ "loss": 1.0014936923980713,
+ "mean_token_accuracy": 0.7382755279541016,
+ "num_tokens": 32866304.0,
+ "step": 2006
+ },
+ {
+ "entropy": 1.3571540117263794,
+ "epoch": 4.054545454545455,
+ "grad_norm": 2.130330801010132,
+ "learning_rate": 9.474747474747476e-07,
+ "loss": 1.3513038158416748,
+ "mean_token_accuracy": 0.6856375336647034,
+ "num_tokens": 32882688.0,
+ "step": 2007
+ },
+ {
+ "entropy": 1.0289682149887085,
+ "epoch": 4.056565656565657,
+ "grad_norm": 2.0878100395202637,
+ "learning_rate": 9.454545454545455e-07,
+ "loss": 1.0360488891601562,
+ "mean_token_accuracy": 0.7443209290504456,
+ "num_tokens": 32899072.0,
+ "step": 2008
+ },
+ {
+ "entropy": 1.078924536705017,
+ "epoch": 4.058585858585858,
+ "grad_norm": 1.9717739820480347,
+ "learning_rate": 9.434343434343435e-07,
+ "loss": 1.0830042362213135,
+ "mean_token_accuracy": 0.7435271143913269,
+ "num_tokens": 32915456.0,
+ "step": 2009
+ },
+ {
+ "entropy": 1.5092878341674805,
+ "epoch": 4.0606060606060606,
+ "grad_norm": 2.406719923019409,
+ "learning_rate": 9.414141414141415e-07,
+ "loss": 1.5087292194366455,
+ "mean_token_accuracy": 0.6460063457489014,
+ "num_tokens": 32931840.0,
+ "step": 2010
+ },
+ {
+ "entropy": 1.427893042564392,
+ "epoch": 4.062626262626263,
+ "grad_norm": 2.3738598823547363,
+ "learning_rate": 9.393939393939395e-07,
+ "loss": 1.4216761589050293,
+ "mean_token_accuracy": 0.6571201682090759,
+ "num_tokens": 32948224.0,
+ "step": 2011
+ },
+ {
+ "entropy": 1.1727535724639893,
+ "epoch": 4.064646464646465,
+ "grad_norm": 1.9937793016433716,
+ "learning_rate": 9.373737373737376e-07,
+ "loss": 1.1711244583129883,
+ "mean_token_accuracy": 0.7261846661567688,
+ "num_tokens": 32964608.0,
+ "step": 2012
+ },
+ {
+ "entropy": 1.634767770767212,
+ "epoch": 4.066666666666666,
+ "grad_norm": 2.2499399185180664,
+ "learning_rate": 9.353535353535354e-07,
+ "loss": 1.6378939151763916,
+ "mean_token_accuracy": 0.6338544487953186,
+ "num_tokens": 32980992.0,
+ "step": 2013
+ },
+ {
+ "entropy": 1.378767967224121,
+ "epoch": 4.068686868686869,
+ "grad_norm": 2.147827386856079,
+ "learning_rate": 9.333333333333334e-07,
+ "loss": 1.3705543279647827,
+ "mean_token_accuracy": 0.6703712940216064,
+ "num_tokens": 32997376.0,
+ "step": 2014
+ },
+ {
+ "entropy": 0.8587742447853088,
+ "epoch": 4.070707070707071,
+ "grad_norm": 2.1850945949554443,
+ "learning_rate": 9.313131313131314e-07,
+ "loss": 0.8584544658660889,
+ "mean_token_accuracy": 0.7740595936775208,
+ "num_tokens": 33013760.0,
+ "step": 2015
+ },
+ {
+ "entropy": 1.180991291999817,
+ "epoch": 4.072727272727272,
+ "grad_norm": 2.3740382194519043,
+ "learning_rate": 9.292929292929294e-07,
+ "loss": 1.1824800968170166,
+ "mean_token_accuracy": 0.6996213793754578,
+ "num_tokens": 33030144.0,
+ "step": 2016
+ },
+ {
+ "entropy": 1.127561092376709,
+ "epoch": 4.074747474747475,
+ "grad_norm": 2.1335041522979736,
+ "learning_rate": 9.272727272727273e-07,
+ "loss": 1.1388006210327148,
+ "mean_token_accuracy": 0.7244137525558472,
+ "num_tokens": 33046528.0,
+ "step": 2017
+ },
+ {
+ "entropy": 1.154600977897644,
+ "epoch": 4.076767676767677,
+ "grad_norm": 2.288207530975342,
+ "learning_rate": 9.252525252525253e-07,
+ "loss": 1.1633810997009277,
+ "mean_token_accuracy": 0.7138495445251465,
+ "num_tokens": 33062912.0,
+ "step": 2018
+ },
+ {
+ "entropy": 1.3016736507415771,
+ "epoch": 4.078787878787879,
+ "grad_norm": 2.6412227153778076,
+ "learning_rate": 9.232323232323233e-07,
+ "loss": 1.3045625686645508,
+ "mean_token_accuracy": 0.6678065657615662,
+ "num_tokens": 33079296.0,
+ "step": 2019
+ },
+ {
+ "entropy": 1.461438536643982,
+ "epoch": 4.08080808080808,
+ "grad_norm": 2.4295499324798584,
+ "learning_rate": 9.212121212121213e-07,
+ "loss": 1.4820640087127686,
+ "mean_token_accuracy": 0.6492428183555603,
+ "num_tokens": 33095680.0,
+ "step": 2020
+ },
+ {
+ "entropy": 1.1305612325668335,
+ "epoch": 4.082828282828283,
+ "grad_norm": 2.3236217498779297,
+ "learning_rate": 9.191919191919192e-07,
+ "loss": 1.1307878494262695,
+ "mean_token_accuracy": 0.7134220600128174,
+ "num_tokens": 33112064.0,
+ "step": 2021
+ },
+ {
+ "entropy": 0.9688110947608948,
+ "epoch": 4.084848484848485,
+ "grad_norm": 2.404463052749634,
+ "learning_rate": 9.171717171717172e-07,
+ "loss": 0.9707077145576477,
+ "mean_token_accuracy": 0.7424889802932739,
+ "num_tokens": 33128448.0,
+ "step": 2022
+ },
+ {
+ "entropy": 1.456017255783081,
+ "epoch": 4.086868686868687,
+ "grad_norm": 2.426558017730713,
+ "learning_rate": 9.151515151515153e-07,
+ "loss": 1.4475734233856201,
+ "mean_token_accuracy": 0.6397166848182678,
+ "num_tokens": 33144832.0,
+ "step": 2023
+ },
+ {
+ "entropy": 1.1142091751098633,
+ "epoch": 4.088888888888889,
+ "grad_norm": 2.5632755756378174,
+ "learning_rate": 9.131313131313133e-07,
+ "loss": 1.1232866048812866,
+ "mean_token_accuracy": 0.7278944849967957,
+ "num_tokens": 33161216.0,
+ "step": 2024
+ },
+ {
+ "entropy": 1.4831620454788208,
+ "epoch": 4.090909090909091,
+ "grad_norm": 2.428093433380127,
+ "learning_rate": 9.111111111111113e-07,
+ "loss": 1.4886207580566406,
+ "mean_token_accuracy": 0.6450904011726379,
+ "num_tokens": 33177600.0,
+ "step": 2025
+ },
+ {
+ "entropy": 1.1659386157989502,
+ "epoch": 4.092929292929293,
+ "grad_norm": 2.404301881790161,
+ "learning_rate": 9.090909090909091e-07,
+ "loss": 1.1533830165863037,
+ "mean_token_accuracy": 0.7195896506309509,
+ "num_tokens": 33193984.0,
+ "step": 2026
+ },
+ {
+ "entropy": 1.560425877571106,
+ "epoch": 4.094949494949495,
+ "grad_norm": 2.2221803665161133,
+ "learning_rate": 9.070707070707071e-07,
+ "loss": 1.5449508428573608,
+ "mean_token_accuracy": 0.6680508255958557,
+ "num_tokens": 33210368.0,
+ "step": 2027
+ },
+ {
+ "entropy": 1.040557861328125,
+ "epoch": 4.096969696969697,
+ "grad_norm": 2.272596597671509,
+ "learning_rate": 9.050505050505051e-07,
+ "loss": 1.0452677011489868,
+ "mean_token_accuracy": 0.7357718348503113,
+ "num_tokens": 33226752.0,
+ "step": 2028
+ },
+ {
+ "entropy": 1.3482027053833008,
+ "epoch": 4.098989898989899,
+ "grad_norm": 2.360107421875,
+ "learning_rate": 9.030303030303031e-07,
+ "loss": 1.3335392475128174,
+ "mean_token_accuracy": 0.6714704632759094,
+ "num_tokens": 33243136.0,
+ "step": 2029
+ },
+ {
+ "entropy": 1.2164970636367798,
+ "epoch": 4.101010101010101,
+ "grad_norm": 2.100684881210327,
+ "learning_rate": 9.01010101010101e-07,
+ "loss": 1.2075049877166748,
+ "mean_token_accuracy": 0.7048729658126831,
+ "num_tokens": 33259520.0,
+ "step": 2030
+ },
+ {
+ "entropy": 0.9130767583847046,
+ "epoch": 4.1030303030303035,
+ "grad_norm": 2.145026922225952,
+ "learning_rate": 8.98989898989899e-07,
+ "loss": 0.9055933356285095,
+ "mean_token_accuracy": 0.758671224117279,
+ "num_tokens": 33275904.0,
+ "step": 2031
+ },
+ {
+ "entropy": 0.9220786094665527,
+ "epoch": 4.105050505050505,
+ "grad_norm": 2.0946104526519775,
+ "learning_rate": 8.96969696969697e-07,
+ "loss": 0.9177285432815552,
+ "mean_token_accuracy": 0.7683805823326111,
+ "num_tokens": 33292288.0,
+ "step": 2032
+ },
+ {
+ "entropy": 1.2093071937561035,
+ "epoch": 4.107070707070707,
+ "grad_norm": 2.0910632610321045,
+ "learning_rate": 8.94949494949495e-07,
+ "loss": 1.2280278205871582,
+ "mean_token_accuracy": 0.7200170755386353,
+ "num_tokens": 33308672.0,
+ "step": 2033
+ },
+ {
+ "entropy": 1.3322206735610962,
+ "epoch": 4.109090909090909,
+ "grad_norm": 2.3038177490234375,
+ "learning_rate": 8.929292929292931e-07,
+ "loss": 1.3241100311279297,
+ "mean_token_accuracy": 0.6911944150924683,
+ "num_tokens": 33325056.0,
+ "step": 2034
+ },
+ {
+ "entropy": 1.2418142557144165,
+ "epoch": 4.111111111111111,
+ "grad_norm": 2.1876115798950195,
+ "learning_rate": 8.90909090909091e-07,
+ "loss": 1.2438215017318726,
+ "mean_token_accuracy": 0.7025524973869324,
+ "num_tokens": 33341440.0,
+ "step": 2035
+ },
+ {
+ "entropy": 1.2280832529067993,
+ "epoch": 4.113131313131313,
+ "grad_norm": 2.4577319622039795,
+ "learning_rate": 8.88888888888889e-07,
+ "loss": 1.202743411064148,
+ "mean_token_accuracy": 0.6992549896240234,
+ "num_tokens": 33357824.0,
+ "step": 2036
+ },
+ {
+ "entropy": 1.132125735282898,
+ "epoch": 4.115151515151515,
+ "grad_norm": 2.29577898979187,
+ "learning_rate": 8.86868686868687e-07,
+ "loss": 1.107844591140747,
+ "mean_token_accuracy": 0.7243527173995972,
+ "num_tokens": 33374208.0,
+ "step": 2037
+ },
+ {
+ "entropy": 1.8715779781341553,
+ "epoch": 4.1171717171717175,
+ "grad_norm": 2.433678388595581,
+ "learning_rate": 8.84848484848485e-07,
+ "loss": 1.8939162492752075,
+ "mean_token_accuracy": 0.611932098865509,
+ "num_tokens": 33390592.0,
+ "step": 2038
+ },
+ {
+ "entropy": 1.3923977613449097,
+ "epoch": 4.119191919191919,
+ "grad_norm": 2.385688066482544,
+ "learning_rate": 8.828282828282829e-07,
+ "loss": 1.3869637250900269,
+ "mean_token_accuracy": 0.6941866278648376,
+ "num_tokens": 33406976.0,
+ "step": 2039
+ },
+ {
+ "entropy": 1.6810003519058228,
+ "epoch": 4.121212121212121,
+ "grad_norm": 2.4660162925720215,
+ "learning_rate": 8.808080808080808e-07,
+ "loss": 1.6879138946533203,
+ "mean_token_accuracy": 0.6045432090759277,
+ "num_tokens": 33423360.0,
+ "step": 2040
+ },
+ {
+ "entropy": 1.1977758407592773,
+ "epoch": 4.123232323232323,
+ "grad_norm": 2.199223518371582,
+ "learning_rate": 8.787878787878788e-07,
+ "loss": 1.2097350358963013,
+ "mean_token_accuracy": 0.7071323990821838,
+ "num_tokens": 33439744.0,
+ "step": 2041
+ },
+ {
+ "entropy": 1.3029590845108032,
+ "epoch": 4.125252525252526,
+ "grad_norm": 2.3923046588897705,
+ "learning_rate": 8.767676767676768e-07,
+ "loss": 1.323540210723877,
+ "mean_token_accuracy": 0.6738519668579102,
+ "num_tokens": 33456128.0,
+ "step": 2042
+ },
+ {
+ "entropy": 0.8669192790985107,
+ "epoch": 4.127272727272727,
+ "grad_norm": 2.2486660480499268,
+ "learning_rate": 8.747474747474747e-07,
+ "loss": 0.8708467483520508,
+ "mean_token_accuracy": 0.7698461413383484,
+ "num_tokens": 33472512.0,
+ "step": 2043
+ },
+ {
+ "entropy": 1.0041810274124146,
+ "epoch": 4.129292929292929,
+ "grad_norm": 2.2023773193359375,
+ "learning_rate": 8.727272727272728e-07,
+ "loss": 0.9956705570220947,
+ "mean_token_accuracy": 0.746275007724762,
+ "num_tokens": 33488896.0,
+ "step": 2044
+ },
+ {
+ "entropy": 0.8258928060531616,
+ "epoch": 4.1313131313131315,
+ "grad_norm": 2.158754825592041,
+ "learning_rate": 8.707070707070708e-07,
+ "loss": 0.8288166522979736,
+ "mean_token_accuracy": 0.7741817235946655,
+ "num_tokens": 33505280.0,
+ "step": 2045
+ },
+ {
+ "entropy": 1.3376224040985107,
+ "epoch": 4.133333333333334,
+ "grad_norm": 2.277294397354126,
+ "learning_rate": 8.686868686868688e-07,
+ "loss": 1.331207036972046,
+ "mean_token_accuracy": 0.6915608048439026,
+ "num_tokens": 33521664.0,
+ "step": 2046
+ },
+ {
+ "entropy": 0.8669873476028442,
+ "epoch": 4.135353535353535,
+ "grad_norm": 2.3317325115203857,
+ "learning_rate": 8.666666666666668e-07,
+ "loss": 0.8673149943351746,
+ "mean_token_accuracy": 0.7650219798088074,
+ "num_tokens": 33538048.0,
+ "step": 2047
+ },
+ {
+ "entropy": 1.4122123718261719,
+ "epoch": 4.137373737373737,
+ "grad_norm": 2.4016566276550293,
+ "learning_rate": 8.646464646464647e-07,
+ "loss": 1.4340333938598633,
+ "mean_token_accuracy": 0.6679286956787109,
+ "num_tokens": 33554432.0,
+ "step": 2048
+ },
+ {
+ "entropy": 1.1499886512756348,
+ "epoch": 4.13939393939394,
+ "grad_norm": 2.3876852989196777,
+ "learning_rate": 8.626262626262627e-07,
+ "loss": 1.1466901302337646,
+ "mean_token_accuracy": 0.7115290760993958,
+ "num_tokens": 33570816.0,
+ "step": 2049
+ },
+ {
+ "entropy": 1.260066270828247,
+ "epoch": 4.141414141414141,
+ "grad_norm": 2.383570909500122,
+ "learning_rate": 8.606060606060607e-07,
+ "loss": 1.2224504947662354,
+ "mean_token_accuracy": 0.6926599740982056,
+ "num_tokens": 33587200.0,
+ "step": 2050
+ },
+ {
+ "epoch": 4.141414141414141,
+ "eval_entropy": 1.2900591329220803,
+ "eval_loss": 1.561437964439392,
+ "eval_mean_token_accuracy": 0.643797178422251,
+ "eval_num_tokens": 33587200.0,
+ "eval_runtime": 43.833,
+ "eval_samples_per_second": 22.586,
+ "eval_steps_per_second": 2.829,
+ "step": 2050
+ },
+ {
+ "entropy": 1.233297348022461,
+ "epoch": 4.143434343434343,
+ "grad_norm": 2.400827407836914,
+ "learning_rate": 8.585858585858587e-07,
+ "loss": 1.2277863025665283,
+ "mean_token_accuracy": 0.696201741695404,
+ "num_tokens": 33603584.0,
+ "step": 2051
+ },
+ {
+ "entropy": 1.1504113674163818,
+ "epoch": 4.1454545454545455,
+ "grad_norm": 2.1206517219543457,
+ "learning_rate": 8.565656565656566e-07,
+ "loss": 1.1498513221740723,
+ "mean_token_accuracy": 0.7126892805099487,
+ "num_tokens": 33619968.0,
+ "step": 2052
+ },
+ {
+ "entropy": 1.2686772346496582,
+ "epoch": 4.147474747474748,
+ "grad_norm": 2.293596029281616,
+ "learning_rate": 8.545454545454546e-07,
+ "loss": 1.271289348602295,
+ "mean_token_accuracy": 0.6941255331039429,
+ "num_tokens": 33636352.0,
+ "step": 2053
+ },
+ {
+ "entropy": 1.0670050382614136,
+ "epoch": 4.149494949494949,
+ "grad_norm": 2.4050474166870117,
+ "learning_rate": 8.525252525252525e-07,
+ "loss": 1.077490210533142,
+ "mean_token_accuracy": 0.7255740165710449,
+ "num_tokens": 33652736.0,
+ "step": 2054
+ },
+ {
+ "entropy": 1.2747689485549927,
+ "epoch": 4.151515151515151,
+ "grad_norm": 2.327498197555542,
+ "learning_rate": 8.505050505050506e-07,
+ "loss": 1.2684743404388428,
+ "mean_token_accuracy": 0.695713222026825,
+ "num_tokens": 33669120.0,
+ "step": 2055
+ },
+ {
+ "entropy": 1.4799714088439941,
+ "epoch": 4.153535353535354,
+ "grad_norm": 2.347869396209717,
+ "learning_rate": 8.484848484848486e-07,
+ "loss": 1.4689722061157227,
+ "mean_token_accuracy": 0.6625549793243408,
+ "num_tokens": 33685504.0,
+ "step": 2056
+ },
+ {
+ "entropy": 1.1450932025909424,
+ "epoch": 4.155555555555556,
+ "grad_norm": 2.486754894256592,
+ "learning_rate": 8.464646464646465e-07,
+ "loss": 1.1476457118988037,
+ "mean_token_accuracy": 0.705483615398407,
+ "num_tokens": 33701888.0,
+ "step": 2057
+ },
+ {
+ "entropy": 1.6324634552001953,
+ "epoch": 4.157575757575757,
+ "grad_norm": 2.251194477081299,
+ "learning_rate": 8.444444444444445e-07,
+ "loss": 1.620585560798645,
+ "mean_token_accuracy": 0.6596848964691162,
+ "num_tokens": 33718272.0,
+ "step": 2058
+ },
+ {
+ "entropy": 1.295936107635498,
+ "epoch": 4.1595959595959595,
+ "grad_norm": 2.140843629837036,
+ "learning_rate": 8.424242424242425e-07,
+ "loss": 1.2875292301177979,
+ "mean_token_accuracy": 0.708109438419342,
+ "num_tokens": 33734656.0,
+ "step": 2059
+ },
+ {
+ "entropy": 1.292955756187439,
+ "epoch": 4.161616161616162,
+ "grad_norm": 2.4827163219451904,
+ "learning_rate": 8.404040404040405e-07,
+ "loss": 1.2855807542800903,
+ "mean_token_accuracy": 0.6802638173103333,
+ "num_tokens": 33751040.0,
+ "step": 2060
+ },
+ {
+ "entropy": 0.9475373029708862,
+ "epoch": 4.163636363636364,
+ "grad_norm": 2.347381114959717,
+ "learning_rate": 8.383838383838384e-07,
+ "loss": 0.934272289276123,
+ "mean_token_accuracy": 0.7487787008285522,
+ "num_tokens": 33767424.0,
+ "step": 2061
+ },
+ {
+ "entropy": 1.2446081638336182,
+ "epoch": 4.165656565656565,
+ "grad_norm": 2.2981362342834473,
+ "learning_rate": 8.363636363636364e-07,
+ "loss": 1.2680091857910156,
+ "mean_token_accuracy": 0.7056668400764465,
+ "num_tokens": 33783808.0,
+ "step": 2062
+ },
+ {
+ "entropy": 1.1058030128479004,
+ "epoch": 4.167676767676768,
+ "grad_norm": 2.0872671604156494,
+ "learning_rate": 8.343434343434344e-07,
+ "loss": 1.093427062034607,
+ "mean_token_accuracy": 0.7307645082473755,
+ "num_tokens": 33800192.0,
+ "step": 2063
+ },
+ {
+ "entropy": 1.4973342418670654,
+ "epoch": 4.16969696969697,
+ "grad_norm": 2.4032182693481445,
+ "learning_rate": 8.323232323232324e-07,
+ "loss": 1.5094172954559326,
+ "mean_token_accuracy": 0.6663410067558289,
+ "num_tokens": 33816576.0,
+ "step": 2064
+ },
+ {
+ "entropy": 0.8914118409156799,
+ "epoch": 4.171717171717171,
+ "grad_norm": 2.370084047317505,
+ "learning_rate": 8.303030303030303e-07,
+ "loss": 0.8813928365707397,
+ "mean_token_accuracy": 0.7614191770553589,
+ "num_tokens": 33832960.0,
+ "step": 2065
+ },
+ {
+ "entropy": 1.1234400272369385,
+ "epoch": 4.1737373737373735,
+ "grad_norm": 2.2805073261260986,
+ "learning_rate": 8.282828282828284e-07,
+ "loss": 1.119314432144165,
+ "mean_token_accuracy": 0.7155593633651733,
+ "num_tokens": 33849344.0,
+ "step": 2066
+ },
+ {
+ "entropy": 0.9295584559440613,
+ "epoch": 4.175757575757576,
+ "grad_norm": 2.3400092124938965,
+ "learning_rate": 8.262626262626264e-07,
+ "loss": 0.925437867641449,
+ "mean_token_accuracy": 0.7456643581390381,
+ "num_tokens": 33865728.0,
+ "step": 2067
+ },
+ {
+ "entropy": 1.2219396829605103,
+ "epoch": 4.177777777777778,
+ "grad_norm": 2.5290143489837646,
+ "learning_rate": 8.242424242424244e-07,
+ "loss": 1.2008111476898193,
+ "mean_token_accuracy": 0.7040180563926697,
+ "num_tokens": 33882112.0,
+ "step": 2068
+ },
+ {
+ "entropy": 1.3869670629501343,
+ "epoch": 4.179797979797979,
+ "grad_norm": 2.4521372318267822,
+ "learning_rate": 8.222222222222223e-07,
+ "loss": 1.3759180307388306,
+ "mean_token_accuracy": 0.6649975776672363,
+ "num_tokens": 33898496.0,
+ "step": 2069
+ },
+ {
+ "entropy": 0.9756981730461121,
+ "epoch": 4.181818181818182,
+ "grad_norm": 2.1221506595611572,
+ "learning_rate": 8.202020202020202e-07,
+ "loss": 0.9636279344558716,
+ "mean_token_accuracy": 0.7576331496238708,
+ "num_tokens": 33914880.0,
+ "step": 2070
+ },
+ {
+ "entropy": 0.8014331459999084,
+ "epoch": 4.183838383838384,
+ "grad_norm": 2.1101298332214355,
+ "learning_rate": 8.181818181818182e-07,
+ "loss": 0.7967961430549622,
+ "mean_token_accuracy": 0.7816316485404968,
+ "num_tokens": 33931264.0,
+ "step": 2071
+ },
+ {
+ "entropy": 1.141619086265564,
+ "epoch": 4.185858585858586,
+ "grad_norm": 2.259256601333618,
+ "learning_rate": 8.161616161616162e-07,
+ "loss": 1.1158604621887207,
+ "mean_token_accuracy": 0.7250854969024658,
+ "num_tokens": 33947648.0,
+ "step": 2072
+ },
+ {
+ "entropy": 0.8742624521255493,
+ "epoch": 4.1878787878787875,
+ "grad_norm": 2.200969934463501,
+ "learning_rate": 8.141414141414142e-07,
+ "loss": 0.8776901364326477,
+ "mean_token_accuracy": 0.7754641175270081,
+ "num_tokens": 33964032.0,
+ "step": 2073
+ },
+ {
+ "entropy": 1.1458066701889038,
+ "epoch": 4.18989898989899,
+ "grad_norm": 2.365546703338623,
+ "learning_rate": 8.121212121212121e-07,
+ "loss": 1.1447772979736328,
+ "mean_token_accuracy": 0.7194064259529114,
+ "num_tokens": 33980416.0,
+ "step": 2074
+ },
+ {
+ "entropy": 1.0582295656204224,
+ "epoch": 4.191919191919192,
+ "grad_norm": 2.2105917930603027,
+ "learning_rate": 8.101010101010101e-07,
+ "loss": 1.0469292402267456,
+ "mean_token_accuracy": 0.7400463819503784,
+ "num_tokens": 33996800.0,
+ "step": 2075
+ },
+ {
+ "entropy": 1.318009376525879,
+ "epoch": 4.193939393939394,
+ "grad_norm": 2.4097955226898193,
+ "learning_rate": 8.080808080808082e-07,
+ "loss": 1.321338415145874,
+ "mean_token_accuracy": 0.6783097386360168,
+ "num_tokens": 34013184.0,
+ "step": 2076
+ },
+ {
+ "entropy": 1.3597925901412964,
+ "epoch": 4.195959595959596,
+ "grad_norm": 2.0832197666168213,
+ "learning_rate": 8.060606060606062e-07,
+ "loss": 1.3505959510803223,
+ "mean_token_accuracy": 0.6894845962524414,
+ "num_tokens": 34029568.0,
+ "step": 2077
+ },
+ {
+ "entropy": 1.1852680444717407,
+ "epoch": 4.197979797979798,
+ "grad_norm": 2.3396389484405518,
+ "learning_rate": 8.040404040404042e-07,
+ "loss": 1.1788443326950073,
+ "mean_token_accuracy": 0.7175134420394897,
+ "num_tokens": 34045952.0,
+ "step": 2078
+ },
+ {
+ "entropy": 1.1703139543533325,
+ "epoch": 4.2,
+ "grad_norm": 2.4051332473754883,
+ "learning_rate": 8.020202020202021e-07,
+ "loss": 1.179924488067627,
+ "mean_token_accuracy": 0.7060942649841309,
+ "num_tokens": 34062336.0,
+ "step": 2079
+ },
+ {
+ "entropy": 0.8304579854011536,
+ "epoch": 4.202020202020202,
+ "grad_norm": 2.030073404312134,
+ "learning_rate": 8.000000000000001e-07,
+ "loss": 0.8240759968757629,
+ "mean_token_accuracy": 0.784745991230011,
+ "num_tokens": 34078720.0,
+ "step": 2080
+ },
+ {
+ "entropy": 0.9552163481712341,
+ "epoch": 4.204040404040404,
+ "grad_norm": 2.1599645614624023,
+ "learning_rate": 7.979797979797981e-07,
+ "loss": 0.9678778648376465,
+ "mean_token_accuracy": 0.753724992275238,
+ "num_tokens": 34095104.0,
+ "step": 2081
+ },
+ {
+ "entropy": 1.700339913368225,
+ "epoch": 4.206060606060606,
+ "grad_norm": 2.778423309326172,
+ "learning_rate": 7.959595959595961e-07,
+ "loss": 1.6714180707931519,
+ "mean_token_accuracy": 0.6206033229827881,
+ "num_tokens": 34111488.0,
+ "step": 2082
+ },
+ {
+ "entropy": 1.2214075326919556,
+ "epoch": 4.208080808080808,
+ "grad_norm": 2.4050512313842773,
+ "learning_rate": 7.939393939393939e-07,
+ "loss": 1.2122421264648438,
+ "mean_token_accuracy": 0.6995603442192078,
+ "num_tokens": 34127872.0,
+ "step": 2083
+ },
+ {
+ "entropy": 1.0523896217346191,
+ "epoch": 4.21010101010101,
+ "grad_norm": 2.1733455657958984,
+ "learning_rate": 7.919191919191919e-07,
+ "loss": 1.0562810897827148,
+ "mean_token_accuracy": 0.7385808229446411,
+ "num_tokens": 34144256.0,
+ "step": 2084
+ },
+ {
+ "entropy": 1.3346856832504272,
+ "epoch": 4.212121212121212,
+ "grad_norm": 2.446331739425659,
+ "learning_rate": 7.898989898989899e-07,
+ "loss": 1.325127363204956,
+ "mean_token_accuracy": 0.6767830848693848,
+ "num_tokens": 34160640.0,
+ "step": 2085
+ },
+ {
+ "entropy": 1.216209888458252,
+ "epoch": 4.214141414141414,
+ "grad_norm": 2.2457573413848877,
+ "learning_rate": 7.878787878787879e-07,
+ "loss": 1.1888031959533691,
+ "mean_token_accuracy": 0.7133610248565674,
+ "num_tokens": 34177024.0,
+ "step": 2086
+ },
+ {
+ "entropy": 1.2156273126602173,
+ "epoch": 4.216161616161616,
+ "grad_norm": 2.2188119888305664,
+ "learning_rate": 7.85858585858586e-07,
+ "loss": 1.2307485342025757,
+ "mean_token_accuracy": 0.7011480331420898,
+ "num_tokens": 34193408.0,
+ "step": 2087
+ },
+ {
+ "entropy": 1.0172580480575562,
+ "epoch": 4.218181818181818,
+ "grad_norm": 1.8507698774337769,
+ "learning_rate": 7.838383838383839e-07,
+ "loss": 0.9992572069168091,
+ "mean_token_accuracy": 0.7587933540344238,
+ "num_tokens": 34209792.0,
+ "step": 2088
+ },
+ {
+ "entropy": 1.3547221422195435,
+ "epoch": 4.22020202020202,
+ "grad_norm": 2.2932472229003906,
+ "learning_rate": 7.818181818181819e-07,
+ "loss": 1.3692772388458252,
+ "mean_token_accuracy": 0.669577419757843,
+ "num_tokens": 34226176.0,
+ "step": 2089
+ },
+ {
+ "entropy": 1.3517241477966309,
+ "epoch": 4.222222222222222,
+ "grad_norm": 2.362355947494507,
+ "learning_rate": 7.797979797979799e-07,
+ "loss": 1.3620293140411377,
+ "mean_token_accuracy": 0.6656692624092102,
+ "num_tokens": 34242560.0,
+ "step": 2090
+ },
+ {
+ "entropy": 1.0812968015670776,
+ "epoch": 4.224242424242425,
+ "grad_norm": 2.288313627243042,
+ "learning_rate": 7.777777777777779e-07,
+ "loss": 1.0882915258407593,
+ "mean_token_accuracy": 0.7280776500701904,
+ "num_tokens": 34258944.0,
+ "step": 2091
+ },
+ {
+ "entropy": 1.2385846376419067,
+ "epoch": 4.226262626262626,
+ "grad_norm": 2.2588205337524414,
+ "learning_rate": 7.757575757575758e-07,
+ "loss": 1.2337942123413086,
+ "mean_token_accuracy": 0.6944919228553772,
+ "num_tokens": 34275328.0,
+ "step": 2092
+ },
+ {
+ "entropy": 0.8685792088508606,
+ "epoch": 4.228282828282828,
+ "grad_norm": 2.193739175796509,
+ "learning_rate": 7.737373737373738e-07,
+ "loss": 0.8698766827583313,
+ "mean_token_accuracy": 0.7663654088973999,
+ "num_tokens": 34291712.0,
+ "step": 2093
+ },
+ {
+ "entropy": 1.2384530305862427,
+ "epoch": 4.2303030303030305,
+ "grad_norm": 2.3214962482452393,
+ "learning_rate": 7.717171717171718e-07,
+ "loss": 1.2307186126708984,
+ "mean_token_accuracy": 0.7134831547737122,
+ "num_tokens": 34308096.0,
+ "step": 2094
+ },
+ {
+ "entropy": 1.4578503370285034,
+ "epoch": 4.232323232323233,
+ "grad_norm": 2.4814023971557617,
+ "learning_rate": 7.696969696969698e-07,
+ "loss": 1.4594008922576904,
+ "mean_token_accuracy": 0.6678676009178162,
+ "num_tokens": 34324480.0,
+ "step": 2095
+ },
+ {
+ "entropy": 0.9257369041442871,
+ "epoch": 4.234343434343434,
+ "grad_norm": 2.2866013050079346,
+ "learning_rate": 7.676767676767677e-07,
+ "loss": 0.933411717414856,
+ "mean_token_accuracy": 0.7463971376419067,
+ "num_tokens": 34340864.0,
+ "step": 2096
+ },
+ {
+ "entropy": 0.8323895335197449,
+ "epoch": 4.236363636363636,
+ "grad_norm": 2.132780075073242,
+ "learning_rate": 7.656565656565656e-07,
+ "loss": 0.822212278842926,
+ "mean_token_accuracy": 0.7714338302612305,
+ "num_tokens": 34357248.0,
+ "step": 2097
+ },
+ {
+ "entropy": 1.3601847887039185,
+ "epoch": 4.238383838383839,
+ "grad_norm": 2.483175039291382,
+ "learning_rate": 7.636363636363637e-07,
+ "loss": 1.3723442554473877,
+ "mean_token_accuracy": 0.6633487939834595,
+ "num_tokens": 34373632.0,
+ "step": 2098
+ },
+ {
+ "entropy": 1.109177589416504,
+ "epoch": 4.240404040404041,
+ "grad_norm": 2.1533427238464355,
+ "learning_rate": 7.616161616161617e-07,
+ "loss": 1.0950508117675781,
+ "mean_token_accuracy": 0.7376037836074829,
+ "num_tokens": 34390016.0,
+ "step": 2099
+ },
+ {
+ "entropy": 0.9974305033683777,
+ "epoch": 4.242424242424242,
+ "grad_norm": 2.475922107696533,
+ "learning_rate": 7.595959595959597e-07,
+ "loss": 0.979351282119751,
+ "mean_token_accuracy": 0.7522594332695007,
+ "num_tokens": 34406400.0,
+ "step": 2100
+ },
+ {
+ "epoch": 4.242424242424242,
+ "eval_entropy": 1.2880813412127956,
+ "eval_loss": 1.562015175819397,
+ "eval_mean_token_accuracy": 0.6436735732901481,
+ "eval_num_tokens": 34406400.0,
+ "eval_runtime": 43.7511,
+ "eval_samples_per_second": 22.628,
+ "eval_steps_per_second": 2.834,
+ "step": 2100
+ },
+ {
+ "entropy": 1.4373509883880615,
+ "epoch": 4.2444444444444445,
+ "grad_norm": 2.398466110229492,
+ "learning_rate": 7.575757575757576e-07,
+ "loss": 1.4485313892364502,
+ "mean_token_accuracy": 0.6607840657234192,
+ "num_tokens": 34422784.0,
+ "step": 2101
+ },
+ {
+ "entropy": 1.1279411315917969,
+ "epoch": 4.246464646464647,
+ "grad_norm": 2.1290903091430664,
+ "learning_rate": 7.555555555555556e-07,
+ "loss": 1.1336861848831177,
+ "mean_token_accuracy": 0.7290546894073486,
+ "num_tokens": 34439168.0,
+ "step": 2102
+ },
+ {
+ "entropy": 1.453575849533081,
+ "epoch": 4.248484848484848,
+ "grad_norm": 2.205562114715576,
+ "learning_rate": 7.535353535353536e-07,
+ "loss": 1.4421193599700928,
+ "mean_token_accuracy": 0.6577308177947998,
+ "num_tokens": 34455552.0,
+ "step": 2103
+ },
+ {
+ "entropy": 0.9789026975631714,
+ "epoch": 4.25050505050505,
+ "grad_norm": 2.220710039138794,
+ "learning_rate": 7.515151515151516e-07,
+ "loss": 0.9738274812698364,
+ "mean_token_accuracy": 0.7520151734352112,
+ "num_tokens": 34471936.0,
+ "step": 2104
+ },
+ {
+ "entropy": 1.12283194065094,
+ "epoch": 4.252525252525253,
+ "grad_norm": 2.2798845767974854,
+ "learning_rate": 7.494949494949495e-07,
+ "loss": 1.111656665802002,
+ "mean_token_accuracy": 0.7236199378967285,
+ "num_tokens": 34488320.0,
+ "step": 2105
+ },
+ {
+ "entropy": 1.0451518297195435,
+ "epoch": 4.254545454545455,
+ "grad_norm": 2.389167547225952,
+ "learning_rate": 7.474747474747475e-07,
+ "loss": 1.045143723487854,
+ "mean_token_accuracy": 0.7235588431358337,
+ "num_tokens": 34504704.0,
+ "step": 2106
+ },
+ {
+ "entropy": 1.3338992595672607,
+ "epoch": 4.256565656565656,
+ "grad_norm": 2.3781392574310303,
+ "learning_rate": 7.454545454545455e-07,
+ "loss": 1.3461604118347168,
+ "mean_token_accuracy": 0.6740351915359497,
+ "num_tokens": 34521088.0,
+ "step": 2107
+ },
+ {
+ "entropy": 1.4721654653549194,
+ "epoch": 4.2585858585858585,
+ "grad_norm": 2.293487071990967,
+ "learning_rate": 7.434343434343436e-07,
+ "loss": 1.4750356674194336,
+ "mean_token_accuracy": 0.6606009006500244,
+ "num_tokens": 34537472.0,
+ "step": 2108
+ },
+ {
+ "entropy": 0.7618300914764404,
+ "epoch": 4.260606060606061,
+ "grad_norm": 2.092986583709717,
+ "learning_rate": 7.414141414141416e-07,
+ "loss": 0.7633351683616638,
+ "mean_token_accuracy": 0.7957987189292908,
+ "num_tokens": 34553856.0,
+ "step": 2109
+ },
+ {
+ "entropy": 0.9247039556503296,
+ "epoch": 4.262626262626263,
+ "grad_norm": 2.023836851119995,
+ "learning_rate": 7.393939393939395e-07,
+ "loss": 0.8996185064315796,
+ "mean_token_accuracy": 0.775036633014679,
+ "num_tokens": 34570240.0,
+ "step": 2110
+ },
+ {
+ "entropy": 1.0806818008422852,
+ "epoch": 4.264646464646464,
+ "grad_norm": 2.447587490081787,
+ "learning_rate": 7.373737373737375e-07,
+ "loss": 1.107656478881836,
+ "mean_token_accuracy": 0.7337567210197449,
+ "num_tokens": 34586624.0,
+ "step": 2111
+ },
+ {
+ "entropy": 1.4007959365844727,
+ "epoch": 4.266666666666667,
+ "grad_norm": 2.2774436473846436,
+ "learning_rate": 7.353535353535354e-07,
+ "loss": 1.3865636587142944,
+ "mean_token_accuracy": 0.6830117106437683,
+ "num_tokens": 34603008.0,
+ "step": 2112
+ },
+ {
+ "entropy": 1.0637953281402588,
+ "epoch": 4.268686868686869,
+ "grad_norm": 2.239159107208252,
+ "learning_rate": 7.333333333333334e-07,
+ "loss": 1.0556224584579468,
+ "mean_token_accuracy": 0.7425500750541687,
+ "num_tokens": 34619392.0,
+ "step": 2113
+ },
+ {
+ "entropy": 0.9682835340499878,
+ "epoch": 4.270707070707071,
+ "grad_norm": 2.5597312450408936,
+ "learning_rate": 7.313131313131313e-07,
+ "loss": 0.9757053256034851,
+ "mean_token_accuracy": 0.7452369332313538,
+ "num_tokens": 34635776.0,
+ "step": 2114
+ },
+ {
+ "entropy": 1.183341383934021,
+ "epoch": 4.2727272727272725,
+ "grad_norm": 2.5379812717437744,
+ "learning_rate": 7.292929292929293e-07,
+ "loss": 1.1930429935455322,
+ "mean_token_accuracy": 0.7052393555641174,
+ "num_tokens": 34652160.0,
+ "step": 2115
+ },
+ {
+ "entropy": 1.0186142921447754,
+ "epoch": 4.274747474747475,
+ "grad_norm": 2.3810384273529053,
+ "learning_rate": 7.272727272727273e-07,
+ "loss": 1.0331826210021973,
+ "mean_token_accuracy": 0.7370542287826538,
+ "num_tokens": 34668544.0,
+ "step": 2116
+ },
+ {
+ "entropy": 1.5887846946716309,
+ "epoch": 4.276767676767677,
+ "grad_norm": 2.179713249206543,
+ "learning_rate": 7.252525252525253e-07,
+ "loss": 1.575054407119751,
+ "mean_token_accuracy": 0.6392892003059387,
+ "num_tokens": 34684928.0,
+ "step": 2117
+ },
+ {
+ "entropy": 1.1686691045761108,
+ "epoch": 4.278787878787879,
+ "grad_norm": 2.100404739379883,
+ "learning_rate": 7.232323232323232e-07,
+ "loss": 1.1814364194869995,
+ "mean_token_accuracy": 0.7209941148757935,
+ "num_tokens": 34701312.0,
+ "step": 2118
+ },
+ {
+ "entropy": 1.2804527282714844,
+ "epoch": 4.280808080808081,
+ "grad_norm": 2.5362112522125244,
+ "learning_rate": 7.212121212121213e-07,
+ "loss": 1.278024673461914,
+ "mean_token_accuracy": 0.6805691123008728,
+ "num_tokens": 34717696.0,
+ "step": 2119
+ },
+ {
+ "entropy": 1.141945481300354,
+ "epoch": 4.282828282828283,
+ "grad_norm": 2.4499824047088623,
+ "learning_rate": 7.191919191919193e-07,
+ "loss": 1.1508493423461914,
+ "mean_token_accuracy": 0.7129335403442383,
+ "num_tokens": 34734080.0,
+ "step": 2120
+ },
+ {
+ "entropy": 1.1163101196289062,
+ "epoch": 4.284848484848485,
+ "grad_norm": 2.3706541061401367,
+ "learning_rate": 7.171717171717173e-07,
+ "loss": 1.0950313806533813,
+ "mean_token_accuracy": 0.7261846661567688,
+ "num_tokens": 34750464.0,
+ "step": 2121
+ },
+ {
+ "entropy": 1.0530426502227783,
+ "epoch": 4.2868686868686865,
+ "grad_norm": 2.1800615787506104,
+ "learning_rate": 7.151515151515153e-07,
+ "loss": 1.0400605201721191,
+ "mean_token_accuracy": 0.744076669216156,
+ "num_tokens": 34766848.0,
+ "step": 2122
+ },
+ {
+ "entropy": 1.4187507629394531,
+ "epoch": 4.288888888888889,
+ "grad_norm": 2.283302068710327,
+ "learning_rate": 7.131313131313132e-07,
+ "loss": 1.4388368129730225,
+ "mean_token_accuracy": 0.659807026386261,
+ "num_tokens": 34783232.0,
+ "step": 2123
+ },
+ {
+ "entropy": 1.04351806640625,
+ "epoch": 4.290909090909091,
+ "grad_norm": 2.2792043685913086,
+ "learning_rate": 7.111111111111112e-07,
+ "loss": 1.0404431819915771,
+ "mean_token_accuracy": 0.7292379140853882,
+ "num_tokens": 34799616.0,
+ "step": 2124
+ },
+ {
+ "entropy": 1.4637887477874756,
+ "epoch": 4.292929292929293,
+ "grad_norm": 2.4931323528289795,
+ "learning_rate": 7.090909090909092e-07,
+ "loss": 1.4716092348098755,
+ "mean_token_accuracy": 0.6443576216697693,
+ "num_tokens": 34816000.0,
+ "step": 2125
+ },
+ {
+ "entropy": 1.133570909500122,
+ "epoch": 4.294949494949495,
+ "grad_norm": 2.2696118354797363,
+ "learning_rate": 7.070707070707071e-07,
+ "loss": 1.138695240020752,
+ "mean_token_accuracy": 0.7142159342765808,
+ "num_tokens": 34832384.0,
+ "step": 2126
+ },
+ {
+ "entropy": 1.057089924812317,
+ "epoch": 4.296969696969697,
+ "grad_norm": 2.3984944820404053,
+ "learning_rate": 7.05050505050505e-07,
+ "loss": 1.0433732271194458,
+ "mean_token_accuracy": 0.7341231107711792,
+ "num_tokens": 34848768.0,
+ "step": 2127
+ },
+ {
+ "entropy": 1.287866234779358,
+ "epoch": 4.298989898989899,
+ "grad_norm": 2.204712152481079,
+ "learning_rate": 7.03030303030303e-07,
+ "loss": 1.3041845560073853,
+ "mean_token_accuracy": 0.687530517578125,
+ "num_tokens": 34865152.0,
+ "step": 2128
+ },
+ {
+ "entropy": 1.1710352897644043,
+ "epoch": 4.301010101010101,
+ "grad_norm": 2.5103063583374023,
+ "learning_rate": 7.01010101010101e-07,
+ "loss": 1.1735949516296387,
+ "mean_token_accuracy": 0.7047508358955383,
+ "num_tokens": 34881536.0,
+ "step": 2129
+ },
+ {
+ "entropy": 0.8662670850753784,
+ "epoch": 4.303030303030303,
+ "grad_norm": 2.1789443492889404,
+ "learning_rate": 6.989898989898991e-07,
+ "loss": 0.8674213290214539,
+ "mean_token_accuracy": 0.7714948654174805,
+ "num_tokens": 34897920.0,
+ "step": 2130
+ },
+ {
+ "entropy": 1.1292206048965454,
+ "epoch": 4.305050505050505,
+ "grad_norm": 2.3685660362243652,
+ "learning_rate": 6.969696969696971e-07,
+ "loss": 1.1365619897842407,
+ "mean_token_accuracy": 0.7210552096366882,
+ "num_tokens": 34914304.0,
+ "step": 2131
+ },
+ {
+ "entropy": 1.0297276973724365,
+ "epoch": 4.307070707070707,
+ "grad_norm": 2.4006168842315674,
+ "learning_rate": 6.94949494949495e-07,
+ "loss": 1.0344676971435547,
+ "mean_token_accuracy": 0.7421836853027344,
+ "num_tokens": 34930688.0,
+ "step": 2132
+ },
+ {
+ "entropy": 0.8545892834663391,
+ "epoch": 4.309090909090909,
+ "grad_norm": 2.171968698501587,
+ "learning_rate": 6.92929292929293e-07,
+ "loss": 0.8570476770401001,
+ "mean_token_accuracy": 0.7680141925811768,
+ "num_tokens": 34947072.0,
+ "step": 2133
+ },
+ {
+ "entropy": 0.9131754040718079,
+ "epoch": 4.311111111111111,
+ "grad_norm": 2.3538546562194824,
+ "learning_rate": 6.90909090909091e-07,
+ "loss": 0.9100440740585327,
+ "mean_token_accuracy": 0.755984365940094,
+ "num_tokens": 34963456.0,
+ "step": 2134
+ },
+ {
+ "entropy": 1.3461705446243286,
+ "epoch": 4.313131313131313,
+ "grad_norm": 2.226499080657959,
+ "learning_rate": 6.88888888888889e-07,
+ "loss": 1.3430912494659424,
+ "mean_token_accuracy": 0.6741573214530945,
+ "num_tokens": 34979840.0,
+ "step": 2135
+ },
+ {
+ "entropy": 1.0766938924789429,
+ "epoch": 4.315151515151515,
+ "grad_norm": 2.382495880126953,
+ "learning_rate": 6.868686868686869e-07,
+ "loss": 1.0890635251998901,
+ "mean_token_accuracy": 0.7278944849967957,
+ "num_tokens": 34996224.0,
+ "step": 2136
+ },
+ {
+ "entropy": 0.8639549016952515,
+ "epoch": 4.317171717171717,
+ "grad_norm": 2.2745895385742188,
+ "learning_rate": 6.848484848484849e-07,
+ "loss": 0.8547804355621338,
+ "mean_token_accuracy": 0.7794944047927856,
+ "num_tokens": 35012608.0,
+ "step": 2137
+ },
+ {
+ "entropy": 1.1669604778289795,
+ "epoch": 4.319191919191919,
+ "grad_norm": 2.2132558822631836,
+ "learning_rate": 6.828282828282829e-07,
+ "loss": 1.1499347686767578,
+ "mean_token_accuracy": 0.7079873085021973,
+ "num_tokens": 35028992.0,
+ "step": 2138
+ },
+ {
+ "entropy": 1.4229295253753662,
+ "epoch": 4.321212121212121,
+ "grad_norm": 2.2925033569335938,
+ "learning_rate": 6.808080808080809e-07,
+ "loss": 1.4052009582519531,
+ "mean_token_accuracy": 0.6652418375015259,
+ "num_tokens": 35045376.0,
+ "step": 2139
+ },
+ {
+ "entropy": 1.175937294960022,
+ "epoch": 4.3232323232323235,
+ "grad_norm": 2.878568172454834,
+ "learning_rate": 6.78787878787879e-07,
+ "loss": 1.173958659172058,
+ "mean_token_accuracy": 0.7080483436584473,
+ "num_tokens": 35061760.0,
+ "step": 2140
+ },
+ {
+ "entropy": 1.1963285207748413,
+ "epoch": 4.325252525252525,
+ "grad_norm": 2.5476808547973633,
+ "learning_rate": 6.767676767676768e-07,
+ "loss": 1.2027249336242676,
+ "mean_token_accuracy": 0.7070713043212891,
+ "num_tokens": 35078144.0,
+ "step": 2141
+ },
+ {
+ "entropy": 1.4109331369400024,
+ "epoch": 4.327272727272727,
+ "grad_norm": 2.547231435775757,
+ "learning_rate": 6.747474747474748e-07,
+ "loss": 1.4095146656036377,
+ "mean_token_accuracy": 0.6638373136520386,
+ "num_tokens": 35094528.0,
+ "step": 2142
+ },
+ {
+ "entropy": 1.2844510078430176,
+ "epoch": 4.329292929292929,
+ "grad_norm": 2.398951768875122,
+ "learning_rate": 6.727272727272728e-07,
+ "loss": 1.268770694732666,
+ "mean_token_accuracy": 0.7037127614021301,
+ "num_tokens": 35110912.0,
+ "step": 2143
+ },
+ {
+ "entropy": 1.3457868099212646,
+ "epoch": 4.331313131313132,
+ "grad_norm": 2.4499454498291016,
+ "learning_rate": 6.707070707070708e-07,
+ "loss": 1.355639934539795,
+ "mean_token_accuracy": 0.6747679710388184,
+ "num_tokens": 35127296.0,
+ "step": 2144
+ },
+ {
+ "entropy": 1.2578988075256348,
+ "epoch": 4.333333333333333,
+ "grad_norm": 2.3502230644226074,
+ "learning_rate": 6.686868686868687e-07,
+ "loss": 1.2514359951019287,
+ "mean_token_accuracy": 0.6861260533332825,
+ "num_tokens": 35143680.0,
+ "step": 2145
+ },
+ {
+ "entropy": 0.9508306384086609,
+ "epoch": 4.335353535353535,
+ "grad_norm": 2.344722270965576,
+ "learning_rate": 6.666666666666667e-07,
+ "loss": 0.9492694139480591,
+ "mean_token_accuracy": 0.7473741769790649,
+ "num_tokens": 35160064.0,
+ "step": 2146
+ },
+ {
+ "entropy": 1.2369167804718018,
+ "epoch": 4.3373737373737375,
+ "grad_norm": 2.4445602893829346,
+ "learning_rate": 6.646464646464647e-07,
+ "loss": 1.2398593425750732,
+ "mean_token_accuracy": 0.6878358721733093,
+ "num_tokens": 35176448.0,
+ "step": 2147
+ },
+ {
+ "entropy": 1.3641530275344849,
+ "epoch": 4.33939393939394,
+ "grad_norm": 2.3283517360687256,
+ "learning_rate": 6.626262626262627e-07,
+ "loss": 1.3664302825927734,
+ "mean_token_accuracy": 0.6764166951179504,
+ "num_tokens": 35192832.0,
+ "step": 2148
+ },
+ {
+ "entropy": 1.5864531993865967,
+ "epoch": 4.341414141414141,
+ "grad_norm": 2.1810457706451416,
+ "learning_rate": 6.606060606060606e-07,
+ "loss": 1.5945943593978882,
+ "mean_token_accuracy": 0.6441743969917297,
+ "num_tokens": 35209216.0,
+ "step": 2149
+ },
+ {
+ "entropy": 1.0947312116622925,
+ "epoch": 4.343434343434343,
+ "grad_norm": 2.3473317623138428,
+ "learning_rate": 6.585858585858586e-07,
+ "loss": 1.0883324146270752,
+ "mean_token_accuracy": 0.7239863276481628,
+ "num_tokens": 35225600.0,
+ "step": 2150
+ },
+ {
+ "epoch": 4.343434343434343,
+ "eval_entropy": 1.2868958841408453,
+ "eval_loss": 1.5624313354492188,
+ "eval_mean_token_accuracy": 0.6437114924192429,
+ "eval_num_tokens": 35225600.0,
+ "eval_runtime": 43.7703,
+ "eval_samples_per_second": 22.618,
+ "eval_steps_per_second": 2.833,
+ "step": 2150
+ },
+ {
+ "entropy": 0.953440248966217,
+ "epoch": 4.345454545454546,
+ "grad_norm": 2.175570249557495,
+ "learning_rate": 6.565656565656567e-07,
+ "loss": 0.9720486998558044,
+ "mean_token_accuracy": 0.7511602640151978,
+ "num_tokens": 35241984.0,
+ "step": 2151
+ },
+ {
+ "entropy": 1.4737303256988525,
+ "epoch": 4.347474747474747,
+ "grad_norm": 2.3273777961730957,
+ "learning_rate": 6.545454545454547e-07,
+ "loss": 1.4861772060394287,
+ "mean_token_accuracy": 0.6781876087188721,
+ "num_tokens": 35258368.0,
+ "step": 2152
+ },
+ {
+ "entropy": 0.9462009072303772,
+ "epoch": 4.349494949494949,
+ "grad_norm": 2.175786256790161,
+ "learning_rate": 6.525252525252527e-07,
+ "loss": 0.949480414390564,
+ "mean_token_accuracy": 0.7550073266029358,
+ "num_tokens": 35274752.0,
+ "step": 2153
+ },
+ {
+ "entropy": 1.2302833795547485,
+ "epoch": 4.351515151515152,
+ "grad_norm": 2.344184398651123,
+ "learning_rate": 6.505050505050506e-07,
+ "loss": 1.2456318140029907,
+ "mean_token_accuracy": 0.7030410170555115,
+ "num_tokens": 35291136.0,
+ "step": 2154
+ },
+ {
+ "entropy": 0.9937285780906677,
+ "epoch": 4.353535353535354,
+ "grad_norm": 2.5566635131835938,
+ "learning_rate": 6.484848484848485e-07,
+ "loss": 1.00638747215271,
+ "mean_token_accuracy": 0.7330239415168762,
+ "num_tokens": 35307520.0,
+ "step": 2155
+ },
+ {
+ "entropy": 1.2210053205490112,
+ "epoch": 4.355555555555555,
+ "grad_norm": 2.367755174636841,
+ "learning_rate": 6.464646464646465e-07,
+ "loss": 1.2135249376296997,
+ "mean_token_accuracy": 0.6969956159591675,
+ "num_tokens": 35323904.0,
+ "step": 2156
+ },
+ {
+ "entropy": 0.9756261110305786,
+ "epoch": 4.357575757575757,
+ "grad_norm": 2.2705581188201904,
+ "learning_rate": 6.444444444444445e-07,
+ "loss": 0.9723948240280151,
+ "mean_token_accuracy": 0.751038134098053,
+ "num_tokens": 35340288.0,
+ "step": 2157
+ },
+ {
+ "entropy": 0.9649578928947449,
+ "epoch": 4.35959595959596,
+ "grad_norm": 2.2868213653564453,
+ "learning_rate": 6.424242424242424e-07,
+ "loss": 0.9617897272109985,
+ "mean_token_accuracy": 0.752137303352356,
+ "num_tokens": 35356672.0,
+ "step": 2158
+ },
+ {
+ "entropy": 1.196886658668518,
+ "epoch": 4.361616161616162,
+ "grad_norm": 2.504281759262085,
+ "learning_rate": 6.404040404040404e-07,
+ "loss": 1.1963926553726196,
+ "mean_token_accuracy": 0.7131778001785278,
+ "num_tokens": 35373056.0,
+ "step": 2159
+ },
+ {
+ "entropy": 0.8882641196250916,
+ "epoch": 4.363636363636363,
+ "grad_norm": 2.2975146770477295,
+ "learning_rate": 6.383838383838384e-07,
+ "loss": 0.8899753093719482,
+ "mean_token_accuracy": 0.7612970471382141,
+ "num_tokens": 35389440.0,
+ "step": 2160
+ },
+ {
+ "entropy": 1.0600148439407349,
+ "epoch": 4.365656565656566,
+ "grad_norm": 2.363847255706787,
+ "learning_rate": 6.363636363636364e-07,
+ "loss": 1.0516207218170166,
+ "mean_token_accuracy": 0.7245969772338867,
+ "num_tokens": 35405824.0,
+ "step": 2161
+ },
+ {
+ "entropy": 1.4234628677368164,
+ "epoch": 4.367676767676768,
+ "grad_norm": 2.414850950241089,
+ "learning_rate": 6.343434343434345e-07,
+ "loss": 1.43110191822052,
+ "mean_token_accuracy": 0.6621885895729065,
+ "num_tokens": 35422208.0,
+ "step": 2162
+ },
+ {
+ "entropy": 0.8017680644989014,
+ "epoch": 4.36969696969697,
+ "grad_norm": 2.2276418209075928,
+ "learning_rate": 6.323232323232324e-07,
+ "loss": 0.7956520318984985,
+ "mean_token_accuracy": 0.783646821975708,
+ "num_tokens": 35438592.0,
+ "step": 2163
+ },
+ {
+ "entropy": 0.9344257712364197,
+ "epoch": 4.3717171717171714,
+ "grad_norm": 2.2372374534606934,
+ "learning_rate": 6.303030303030304e-07,
+ "loss": 0.9264242649078369,
+ "mean_token_accuracy": 0.7545188069343567,
+ "num_tokens": 35454976.0,
+ "step": 2164
+ },
+ {
+ "entropy": 1.2193312644958496,
+ "epoch": 4.373737373737374,
+ "grad_norm": 2.213578462600708,
+ "learning_rate": 6.282828282828284e-07,
+ "loss": 1.2413791418075562,
+ "mean_token_accuracy": 0.7096971273422241,
+ "num_tokens": 35471360.0,
+ "step": 2165
+ },
+ {
+ "entropy": 1.2569602727890015,
+ "epoch": 4.375757575757576,
+ "grad_norm": 2.372586250305176,
+ "learning_rate": 6.262626262626264e-07,
+ "loss": 1.2514206171035767,
+ "mean_token_accuracy": 0.7004762887954712,
+ "num_tokens": 35487744.0,
+ "step": 2166
+ },
+ {
+ "entropy": 0.9451959729194641,
+ "epoch": 4.377777777777778,
+ "grad_norm": 2.2225098609924316,
+ "learning_rate": 6.242424242424243e-07,
+ "loss": 0.9277691841125488,
+ "mean_token_accuracy": 0.7629457712173462,
+ "num_tokens": 35504128.0,
+ "step": 2167
+ },
+ {
+ "entropy": 1.2604646682739258,
+ "epoch": 4.37979797979798,
+ "grad_norm": 2.3232014179229736,
+ "learning_rate": 6.222222222222223e-07,
+ "loss": 1.2507843971252441,
+ "mean_token_accuracy": 0.694614052772522,
+ "num_tokens": 35520512.0,
+ "step": 2168
+ },
+ {
+ "entropy": 0.9686046838760376,
+ "epoch": 4.381818181818182,
+ "grad_norm": 2.1185109615325928,
+ "learning_rate": 6.202020202020202e-07,
+ "loss": 0.9650119543075562,
+ "mean_token_accuracy": 0.7553126811981201,
+ "num_tokens": 35536896.0,
+ "step": 2169
+ },
+ {
+ "entropy": 1.117854118347168,
+ "epoch": 4.383838383838384,
+ "grad_norm": 2.6666815280914307,
+ "learning_rate": 6.181818181818182e-07,
+ "loss": 1.1222784519195557,
+ "mean_token_accuracy": 0.7167195677757263,
+ "num_tokens": 35553280.0,
+ "step": 2170
+ },
+ {
+ "entropy": 1.2812732458114624,
+ "epoch": 4.3858585858585855,
+ "grad_norm": 2.3317956924438477,
+ "learning_rate": 6.161616161616162e-07,
+ "loss": 1.304545283317566,
+ "mean_token_accuracy": 0.6802027225494385,
+ "num_tokens": 35569664.0,
+ "step": 2171
+ },
+ {
+ "entropy": 1.1000746488571167,
+ "epoch": 4.387878787878788,
+ "grad_norm": 2.411543369293213,
+ "learning_rate": 6.141414141414142e-07,
+ "loss": 1.0838086605072021,
+ "mean_token_accuracy": 0.7247801423072815,
+ "num_tokens": 35586048.0,
+ "step": 2172
+ },
+ {
+ "entropy": 1.5119904279708862,
+ "epoch": 4.38989898989899,
+ "grad_norm": 2.3798773288726807,
+ "learning_rate": 6.121212121212121e-07,
+ "loss": 1.5091912746429443,
+ "mean_token_accuracy": 0.644052267074585,
+ "num_tokens": 35602432.0,
+ "step": 2173
+ },
+ {
+ "entropy": 1.3415027856826782,
+ "epoch": 4.391919191919192,
+ "grad_norm": 2.361830472946167,
+ "learning_rate": 6.101010101010101e-07,
+ "loss": 1.3209997415542603,
+ "mean_token_accuracy": 0.6723253726959229,
+ "num_tokens": 35618816.0,
+ "step": 2174
+ },
+ {
+ "entropy": 1.2479910850524902,
+ "epoch": 4.393939393939394,
+ "grad_norm": 2.24318265914917,
+ "learning_rate": 6.080808080808082e-07,
+ "loss": 1.2585164308547974,
+ "mean_token_accuracy": 0.7048729658126831,
+ "num_tokens": 35635200.0,
+ "step": 2175
+ },
+ {
+ "entropy": 1.430103063583374,
+ "epoch": 4.395959595959596,
+ "grad_norm": 2.334134101867676,
+ "learning_rate": 6.060606060606061e-07,
+ "loss": 1.4244396686553955,
+ "mean_token_accuracy": 0.6552271842956543,
+ "num_tokens": 35651584.0,
+ "step": 2176
+ },
+ {
+ "entropy": 1.7399390935897827,
+ "epoch": 4.397979797979798,
+ "grad_norm": 2.362614393234253,
+ "learning_rate": 6.040404040404041e-07,
+ "loss": 1.7584240436553955,
+ "mean_token_accuracy": 0.6219467520713806,
+ "num_tokens": 35667968.0,
+ "step": 2177
+ },
+ {
+ "entropy": 1.078808307647705,
+ "epoch": 4.4,
+ "grad_norm": 2.210782289505005,
+ "learning_rate": 6.020202020202021e-07,
+ "loss": 1.0720242261886597,
+ "mean_token_accuracy": 0.7338788509368896,
+ "num_tokens": 35684352.0,
+ "step": 2178
+ },
+ {
+ "entropy": 1.418910026550293,
+ "epoch": 4.402020202020202,
+ "grad_norm": 2.601738929748535,
+ "learning_rate": 6.000000000000001e-07,
+ "loss": 1.4044787883758545,
+ "mean_token_accuracy": 0.6562652587890625,
+ "num_tokens": 35700736.0,
+ "step": 2179
+ },
+ {
+ "entropy": 1.1740971803665161,
+ "epoch": 4.404040404040404,
+ "grad_norm": 2.2930097579956055,
+ "learning_rate": 5.979797979797981e-07,
+ "loss": 1.1786755323410034,
+ "mean_token_accuracy": 0.7087811231613159,
+ "num_tokens": 35717120.0,
+ "step": 2180
+ },
+ {
+ "entropy": 0.9672836661338806,
+ "epoch": 4.406060606060606,
+ "grad_norm": 2.149219274520874,
+ "learning_rate": 5.959595959595961e-07,
+ "loss": 0.9852940440177917,
+ "mean_token_accuracy": 0.7513434290885925,
+ "num_tokens": 35733504.0,
+ "step": 2181
+ },
+ {
+ "entropy": 1.3657665252685547,
+ "epoch": 4.4080808080808085,
+ "grad_norm": 2.5746331214904785,
+ "learning_rate": 5.93939393939394e-07,
+ "loss": 1.3388352394104004,
+ "mean_token_accuracy": 0.6656082272529602,
+ "num_tokens": 35749888.0,
+ "step": 2182
+ },
+ {
+ "entropy": 1.6251881122589111,
+ "epoch": 4.41010101010101,
+ "grad_norm": 2.2786130905151367,
+ "learning_rate": 5.91919191919192e-07,
+ "loss": 1.6205543279647827,
+ "mean_token_accuracy": 0.6703712940216064,
+ "num_tokens": 35766272.0,
+ "step": 2183
+ },
+ {
+ "entropy": 1.0865193605422974,
+ "epoch": 4.412121212121212,
+ "grad_norm": 2.426384687423706,
+ "learning_rate": 5.898989898989899e-07,
+ "loss": 1.094101905822754,
+ "mean_token_accuracy": 0.715254008769989,
+ "num_tokens": 35782656.0,
+ "step": 2184
+ },
+ {
+ "entropy": 1.110110878944397,
+ "epoch": 4.414141414141414,
+ "grad_norm": 2.1766345500946045,
+ "learning_rate": 5.878787878787879e-07,
+ "loss": 1.09341299533844,
+ "mean_token_accuracy": 0.739130437374115,
+ "num_tokens": 35799040.0,
+ "step": 2185
+ },
+ {
+ "entropy": 1.2953979969024658,
+ "epoch": 4.416161616161617,
+ "grad_norm": 2.149765729904175,
+ "learning_rate": 5.858585858585859e-07,
+ "loss": 1.2975831031799316,
+ "mean_token_accuracy": 0.7040791511535645,
+ "num_tokens": 35815424.0,
+ "step": 2186
+ },
+ {
+ "entropy": 0.7426241636276245,
+ "epoch": 4.418181818181818,
+ "grad_norm": 2.299720287322998,
+ "learning_rate": 5.838383838383839e-07,
+ "loss": 0.7390038967132568,
+ "mean_token_accuracy": 0.7927454710006714,
+ "num_tokens": 35831808.0,
+ "step": 2187
+ },
+ {
+ "entropy": 1.1807751655578613,
+ "epoch": 4.42020202020202,
+ "grad_norm": 2.2330236434936523,
+ "learning_rate": 5.818181818181819e-07,
+ "loss": 1.1834851503372192,
+ "mean_token_accuracy": 0.7061553597450256,
+ "num_tokens": 35848192.0,
+ "step": 2188
+ },
+ {
+ "entropy": 1.2835958003997803,
+ "epoch": 4.4222222222222225,
+ "grad_norm": 2.284749746322632,
+ "learning_rate": 5.797979797979798e-07,
+ "loss": 1.2744988203048706,
+ "mean_token_accuracy": 0.6910722851753235,
+ "num_tokens": 35864576.0,
+ "step": 2189
+ },
+ {
+ "entropy": 0.7347697019577026,
+ "epoch": 4.424242424242424,
+ "grad_norm": 2.1876397132873535,
+ "learning_rate": 5.777777777777778e-07,
+ "loss": 0.7683783769607544,
+ "mean_token_accuracy": 0.7909135222434998,
+ "num_tokens": 35880960.0,
+ "step": 2190
+ },
+ {
+ "entropy": 1.0840578079223633,
+ "epoch": 4.426262626262626,
+ "grad_norm": 2.3153281211853027,
+ "learning_rate": 5.757575757575758e-07,
+ "loss": 1.0639472007751465,
+ "mean_token_accuracy": 0.7304592132568359,
+ "num_tokens": 35897344.0,
+ "step": 2191
+ },
+ {
+ "entropy": 1.019670009613037,
+ "epoch": 4.428282828282828,
+ "grad_norm": 2.3929173946380615,
+ "learning_rate": 5.737373737373738e-07,
+ "loss": 1.007870078086853,
+ "mean_token_accuracy": 0.7322300672531128,
+ "num_tokens": 35913728.0,
+ "step": 2192
+ },
+ {
+ "entropy": 1.3530149459838867,
+ "epoch": 4.430303030303031,
+ "grad_norm": 2.312758445739746,
+ "learning_rate": 5.717171717171718e-07,
+ "loss": 1.3727014064788818,
+ "mean_token_accuracy": 0.6725696325302124,
+ "num_tokens": 35930112.0,
+ "step": 2193
+ },
+ {
+ "entropy": 0.8414053320884705,
+ "epoch": 4.432323232323232,
+ "grad_norm": 2.2400619983673096,
+ "learning_rate": 5.696969696969698e-07,
+ "loss": 0.8350386619567871,
+ "mean_token_accuracy": 0.7728993892669678,
+ "num_tokens": 35946496.0,
+ "step": 2194
+ },
+ {
+ "entropy": 1.4875850677490234,
+ "epoch": 4.434343434343434,
+ "grad_norm": 2.376569986343384,
+ "learning_rate": 5.676767676767677e-07,
+ "loss": 1.4847931861877441,
+ "mean_token_accuracy": 0.6506472826004028,
+ "num_tokens": 35962880.0,
+ "step": 2195
+ },
+ {
+ "entropy": 0.9875659942626953,
+ "epoch": 4.4363636363636365,
+ "grad_norm": 2.1472887992858887,
+ "learning_rate": 5.656565656565658e-07,
+ "loss": 0.9780405759811401,
+ "mean_token_accuracy": 0.7515876889228821,
+ "num_tokens": 35979264.0,
+ "step": 2196
+ },
+ {
+ "entropy": 1.5663740634918213,
+ "epoch": 4.438383838383839,
+ "grad_norm": 2.605659246444702,
+ "learning_rate": 5.636363636363638e-07,
+ "loss": 1.55465829372406,
+ "mean_token_accuracy": 0.6351978778839111,
+ "num_tokens": 35995648.0,
+ "step": 2197
+ },
+ {
+ "entropy": 1.0373486280441284,
+ "epoch": 4.44040404040404,
+ "grad_norm": 2.259995937347412,
+ "learning_rate": 5.616161616161616e-07,
+ "loss": 1.0467290878295898,
+ "mean_token_accuracy": 0.735344409942627,
+ "num_tokens": 36012032.0,
+ "step": 2198
+ },
+ {
+ "entropy": 0.9681136012077332,
+ "epoch": 4.442424242424242,
+ "grad_norm": 2.393617630004883,
+ "learning_rate": 5.595959595959596e-07,
+ "loss": 0.9561377763748169,
+ "mean_token_accuracy": 0.7390693426132202,
+ "num_tokens": 36028416.0,
+ "step": 2199
+ },
+ {
+ "entropy": 1.3843019008636475,
+ "epoch": 4.444444444444445,
+ "grad_norm": 2.3265609741210938,
+ "learning_rate": 5.575757575757576e-07,
+ "loss": 1.372997522354126,
+ "mean_token_accuracy": 0.6861870884895325,
+ "num_tokens": 36044800.0,
+ "step": 2200
+ },
+ {
+ "epoch": 4.444444444444445,
+ "eval_entropy": 1.2872899544815863,
+ "eval_loss": 1.5627764463424683,
+ "eval_mean_token_accuracy": 0.6437114914578776,
+ "eval_num_tokens": 36044800.0,
+ "eval_runtime": 43.7611,
+ "eval_samples_per_second": 22.623,
+ "eval_steps_per_second": 2.834,
+ "step": 2200
+ },
+ {
+ "entropy": 0.9683943390846252,
+ "epoch": 4.446464646464646,
+ "grad_norm": 2.1208672523498535,
+ "learning_rate": 5.555555555555555e-07,
+ "loss": 0.9520723819732666,
+ "mean_token_accuracy": 0.7614191770553589,
+ "num_tokens": 36061184.0,
+ "step": 2201
+ },
+ {
+ "entropy": 1.2325289249420166,
+ "epoch": 4.448484848484848,
+ "grad_norm": 2.3613944053649902,
+ "learning_rate": 5.535353535353536e-07,
+ "loss": 1.2266604900360107,
+ "mean_token_accuracy": 0.7127503752708435,
+ "num_tokens": 36077568.0,
+ "step": 2202
+ },
+ {
+ "entropy": 1.2971447706222534,
+ "epoch": 4.4505050505050505,
+ "grad_norm": 2.4161171913146973,
+ "learning_rate": 5.515151515151516e-07,
+ "loss": 1.298982858657837,
+ "mean_token_accuracy": 0.6858817934989929,
+ "num_tokens": 36093952.0,
+ "step": 2203
+ },
+ {
+ "entropy": 1.2562965154647827,
+ "epoch": 4.452525252525253,
+ "grad_norm": 2.4957022666931152,
+ "learning_rate": 5.494949494949495e-07,
+ "loss": 1.2517296075820923,
+ "mean_token_accuracy": 0.6927821040153503,
+ "num_tokens": 36110336.0,
+ "step": 2204
+ },
+ {
+ "entropy": 1.2625181674957275,
+ "epoch": 4.454545454545454,
+ "grad_norm": 2.1725847721099854,
+ "learning_rate": 5.474747474747475e-07,
+ "loss": 1.2470554113388062,
+ "mean_token_accuracy": 0.7031021118164062,
+ "num_tokens": 36126720.0,
+ "step": 2205
+ },
+ {
+ "entropy": 1.1061078310012817,
+ "epoch": 4.456565656565656,
+ "grad_norm": 2.4533944129943848,
+ "learning_rate": 5.454545454545455e-07,
+ "loss": 1.0851916074752808,
+ "mean_token_accuracy": 0.7231924533843994,
+ "num_tokens": 36143104.0,
+ "step": 2206
+ },
+ {
+ "entropy": 1.125205636024475,
+ "epoch": 4.458585858585859,
+ "grad_norm": 2.3659775257110596,
+ "learning_rate": 5.434343434343435e-07,
+ "loss": 1.1383261680603027,
+ "mean_token_accuracy": 0.7042623162269592,
+ "num_tokens": 36159488.0,
+ "step": 2207
+ },
+ {
+ "entropy": 1.3721956014633179,
+ "epoch": 4.460606060606061,
+ "grad_norm": 2.4929234981536865,
+ "learning_rate": 5.414141414141415e-07,
+ "loss": 1.3432800769805908,
+ "mean_token_accuracy": 0.67666095495224,
+ "num_tokens": 36175872.0,
+ "step": 2208
+ },
+ {
+ "entropy": 0.9716646671295166,
+ "epoch": 4.462626262626262,
+ "grad_norm": 2.1126415729522705,
+ "learning_rate": 5.393939393939395e-07,
+ "loss": 0.9754329919815063,
+ "mean_token_accuracy": 0.7575720548629761,
+ "num_tokens": 36192256.0,
+ "step": 2209
+ },
+ {
+ "entropy": 1.1441161632537842,
+ "epoch": 4.4646464646464645,
+ "grad_norm": 2.329407215118408,
+ "learning_rate": 5.373737373737374e-07,
+ "loss": 1.1453542709350586,
+ "mean_token_accuracy": 0.7070713043212891,
+ "num_tokens": 36208640.0,
+ "step": 2210
+ },
+ {
+ "entropy": 1.3255770206451416,
+ "epoch": 4.466666666666667,
+ "grad_norm": 2.0918211936950684,
+ "learning_rate": 5.353535353535354e-07,
+ "loss": 1.3274650573730469,
+ "mean_token_accuracy": 0.6834391951560974,
+ "num_tokens": 36225024.0,
+ "step": 2211
+ },
+ {
+ "entropy": 1.4749172925949097,
+ "epoch": 4.468686868686869,
+ "grad_norm": 2.1065104007720947,
+ "learning_rate": 5.333333333333335e-07,
+ "loss": 1.4526000022888184,
+ "mean_token_accuracy": 0.6637151837348938,
+ "num_tokens": 36241408.0,
+ "step": 2212
+ },
+ {
+ "entropy": 1.3020631074905396,
+ "epoch": 4.47070707070707,
+ "grad_norm": 2.435131788253784,
+ "learning_rate": 5.313131313131313e-07,
+ "loss": 1.3230618238449097,
+ "mean_token_accuracy": 0.683744490146637,
+ "num_tokens": 36257792.0,
+ "step": 2213
+ },
+ {
+ "entropy": 1.2354505062103271,
+ "epoch": 4.472727272727273,
+ "grad_norm": 2.2865445613861084,
+ "learning_rate": 5.292929292929293e-07,
+ "loss": 1.2283403873443604,
+ "mean_token_accuracy": 0.695713222026825,
+ "num_tokens": 36274176.0,
+ "step": 2214
+ },
+ {
+ "entropy": 0.9121840596199036,
+ "epoch": 4.474747474747475,
+ "grad_norm": 2.379857063293457,
+ "learning_rate": 5.272727272727273e-07,
+ "loss": 0.909171462059021,
+ "mean_token_accuracy": 0.755923330783844,
+ "num_tokens": 36290560.0,
+ "step": 2215
+ },
+ {
+ "entropy": 1.3496302366256714,
+ "epoch": 4.476767676767677,
+ "grad_norm": 2.101379871368408,
+ "learning_rate": 5.252525252525253e-07,
+ "loss": 1.3399009704589844,
+ "mean_token_accuracy": 0.69840008020401,
+ "num_tokens": 36306944.0,
+ "step": 2216
+ },
+ {
+ "entropy": 1.609591007232666,
+ "epoch": 4.4787878787878785,
+ "grad_norm": 2.506319284439087,
+ "learning_rate": 5.232323232323232e-07,
+ "loss": 1.6377949714660645,
+ "mean_token_accuracy": 0.6235954761505127,
+ "num_tokens": 36323328.0,
+ "step": 2217
+ },
+ {
+ "entropy": 1.0830663442611694,
+ "epoch": 4.480808080808081,
+ "grad_norm": 2.206979274749756,
+ "learning_rate": 5.212121212121213e-07,
+ "loss": 1.0871551036834717,
+ "mean_token_accuracy": 0.7242916226387024,
+ "num_tokens": 36339712.0,
+ "step": 2218
+ },
+ {
+ "entropy": 1.0723860263824463,
+ "epoch": 4.482828282828283,
+ "grad_norm": 2.9275877475738525,
+ "learning_rate": 5.191919191919192e-07,
+ "loss": 1.0785467624664307,
+ "mean_token_accuracy": 0.7305202484130859,
+ "num_tokens": 36356096.0,
+ "step": 2219
+ },
+ {
+ "entropy": 1.3322672843933105,
+ "epoch": 4.484848484848484,
+ "grad_norm": 2.3471016883850098,
+ "learning_rate": 5.171717171717172e-07,
+ "loss": 1.311643123626709,
+ "mean_token_accuracy": 0.6893014311790466,
+ "num_tokens": 36372480.0,
+ "step": 2220
+ },
+ {
+ "entropy": 1.29927396774292,
+ "epoch": 4.486868686868687,
+ "grad_norm": 2.427845001220703,
+ "learning_rate": 5.151515151515152e-07,
+ "loss": 1.3044838905334473,
+ "mean_token_accuracy": 0.697239875793457,
+ "num_tokens": 36388864.0,
+ "step": 2221
+ },
+ {
+ "entropy": 1.341801404953003,
+ "epoch": 4.488888888888889,
+ "grad_norm": 2.4492251873016357,
+ "learning_rate": 5.131313131313132e-07,
+ "loss": 1.341687560081482,
+ "mean_token_accuracy": 0.6740351915359497,
+ "num_tokens": 36405248.0,
+ "step": 2222
+ },
+ {
+ "entropy": 0.7857770323753357,
+ "epoch": 4.490909090909091,
+ "grad_norm": 2.296898365020752,
+ "learning_rate": 5.111111111111112e-07,
+ "loss": 0.8039225935935974,
+ "mean_token_accuracy": 0.7828529477119446,
+ "num_tokens": 36421632.0,
+ "step": 2223
+ },
+ {
+ "entropy": 0.8067459464073181,
+ "epoch": 4.4929292929292926,
+ "grad_norm": 2.2211732864379883,
+ "learning_rate": 5.090909090909092e-07,
+ "loss": 0.8270186185836792,
+ "mean_token_accuracy": 0.7794944047927856,
+ "num_tokens": 36438016.0,
+ "step": 2224
+ },
+ {
+ "entropy": 1.1513580083847046,
+ "epoch": 4.494949494949495,
+ "grad_norm": 2.4064536094665527,
+ "learning_rate": 5.070707070707072e-07,
+ "loss": 1.1496974229812622,
+ "mean_token_accuracy": 0.7101245522499084,
+ "num_tokens": 36454400.0,
+ "step": 2225
+ },
+ {
+ "entropy": 1.0617595911026,
+ "epoch": 4.496969696969697,
+ "grad_norm": 2.2687058448791504,
+ "learning_rate": 5.05050505050505e-07,
+ "loss": 1.069061040878296,
+ "mean_token_accuracy": 0.7382144331932068,
+ "num_tokens": 36470784.0,
+ "step": 2226
+ },
+ {
+ "entropy": 1.009508490562439,
+ "epoch": 4.498989898989899,
+ "grad_norm": 2.3550171852111816,
+ "learning_rate": 5.03030303030303e-07,
+ "loss": 1.0016307830810547,
+ "mean_token_accuracy": 0.7346116304397583,
+ "num_tokens": 36487168.0,
+ "step": 2227
+ },
+ {
+ "entropy": 0.9507298469543457,
+ "epoch": 4.501010101010101,
+ "grad_norm": 2.431490182876587,
+ "learning_rate": 5.01010101010101e-07,
+ "loss": 0.9455212950706482,
+ "mean_token_accuracy": 0.7419394254684448,
+ "num_tokens": 36503552.0,
+ "step": 2228
+ },
+ {
+ "entropy": 1.0880755186080933,
+ "epoch": 4.503030303030303,
+ "grad_norm": 2.223511219024658,
+ "learning_rate": 4.98989898989899e-07,
+ "loss": 1.0870615243911743,
+ "mean_token_accuracy": 0.733207106590271,
+ "num_tokens": 36519936.0,
+ "step": 2229
+ },
+ {
+ "entropy": 1.468039870262146,
+ "epoch": 4.505050505050505,
+ "grad_norm": 2.192481517791748,
+ "learning_rate": 4.96969696969697e-07,
+ "loss": 1.4589731693267822,
+ "mean_token_accuracy": 0.6582193374633789,
+ "num_tokens": 36536320.0,
+ "step": 2230
+ },
+ {
+ "entropy": 1.124917984008789,
+ "epoch": 4.5070707070707074,
+ "grad_norm": 2.2918739318847656,
+ "learning_rate": 4.94949494949495e-07,
+ "loss": 1.120725393295288,
+ "mean_token_accuracy": 0.7203224301338196,
+ "num_tokens": 36552704.0,
+ "step": 2231
+ },
+ {
+ "entropy": 0.9215601682662964,
+ "epoch": 4.509090909090909,
+ "grad_norm": 2.1858043670654297,
+ "learning_rate": 4.929292929292929e-07,
+ "loss": 0.9341068863868713,
+ "mean_token_accuracy": 0.7614191770553589,
+ "num_tokens": 36569088.0,
+ "step": 2232
+ },
+ {
+ "entropy": 1.160864233970642,
+ "epoch": 4.511111111111111,
+ "grad_norm": 2.378822088241577,
+ "learning_rate": 4.909090909090909e-07,
+ "loss": 1.165703296661377,
+ "mean_token_accuracy": 0.7143380641937256,
+ "num_tokens": 36585472.0,
+ "step": 2233
+ },
+ {
+ "entropy": 1.1323046684265137,
+ "epoch": 4.513131313131313,
+ "grad_norm": 2.0554256439208984,
+ "learning_rate": 4.88888888888889e-07,
+ "loss": 1.127324104309082,
+ "mean_token_accuracy": 0.7274059653282166,
+ "num_tokens": 36601856.0,
+ "step": 2234
+ },
+ {
+ "entropy": 1.160245418548584,
+ "epoch": 4.515151515151516,
+ "grad_norm": 2.190343141555786,
+ "learning_rate": 4.868686868686869e-07,
+ "loss": 1.178252935409546,
+ "mean_token_accuracy": 0.7177577018737793,
+ "num_tokens": 36618240.0,
+ "step": 2235
+ },
+ {
+ "entropy": 1.0734480619430542,
+ "epoch": 4.517171717171717,
+ "grad_norm": 2.3177547454833984,
+ "learning_rate": 4.848484848484849e-07,
+ "loss": 1.0701775550842285,
+ "mean_token_accuracy": 0.7159867882728577,
+ "num_tokens": 36634624.0,
+ "step": 2236
+ },
+ {
+ "entropy": 1.0354589223861694,
+ "epoch": 4.519191919191919,
+ "grad_norm": 2.433401584625244,
+ "learning_rate": 4.828282828282829e-07,
+ "loss": 1.0339555740356445,
+ "mean_token_accuracy": 0.7307034730911255,
+ "num_tokens": 36651008.0,
+ "step": 2237
+ },
+ {
+ "entropy": 1.7667415142059326,
+ "epoch": 4.5212121212121215,
+ "grad_norm": 2.2882981300354004,
+ "learning_rate": 4.808080808080809e-07,
+ "loss": 1.7917096614837646,
+ "mean_token_accuracy": 0.619015634059906,
+ "num_tokens": 36667392.0,
+ "step": 2238
+ },
+ {
+ "entropy": 1.3639158010482788,
+ "epoch": 4.523232323232323,
+ "grad_norm": 2.151674747467041,
+ "learning_rate": 4.787878787878789e-07,
+ "loss": 1.3818566799163818,
+ "mean_token_accuracy": 0.6798974275588989,
+ "num_tokens": 36683776.0,
+ "step": 2239
+ },
+ {
+ "entropy": 1.5619690418243408,
+ "epoch": 4.525252525252525,
+ "grad_norm": 2.41402006149292,
+ "learning_rate": 4.767676767676768e-07,
+ "loss": 1.5720285177230835,
+ "mean_token_accuracy": 0.622801661491394,
+ "num_tokens": 36700160.0,
+ "step": 2240
+ },
+ {
+ "entropy": 1.1401708126068115,
+ "epoch": 4.527272727272727,
+ "grad_norm": 2.5265958309173584,
+ "learning_rate": 4.747474747474748e-07,
+ "loss": 1.1326185464859009,
+ "mean_token_accuracy": 0.7091475129127502,
+ "num_tokens": 36716544.0,
+ "step": 2241
+ },
+ {
+ "entropy": 1.32151460647583,
+ "epoch": 4.52929292929293,
+ "grad_norm": 2.58975887298584,
+ "learning_rate": 4.7272727272727273e-07,
+ "loss": 1.3325283527374268,
+ "mean_token_accuracy": 0.678859293460846,
+ "num_tokens": 36732928.0,
+ "step": 2242
+ },
+ {
+ "entropy": 0.9835161566734314,
+ "epoch": 4.531313131313131,
+ "grad_norm": 2.2142131328582764,
+ "learning_rate": 4.7070707070707073e-07,
+ "loss": 0.9954148530960083,
+ "mean_token_accuracy": 0.7529922127723694,
+ "num_tokens": 36749312.0,
+ "step": 2243
+ },
+ {
+ "entropy": 1.6663132905960083,
+ "epoch": 4.533333333333333,
+ "grad_norm": 2.3992927074432373,
+ "learning_rate": 4.686868686868688e-07,
+ "loss": 1.6849396228790283,
+ "mean_token_accuracy": 0.6166340708732605,
+ "num_tokens": 36765696.0,
+ "step": 2244
+ },
+ {
+ "entropy": 1.2209111452102661,
+ "epoch": 4.5353535353535355,
+ "grad_norm": 2.415360689163208,
+ "learning_rate": 4.666666666666667e-07,
+ "loss": 1.222484827041626,
+ "mean_token_accuracy": 0.7084758281707764,
+ "num_tokens": 36782080.0,
+ "step": 2245
+ },
+ {
+ "entropy": 1.1928186416625977,
+ "epoch": 4.537373737373738,
+ "grad_norm": 2.3891279697418213,
+ "learning_rate": 4.646464646464647e-07,
+ "loss": 1.2071551084518433,
+ "mean_token_accuracy": 0.6979726552963257,
+ "num_tokens": 36798464.0,
+ "step": 2246
+ },
+ {
+ "entropy": 1.0771721601486206,
+ "epoch": 4.539393939393939,
+ "grad_norm": 2.1842703819274902,
+ "learning_rate": 4.6262626262626265e-07,
+ "loss": 1.0782114267349243,
+ "mean_token_accuracy": 0.7377259135246277,
+ "num_tokens": 36814848.0,
+ "step": 2247
+ },
+ {
+ "entropy": 0.7138144969940186,
+ "epoch": 4.541414141414141,
+ "grad_norm": 2.052765130996704,
+ "learning_rate": 4.6060606060606064e-07,
+ "loss": 0.7110669612884521,
+ "mean_token_accuracy": 0.803798258304596,
+ "num_tokens": 36831232.0,
+ "step": 2248
+ },
+ {
+ "entropy": 1.0407626628875732,
+ "epoch": 4.543434343434344,
+ "grad_norm": 2.396315813064575,
+ "learning_rate": 4.585858585858586e-07,
+ "loss": 1.0258148908615112,
+ "mean_token_accuracy": 0.739130437374115,
+ "num_tokens": 36847616.0,
+ "step": 2249
+ },
+ {
+ "entropy": 1.3311498165130615,
+ "epoch": 4.545454545454545,
+ "grad_norm": 2.403750419616699,
+ "learning_rate": 4.5656565656565663e-07,
+ "loss": 1.3521225452423096,
+ "mean_token_accuracy": 0.680446982383728,
+ "num_tokens": 36864000.0,
+ "step": 2250
+ },
+ {
+ "epoch": 4.545454545454545,
+ "eval_entropy": 1.2901163678015433,
+ "eval_loss": 1.56230628490448,
+ "eval_mean_token_accuracy": 0.6438065368321634,
+ "eval_num_tokens": 36864000.0,
+ "eval_runtime": 43.7396,
+ "eval_samples_per_second": 22.634,
+ "eval_steps_per_second": 2.835,
+ "step": 2250
+ },
+ {
+ "entropy": 1.0132540464401245,
+ "epoch": 4.547474747474747,
+ "grad_norm": 2.424539089202881,
+ "learning_rate": 4.5454545454545457e-07,
+ "loss": 0.9965545535087585,
+ "mean_token_accuracy": 0.7355886697769165,
+ "num_tokens": 36880384.0,
+ "step": 2251
+ },
+ {
+ "entropy": 1.566239833831787,
+ "epoch": 4.5494949494949495,
+ "grad_norm": 2.498107671737671,
+ "learning_rate": 4.5252525252525257e-07,
+ "loss": 1.5575640201568604,
+ "mean_token_accuracy": 0.6338544487953186,
+ "num_tokens": 36896768.0,
+ "step": 2252
+ },
+ {
+ "entropy": 1.1055033206939697,
+ "epoch": 4.551515151515152,
+ "grad_norm": 2.2892208099365234,
+ "learning_rate": 4.505050505050505e-07,
+ "loss": 1.0803035497665405,
+ "mean_token_accuracy": 0.7255129218101501,
+ "num_tokens": 36913152.0,
+ "step": 2253
+ },
+ {
+ "entropy": 1.2764747142791748,
+ "epoch": 4.553535353535354,
+ "grad_norm": 2.2250852584838867,
+ "learning_rate": 4.484848484848485e-07,
+ "loss": 1.2709777355194092,
+ "mean_token_accuracy": 0.7001709938049316,
+ "num_tokens": 36929536.0,
+ "step": 2254
+ },
+ {
+ "entropy": 1.082503318786621,
+ "epoch": 4.555555555555555,
+ "grad_norm": 2.4460432529449463,
+ "learning_rate": 4.4646464646464655e-07,
+ "loss": 1.072137475013733,
+ "mean_token_accuracy": 0.7195896506309509,
+ "num_tokens": 36945920.0,
+ "step": 2255
+ },
+ {
+ "entropy": 1.116898775100708,
+ "epoch": 4.557575757575758,
+ "grad_norm": 2.265634536743164,
+ "learning_rate": 4.444444444444445e-07,
+ "loss": 1.1262788772583008,
+ "mean_token_accuracy": 0.7217268943786621,
+ "num_tokens": 36962304.0,
+ "step": 2256
+ },
+ {
+ "entropy": 1.7029410600662231,
+ "epoch": 4.55959595959596,
+ "grad_norm": 2.507423162460327,
+ "learning_rate": 4.424242424242425e-07,
+ "loss": 1.7347257137298584,
+ "mean_token_accuracy": 0.6124816536903381,
+ "num_tokens": 36978688.0,
+ "step": 2257
+ },
+ {
+ "entropy": 1.312424898147583,
+ "epoch": 4.561616161616161,
+ "grad_norm": 2.5053603649139404,
+ "learning_rate": 4.404040404040404e-07,
+ "loss": 1.302088975906372,
+ "mean_token_accuracy": 0.6765388250350952,
+ "num_tokens": 36995072.0,
+ "step": 2258
+ },
+ {
+ "entropy": 1.0829205513000488,
+ "epoch": 4.5636363636363635,
+ "grad_norm": 2.2573883533477783,
+ "learning_rate": 4.383838383838384e-07,
+ "loss": 1.0810492038726807,
+ "mean_token_accuracy": 0.7468856573104858,
+ "num_tokens": 37011456.0,
+ "step": 2259
+ },
+ {
+ "entropy": 1.1587903499603271,
+ "epoch": 4.565656565656566,
+ "grad_norm": 2.3444430828094482,
+ "learning_rate": 4.363636363636364e-07,
+ "loss": 1.1578619480133057,
+ "mean_token_accuracy": 0.7051172256469727,
+ "num_tokens": 37027840.0,
+ "step": 2260
+ },
+ {
+ "entropy": 1.1421524286270142,
+ "epoch": 4.567676767676768,
+ "grad_norm": 2.4562175273895264,
+ "learning_rate": 4.343434343434344e-07,
+ "loss": 1.1468560695648193,
+ "mean_token_accuracy": 0.7044455409049988,
+ "num_tokens": 37044224.0,
+ "step": 2261
+ },
+ {
+ "entropy": 0.8396296501159668,
+ "epoch": 4.569696969696969,
+ "grad_norm": 2.081439733505249,
+ "learning_rate": 4.3232323232323235e-07,
+ "loss": 0.846064031124115,
+ "mean_token_accuracy": 0.7702735662460327,
+ "num_tokens": 37060608.0,
+ "step": 2262
+ },
+ {
+ "entropy": 1.3769422769546509,
+ "epoch": 4.571717171717172,
+ "grad_norm": 2.249382972717285,
+ "learning_rate": 4.3030303030303034e-07,
+ "loss": 1.391121745109558,
+ "mean_token_accuracy": 0.6820957660675049,
+ "num_tokens": 37076992.0,
+ "step": 2263
+ },
+ {
+ "entropy": 1.8412320613861084,
+ "epoch": 4.573737373737374,
+ "grad_norm": 2.3502392768859863,
+ "learning_rate": 4.282828282828283e-07,
+ "loss": 1.8429474830627441,
+ "mean_token_accuracy": 0.5960552096366882,
+ "num_tokens": 37093376.0,
+ "step": 2264
+ },
+ {
+ "entropy": 1.3433306217193604,
+ "epoch": 4.575757575757576,
+ "grad_norm": 2.2805447578430176,
+ "learning_rate": 4.262626262626263e-07,
+ "loss": 1.3156907558441162,
+ "mean_token_accuracy": 0.6905227303504944,
+ "num_tokens": 37109760.0,
+ "step": 2265
+ },
+ {
+ "entropy": 0.8410167694091797,
+ "epoch": 4.5777777777777775,
+ "grad_norm": 2.1443328857421875,
+ "learning_rate": 4.242424242424243e-07,
+ "loss": 0.8473227024078369,
+ "mean_token_accuracy": 0.7844406366348267,
+ "num_tokens": 37126144.0,
+ "step": 2266
+ },
+ {
+ "entropy": 1.2433832883834839,
+ "epoch": 4.57979797979798,
+ "grad_norm": 2.4752817153930664,
+ "learning_rate": 4.2222222222222226e-07,
+ "loss": 1.2559361457824707,
+ "mean_token_accuracy": 0.6909501552581787,
+ "num_tokens": 37142528.0,
+ "step": 2267
+ },
+ {
+ "entropy": 1.2694175243377686,
+ "epoch": 4.581818181818182,
+ "grad_norm": 2.369257688522339,
+ "learning_rate": 4.2020202020202026e-07,
+ "loss": 1.2467796802520752,
+ "mean_token_accuracy": 0.6978505253791809,
+ "num_tokens": 37158912.0,
+ "step": 2268
+ },
+ {
+ "entropy": 1.281100869178772,
+ "epoch": 4.583838383838383,
+ "grad_norm": 2.594801664352417,
+ "learning_rate": 4.181818181818182e-07,
+ "loss": 1.2897577285766602,
+ "mean_token_accuracy": 0.6805691123008728,
+ "num_tokens": 37175296.0,
+ "step": 2269
+ },
+ {
+ "entropy": 1.2010201215744019,
+ "epoch": 4.585858585858586,
+ "grad_norm": 2.5778253078460693,
+ "learning_rate": 4.161616161616162e-07,
+ "loss": 1.2004069089889526,
+ "mean_token_accuracy": 0.7007816433906555,
+ "num_tokens": 37191680.0,
+ "step": 2270
+ },
+ {
+ "entropy": 1.1471829414367676,
+ "epoch": 4.587878787878788,
+ "grad_norm": 2.2475953102111816,
+ "learning_rate": 4.141414141414142e-07,
+ "loss": 1.1470632553100586,
+ "mean_token_accuracy": 0.7180629968643188,
+ "num_tokens": 37208064.0,
+ "step": 2271
+ },
+ {
+ "entropy": 1.2060309648513794,
+ "epoch": 4.58989898989899,
+ "grad_norm": 2.556947946548462,
+ "learning_rate": 4.121212121212122e-07,
+ "loss": 1.2037420272827148,
+ "mean_token_accuracy": 0.7006595134735107,
+ "num_tokens": 37224448.0,
+ "step": 2272
+ },
+ {
+ "entropy": 1.3802626132965088,
+ "epoch": 4.591919191919192,
+ "grad_norm": 2.199343681335449,
+ "learning_rate": 4.101010101010101e-07,
+ "loss": 1.3555655479431152,
+ "mean_token_accuracy": 0.6707376837730408,
+ "num_tokens": 37240832.0,
+ "step": 2273
+ },
+ {
+ "entropy": 1.1409947872161865,
+ "epoch": 4.593939393939394,
+ "grad_norm": 2.383211612701416,
+ "learning_rate": 4.080808080808081e-07,
+ "loss": 1.1248835325241089,
+ "mean_token_accuracy": 0.7173302173614502,
+ "num_tokens": 37257216.0,
+ "step": 2274
+ },
+ {
+ "entropy": 0.811996340751648,
+ "epoch": 4.595959595959596,
+ "grad_norm": 2.3031976222991943,
+ "learning_rate": 4.0606060606060605e-07,
+ "loss": 0.8029609322547913,
+ "mean_token_accuracy": 0.7873107194900513,
+ "num_tokens": 37273600.0,
+ "step": 2275
+ },
+ {
+ "entropy": 1.4265928268432617,
+ "epoch": 4.597979797979798,
+ "grad_norm": 2.5571768283843994,
+ "learning_rate": 4.040404040404041e-07,
+ "loss": 1.4269866943359375,
+ "mean_token_accuracy": 0.6546165347099304,
+ "num_tokens": 37289984.0,
+ "step": 2276
+ },
+ {
+ "entropy": 1.6510447263717651,
+ "epoch": 4.6,
+ "grad_norm": 2.3172078132629395,
+ "learning_rate": 4.020202020202021e-07,
+ "loss": 1.6616134643554688,
+ "mean_token_accuracy": 0.6251832246780396,
+ "num_tokens": 37306368.0,
+ "step": 2277
+ },
+ {
+ "entropy": 1.0235978364944458,
+ "epoch": 4.602020202020202,
+ "grad_norm": 2.3221075534820557,
+ "learning_rate": 4.0000000000000003e-07,
+ "loss": 1.0012810230255127,
+ "mean_token_accuracy": 0.7437713742256165,
+ "num_tokens": 37322752.0,
+ "step": 2278
+ },
+ {
+ "entropy": 1.029131531715393,
+ "epoch": 4.604040404040404,
+ "grad_norm": 2.189196825027466,
+ "learning_rate": 3.9797979797979803e-07,
+ "loss": 1.0218143463134766,
+ "mean_token_accuracy": 0.7482901811599731,
+ "num_tokens": 37339136.0,
+ "step": 2279
+ },
+ {
+ "entropy": 1.329385757446289,
+ "epoch": 4.606060606060606,
+ "grad_norm": 2.4340617656707764,
+ "learning_rate": 3.9595959595959597e-07,
+ "loss": 1.3449084758758545,
+ "mean_token_accuracy": 0.6770884394645691,
+ "num_tokens": 37355520.0,
+ "step": 2280
+ },
+ {
+ "entropy": 0.7388880848884583,
+ "epoch": 4.608080808080808,
+ "grad_norm": 2.2112226486206055,
+ "learning_rate": 3.9393939393939396e-07,
+ "loss": 0.7312659025192261,
+ "mean_token_accuracy": 0.7933561205863953,
+ "num_tokens": 37371904.0,
+ "step": 2281
+ },
+ {
+ "entropy": 1.159648060798645,
+ "epoch": 4.61010101010101,
+ "grad_norm": 2.5980238914489746,
+ "learning_rate": 3.9191919191919196e-07,
+ "loss": 1.2006101608276367,
+ "mean_token_accuracy": 0.7051783204078674,
+ "num_tokens": 37388288.0,
+ "step": 2282
+ },
+ {
+ "entropy": 1.2951687574386597,
+ "epoch": 4.612121212121212,
+ "grad_norm": 2.244856119155884,
+ "learning_rate": 3.8989898989898995e-07,
+ "loss": 1.2880356311798096,
+ "mean_token_accuracy": 0.6805080771446228,
+ "num_tokens": 37404672.0,
+ "step": 2283
+ },
+ {
+ "entropy": 1.1866800785064697,
+ "epoch": 4.6141414141414145,
+ "grad_norm": 2.307337999343872,
+ "learning_rate": 3.878787878787879e-07,
+ "loss": 1.1886307001113892,
+ "mean_token_accuracy": 0.7155593633651733,
+ "num_tokens": 37421056.0,
+ "step": 2284
+ },
+ {
+ "entropy": 1.3319196701049805,
+ "epoch": 4.616161616161616,
+ "grad_norm": 2.263145923614502,
+ "learning_rate": 3.858585858585859e-07,
+ "loss": 1.317441701889038,
+ "mean_token_accuracy": 0.6865534782409668,
+ "num_tokens": 37437440.0,
+ "step": 2285
+ },
+ {
+ "entropy": 2.135301351547241,
+ "epoch": 4.618181818181818,
+ "grad_norm": 2.3971385955810547,
+ "learning_rate": 3.838383838383838e-07,
+ "loss": 2.114279270172119,
+ "mean_token_accuracy": 0.541829526424408,
+ "num_tokens": 37453824.0,
+ "step": 2286
+ },
+ {
+ "entropy": 1.0097599029541016,
+ "epoch": 4.62020202020202,
+ "grad_norm": 2.405428171157837,
+ "learning_rate": 3.8181818181818187e-07,
+ "loss": 1.007436990737915,
+ "mean_token_accuracy": 0.7344895005226135,
+ "num_tokens": 37470208.0,
+ "step": 2287
+ },
+ {
+ "entropy": 1.4621717929840088,
+ "epoch": 4.622222222222222,
+ "grad_norm": 2.3339080810546875,
+ "learning_rate": 3.7979797979797987e-07,
+ "loss": 1.4692507982254028,
+ "mean_token_accuracy": 0.653761625289917,
+ "num_tokens": 37486592.0,
+ "step": 2288
+ },
+ {
+ "entropy": 1.4064738750457764,
+ "epoch": 4.624242424242424,
+ "grad_norm": 2.5219573974609375,
+ "learning_rate": 3.777777777777778e-07,
+ "loss": 1.4030996561050415,
+ "mean_token_accuracy": 0.6621885895729065,
+ "num_tokens": 37502976.0,
+ "step": 2289
+ },
+ {
+ "entropy": 0.9653409719467163,
+ "epoch": 4.626262626262626,
+ "grad_norm": 2.309343099594116,
+ "learning_rate": 3.757575757575758e-07,
+ "loss": 0.9549931287765503,
+ "mean_token_accuracy": 0.7438324093818665,
+ "num_tokens": 37519360.0,
+ "step": 2290
+ },
+ {
+ "entropy": 1.439044713973999,
+ "epoch": 4.6282828282828286,
+ "grad_norm": 2.388458013534546,
+ "learning_rate": 3.7373737373737374e-07,
+ "loss": 1.4296658039093018,
+ "mean_token_accuracy": 0.6632266640663147,
+ "num_tokens": 37535744.0,
+ "step": 2291
+ },
+ {
+ "entropy": 1.1554685831069946,
+ "epoch": 4.63030303030303,
+ "grad_norm": 2.307220935821533,
+ "learning_rate": 3.717171717171718e-07,
+ "loss": 1.1408238410949707,
+ "mean_token_accuracy": 0.7106741666793823,
+ "num_tokens": 37552128.0,
+ "step": 2292
+ },
+ {
+ "entropy": 1.3482651710510254,
+ "epoch": 4.632323232323232,
+ "grad_norm": 2.4873476028442383,
+ "learning_rate": 3.6969696969696973e-07,
+ "loss": 1.3547667264938354,
+ "mean_token_accuracy": 0.6782486438751221,
+ "num_tokens": 37568512.0,
+ "step": 2293
+ },
+ {
+ "entropy": 1.2003843784332275,
+ "epoch": 4.634343434343434,
+ "grad_norm": 2.6622185707092285,
+ "learning_rate": 3.676767676767677e-07,
+ "loss": 1.220603585243225,
+ "mean_token_accuracy": 0.7004152536392212,
+ "num_tokens": 37584896.0,
+ "step": 2294
+ },
+ {
+ "entropy": 1.4963154792785645,
+ "epoch": 4.636363636363637,
+ "grad_norm": 2.388099193572998,
+ "learning_rate": 3.6565656565656566e-07,
+ "loss": 1.4815962314605713,
+ "mean_token_accuracy": 0.6540058851242065,
+ "num_tokens": 37601280.0,
+ "step": 2295
+ },
+ {
+ "entropy": 0.8894418478012085,
+ "epoch": 4.638383838383838,
+ "grad_norm": 2.2750332355499268,
+ "learning_rate": 3.6363636363636366e-07,
+ "loss": 0.8816108703613281,
+ "mean_token_accuracy": 0.7600757479667664,
+ "num_tokens": 37617664.0,
+ "step": 2296
+ },
+ {
+ "entropy": 1.4483006000518799,
+ "epoch": 4.64040404040404,
+ "grad_norm": 2.4239723682403564,
+ "learning_rate": 3.616161616161616e-07,
+ "loss": 1.4309420585632324,
+ "mean_token_accuracy": 0.6664631366729736,
+ "num_tokens": 37634048.0,
+ "step": 2297
+ },
+ {
+ "entropy": 1.0523942708969116,
+ "epoch": 4.642424242424243,
+ "grad_norm": 2.5061404705047607,
+ "learning_rate": 3.5959595959595965e-07,
+ "loss": 1.0694353580474854,
+ "mean_token_accuracy": 0.7223986387252808,
+ "num_tokens": 37650432.0,
+ "step": 2298
+ },
+ {
+ "entropy": 1.023050308227539,
+ "epoch": 4.644444444444445,
+ "grad_norm": 2.3206429481506348,
+ "learning_rate": 3.5757575757575764e-07,
+ "loss": 1.0274075269699097,
+ "mean_token_accuracy": 0.736932098865509,
+ "num_tokens": 37666816.0,
+ "step": 2299
+ },
+ {
+ "entropy": 1.237621784210205,
+ "epoch": 4.646464646464646,
+ "grad_norm": 2.2803795337677,
+ "learning_rate": 3.555555555555556e-07,
+ "loss": 1.2464512586593628,
+ "mean_token_accuracy": 0.704323410987854,
+ "num_tokens": 37683200.0,
+ "step": 2300
+ },
+ {
+ "epoch": 4.646464646464646,
+ "eval_entropy": 1.28836673738495,
+ "eval_loss": 1.5621296167373657,
+ "eval_mean_token_accuracy": 0.6437666464236474,
+ "eval_num_tokens": 37683200.0,
+ "eval_runtime": 43.76,
+ "eval_samples_per_second": 22.623,
+ "eval_steps_per_second": 2.834,
+ "step": 2300
+ },
+ {
+ "entropy": 1.0633931159973145,
+ "epoch": 4.648484848484848,
+ "grad_norm": 2.4361109733581543,
+ "learning_rate": 3.535353535353536e-07,
+ "loss": 1.0744309425354004,
+ "mean_token_accuracy": 0.7197728157043457,
+ "num_tokens": 37699584.0,
+ "step": 2301
+ },
+ {
+ "entropy": 1.3343605995178223,
+ "epoch": 4.650505050505051,
+ "grad_norm": 2.421180486679077,
+ "learning_rate": 3.515151515151515e-07,
+ "loss": 1.3187456130981445,
+ "mean_token_accuracy": 0.6835613250732422,
+ "num_tokens": 37715968.0,
+ "step": 2302
+ },
+ {
+ "entropy": 1.0245436429977417,
+ "epoch": 4.652525252525253,
+ "grad_norm": 2.140960693359375,
+ "learning_rate": 3.4949494949494956e-07,
+ "loss": 1.0146875381469727,
+ "mean_token_accuracy": 0.7372373938560486,
+ "num_tokens": 37732352.0,
+ "step": 2303
+ },
+ {
+ "entropy": 1.1643346548080444,
+ "epoch": 4.654545454545454,
+ "grad_norm": 2.2306506633758545,
+ "learning_rate": 3.474747474747475e-07,
+ "loss": 1.1504415273666382,
+ "mean_token_accuracy": 0.7142769694328308,
+ "num_tokens": 37748736.0,
+ "step": 2304
+ },
+ {
+ "entropy": 0.7440503835678101,
+ "epoch": 4.656565656565657,
+ "grad_norm": 2.0574123859405518,
+ "learning_rate": 3.454545454545455e-07,
+ "loss": 0.7455213069915771,
+ "mean_token_accuracy": 0.8050195574760437,
+ "num_tokens": 37765120.0,
+ "step": 2305
+ },
+ {
+ "entropy": 1.4619858264923096,
+ "epoch": 4.658585858585859,
+ "grad_norm": 2.6212973594665527,
+ "learning_rate": 3.4343434343434344e-07,
+ "loss": 1.469425916671753,
+ "mean_token_accuracy": 0.6445407867431641,
+ "num_tokens": 37781504.0,
+ "step": 2306
+ },
+ {
+ "entropy": 0.9174419641494751,
+ "epoch": 4.66060606060606,
+ "grad_norm": 2.220879316329956,
+ "learning_rate": 3.4141414141414143e-07,
+ "loss": 0.9032682180404663,
+ "mean_token_accuracy": 0.757022500038147,
+ "num_tokens": 37797888.0,
+ "step": 2307
+ },
+ {
+ "entropy": 1.4366378784179688,
+ "epoch": 4.6626262626262625,
+ "grad_norm": 2.5851221084594727,
+ "learning_rate": 3.393939393939395e-07,
+ "loss": 1.4307548999786377,
+ "mean_token_accuracy": 0.6482657790184021,
+ "num_tokens": 37814272.0,
+ "step": 2308
+ },
+ {
+ "entropy": 1.2210568189620972,
+ "epoch": 4.664646464646465,
+ "grad_norm": 2.420168161392212,
+ "learning_rate": 3.373737373737374e-07,
+ "loss": 1.2302658557891846,
+ "mean_token_accuracy": 0.7040791511535645,
+ "num_tokens": 37830656.0,
+ "step": 2309
+ },
+ {
+ "entropy": 1.6781017780303955,
+ "epoch": 4.666666666666667,
+ "grad_norm": 2.405329704284668,
+ "learning_rate": 3.353535353535354e-07,
+ "loss": 1.7171658277511597,
+ "mean_token_accuracy": 0.6167562007904053,
+ "num_tokens": 37847040.0,
+ "step": 2310
+ },
+ {
+ "entropy": 1.555785894393921,
+ "epoch": 4.668686868686868,
+ "grad_norm": 2.4297945499420166,
+ "learning_rate": 3.3333333333333335e-07,
+ "loss": 1.5461559295654297,
+ "mean_token_accuracy": 0.6530898809432983,
+ "num_tokens": 37863424.0,
+ "step": 2311
+ },
+ {
+ "entropy": 1.2855968475341797,
+ "epoch": 4.670707070707071,
+ "grad_norm": 2.272463798522949,
+ "learning_rate": 3.3131313131313135e-07,
+ "loss": 1.259823203086853,
+ "mean_token_accuracy": 0.6882022619247437,
+ "num_tokens": 37879808.0,
+ "step": 2312
+ },
+ {
+ "entropy": 1.4039292335510254,
+ "epoch": 4.672727272727273,
+ "grad_norm": 2.327253818511963,
+ "learning_rate": 3.292929292929293e-07,
+ "loss": 1.4226019382476807,
+ "mean_token_accuracy": 0.6649364829063416,
+ "num_tokens": 37896192.0,
+ "step": 2313
+ },
+ {
+ "entropy": 1.3019342422485352,
+ "epoch": 4.674747474747475,
+ "grad_norm": 2.5177788734436035,
+ "learning_rate": 3.2727272727272733e-07,
+ "loss": 1.3209342956542969,
+ "mean_token_accuracy": 0.6801416873931885,
+ "num_tokens": 37912576.0,
+ "step": 2314
+ },
+ {
+ "entropy": 1.002327561378479,
+ "epoch": 4.6767676767676765,
+ "grad_norm": 2.3929429054260254,
+ "learning_rate": 3.252525252525253e-07,
+ "loss": 1.0002248287200928,
+ "mean_token_accuracy": 0.735832929611206,
+ "num_tokens": 37928960.0,
+ "step": 2315
+ },
+ {
+ "entropy": 1.0516724586486816,
+ "epoch": 4.678787878787879,
+ "grad_norm": 2.405729293823242,
+ "learning_rate": 3.2323232323232327e-07,
+ "loss": 1.05308198928833,
+ "mean_token_accuracy": 0.7308256030082703,
+ "num_tokens": 37945344.0,
+ "step": 2316
+ },
+ {
+ "entropy": 1.1786670684814453,
+ "epoch": 4.680808080808081,
+ "grad_norm": 2.480250120162964,
+ "learning_rate": 3.212121212121212e-07,
+ "loss": 1.1930567026138306,
+ "mean_token_accuracy": 0.7120786309242249,
+ "num_tokens": 37961728.0,
+ "step": 2317
+ },
+ {
+ "entropy": 1.6102875471115112,
+ "epoch": 4.682828282828282,
+ "grad_norm": 2.2855021953582764,
+ "learning_rate": 3.191919191919192e-07,
+ "loss": 1.5947301387786865,
+ "mean_token_accuracy": 0.6359916925430298,
+ "num_tokens": 37978112.0,
+ "step": 2318
+ },
+ {
+ "entropy": 1.3153326511383057,
+ "epoch": 4.684848484848485,
+ "grad_norm": 2.46415638923645,
+ "learning_rate": 3.1717171717171725e-07,
+ "loss": 1.3392620086669922,
+ "mean_token_accuracy": 0.6741573214530945,
+ "num_tokens": 37994496.0,
+ "step": 2319
+ },
+ {
+ "entropy": 1.0658379793167114,
+ "epoch": 4.686868686868687,
+ "grad_norm": 2.5052623748779297,
+ "learning_rate": 3.151515151515152e-07,
+ "loss": 1.078431248664856,
+ "mean_token_accuracy": 0.7183072566986084,
+ "num_tokens": 38010880.0,
+ "step": 2320
+ },
+ {
+ "entropy": 1.589642882347107,
+ "epoch": 4.688888888888889,
+ "grad_norm": 2.3059611320495605,
+ "learning_rate": 3.131313131313132e-07,
+ "loss": 1.592090129852295,
+ "mean_token_accuracy": 0.6395334601402283,
+ "num_tokens": 38027264.0,
+ "step": 2321
+ },
+ {
+ "entropy": 1.2305296659469604,
+ "epoch": 4.690909090909091,
+ "grad_norm": 2.271082639694214,
+ "learning_rate": 3.111111111111111e-07,
+ "loss": 1.2300341129302979,
+ "mean_token_accuracy": 0.686431348323822,
+ "num_tokens": 38043648.0,
+ "step": 2322
+ },
+ {
+ "entropy": 1.054584264755249,
+ "epoch": 4.692929292929293,
+ "grad_norm": 2.352559804916382,
+ "learning_rate": 3.090909090909091e-07,
+ "loss": 1.0419890880584717,
+ "mean_token_accuracy": 0.7354665398597717,
+ "num_tokens": 38060032.0,
+ "step": 2323
+ },
+ {
+ "entropy": 1.6653000116348267,
+ "epoch": 4.694949494949495,
+ "grad_norm": 2.4491634368896484,
+ "learning_rate": 3.070707070707071e-07,
+ "loss": 1.6999539136886597,
+ "mean_token_accuracy": 0.6111382246017456,
+ "num_tokens": 38076416.0,
+ "step": 2324
+ },
+ {
+ "entropy": 1.2883809804916382,
+ "epoch": 4.696969696969697,
+ "grad_norm": 2.5294644832611084,
+ "learning_rate": 3.0505050505050505e-07,
+ "loss": 1.2754499912261963,
+ "mean_token_accuracy": 0.6855764389038086,
+ "num_tokens": 38092800.0,
+ "step": 2325
+ },
+ {
+ "entropy": 0.8978599905967712,
+ "epoch": 4.698989898989899,
+ "grad_norm": 2.085676908493042,
+ "learning_rate": 3.0303030303030305e-07,
+ "loss": 0.8723149299621582,
+ "mean_token_accuracy": 0.7736321687698364,
+ "num_tokens": 38109184.0,
+ "step": 2326
+ },
+ {
+ "entropy": 1.1231838464736938,
+ "epoch": 4.701010101010101,
+ "grad_norm": 2.4599316120147705,
+ "learning_rate": 3.0101010101010104e-07,
+ "loss": 1.125774621963501,
+ "mean_token_accuracy": 0.7134220600128174,
+ "num_tokens": 38125568.0,
+ "step": 2327
+ },
+ {
+ "entropy": 0.8676767349243164,
+ "epoch": 4.703030303030303,
+ "grad_norm": 2.30765438079834,
+ "learning_rate": 2.9898989898989904e-07,
+ "loss": 0.8640745878219604,
+ "mean_token_accuracy": 0.7744259834289551,
+ "num_tokens": 38141952.0,
+ "step": 2328
+ },
+ {
+ "entropy": 1.2736501693725586,
+ "epoch": 4.705050505050505,
+ "grad_norm": 2.3648366928100586,
+ "learning_rate": 2.96969696969697e-07,
+ "loss": 1.2754569053649902,
+ "mean_token_accuracy": 0.6851490139961243,
+ "num_tokens": 38158336.0,
+ "step": 2329
+ },
+ {
+ "entropy": 0.9255363941192627,
+ "epoch": 4.707070707070707,
+ "grad_norm": 2.335263967514038,
+ "learning_rate": 2.9494949494949497e-07,
+ "loss": 0.9199962019920349,
+ "mean_token_accuracy": 0.7528700828552246,
+ "num_tokens": 38174720.0,
+ "step": 2330
+ },
+ {
+ "entropy": 0.9790836572647095,
+ "epoch": 4.709090909090909,
+ "grad_norm": 2.2479305267333984,
+ "learning_rate": 2.9292929292929296e-07,
+ "loss": 0.9697257280349731,
+ "mean_token_accuracy": 0.7452369332313538,
+ "num_tokens": 38191104.0,
+ "step": 2331
+ },
+ {
+ "entropy": 1.099324345588684,
+ "epoch": 4.711111111111111,
+ "grad_norm": 2.2268218994140625,
+ "learning_rate": 2.9090909090909096e-07,
+ "loss": 1.1064157485961914,
+ "mean_token_accuracy": 0.731008768081665,
+ "num_tokens": 38207488.0,
+ "step": 2332
+ },
+ {
+ "entropy": 1.2142791748046875,
+ "epoch": 4.7131313131313135,
+ "grad_norm": 2.3290510177612305,
+ "learning_rate": 2.888888888888889e-07,
+ "loss": 1.1970348358154297,
+ "mean_token_accuracy": 0.6973620057106018,
+ "num_tokens": 38223872.0,
+ "step": 2333
+ },
+ {
+ "entropy": 1.235068440437317,
+ "epoch": 4.715151515151515,
+ "grad_norm": 2.636742353439331,
+ "learning_rate": 2.868686868686869e-07,
+ "loss": 1.2286301851272583,
+ "mean_token_accuracy": 0.6879580020904541,
+ "num_tokens": 38240256.0,
+ "step": 2334
+ },
+ {
+ "entropy": 1.0521979331970215,
+ "epoch": 4.717171717171717,
+ "grad_norm": 2.3975024223327637,
+ "learning_rate": 2.848484848484849e-07,
+ "loss": 1.0421833992004395,
+ "mean_token_accuracy": 0.7196506857872009,
+ "num_tokens": 38256640.0,
+ "step": 2335
+ },
+ {
+ "entropy": 0.9967750906944275,
+ "epoch": 4.719191919191919,
+ "grad_norm": 2.463973045349121,
+ "learning_rate": 2.828282828282829e-07,
+ "loss": 0.9987051486968994,
+ "mean_token_accuracy": 0.7416951656341553,
+ "num_tokens": 38273024.0,
+ "step": 2336
+ },
+ {
+ "entropy": 1.0743719339370728,
+ "epoch": 4.721212121212121,
+ "grad_norm": 2.17183518409729,
+ "learning_rate": 2.808080808080808e-07,
+ "loss": 1.0761301517486572,
+ "mean_token_accuracy": 0.7257571816444397,
+ "num_tokens": 38289408.0,
+ "step": 2337
+ },
+ {
+ "entropy": 1.3123915195465088,
+ "epoch": 4.723232323232323,
+ "grad_norm": 2.502078056335449,
+ "learning_rate": 2.787878787878788e-07,
+ "loss": 1.3049037456512451,
+ "mean_token_accuracy": 0.6780654788017273,
+ "num_tokens": 38305792.0,
+ "step": 2338
+ },
+ {
+ "entropy": 0.9125492572784424,
+ "epoch": 4.725252525252525,
+ "grad_norm": 2.288921594619751,
+ "learning_rate": 2.767676767676768e-07,
+ "loss": 0.9170413017272949,
+ "mean_token_accuracy": 0.7563507556915283,
+ "num_tokens": 38322176.0,
+ "step": 2339
+ },
+ {
+ "entropy": 0.9668527245521545,
+ "epoch": 4.7272727272727275,
+ "grad_norm": 2.2450435161590576,
+ "learning_rate": 2.7474747474747475e-07,
+ "loss": 0.9799595475196838,
+ "mean_token_accuracy": 0.7448094487190247,
+ "num_tokens": 38338560.0,
+ "step": 2340
+ },
+ {
+ "entropy": 1.2308948040008545,
+ "epoch": 4.72929292929293,
+ "grad_norm": 2.4422991275787354,
+ "learning_rate": 2.7272727272727274e-07,
+ "loss": 1.2512444257736206,
+ "mean_token_accuracy": 0.6996824741363525,
+ "num_tokens": 38354944.0,
+ "step": 2341
+ },
+ {
+ "entropy": 1.3918697834014893,
+ "epoch": 4.731313131313131,
+ "grad_norm": 2.560283660888672,
+ "learning_rate": 2.7070707070707074e-07,
+ "loss": 1.3936901092529297,
+ "mean_token_accuracy": 0.6690889000892639,
+ "num_tokens": 38371328.0,
+ "step": 2342
+ },
+ {
+ "entropy": 0.951773464679718,
+ "epoch": 4.733333333333333,
+ "grad_norm": 2.337327241897583,
+ "learning_rate": 2.686868686868687e-07,
+ "loss": 0.9381144046783447,
+ "mean_token_accuracy": 0.7452979683876038,
+ "num_tokens": 38387712.0,
+ "step": 2343
+ },
+ {
+ "entropy": 1.1627814769744873,
+ "epoch": 4.735353535353536,
+ "grad_norm": 2.3551900386810303,
+ "learning_rate": 2.666666666666667e-07,
+ "loss": 1.1717885732650757,
+ "mean_token_accuracy": 0.7102466821670532,
+ "num_tokens": 38404096.0,
+ "step": 2344
+ },
+ {
+ "entropy": 1.223041296005249,
+ "epoch": 4.737373737373737,
+ "grad_norm": 2.2816755771636963,
+ "learning_rate": 2.6464646464646467e-07,
+ "loss": 1.2163041830062866,
+ "mean_token_accuracy": 0.7085979580879211,
+ "num_tokens": 38420480.0,
+ "step": 2345
+ },
+ {
+ "entropy": 0.9747529625892639,
+ "epoch": 4.739393939393939,
+ "grad_norm": 2.310767889022827,
+ "learning_rate": 2.6262626262626266e-07,
+ "loss": 0.973397970199585,
+ "mean_token_accuracy": 0.7438935041427612,
+ "num_tokens": 38436864.0,
+ "step": 2346
+ },
+ {
+ "entropy": 1.3344645500183105,
+ "epoch": 4.7414141414141415,
+ "grad_norm": 2.2414627075195312,
+ "learning_rate": 2.6060606060606065e-07,
+ "loss": 1.348139762878418,
+ "mean_token_accuracy": 0.6953468322753906,
+ "num_tokens": 38453248.0,
+ "step": 2347
+ },
+ {
+ "entropy": 1.1264088153839111,
+ "epoch": 4.743434343434344,
+ "grad_norm": 2.222374677658081,
+ "learning_rate": 2.585858585858586e-07,
+ "loss": 1.1195029020309448,
+ "mean_token_accuracy": 0.7311919927597046,
+ "num_tokens": 38469632.0,
+ "step": 2348
+ },
+ {
+ "entropy": 1.1974331140518188,
+ "epoch": 4.745454545454545,
+ "grad_norm": 2.448791265487671,
+ "learning_rate": 2.565656565656566e-07,
+ "loss": 1.1753729581832886,
+ "mean_token_accuracy": 0.7039570212364197,
+ "num_tokens": 38486016.0,
+ "step": 2349
+ },
+ {
+ "entropy": 1.1791099309921265,
+ "epoch": 4.747474747474747,
+ "grad_norm": 2.425783634185791,
+ "learning_rate": 2.545454545454546e-07,
+ "loss": 1.1861391067504883,
+ "mean_token_accuracy": 0.6938812732696533,
+ "num_tokens": 38502400.0,
+ "step": 2350
+ },
+ {
+ "epoch": 4.747474747474747,
+ "eval_entropy": 1.2881583263797145,
+ "eval_loss": 1.5621256828308105,
+ "eval_mean_token_accuracy": 0.6438006249166304,
+ "eval_num_tokens": 38502400.0,
+ "eval_runtime": 43.7945,
+ "eval_samples_per_second": 22.606,
+ "eval_steps_per_second": 2.831,
+ "step": 2350
+ },
+ {
+ "entropy": 1.0894598960876465,
+ "epoch": 4.74949494949495,
+ "grad_norm": 2.1677145957946777,
+ "learning_rate": 2.525252525252525e-07,
+ "loss": 1.1142408847808838,
+ "mean_token_accuracy": 0.7368099689483643,
+ "num_tokens": 38518784.0,
+ "step": 2351
+ },
+ {
+ "entropy": 1.2555266618728638,
+ "epoch": 4.751515151515152,
+ "grad_norm": 2.330214500427246,
+ "learning_rate": 2.505050505050505e-07,
+ "loss": 1.241432785987854,
+ "mean_token_accuracy": 0.7002931237220764,
+ "num_tokens": 38535168.0,
+ "step": 2352
+ },
+ {
+ "entropy": 1.2537733316421509,
+ "epoch": 4.753535353535353,
+ "grad_norm": 2.2559266090393066,
+ "learning_rate": 2.484848484848485e-07,
+ "loss": 1.25982666015625,
+ "mean_token_accuracy": 0.6988885998725891,
+ "num_tokens": 38551552.0,
+ "step": 2353
+ },
+ {
+ "entropy": 1.3228288888931274,
+ "epoch": 4.7555555555555555,
+ "grad_norm": 2.329820156097412,
+ "learning_rate": 2.4646464646464645e-07,
+ "loss": 1.32167387008667,
+ "mean_token_accuracy": 0.67659991979599,
+ "num_tokens": 38567936.0,
+ "step": 2354
+ },
+ {
+ "entropy": 1.093949556350708,
+ "epoch": 4.757575757575758,
+ "grad_norm": 2.358074903488159,
+ "learning_rate": 2.444444444444445e-07,
+ "loss": 1.0948805809020996,
+ "mean_token_accuracy": 0.7231924533843994,
+ "num_tokens": 38584320.0,
+ "step": 2355
+ },
+ {
+ "entropy": 1.0497996807098389,
+ "epoch": 4.759595959595959,
+ "grad_norm": 2.2141003608703613,
+ "learning_rate": 2.4242424242424244e-07,
+ "loss": 1.0230276584625244,
+ "mean_token_accuracy": 0.742977499961853,
+ "num_tokens": 38600704.0,
+ "step": 2356
+ },
+ {
+ "entropy": 1.1963584423065186,
+ "epoch": 4.761616161616161,
+ "grad_norm": 2.2645151615142822,
+ "learning_rate": 2.4040404040404043e-07,
+ "loss": 1.181758165359497,
+ "mean_token_accuracy": 0.7053614854812622,
+ "num_tokens": 38617088.0,
+ "step": 2357
+ },
+ {
+ "entropy": 1.4849578142166138,
+ "epoch": 4.763636363636364,
+ "grad_norm": 2.188263177871704,
+ "learning_rate": 2.383838383838384e-07,
+ "loss": 1.466001033782959,
+ "mean_token_accuracy": 0.6846604943275452,
+ "num_tokens": 38633472.0,
+ "step": 2358
+ },
+ {
+ "entropy": 1.535861849784851,
+ "epoch": 4.765656565656566,
+ "grad_norm": 2.2588133811950684,
+ "learning_rate": 2.3636363636363637e-07,
+ "loss": 1.5331593751907349,
+ "mean_token_accuracy": 0.6493038535118103,
+ "num_tokens": 38649856.0,
+ "step": 2359
+ },
+ {
+ "entropy": 1.0385324954986572,
+ "epoch": 4.767676767676767,
+ "grad_norm": 2.2858803272247314,
+ "learning_rate": 2.343434343434344e-07,
+ "loss": 1.049726963043213,
+ "mean_token_accuracy": 0.7335124611854553,
+ "num_tokens": 38666240.0,
+ "step": 2360
+ },
+ {
+ "entropy": 1.3631771802902222,
+ "epoch": 4.7696969696969695,
+ "grad_norm": 2.2661380767822266,
+ "learning_rate": 2.3232323232323235e-07,
+ "loss": 1.361598014831543,
+ "mean_token_accuracy": 0.6767220497131348,
+ "num_tokens": 38682624.0,
+ "step": 2361
+ },
+ {
+ "entropy": 0.8404029011726379,
+ "epoch": 4.771717171717172,
+ "grad_norm": 2.2505478858947754,
+ "learning_rate": 2.3030303030303032e-07,
+ "loss": 0.8151557445526123,
+ "mean_token_accuracy": 0.7870664596557617,
+ "num_tokens": 38699008.0,
+ "step": 2362
+ },
+ {
+ "entropy": 1.173842191696167,
+ "epoch": 4.773737373737374,
+ "grad_norm": 2.144379138946533,
+ "learning_rate": 2.2828282828282832e-07,
+ "loss": 1.1829612255096436,
+ "mean_token_accuracy": 0.7094528675079346,
+ "num_tokens": 38715392.0,
+ "step": 2363
+ },
+ {
+ "entropy": 1.5040391683578491,
+ "epoch": 4.775757575757575,
+ "grad_norm": 2.397090196609497,
+ "learning_rate": 2.2626262626262628e-07,
+ "loss": 1.5017142295837402,
+ "mean_token_accuracy": 0.6525403261184692,
+ "num_tokens": 38731776.0,
+ "step": 2364
+ },
+ {
+ "entropy": 1.0069396495819092,
+ "epoch": 4.777777777777778,
+ "grad_norm": 2.2198808193206787,
+ "learning_rate": 2.2424242424242425e-07,
+ "loss": 1.0101712942123413,
+ "mean_token_accuracy": 0.7387640476226807,
+ "num_tokens": 38748160.0,
+ "step": 2365
+ },
+ {
+ "entropy": 1.0643916130065918,
+ "epoch": 4.77979797979798,
+ "grad_norm": 2.1866135597229004,
+ "learning_rate": 2.2222222222222224e-07,
+ "loss": 1.0893548727035522,
+ "mean_token_accuracy": 0.7278944849967957,
+ "num_tokens": 38764544.0,
+ "step": 2366
+ },
+ {
+ "entropy": 1.093658208847046,
+ "epoch": 4.781818181818182,
+ "grad_norm": 2.309128761291504,
+ "learning_rate": 2.202020202020202e-07,
+ "loss": 1.0758569240570068,
+ "mean_token_accuracy": 0.7228871583938599,
+ "num_tokens": 38780928.0,
+ "step": 2367
+ },
+ {
+ "entropy": 1.0118215084075928,
+ "epoch": 4.783838383838384,
+ "grad_norm": 2.3465147018432617,
+ "learning_rate": 2.181818181818182e-07,
+ "loss": 1.0125476121902466,
+ "mean_token_accuracy": 0.7398021221160889,
+ "num_tokens": 38797312.0,
+ "step": 2368
+ },
+ {
+ "entropy": 1.4553667306900024,
+ "epoch": 4.785858585858586,
+ "grad_norm": 2.311668872833252,
+ "learning_rate": 2.1616161616161617e-07,
+ "loss": 1.4538980722427368,
+ "mean_token_accuracy": 0.6701270341873169,
+ "num_tokens": 38813696.0,
+ "step": 2369
+ },
+ {
+ "entropy": 1.4284709692001343,
+ "epoch": 4.787878787878788,
+ "grad_norm": 2.2102832794189453,
+ "learning_rate": 2.1414141414141414e-07,
+ "loss": 1.445950984954834,
+ "mean_token_accuracy": 0.674401581287384,
+ "num_tokens": 38830080.0,
+ "step": 2370
+ },
+ {
+ "entropy": 1.3425463438034058,
+ "epoch": 4.78989898989899,
+ "grad_norm": 2.264972448348999,
+ "learning_rate": 2.1212121212121216e-07,
+ "loss": 1.3158369064331055,
+ "mean_token_accuracy": 0.685271143913269,
+ "num_tokens": 38846464.0,
+ "step": 2371
+ },
+ {
+ "entropy": 1.0617026090621948,
+ "epoch": 4.791919191919192,
+ "grad_norm": 2.3085737228393555,
+ "learning_rate": 2.1010101010101013e-07,
+ "loss": 1.0665439367294312,
+ "mean_token_accuracy": 0.7296653389930725,
+ "num_tokens": 38862848.0,
+ "step": 2372
+ },
+ {
+ "entropy": 1.580702543258667,
+ "epoch": 4.793939393939394,
+ "grad_norm": 2.872378349304199,
+ "learning_rate": 2.080808080808081e-07,
+ "loss": 1.5895251035690308,
+ "mean_token_accuracy": 0.6335490942001343,
+ "num_tokens": 38879232.0,
+ "step": 2373
+ },
+ {
+ "entropy": 1.4491817951202393,
+ "epoch": 4.795959595959596,
+ "grad_norm": 2.2542786598205566,
+ "learning_rate": 2.060606060606061e-07,
+ "loss": 1.4576756954193115,
+ "mean_token_accuracy": 0.669577419757843,
+ "num_tokens": 38895616.0,
+ "step": 2374
+ },
+ {
+ "entropy": 1.1148335933685303,
+ "epoch": 4.797979797979798,
+ "grad_norm": 2.310889959335327,
+ "learning_rate": 2.0404040404040406e-07,
+ "loss": 1.1072784662246704,
+ "mean_token_accuracy": 0.724963366985321,
+ "num_tokens": 38912000.0,
+ "step": 2375
+ },
+ {
+ "entropy": 1.1296288967132568,
+ "epoch": 4.8,
+ "grad_norm": 2.3222877979278564,
+ "learning_rate": 2.0202020202020205e-07,
+ "loss": 1.1322218179702759,
+ "mean_token_accuracy": 0.7090253829956055,
+ "num_tokens": 38928384.0,
+ "step": 2376
+ },
+ {
+ "entropy": 1.0851423740386963,
+ "epoch": 4.802020202020202,
+ "grad_norm": 2.2862296104431152,
+ "learning_rate": 2.0000000000000002e-07,
+ "loss": 1.0714319944381714,
+ "mean_token_accuracy": 0.732046902179718,
+ "num_tokens": 38944768.0,
+ "step": 2377
+ },
+ {
+ "entropy": 1.4850348234176636,
+ "epoch": 4.804040404040404,
+ "grad_norm": 2.2243518829345703,
+ "learning_rate": 1.9797979797979798e-07,
+ "loss": 1.483095407485962,
+ "mean_token_accuracy": 0.6584025621414185,
+ "num_tokens": 38961152.0,
+ "step": 2378
+ },
+ {
+ "entropy": 1.0152039527893066,
+ "epoch": 4.806060606060606,
+ "grad_norm": 2.3812599182128906,
+ "learning_rate": 1.9595959595959598e-07,
+ "loss": 1.0178192853927612,
+ "mean_token_accuracy": 0.7494503855705261,
+ "num_tokens": 38977536.0,
+ "step": 2379
+ },
+ {
+ "entropy": 1.606149435043335,
+ "epoch": 4.808080808080808,
+ "grad_norm": 2.578829050064087,
+ "learning_rate": 1.9393939393939395e-07,
+ "loss": 1.6279492378234863,
+ "mean_token_accuracy": 0.6199926733970642,
+ "num_tokens": 38993920.0,
+ "step": 2380
+ },
+ {
+ "entropy": 1.254974126815796,
+ "epoch": 4.81010101010101,
+ "grad_norm": 2.365147590637207,
+ "learning_rate": 1.919191919191919e-07,
+ "loss": 1.2483952045440674,
+ "mean_token_accuracy": 0.6998656392097473,
+ "num_tokens": 39010304.0,
+ "step": 2381
+ },
+ {
+ "entropy": 1.0788639783859253,
+ "epoch": 4.8121212121212125,
+ "grad_norm": 2.290659189224243,
+ "learning_rate": 1.8989898989898993e-07,
+ "loss": 1.1055562496185303,
+ "mean_token_accuracy": 0.7249022722244263,
+ "num_tokens": 39026688.0,
+ "step": 2382
+ },
+ {
+ "entropy": 1.4539564847946167,
+ "epoch": 4.814141414141414,
+ "grad_norm": 2.395479440689087,
+ "learning_rate": 1.878787878787879e-07,
+ "loss": 1.4771209955215454,
+ "mean_token_accuracy": 0.6563263535499573,
+ "num_tokens": 39043072.0,
+ "step": 2383
+ },
+ {
+ "entropy": 0.9699754118919373,
+ "epoch": 4.816161616161616,
+ "grad_norm": 2.471874713897705,
+ "learning_rate": 1.858585858585859e-07,
+ "loss": 0.9586566686630249,
+ "mean_token_accuracy": 0.7446262836456299,
+ "num_tokens": 39059456.0,
+ "step": 2384
+ },
+ {
+ "entropy": 1.3050302267074585,
+ "epoch": 4.818181818181818,
+ "grad_norm": 2.2205452919006348,
+ "learning_rate": 1.8383838383838386e-07,
+ "loss": 1.3049702644348145,
+ "mean_token_accuracy": 0.6822789311408997,
+ "num_tokens": 39075840.0,
+ "step": 2385
+ },
+ {
+ "entropy": 1.1255296468734741,
+ "epoch": 4.82020202020202,
+ "grad_norm": 2.274259328842163,
+ "learning_rate": 1.8181818181818183e-07,
+ "loss": 1.1260244846343994,
+ "mean_token_accuracy": 0.7134220600128174,
+ "num_tokens": 39092224.0,
+ "step": 2386
+ },
+ {
+ "entropy": 0.9318177103996277,
+ "epoch": 4.822222222222222,
+ "grad_norm": 2.0648000240325928,
+ "learning_rate": 1.7979797979797982e-07,
+ "loss": 0.9120785593986511,
+ "mean_token_accuracy": 0.7661211490631104,
+ "num_tokens": 39108608.0,
+ "step": 2387
+ },
+ {
+ "entropy": 1.0134271383285522,
+ "epoch": 4.824242424242424,
+ "grad_norm": 2.260668992996216,
+ "learning_rate": 1.777777777777778e-07,
+ "loss": 1.0051692724227905,
+ "mean_token_accuracy": 0.7439545392990112,
+ "num_tokens": 39124992.0,
+ "step": 2388
+ },
+ {
+ "entropy": 1.201913833618164,
+ "epoch": 4.8262626262626265,
+ "grad_norm": 2.3676605224609375,
+ "learning_rate": 1.7575757575757576e-07,
+ "loss": 1.2044906616210938,
+ "mean_token_accuracy": 0.7144601941108704,
+ "num_tokens": 39141376.0,
+ "step": 2389
+ },
+ {
+ "entropy": 1.2177425622940063,
+ "epoch": 4.828282828282829,
+ "grad_norm": 2.068251609802246,
+ "learning_rate": 1.7373737373737375e-07,
+ "loss": 1.2142643928527832,
+ "mean_token_accuracy": 0.7080483436584473,
+ "num_tokens": 39157760.0,
+ "step": 2390
+ },
+ {
+ "entropy": 1.4004939794540405,
+ "epoch": 4.83030303030303,
+ "grad_norm": 2.4678943157196045,
+ "learning_rate": 1.7171717171717172e-07,
+ "loss": 1.382983922958374,
+ "mean_token_accuracy": 0.6609672904014587,
+ "num_tokens": 39174144.0,
+ "step": 2391
+ },
+ {
+ "entropy": 1.2480416297912598,
+ "epoch": 4.832323232323232,
+ "grad_norm": 2.2275490760803223,
+ "learning_rate": 1.6969696969696974e-07,
+ "loss": 1.2442829608917236,
+ "mean_token_accuracy": 0.6988885998725891,
+ "num_tokens": 39190528.0,
+ "step": 2392
+ },
+ {
+ "entropy": 1.175554633140564,
+ "epoch": 4.834343434343435,
+ "grad_norm": 2.3274128437042236,
+ "learning_rate": 1.676767676767677e-07,
+ "loss": 1.1996537446975708,
+ "mean_token_accuracy": 0.7000488638877869,
+ "num_tokens": 39206912.0,
+ "step": 2393
+ },
+ {
+ "entropy": 1.129797339439392,
+ "epoch": 4.836363636363636,
+ "grad_norm": 2.0076446533203125,
+ "learning_rate": 1.6565656565656567e-07,
+ "loss": 1.1391470432281494,
+ "mean_token_accuracy": 0.7203224301338196,
+ "num_tokens": 39223296.0,
+ "step": 2394
+ },
+ {
+ "entropy": 1.3851672410964966,
+ "epoch": 4.838383838383838,
+ "grad_norm": 2.322408676147461,
+ "learning_rate": 1.6363636363636367e-07,
+ "loss": 1.3847455978393555,
+ "mean_token_accuracy": 0.6731191873550415,
+ "num_tokens": 39239680.0,
+ "step": 2395
+ },
+ {
+ "entropy": 0.807838499546051,
+ "epoch": 4.8404040404040405,
+ "grad_norm": 2.1578664779663086,
+ "learning_rate": 1.6161616161616163e-07,
+ "loss": 0.790963888168335,
+ "mean_token_accuracy": 0.7919516563415527,
+ "num_tokens": 39256064.0,
+ "step": 2396
+ },
+ {
+ "entropy": 1.3650189638137817,
+ "epoch": 4.842424242424243,
+ "grad_norm": 2.470627546310425,
+ "learning_rate": 1.595959595959596e-07,
+ "loss": 1.352724552154541,
+ "mean_token_accuracy": 0.6680508255958557,
+ "num_tokens": 39272448.0,
+ "step": 2397
+ },
+ {
+ "entropy": 0.9988412261009216,
+ "epoch": 4.844444444444444,
+ "grad_norm": 2.3024678230285645,
+ "learning_rate": 1.575757575757576e-07,
+ "loss": 1.0097228288650513,
+ "mean_token_accuracy": 0.7401074767112732,
+ "num_tokens": 39288832.0,
+ "step": 2398
+ },
+ {
+ "entropy": 1.2273977994918823,
+ "epoch": 4.846464646464646,
+ "grad_norm": 2.496877908706665,
+ "learning_rate": 1.5555555555555556e-07,
+ "loss": 1.2331998348236084,
+ "mean_token_accuracy": 0.6988885998725891,
+ "num_tokens": 39305216.0,
+ "step": 2399
+ },
+ {
+ "entropy": 1.1379625797271729,
+ "epoch": 4.848484848484849,
+ "grad_norm": 2.2165470123291016,
+ "learning_rate": 1.5353535353535356e-07,
+ "loss": 1.1311036348342896,
+ "mean_token_accuracy": 0.7255129218101501,
+ "num_tokens": 39321600.0,
+ "step": 2400
+ },
+ {
+ "epoch": 4.848484848484849,
+ "eval_entropy": 1.287453404838039,
+ "eval_loss": 1.5620986223220825,
+ "eval_mean_token_accuracy": 0.6438380523074058,
+ "eval_num_tokens": 39321600.0,
+ "eval_runtime": 43.7382,
+ "eval_samples_per_second": 22.635,
+ "eval_steps_per_second": 2.835,
+ "step": 2400
+ },
+ {
+ "entropy": 0.8480889797210693,
+ "epoch": 4.850505050505051,
+ "grad_norm": 2.2270820140838623,
+ "learning_rate": 1.5151515151515152e-07,
+ "loss": 0.8555670976638794,
+ "mean_token_accuracy": 0.7665486335754395,
+ "num_tokens": 39337984.0,
+ "step": 2401
+ },
+ {
+ "entropy": 0.9117488265037537,
+ "epoch": 4.852525252525252,
+ "grad_norm": 2.1996190547943115,
+ "learning_rate": 1.4949494949494952e-07,
+ "loss": 0.9289630651473999,
+ "mean_token_accuracy": 0.7600146532058716,
+ "num_tokens": 39354368.0,
+ "step": 2402
+ },
+ {
+ "entropy": 1.3071238994598389,
+ "epoch": 4.8545454545454545,
+ "grad_norm": 2.178358793258667,
+ "learning_rate": 1.4747474747474749e-07,
+ "loss": 1.3033668994903564,
+ "mean_token_accuracy": 0.6831338405609131,
+ "num_tokens": 39370752.0,
+ "step": 2403
+ },
+ {
+ "entropy": 1.4874638319015503,
+ "epoch": 4.856565656565657,
+ "grad_norm": 2.360041379928589,
+ "learning_rate": 1.4545454545454548e-07,
+ "loss": 1.4955557584762573,
+ "mean_token_accuracy": 0.6526013612747192,
+ "num_tokens": 39387136.0,
+ "step": 2404
+ },
+ {
+ "entropy": 1.1542364358901978,
+ "epoch": 4.858585858585858,
+ "grad_norm": 2.1776201725006104,
+ "learning_rate": 1.4343434343434345e-07,
+ "loss": 1.1404229402542114,
+ "mean_token_accuracy": 0.7199560403823853,
+ "num_tokens": 39403520.0,
+ "step": 2405
+ },
+ {
+ "entropy": 1.2594319581985474,
+ "epoch": 4.86060606060606,
+ "grad_norm": 2.144207715988159,
+ "learning_rate": 1.4141414141414144e-07,
+ "loss": 1.272618055343628,
+ "mean_token_accuracy": 0.7062774896621704,
+ "num_tokens": 39419904.0,
+ "step": 2406
+ },
+ {
+ "entropy": 1.4888932704925537,
+ "epoch": 4.862626262626263,
+ "grad_norm": 2.461423873901367,
+ "learning_rate": 1.393939393939394e-07,
+ "loss": 1.4978430271148682,
+ "mean_token_accuracy": 0.648937463760376,
+ "num_tokens": 39436288.0,
+ "step": 2407
+ },
+ {
+ "entropy": 1.2378987073898315,
+ "epoch": 4.864646464646465,
+ "grad_norm": 2.3147332668304443,
+ "learning_rate": 1.3737373737373738e-07,
+ "loss": 1.251128077507019,
+ "mean_token_accuracy": 0.7049340605735779,
+ "num_tokens": 39452672.0,
+ "step": 2408
+ },
+ {
+ "entropy": 1.0804370641708374,
+ "epoch": 4.866666666666667,
+ "grad_norm": 2.2359390258789062,
+ "learning_rate": 1.3535353535353537e-07,
+ "loss": 1.0663418769836426,
+ "mean_token_accuracy": 0.7359550595283508,
+ "num_tokens": 39469056.0,
+ "step": 2409
+ },
+ {
+ "entropy": 0.9537230730056763,
+ "epoch": 4.8686868686868685,
+ "grad_norm": 2.1780338287353516,
+ "learning_rate": 1.3333333333333336e-07,
+ "loss": 0.9360350370407104,
+ "mean_token_accuracy": 0.755984365940094,
+ "num_tokens": 39485440.0,
+ "step": 2410
+ },
+ {
+ "entropy": 1.2282828092575073,
+ "epoch": 4.870707070707071,
+ "grad_norm": 2.4646949768066406,
+ "learning_rate": 1.3131313131313133e-07,
+ "loss": 1.2513031959533691,
+ "mean_token_accuracy": 0.6998656392097473,
+ "num_tokens": 39501824.0,
+ "step": 2411
+ },
+ {
+ "entropy": 0.9471758008003235,
+ "epoch": 4.872727272727273,
+ "grad_norm": 2.4472343921661377,
+ "learning_rate": 1.292929292929293e-07,
+ "loss": 0.9313589334487915,
+ "mean_token_accuracy": 0.745175838470459,
+ "num_tokens": 39518208.0,
+ "step": 2412
+ },
+ {
+ "entropy": 1.5314515829086304,
+ "epoch": 4.874747474747474,
+ "grad_norm": 2.40248703956604,
+ "learning_rate": 1.272727272727273e-07,
+ "loss": 1.513268232345581,
+ "mean_token_accuracy": 0.6563873887062073,
+ "num_tokens": 39534592.0,
+ "step": 2413
+ },
+ {
+ "entropy": 1.4170082807540894,
+ "epoch": 4.876767676767677,
+ "grad_norm": 2.513214588165283,
+ "learning_rate": 1.2525252525252526e-07,
+ "loss": 1.4214344024658203,
+ "mean_token_accuracy": 0.6609672904014587,
+ "num_tokens": 39550976.0,
+ "step": 2414
+ },
+ {
+ "entropy": 1.2669848203659058,
+ "epoch": 4.878787878787879,
+ "grad_norm": 2.3023788928985596,
+ "learning_rate": 1.2323232323232323e-07,
+ "loss": 1.2646489143371582,
+ "mean_token_accuracy": 0.6824621558189392,
+ "num_tokens": 39567360.0,
+ "step": 2415
+ },
+ {
+ "entropy": 1.4063810110092163,
+ "epoch": 4.880808080808081,
+ "grad_norm": 2.303424596786499,
+ "learning_rate": 1.2121212121212122e-07,
+ "loss": 1.399657964706421,
+ "mean_token_accuracy": 0.6720811128616333,
+ "num_tokens": 39583744.0,
+ "step": 2416
+ },
+ {
+ "entropy": 1.4641772508621216,
+ "epoch": 4.8828282828282825,
+ "grad_norm": 2.468921661376953,
+ "learning_rate": 1.191919191919192e-07,
+ "loss": 1.46146559715271,
+ "mean_token_accuracy": 0.65492182970047,
+ "num_tokens": 39600128.0,
+ "step": 2417
+ },
+ {
+ "entropy": 1.2154735326766968,
+ "epoch": 4.884848484848485,
+ "grad_norm": 2.429311752319336,
+ "learning_rate": 1.171717171717172e-07,
+ "loss": 1.2101836204528809,
+ "mean_token_accuracy": 0.6988885998725891,
+ "num_tokens": 39616512.0,
+ "step": 2418
+ },
+ {
+ "entropy": 0.8925865888595581,
+ "epoch": 4.886868686868687,
+ "grad_norm": 2.281524419784546,
+ "learning_rate": 1.1515151515151516e-07,
+ "loss": 0.884310245513916,
+ "mean_token_accuracy": 0.7589154839515686,
+ "num_tokens": 39632896.0,
+ "step": 2419
+ },
+ {
+ "entropy": 1.2473207712173462,
+ "epoch": 4.888888888888889,
+ "grad_norm": 2.0147485733032227,
+ "learning_rate": 1.1313131313131314e-07,
+ "loss": 1.2424726486206055,
+ "mean_token_accuracy": 0.7096971273422241,
+ "num_tokens": 39649280.0,
+ "step": 2420
+ },
+ {
+ "entropy": 1.176668405532837,
+ "epoch": 4.890909090909091,
+ "grad_norm": 2.4631848335266113,
+ "learning_rate": 1.1111111111111112e-07,
+ "loss": 1.173363208770752,
+ "mean_token_accuracy": 0.6982779502868652,
+ "num_tokens": 39665664.0,
+ "step": 2421
+ },
+ {
+ "entropy": 1.1733225584030151,
+ "epoch": 4.892929292929293,
+ "grad_norm": 2.567352294921875,
+ "learning_rate": 1.090909090909091e-07,
+ "loss": 1.1714212894439697,
+ "mean_token_accuracy": 0.7132999300956726,
+ "num_tokens": 39682048.0,
+ "step": 2422
+ },
+ {
+ "entropy": 0.9787822961807251,
+ "epoch": 4.894949494949495,
+ "grad_norm": 2.5099260807037354,
+ "learning_rate": 1.0707070707070707e-07,
+ "loss": 0.9771278500556946,
+ "mean_token_accuracy": 0.7329018115997314,
+ "num_tokens": 39698432.0,
+ "step": 2423
+ },
+ {
+ "entropy": 0.9620134234428406,
+ "epoch": 4.8969696969696965,
+ "grad_norm": 2.085623025894165,
+ "learning_rate": 1.0505050505050506e-07,
+ "loss": 0.9678645133972168,
+ "mean_token_accuracy": 0.7519540786743164,
+ "num_tokens": 39714816.0,
+ "step": 2424
+ },
+ {
+ "entropy": 1.3669517040252686,
+ "epoch": 4.898989898989899,
+ "grad_norm": 2.2397220134735107,
+ "learning_rate": 1.0303030303030304e-07,
+ "loss": 1.3635847568511963,
+ "mean_token_accuracy": 0.6789814233779907,
+ "num_tokens": 39731200.0,
+ "step": 2425
+ },
+ {
+ "entropy": 1.088536262512207,
+ "epoch": 4.901010101010101,
+ "grad_norm": 2.311772108078003,
+ "learning_rate": 1.0101010101010103e-07,
+ "loss": 1.074340581893921,
+ "mean_token_accuracy": 0.7220932841300964,
+ "num_tokens": 39747584.0,
+ "step": 2426
+ },
+ {
+ "entropy": 1.3117040395736694,
+ "epoch": 4.903030303030303,
+ "grad_norm": 2.3678650856018066,
+ "learning_rate": 9.898989898989899e-08,
+ "loss": 1.3200736045837402,
+ "mean_token_accuracy": 0.6888129115104675,
+ "num_tokens": 39763968.0,
+ "step": 2427
+ },
+ {
+ "entropy": 1.0334731340408325,
+ "epoch": 4.9050505050505055,
+ "grad_norm": 2.244715452194214,
+ "learning_rate": 9.696969696969697e-08,
+ "loss": 1.0399707555770874,
+ "mean_token_accuracy": 0.7382755279541016,
+ "num_tokens": 39780352.0,
+ "step": 2428
+ },
+ {
+ "entropy": 1.2658854722976685,
+ "epoch": 4.907070707070707,
+ "grad_norm": 2.3209128379821777,
+ "learning_rate": 9.494949494949497e-08,
+ "loss": 1.2778537273406982,
+ "mean_token_accuracy": 0.6900952458381653,
+ "num_tokens": 39796736.0,
+ "step": 2429
+ },
+ {
+ "entropy": 1.4913005828857422,
+ "epoch": 4.909090909090909,
+ "grad_norm": 2.4210398197174072,
+ "learning_rate": 9.292929292929295e-08,
+ "loss": 1.5127770900726318,
+ "mean_token_accuracy": 0.641792893409729,
+ "num_tokens": 39813120.0,
+ "step": 2430
+ },
+ {
+ "entropy": 1.7913345098495483,
+ "epoch": 4.911111111111111,
+ "grad_norm": 2.3230996131896973,
+ "learning_rate": 9.090909090909091e-08,
+ "loss": 1.806335687637329,
+ "mean_token_accuracy": 0.5915364027023315,
+ "num_tokens": 39829504.0,
+ "step": 2431
+ },
+ {
+ "entropy": 1.3686211109161377,
+ "epoch": 4.913131313131313,
+ "grad_norm": 2.3728842735290527,
+ "learning_rate": 8.88888888888889e-08,
+ "loss": 1.3729017972946167,
+ "mean_token_accuracy": 0.6770273447036743,
+ "num_tokens": 39845888.0,
+ "step": 2432
+ },
+ {
+ "entropy": 1.532094120979309,
+ "epoch": 4.915151515151515,
+ "grad_norm": 2.3725192546844482,
+ "learning_rate": 8.686868686868688e-08,
+ "loss": 1.5053999423980713,
+ "mean_token_accuracy": 0.6447240114212036,
+ "num_tokens": 39862272.0,
+ "step": 2433
+ },
+ {
+ "entropy": 1.1579054594039917,
+ "epoch": 4.917171717171717,
+ "grad_norm": 2.4051802158355713,
+ "learning_rate": 8.484848484848487e-08,
+ "loss": 1.1492040157318115,
+ "mean_token_accuracy": 0.7176966071128845,
+ "num_tokens": 39878656.0,
+ "step": 2434
+ },
+ {
+ "entropy": 1.1870861053466797,
+ "epoch": 4.91919191919192,
+ "grad_norm": 2.2848308086395264,
+ "learning_rate": 8.282828282828284e-08,
+ "loss": 1.161110281944275,
+ "mean_token_accuracy": 0.7132999300956726,
+ "num_tokens": 39895040.0,
+ "step": 2435
+ },
+ {
+ "entropy": 0.993489682674408,
+ "epoch": 4.921212121212121,
+ "grad_norm": 2.209522247314453,
+ "learning_rate": 8.080808080808082e-08,
+ "loss": 0.9955021142959595,
+ "mean_token_accuracy": 0.7460307478904724,
+ "num_tokens": 39911424.0,
+ "step": 2436
+ },
+ {
+ "entropy": 0.8698616027832031,
+ "epoch": 4.923232323232323,
+ "grad_norm": 2.0848515033721924,
+ "learning_rate": 7.87878787878788e-08,
+ "loss": 0.8670492172241211,
+ "mean_token_accuracy": 0.7761968970298767,
+ "num_tokens": 39927808.0,
+ "step": 2437
+ },
+ {
+ "entropy": 1.2253624200820923,
+ "epoch": 4.925252525252525,
+ "grad_norm": 2.166091203689575,
+ "learning_rate": 7.676767676767678e-08,
+ "loss": 1.1965932846069336,
+ "mean_token_accuracy": 0.7053004503250122,
+ "num_tokens": 39944192.0,
+ "step": 2438
+ },
+ {
+ "entropy": 1.1075690984725952,
+ "epoch": 4.927272727272728,
+ "grad_norm": 2.2962300777435303,
+ "learning_rate": 7.474747474747476e-08,
+ "loss": 1.1296148300170898,
+ "mean_token_accuracy": 0.7225818037986755,
+ "num_tokens": 39960576.0,
+ "step": 2439
+ },
+ {
+ "entropy": 1.234410047531128,
+ "epoch": 4.929292929292929,
+ "grad_norm": 2.331155776977539,
+ "learning_rate": 7.272727272727274e-08,
+ "loss": 1.2488322257995605,
+ "mean_token_accuracy": 0.6974230408668518,
+ "num_tokens": 39976960.0,
+ "step": 2440
+ },
+ {
+ "entropy": 1.4327800273895264,
+ "epoch": 4.931313131313131,
+ "grad_norm": 2.460622787475586,
+ "learning_rate": 7.070707070707072e-08,
+ "loss": 1.4103214740753174,
+ "mean_token_accuracy": 0.6623717546463013,
+ "num_tokens": 39993344.0,
+ "step": 2441
+ },
+ {
+ "entropy": 1.3475353717803955,
+ "epoch": 4.933333333333334,
+ "grad_norm": 2.5242390632629395,
+ "learning_rate": 6.868686868686869e-08,
+ "loss": 1.3579360246658325,
+ "mean_token_accuracy": 0.6707987189292908,
+ "num_tokens": 40009728.0,
+ "step": 2442
+ },
+ {
+ "entropy": 1.259186029434204,
+ "epoch": 4.935353535353535,
+ "grad_norm": 2.615501642227173,
+ "learning_rate": 6.666666666666668e-08,
+ "loss": 1.2570360898971558,
+ "mean_token_accuracy": 0.6861870884895325,
+ "num_tokens": 40026112.0,
+ "step": 2443
+ },
+ {
+ "entropy": 1.1449041366577148,
+ "epoch": 4.937373737373737,
+ "grad_norm": 2.3570308685302734,
+ "learning_rate": 6.464646464646465e-08,
+ "loss": 1.1318464279174805,
+ "mean_token_accuracy": 0.7102466821670532,
+ "num_tokens": 40042496.0,
+ "step": 2444
+ },
+ {
+ "entropy": 0.9617025852203369,
+ "epoch": 4.9393939393939394,
+ "grad_norm": 2.316823959350586,
+ "learning_rate": 6.262626262626263e-08,
+ "loss": 0.9381533861160278,
+ "mean_token_accuracy": 0.7504274845123291,
+ "num_tokens": 40058880.0,
+ "step": 2445
+ },
+ {
+ "entropy": 1.0141503810882568,
+ "epoch": 4.941414141414142,
+ "grad_norm": 2.1891255378723145,
+ "learning_rate": 6.060606060606061e-08,
+ "loss": 1.015519618988037,
+ "mean_token_accuracy": 0.738092303276062,
+ "num_tokens": 40075264.0,
+ "step": 2446
+ },
+ {
+ "entropy": 1.2403353452682495,
+ "epoch": 4.943434343434343,
+ "grad_norm": 2.4201672077178955,
+ "learning_rate": 5.85858585858586e-08,
+ "loss": 1.2536835670471191,
+ "mean_token_accuracy": 0.6921714544296265,
+ "num_tokens": 40091648.0,
+ "step": 2447
+ },
+ {
+ "entropy": 1.2879153490066528,
+ "epoch": 4.945454545454545,
+ "grad_norm": 2.2885098457336426,
+ "learning_rate": 5.656565656565657e-08,
+ "loss": 1.2876033782958984,
+ "mean_token_accuracy": 0.7035295367240906,
+ "num_tokens": 40108032.0,
+ "step": 2448
+ },
+ {
+ "entropy": 1.6080931425094604,
+ "epoch": 4.947474747474748,
+ "grad_norm": 2.2233543395996094,
+ "learning_rate": 5.454545454545455e-08,
+ "loss": 1.6212408542633057,
+ "mean_token_accuracy": 0.63312166929245,
+ "num_tokens": 40124416.0,
+ "step": 2449
+ },
+ {
+ "entropy": 1.1142158508300781,
+ "epoch": 4.94949494949495,
+ "grad_norm": 2.270122766494751,
+ "learning_rate": 5.252525252525253e-08,
+ "loss": 1.1052987575531006,
+ "mean_token_accuracy": 0.7298485636711121,
+ "num_tokens": 40140800.0,
+ "step": 2450
+ },
+ {
+ "epoch": 4.94949494949495,
+ "eval_entropy": 1.2876735839151567,
+ "eval_loss": 1.562165379524231,
+ "eval_mean_token_accuracy": 0.6437897917724424,
+ "eval_num_tokens": 40140800.0,
+ "eval_runtime": 43.7403,
+ "eval_samples_per_second": 22.634,
+ "eval_steps_per_second": 2.835,
+ "step": 2450
+ },
+ {
+ "entropy": 1.339135766029358,
+ "epoch": 4.951515151515151,
+ "grad_norm": 2.068509101867676,
+ "learning_rate": 5.050505050505051e-08,
+ "loss": 1.3422555923461914,
+ "mean_token_accuracy": 0.6739740967750549,
+ "num_tokens": 40157184.0,
+ "step": 2451
+ },
+ {
+ "entropy": 1.086835265159607,
+ "epoch": 4.9535353535353535,
+ "grad_norm": 2.3591971397399902,
+ "learning_rate": 4.8484848484848486e-08,
+ "loss": 1.0669522285461426,
+ "mean_token_accuracy": 0.7236199378967285,
+ "num_tokens": 40173568.0,
+ "step": 2452
+ },
+ {
+ "entropy": 1.2980481386184692,
+ "epoch": 4.955555555555556,
+ "grad_norm": 2.3696742057800293,
+ "learning_rate": 4.6464646464646474e-08,
+ "loss": 1.313071370124817,
+ "mean_token_accuracy": 0.6906448602676392,
+ "num_tokens": 40189952.0,
+ "step": 2453
+ },
+ {
+ "entropy": 1.1795636415481567,
+ "epoch": 4.957575757575757,
+ "grad_norm": 2.3436481952667236,
+ "learning_rate": 4.444444444444445e-08,
+ "loss": 1.1898691654205322,
+ "mean_token_accuracy": 0.7040791511535645,
+ "num_tokens": 40206336.0,
+ "step": 2454
+ },
+ {
+ "entropy": 0.8319778442382812,
+ "epoch": 4.959595959595959,
+ "grad_norm": 2.1321425437927246,
+ "learning_rate": 4.2424242424242435e-08,
+ "loss": 0.8329704999923706,
+ "mean_token_accuracy": 0.7830972075462341,
+ "num_tokens": 40222720.0,
+ "step": 2455
+ },
+ {
+ "entropy": 1.052415132522583,
+ "epoch": 4.961616161616162,
+ "grad_norm": 2.2577855587005615,
+ "learning_rate": 4.040404040404041e-08,
+ "loss": 1.0562504529953003,
+ "mean_token_accuracy": 0.733146071434021,
+ "num_tokens": 40239104.0,
+ "step": 2456
+ },
+ {
+ "entropy": 1.0988104343414307,
+ "epoch": 4.963636363636364,
+ "grad_norm": 2.401578187942505,
+ "learning_rate": 3.838383838383839e-08,
+ "loss": 1.1040318012237549,
+ "mean_token_accuracy": 0.72013920545578,
+ "num_tokens": 40255488.0,
+ "step": 2457
+ },
+ {
+ "entropy": 1.3352779150009155,
+ "epoch": 4.965656565656566,
+ "grad_norm": 2.185018301010132,
+ "learning_rate": 3.636363636363637e-08,
+ "loss": 1.32777738571167,
+ "mean_token_accuracy": 0.6927210688591003,
+ "num_tokens": 40271872.0,
+ "step": 2458
+ },
+ {
+ "entropy": 1.740423560142517,
+ "epoch": 4.9676767676767675,
+ "grad_norm": 2.3727855682373047,
+ "learning_rate": 3.4343434343434344e-08,
+ "loss": 1.7246336936950684,
+ "mean_token_accuracy": 0.6031387448310852,
+ "num_tokens": 40288256.0,
+ "step": 2459
+ },
+ {
+ "entropy": 1.2249867916107178,
+ "epoch": 4.96969696969697,
+ "grad_norm": 2.190229654312134,
+ "learning_rate": 3.2323232323232324e-08,
+ "loss": 1.2399539947509766,
+ "mean_token_accuracy": 0.7046898007392883,
+ "num_tokens": 40304640.0,
+ "step": 2460
+ },
+ {
+ "entropy": 1.051764726638794,
+ "epoch": 4.971717171717172,
+ "grad_norm": 2.1387240886688232,
+ "learning_rate": 3.0303030303030305e-08,
+ "loss": 1.0502032041549683,
+ "mean_token_accuracy": 0.7348558902740479,
+ "num_tokens": 40321024.0,
+ "step": 2461
+ },
+ {
+ "entropy": 0.8195315599441528,
+ "epoch": 4.973737373737373,
+ "grad_norm": 2.1464476585388184,
+ "learning_rate": 2.8282828282828285e-08,
+ "loss": 0.8101259469985962,
+ "mean_token_accuracy": 0.77992182970047,
+ "num_tokens": 40337408.0,
+ "step": 2462
+ },
+ {
+ "entropy": 0.9888697266578674,
+ "epoch": 4.975757575757576,
+ "grad_norm": 2.357499837875366,
+ "learning_rate": 2.6262626262626266e-08,
+ "loss": 0.9762573838233948,
+ "mean_token_accuracy": 0.7430996298789978,
+ "num_tokens": 40353792.0,
+ "step": 2463
+ },
+ {
+ "entropy": 1.4075063467025757,
+ "epoch": 4.977777777777778,
+ "grad_norm": 2.2885241508483887,
+ "learning_rate": 2.4242424242424243e-08,
+ "loss": 1.3647881746292114,
+ "mean_token_accuracy": 0.6737909317016602,
+ "num_tokens": 40370176.0,
+ "step": 2464
+ },
+ {
+ "entropy": 0.6581518054008484,
+ "epoch": 4.97979797979798,
+ "grad_norm": 2.228896379470825,
+ "learning_rate": 2.2222222222222224e-08,
+ "loss": 0.6321805715560913,
+ "mean_token_accuracy": 0.8178431987762451,
+ "num_tokens": 40386560.0,
+ "step": 2465
+ },
+ {
+ "entropy": 1.0636364221572876,
+ "epoch": 4.9818181818181815,
+ "grad_norm": 2.3927156925201416,
+ "learning_rate": 2.0202020202020204e-08,
+ "loss": 1.0664286613464355,
+ "mean_token_accuracy": 0.7248412370681763,
+ "num_tokens": 40402944.0,
+ "step": 2466
+ },
+ {
+ "entropy": 0.9568659663200378,
+ "epoch": 4.983838383838384,
+ "grad_norm": 2.3572115898132324,
+ "learning_rate": 1.8181818181818185e-08,
+ "loss": 0.9277185201644897,
+ "mean_token_accuracy": 0.7493282556533813,
+ "num_tokens": 40419328.0,
+ "step": 2467
+ },
+ {
+ "entropy": 1.0510457754135132,
+ "epoch": 4.985858585858586,
+ "grad_norm": 2.425138473510742,
+ "learning_rate": 1.6161616161616162e-08,
+ "loss": 1.0535608530044556,
+ "mean_token_accuracy": 0.7256961464881897,
+ "num_tokens": 40435712.0,
+ "step": 2468
+ },
+ {
+ "entropy": 0.7372819185256958,
+ "epoch": 4.987878787878788,
+ "grad_norm": 2.304194688796997,
+ "learning_rate": 1.4141414141414143e-08,
+ "loss": 0.7333900332450867,
+ "mean_token_accuracy": 0.7998901009559631,
+ "num_tokens": 40452096.0,
+ "step": 2469
+ },
+ {
+ "entropy": 1.5864061117172241,
+ "epoch": 4.98989898989899,
+ "grad_norm": 2.5311288833618164,
+ "learning_rate": 1.2121212121212122e-08,
+ "loss": 1.5565006732940674,
+ "mean_token_accuracy": 0.6236565709114075,
+ "num_tokens": 40468480.0,
+ "step": 2470
+ },
+ {
+ "entropy": 1.3043371438980103,
+ "epoch": 4.991919191919192,
+ "grad_norm": 2.3082878589630127,
+ "learning_rate": 1.0101010101010102e-08,
+ "loss": 1.3108078241348267,
+ "mean_token_accuracy": 0.6911944150924683,
+ "num_tokens": 40484864.0,
+ "step": 2471
+ },
+ {
+ "entropy": 0.968794584274292,
+ "epoch": 4.993939393939394,
+ "grad_norm": 2.276085376739502,
+ "learning_rate": 8.080808080808081e-09,
+ "loss": 0.9717127084732056,
+ "mean_token_accuracy": 0.7609916925430298,
+ "num_tokens": 40501248.0,
+ "step": 2472
+ },
+ {
+ "entropy": 1.117543339729309,
+ "epoch": 4.9959595959595955,
+ "grad_norm": 2.148294448852539,
+ "learning_rate": 6.060606060606061e-09,
+ "loss": 1.1264331340789795,
+ "mean_token_accuracy": 0.727161705493927,
+ "num_tokens": 40517632.0,
+ "step": 2473
+ },
+ {
+ "entropy": 1.251431941986084,
+ "epoch": 4.997979797979798,
+ "grad_norm": 2.5389761924743652,
+ "learning_rate": 4.0404040404040405e-09,
+ "loss": 1.261974573135376,
+ "mean_token_accuracy": 0.689728856086731,
+ "num_tokens": 40534016.0,
+ "step": 2474
+ },
+ {
+ "entropy": 1.1107192039489746,
+ "epoch": 5.0,
+ "grad_norm": 2.3861618041992188,
+ "learning_rate": 2.0202020202020203e-09,
+ "loss": 1.0901215076446533,
+ "mean_token_accuracy": 0.7343673706054688,
+ "num_tokens": 40550400.0,
+ "step": 2475
+ },
+ {
+ "epoch": 5.0,
+ "eval_entropy": 1.287564484342452,
+ "eval_loss": 1.5621511936187744,
+ "eval_mean_token_accuracy": 0.6438104760262274,
+ "eval_num_tokens": 40550400.0,
+ "eval_runtime": 43.7155,
+ "eval_samples_per_second": 22.646,
+ "eval_steps_per_second": 2.837,
+ "step": 2475
+ }
+ ],
+ "logging_steps": 1,
+ "max_steps": 2475,
+ "num_input_tokens_seen": 0,
+ "num_train_epochs": 5,
+ "save_steps": 500,
+ "stateful_callbacks": {
+ "TrainerControl": {
+ "args": {
+ "should_epoch_stop": false,
+ "should_evaluate": false,
+ "should_log": false,
+ "should_save": true,
+ "should_training_stop": true
+ },
+ "attributes": {}
+ }
+ },
+ "total_flos": 3.429128472625152e+17,
+ "train_batch_size": 4,
+ "trial_name": null,
+ "trial_params": null
+}
diff --git a/qwen3-1.7b-teutonic-5e6/checkpoint-2475/training_args.bin b/qwen3-1.7b-teutonic-5e6/checkpoint-2475/training_args.bin
new file mode 100644
index 0000000000000000000000000000000000000000..37649d4bae96a1df88666daf20b4e5b6e092d976
--- /dev/null
+++ b/qwen3-1.7b-teutonic-5e6/checkpoint-2475/training_args.bin
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:58bbcabfe99a1cf24f2e76aab66b507c2b720d9271dc9f17c8208d741a9c3a65
+size 7121
diff --git a/qwen3-1.7b-teutonic-5e6/checkpoint-2475/zero_to_fp32.py b/qwen3-1.7b-teutonic-5e6/checkpoint-2475/zero_to_fp32.py
new file mode 100644
index 0000000000000000000000000000000000000000..3970548c400fd4361bd923b763db90e963ddaf8c
--- /dev/null
+++ b/qwen3-1.7b-teutonic-5e6/checkpoint-2475/zero_to_fp32.py
@@ -0,0 +1,854 @@
+#!/usr/bin/env python
+
+# Copyright (c) Microsoft Corporation.
+# SPDX-License-Identifier: Apache-2.0
+
+# DeepSpeed Team
+
+# This script extracts fp32 consolidated weights from a zero 1, 2 and 3 DeepSpeed checkpoints. It gets
+# copied into the top level checkpoint dir, so the user can easily do the conversion at any point in
+# the future. Once extracted, the weights don't require DeepSpeed and can be used in any
+# application.
+#
+# example:
+# python zero_to_fp32.py . output_dir/
+# or
+# python zero_to_fp32.py . output_dir/ --safe_serialization
+
+import argparse
+import torch
+import glob
+import math
+import os
+import re
+import gc
+import json
+import numpy as np
+from tqdm import tqdm
+from collections import OrderedDict
+from dataclasses import dataclass
+
+# while this script doesn't use deepspeed to recover data, since the checkpoints are pickled with
+# DeepSpeed data structures it has to be available in the current python environment.
+from deepspeed.utils import logger
+from deepspeed.checkpoint.constants import (DS_VERSION, OPTIMIZER_STATE_DICT, SINGLE_PARTITION_OF_FP32_GROUPS,
+ FP32_FLAT_GROUPS, ZERO_STAGE, PARTITION_COUNT, PARAM_SHAPES, BUFFER_NAMES,
+ FROZEN_PARAM_SHAPES, FROZEN_PARAM_FRAGMENTS, AUTOEP_LAYERS_KEY,
+ AUTOEP_LAYERS_KEY_LEGACY, AUTOEP_ZERO3_EXPERT_STATE_FORMAT_KEY,
+ AUTOEP_ZERO3_PARTITIONED_EXPERT_STATE_FORMAT, PARAM_ALIGNMENT_PADDINGS)
+
+
+@dataclass
+class zero_model_state:
+ buffers: dict()
+ param_shapes: dict()
+ shared_params: list
+ ds_version: int
+ frozen_param_shapes: dict()
+ frozen_param_fragments: dict()
+
+
+debug = 0
+
+# load to cpu
+device = torch.device('cpu')
+
+OUTPUT_DTYPE_NAMES = {
+ 'float32': torch.float32,
+ 'fp32': torch.float32,
+ 'float16': torch.float16,
+ 'fp16': torch.float16,
+ 'bfloat16': torch.bfloat16,
+ 'bf16': torch.bfloat16,
+}
+
+
+def _resolve_output_dtype(dtype):
+ requested_dtype = dtype
+ if isinstance(dtype, str):
+ dtype_name = dtype[6:] if dtype.startswith('torch.') else dtype
+ dtype = OUTPUT_DTYPE_NAMES.get(dtype_name.lower())
+ if dtype not in set(OUTPUT_DTYPE_NAMES.values()):
+ supported = ', '.join(sorted(OUTPUT_DTYPE_NAMES))
+ raise ValueError(f"Unsupported output dtype {requested_dtype!r}. Choose one of: {supported}")
+ return dtype
+
+
+def atoi(text):
+ return int(text) if text.isdigit() else text
+
+
+def natural_keys(text):
+ '''
+ alist.sort(key=natural_keys) sorts in human order
+ http://nedbatchelder.com/blog/200712/human_sorting.html
+ (See Toothy's implementation in the comments)
+ '''
+ return [atoi(c) for c in re.split(r'(\d+)', text)]
+
+
+def get_model_state_file(checkpoint_dir, zero_stage):
+ if not os.path.isdir(checkpoint_dir):
+ raise FileNotFoundError(f"Directory '{checkpoint_dir}' doesn't exist")
+
+ # there should be only one file
+ if zero_stage <= 2:
+ file = os.path.join(checkpoint_dir, "mp_rank_00_model_states.pt")
+ elif zero_stage == 3:
+ file = os.path.join(checkpoint_dir, "zero_pp_rank_0_mp_rank_00_model_states.pt")
+
+ if not os.path.exists(file):
+ raise FileNotFoundError(f"can't find model states file at '{file}'")
+
+ return file
+
+
+def get_checkpoint_files(checkpoint_dir, glob_pattern):
+ # XXX: need to test that this simple glob rule works for multi-node setup too
+ ckpt_files = sorted(glob.glob(os.path.join(checkpoint_dir, glob_pattern)), key=natural_keys)
+
+ if len(ckpt_files) == 0:
+ raise FileNotFoundError(f"can't find {glob_pattern} files in directory '{checkpoint_dir}'")
+
+ return ckpt_files
+
+
+def get_optim_files(checkpoint_dir):
+ return get_checkpoint_files(checkpoint_dir, "*_optim_states.pt")
+
+
+def get_model_state_files(checkpoint_dir):
+ return get_checkpoint_files(checkpoint_dir, "*_model_states.pt")
+
+
+def _has_autoep_zero3_partitioned_metadata(state_dict):
+ autoep_layers = state_dict.get(AUTOEP_LAYERS_KEY)
+ if autoep_layers is None:
+ autoep_layers = state_dict.get(AUTOEP_LAYERS_KEY_LEGACY)
+ if not isinstance(autoep_layers, list):
+ return False
+ return any(
+ isinstance(entry, dict)
+ and entry.get(AUTOEP_ZERO3_EXPERT_STATE_FORMAT_KEY) == AUTOEP_ZERO3_PARTITIONED_EXPERT_STATE_FORMAT
+ for entry in autoep_layers)
+
+
+def _raise_if_autoep_zero3_partitioned_state(state_dict):
+ if _has_autoep_zero3_partitioned_metadata(state_dict):
+ raise NotImplementedError("zero_to_fp32 does not support AutoEP ZeRO-3 partition-native checkpoints. "
+ "AutoEP expert parameters are partitioned over expert replica groups, so "
+ "global data-parallel consolidation would produce incomplete expert tensors. "
+ "Use ds_to_universal.py for expert-aware conversion.")
+
+
+def _raise_if_autoep_zero3_partitioned_checkpoint(model_files):
+ for file in model_files:
+ state_dict = torch.load(file, map_location=device, weights_only=False)
+ _raise_if_autoep_zero3_partitioned_state(state_dict)
+
+
+def parse_model_states(files):
+ zero_model_states = []
+ for file in files:
+ state_dict = torch.load(file, map_location=device, weights_only=False)
+ _raise_if_autoep_zero3_partitioned_state(state_dict)
+
+ if BUFFER_NAMES not in state_dict:
+ raise ValueError(f"{file} is not a model state checkpoint")
+ buffer_names = state_dict[BUFFER_NAMES]
+ if debug:
+ print("Found buffers:", buffer_names)
+
+ # recover just the buffers while restoring them to fp32 if they were saved in fp16
+ buffers = {k: v.float() for k, v in state_dict["module"].items() if k in buffer_names}
+ param_shapes = state_dict[PARAM_SHAPES]
+
+ # collect parameters that are included in param_shapes
+ param_names = []
+ for s in param_shapes:
+ for name in s.keys():
+ param_names.append(name)
+
+ # update with frozen parameters
+ frozen_param_shapes = state_dict.get(FROZEN_PARAM_SHAPES, None)
+ if frozen_param_shapes is not None:
+ if debug:
+ print(f"Found frozen_param_shapes: {frozen_param_shapes}")
+ param_names += list(frozen_param_shapes.keys())
+
+ # handle shared params
+ shared_params = [[k, v] for k, v in state_dict["shared_params"].items()]
+
+ ds_version = state_dict.get(DS_VERSION, None)
+
+ frozen_param_fragments = state_dict.get(FROZEN_PARAM_FRAGMENTS, None)
+
+ z_model_state = zero_model_state(buffers=buffers,
+ param_shapes=param_shapes,
+ shared_params=shared_params,
+ ds_version=ds_version,
+ frozen_param_shapes=frozen_param_shapes,
+ frozen_param_fragments=frozen_param_fragments)
+ zero_model_states.append(z_model_state)
+
+ return zero_model_states
+
+
+def parse_optim_states(files, ds_checkpoint_dir):
+ total_files = len(files)
+ state_dicts = []
+ for f in tqdm(files, desc='Loading checkpoint shards'):
+ state_dict = torch.load(f, map_location=device, mmap=True, weights_only=False)
+ # immediately discard the potentially huge 2 optimizer states as we only care for fp32 master weights
+ # and also handle the case where it was already removed by another helper script
+ state_dict["optimizer_state_dict"].pop("optimizer_state_dict", None)
+ state_dicts.append(state_dict)
+
+ if ZERO_STAGE not in state_dicts[0][OPTIMIZER_STATE_DICT]:
+ raise ValueError(f"{files[0]} is not a zero checkpoint")
+ zero_stage = state_dicts[0][OPTIMIZER_STATE_DICT][ZERO_STAGE]
+ world_size = state_dicts[0][OPTIMIZER_STATE_DICT][PARTITION_COUNT]
+
+ # For ZeRO-2 each param group can have different partition_count as data parallelism for expert
+ # parameters can be different from data parallelism for non-expert parameters. So we can just
+ # use the max of the partition_count to get the dp world_size.
+
+ if type(world_size) is list:
+ world_size = max(world_size)
+
+ if world_size != total_files:
+ raise ValueError(
+ f"Expected {world_size} of '*_optim_states.pt' under '{ds_checkpoint_dir}' but found {total_files} files. "
+ "Possibly due to an overwrite of an old checkpoint, or a checkpoint didn't get saved by one or more processes."
+ )
+
+ # the groups are named differently in each stage
+ if zero_stage <= 2:
+ fp32_groups_key = SINGLE_PARTITION_OF_FP32_GROUPS
+ elif zero_stage == 3:
+ fp32_groups_key = FP32_FLAT_GROUPS
+ else:
+ raise ValueError(f"unknown zero stage {zero_stage}")
+
+ fp32_flat_groups = [state_dicts[i][OPTIMIZER_STATE_DICT][fp32_groups_key] for i in range(len(state_dicts))]
+ param_alignment_paddings = state_dicts[0][OPTIMIZER_STATE_DICT].get(PARAM_ALIGNMENT_PADDINGS)
+ return zero_stage, world_size, fp32_flat_groups, param_alignment_paddings
+
+
+def _get_fp32_state_dict_from_zero_checkpoint(ds_checkpoint_dir, exclude_frozen_parameters):
+ """
+ Returns fp32 state_dict reconstructed from ds checkpoint
+
+ Args:
+ - ``ds_checkpoint_dir``: path to the deepspeed checkpoint folder (where the optimizer files are)
+
+ """
+ print(f"Processing zero checkpoint '{ds_checkpoint_dir}'")
+
+ # parse_model_states rejects AutoEP ZeRO-3 partition-native checkpoints
+ # before the expensive optimizer-shard load below.
+ model_files = get_model_state_files(ds_checkpoint_dir)
+ zero_model_states = parse_model_states(model_files)
+ print(f'Parsing checkpoint created by deepspeed=={zero_model_states[0].ds_version}')
+
+ optim_files = get_optim_files(ds_checkpoint_dir)
+ zero_stage, world_size, fp32_flat_groups, param_alignment_paddings = parse_optim_states(
+ optim_files, ds_checkpoint_dir)
+ print(f"Detected checkpoint of type zero stage {zero_stage}, world_size: {world_size}")
+
+ if zero_stage <= 2:
+ return _get_fp32_state_dict_from_zero2_checkpoint(world_size, fp32_flat_groups, zero_model_states,
+ exclude_frozen_parameters, param_alignment_paddings)
+ elif zero_stage == 3:
+ return _get_fp32_state_dict_from_zero3_checkpoint(world_size, fp32_flat_groups, zero_model_states,
+ exclude_frozen_parameters)
+
+
+def _zero2_merge_frozen_params(state_dict, zero_model_states):
+ if zero_model_states[0].frozen_param_shapes is None or len(zero_model_states[0].frozen_param_shapes) == 0:
+ return
+
+ frozen_param_shapes = zero_model_states[0].frozen_param_shapes
+ frozen_param_fragments = zero_model_states[0].frozen_param_fragments
+
+ if debug:
+ num_elem = sum(s.numel() for s in frozen_param_shapes.values())
+ print(f'rank 0: {FROZEN_PARAM_SHAPES}.numel = {num_elem}')
+
+ wanted_params = len(frozen_param_shapes)
+ wanted_numel = sum(s.numel() for s in frozen_param_shapes.values())
+ avail_numel = sum([p.numel() for p in frozen_param_fragments.values()])
+ print(f'Frozen params: Have {avail_numel} numels to process.')
+ print(f'Frozen params: Need {wanted_numel} numels in {wanted_params} params')
+
+ total_params = 0
+ total_numel = 0
+ for name, shape in frozen_param_shapes.items():
+ total_params += 1
+ unpartitioned_numel = shape.numel()
+ total_numel += unpartitioned_numel
+
+ state_dict[name] = frozen_param_fragments[name]
+
+ if debug:
+ print(f"{name} full shape: {shape} unpartitioned numel {unpartitioned_numel} ")
+
+ print(f"Reconstructed Frozen fp32 state dict with {total_params} params {total_numel} elements")
+
+
+def _has_callable(obj, fn):
+ attr = getattr(obj, fn, None)
+ return callable(attr)
+
+
+def _zero2_merge_trainable_params(state_dict,
+ world_size,
+ fp32_flat_groups,
+ zero_model_states,
+ param_alignment_paddings=None):
+ param_shapes = zero_model_states[0].param_shapes
+
+ # Reconstruction protocol:
+ #
+ # XXX: document this
+
+ if debug:
+ for i in range(world_size):
+ for j in range(len(fp32_flat_groups[0])):
+ print(f"{FP32_FLAT_GROUPS}[{i}][{j}].shape={fp32_flat_groups[i][j].shape}")
+
+ # XXX: memory usage doubles here (zero2)
+ num_param_groups = len(fp32_flat_groups[0])
+ merged_single_partition_of_fp32_groups = []
+ for i in range(num_param_groups):
+ merged_partitions = [sd[i] for sd in fp32_flat_groups]
+ full_single_fp32_vector = torch.cat(merged_partitions, 0)
+ merged_single_partition_of_fp32_groups.append(full_single_fp32_vector)
+ avail_numel = sum(
+ [full_single_fp32_vector.numel() for full_single_fp32_vector in merged_single_partition_of_fp32_groups])
+
+ if debug:
+ wanted_params = sum([len(shapes) for shapes in param_shapes])
+ wanted_numel = sum([sum(shape.numel() for shape in shapes.values()) for shapes in param_shapes])
+ # not asserting if there is a mismatch due to possible padding
+ print(f"Have {avail_numel} numels to process.")
+ print(f"Need {wanted_numel} numels in {wanted_params} params.")
+
+ # params
+ # XXX: for huge models that can't fit into the host's RAM we will have to recode this to support
+ # out-of-core computing solution
+ total_numel = 0
+ total_params = 0
+ for group_idx, (shapes,
+ full_single_fp32_vector) in enumerate(zip(param_shapes, merged_single_partition_of_fp32_groups)):
+ offset = 0
+ avail_numel = full_single_fp32_vector.numel()
+ group_alignment_paddings = None
+ if param_alignment_paddings is not None:
+ group_alignment_paddings = param_alignment_paddings[group_idx]
+ if len(group_alignment_paddings) != len(shapes):
+ raise ValueError(f"Expected {len(shapes)} parameter alignment paddings for group {group_idx}, "
+ f"but found {len(group_alignment_paddings)}")
+
+ for param_idx, (name, shape) in enumerate(shapes.items()):
+
+ unpartitioned_numel = shape.numel() if _has_callable(shape, 'numel') else math.prod(shape)
+ total_numel += unpartitioned_numel
+ total_params += 1
+
+ if debug:
+ print(f"{name} full shape: {shape} unpartitioned numel {unpartitioned_numel} ")
+ state_dict[name] = full_single_fp32_vector.narrow(0, offset, unpartitioned_numel).view(shape)
+ offset += unpartitioned_numel
+ if group_alignment_paddings is not None:
+ offset += group_alignment_paddings[param_idx]
+
+ # Z2 started to align to 2*world_size to improve nccl performance. Therefore both offset and
+ # avail_numel can differ by anywhere between 0..2*world_size. Due to two unrelated complex
+ # paddings performed in the code it's almost impossible to predict the exact numbers w/o the
+ # live optimizer object, so we are checking that the numbers are within the right range
+ align_to = 2 * world_size
+
+ def zero2_align(x):
+ return align_to * math.ceil(x / align_to)
+
+ if debug:
+ print(f"original offset={offset}, avail_numel={avail_numel}")
+
+ offset = zero2_align(offset)
+ avail_numel = zero2_align(avail_numel)
+
+ if debug:
+ print(f"aligned offset={offset}, avail_numel={avail_numel}")
+
+ # Sanity check
+ if offset != avail_numel:
+ raise ValueError(f"consumed {offset} numels out of {avail_numel} - something is wrong")
+
+ print(f"Reconstructed fp32 state dict with {total_params} params {total_numel} elements")
+
+
+def _get_fp32_state_dict_from_zero2_checkpoint(world_size,
+ fp32_flat_groups,
+ zero_model_states,
+ exclude_frozen_parameters,
+ param_alignment_paddings=None):
+ state_dict = OrderedDict()
+
+ # buffers
+ buffers = zero_model_states[0].buffers
+ state_dict.update(buffers)
+ if debug:
+ print(f"added {len(buffers)} buffers")
+
+ if not exclude_frozen_parameters:
+ _zero2_merge_frozen_params(state_dict, zero_model_states)
+
+ _zero2_merge_trainable_params(state_dict, world_size, fp32_flat_groups, zero_model_states,
+ param_alignment_paddings)
+
+ # recover shared parameters
+ for pair in zero_model_states[0].shared_params:
+ if pair[1] in state_dict:
+ state_dict[pair[0]] = state_dict[pair[1]]
+
+ return state_dict
+
+
+def zero3_partitioned_param_info(unpartitioned_numel, world_size):
+ remainder = unpartitioned_numel % world_size
+ padding_numel = (world_size - remainder) if remainder else 0
+ partitioned_numel = math.ceil(unpartitioned_numel / world_size)
+ return partitioned_numel, padding_numel
+
+
+def _zero3_merge_frozen_params(state_dict, world_size, zero_model_states):
+ if zero_model_states[0].frozen_param_shapes is None or len(zero_model_states[0].frozen_param_shapes) == 0:
+ return
+
+ if debug:
+ for i in range(world_size):
+ num_elem = sum(s.numel() for s in zero_model_states[i].frozen_param_fragments.values())
+ print(f'rank {i}: {FROZEN_PARAM_SHAPES}.numel = {num_elem}')
+
+ frozen_param_shapes = zero_model_states[0].frozen_param_shapes
+ wanted_params = len(frozen_param_shapes)
+ wanted_numel = sum(s.numel() for s in frozen_param_shapes.values())
+ avail_numel = sum([p.numel() for p in zero_model_states[0].frozen_param_fragments.values()]) * world_size
+ print(f'Frozen params: Have {avail_numel} numels to process.')
+ print(f'Frozen params: Need {wanted_numel} numels in {wanted_params} params')
+
+ total_params = 0
+ total_numel = 0
+ for name, shape in zero_model_states[0].frozen_param_shapes.items():
+ total_params += 1
+ unpartitioned_numel = shape.numel()
+ total_numel += unpartitioned_numel
+
+ param_frags = tuple(model_state.frozen_param_fragments[name] for model_state in zero_model_states)
+ state_dict[name] = torch.cat(param_frags, 0).narrow(0, 0, unpartitioned_numel).view(shape)
+
+ partitioned_numel, partitioned_padding_numel = zero3_partitioned_param_info(unpartitioned_numel, world_size)
+
+ if debug:
+ print(
+ f"Frozen params: {total_params} {name} full shape: {shape} partition0 numel={partitioned_numel} partitioned_padding_numel={partitioned_padding_numel}"
+ )
+
+ print(f"Reconstructed Frozen fp32 state dict with {total_params} params {total_numel} elements")
+
+
+class GatheredTensor:
+ """
+ A pseudo tensor that collects partitioned weights.
+ It is more memory efficient when there are multiple groups.
+ """
+
+ def __init__(self, flat_groups, flat_groups_offset, offset, partitioned_numel, shape):
+ self.flat_groups = flat_groups
+ self.flat_groups_offset = flat_groups_offset
+ self.offset = offset
+ self.partitioned_numel = partitioned_numel
+ self.shape = shape
+ self.dtype = self.flat_groups[0][0].dtype
+
+ def contiguous(self):
+ """
+ Merge partitioned weights from flat_groups into a single tensor.
+ """
+ end_idx = self.offset + self.partitioned_numel
+ world_size = len(self.flat_groups)
+ pad_flat_param_chunks = []
+
+ for rank_i in range(world_size):
+ # for each rank, we need to collect weights from related group/groups
+ flat_groups_at_rank_i = self.flat_groups[rank_i]
+ start_group_id = None
+ end_group_id = None
+ for group_id in range(len(self.flat_groups_offset)):
+ if self.flat_groups_offset[group_id] <= self.offset < self.flat_groups_offset[group_id + 1]:
+ start_group_id = group_id
+ if self.flat_groups_offset[group_id] < end_idx <= self.flat_groups_offset[group_id + 1]:
+ end_group_id = group_id
+ break
+ # collect weights from related group/groups
+ for group_id in range(start_group_id, end_group_id + 1):
+ flat_tensor = flat_groups_at_rank_i[group_id]
+ start_offset = self.offset - self.flat_groups_offset[group_id]
+ end_offset = min(end_idx, self.flat_groups_offset[group_id + 1]) - self.flat_groups_offset[group_id]
+ pad_flat_param_chunks.append(flat_tensor[start_offset:end_offset])
+
+ # collect weights from all ranks
+ pad_flat_param = torch.cat(pad_flat_param_chunks, dim=0)
+ param = pad_flat_param[:self.shape.numel()].view(self.shape).contiguous()
+ return param
+
+
+def _zero3_merge_trainable_params(state_dict, world_size, fp32_flat_groups, zero_model_states):
+ param_shapes = zero_model_states[0].param_shapes
+ avail_numel = sum([flat_group.numel() for flat_group in fp32_flat_groups[0]]) * world_size
+
+ # Reconstruction protocol: For zero3 we need to zip the partitions together at boundary of each
+ # param, re-consolidating each param, while dealing with padding if any
+
+ # merge list of dicts, preserving order
+ param_shapes = {k: v for d in param_shapes for k, v in d.items()}
+
+ if debug:
+ for i in range(world_size):
+ print(f"{FP32_FLAT_GROUPS}[{i}].shape={fp32_flat_groups[i].shape}")
+
+ wanted_params = len(param_shapes)
+ wanted_numel = sum(shape.numel() for shape in param_shapes.values())
+ # not asserting if there is a mismatch due to possible padding
+ avail_numel = fp32_flat_groups[0].numel() * world_size
+ print(f"Trainable params: Have {avail_numel} numels to process.")
+ print(f"Trainable params: Need {wanted_numel} numels in {wanted_params} params.")
+
+ # params
+ # XXX: for huge models that can't fit into the host's RAM we will have to recode this to support
+ # out-of-core computing solution
+ offset = 0
+ total_numel = 0
+ total_params = 0
+ flat_groups_offset = [0] + list(np.cumsum([flat_tensor.numel() for flat_tensor in fp32_flat_groups[0]]))
+ for name, shape in tqdm(param_shapes.items(), desc='Gathering sharded weights'):
+ unpartitioned_numel = shape.numel()
+ total_numel += unpartitioned_numel
+ total_params += 1
+ partitioned_numel, partitioned_padding_numel = zero3_partitioned_param_info(unpartitioned_numel, world_size)
+
+ if debug:
+ print(
+ f"Trainable params: {total_params} {name} full shape: {shape} partition0 numel={partitioned_numel} partitioned_padding_numel={partitioned_padding_numel}"
+ )
+
+ # memory efficient tensor
+ tensor = GatheredTensor(fp32_flat_groups, flat_groups_offset, offset, partitioned_numel, shape)
+ state_dict[name] = tensor
+ offset += partitioned_numel
+
+ offset *= world_size
+
+ # Sanity check
+ if offset != avail_numel:
+ raise ValueError(f"consumed {offset} numels out of {avail_numel} - something is wrong")
+
+ print(f"Reconstructed Trainable fp32 state dict with {total_params} params {total_numel} elements")
+
+
+def _get_fp32_state_dict_from_zero3_checkpoint(world_size, fp32_flat_groups, zero_model_states,
+ exclude_frozen_parameters):
+ state_dict = OrderedDict()
+
+ # buffers
+ buffers = zero_model_states[0].buffers
+ state_dict.update(buffers)
+ if debug:
+ print(f"added {len(buffers)} buffers")
+
+ if not exclude_frozen_parameters:
+ _zero3_merge_frozen_params(state_dict, world_size, zero_model_states)
+
+ _zero3_merge_trainable_params(state_dict, world_size, fp32_flat_groups, zero_model_states)
+
+ # recover shared parameters
+ for pair in zero_model_states[0].shared_params:
+ if pair[1] in state_dict:
+ state_dict[pair[0]] = state_dict[pair[1]]
+
+ return state_dict
+
+
+def to_torch_tensor(state_dict, return_empty_tensor=False, dtype=None):
+ """
+ Convert state_dict of GatheredTensor to torch tensor
+ """
+ if dtype is not None:
+ dtype = _resolve_output_dtype(dtype)
+
+ torch_state_dict = {}
+ converted_tensors = {}
+ for name, tensor in state_dict.items():
+ tensor_id = id(tensor)
+ if tensor_id in converted_tensors: # shared tensors
+ shared_tensor = torch_state_dict[converted_tensors[tensor_id]]
+ torch_state_dict[name] = shared_tensor
+ else:
+ converted_tensors[tensor_id] = name
+ if return_empty_tensor:
+ torch_state_dict[name] = torch.empty(tensor.shape, dtype=dtype or tensor.dtype)
+ else:
+ contiguous_tensor = tensor.contiguous()
+ torch_state_dict[name] = contiguous_tensor.to(dtype=dtype) if dtype else contiguous_tensor
+ return torch_state_dict
+
+
+def get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir,
+ tag=None,
+ exclude_frozen_parameters=False,
+ lazy_mode=False):
+ """
+ Convert ZeRO 2 or 3 checkpoint into a single fp32 consolidated state_dict that can be loaded with
+ ``load_state_dict()`` and used for training without DeepSpeed or shared with others, for example
+ via a model hub.
+
+ Args:
+ - ``checkpoint_dir``: path to the desired checkpoint folder
+ - ``tag``: checkpoint tag used as a unique identifier for checkpoint. If not provided will attempt to load tag in 'latest' file. e.g., ``global_step14``
+ - ``exclude_frozen_parameters``: exclude frozen parameters
+ - ``lazy_mode``: get state_dict in lazy mode. It returns a dict of pesduo tensor instead of torch tensor, which is more memory efficient.
+ Convert the pesduo tensor to torch tensor by ``.contiguous()``
+
+ Returns:
+ - pytorch ``state_dict``
+
+ A typical usage might be ::
+
+ from deepspeed.utils.zero_to_fp32 import get_fp32_state_dict_from_zero_checkpoint
+ # do the training and checkpoint saving
+ state_dict = get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir) # already on cpu
+ model = model.cpu() # move to cpu
+ model.load_state_dict(state_dict)
+ # submit to model hub or save the model to share with others
+
+ In this example the ``model`` will no longer be usable in the deepspeed context of the same
+ application. i.e. you will need to re-initialize the deepspeed engine, since
+ ``model.load_state_dict(state_dict)`` will remove all the deepspeed magic from it.
+
+ If you want it all done for you, use ``load_state_dict_from_zero_checkpoint`` instead.
+
+ Note: the above usage may not work if your application doesn't have sufficient free CPU memory.
+ You may need to use the offline approach using the ``zero_to_fp32.py`` script that is saved with
+ the checkpoint. Or you can load state_dict in lazy mode ::
+
+ from deepspeed.utils.zero_to_fp32 import get_fp32_state_dict_from_zero_checkpoint
+ state_dict = get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir, lazy_mode=True) # not on cpu
+ for name, lazy_tensor in state_dict.item():
+ tensor = lazy_tensor.contiguous() # to cpu
+ print(name, tensor)
+ # del tensor to release memory if it no longer in use
+ """
+ if tag is None:
+ latest_path = os.path.join(checkpoint_dir, 'latest')
+ if os.path.isfile(latest_path):
+ with open(latest_path, 'r') as fd:
+ tag = fd.read().strip()
+ else:
+ raise ValueError(f"Unable to find 'latest' file at {latest_path}")
+
+ ds_checkpoint_dir = os.path.join(checkpoint_dir, tag)
+
+ if not os.path.isdir(ds_checkpoint_dir):
+ raise FileNotFoundError(f"Directory '{ds_checkpoint_dir}' doesn't exist")
+
+ state_dict = _get_fp32_state_dict_from_zero_checkpoint(ds_checkpoint_dir, exclude_frozen_parameters)
+ if lazy_mode:
+ return state_dict
+ else:
+ return to_torch_tensor(state_dict)
+
+
+def convert_zero_checkpoint_to_state_dict(checkpoint_dir,
+ output_dir,
+ dtype=torch.float32,
+ max_shard_size="5GB",
+ safe_serialization=False,
+ tag=None,
+ exclude_frozen_parameters=False):
+ """
+ Convert ZeRO 2 or 3 checkpoint into a consolidated ``state_dict`` file that can be
+ loaded with ``torch.load(file)`` + ``load_state_dict()`` and used for training without DeepSpeed.
+
+ Args:
+ - ``checkpoint_dir``: path to the desired checkpoint folder. (one that contains the tag-folder, like ``global_step14``)
+ - ``output_dir``: directory for the PyTorch state_dict output files
+ - ``dtype``: output tensor dtype. Supports float32, float16, and bfloat16 as strings or torch dtypes.
+ - ``max_shard_size``: the maximum size for a checkpoint before being sharded, default value is 5GB
+ - ``safe_serialization``: whether to save the model using `safetensors` or the traditional PyTorch way (that uses `pickle`).
+ - ``tag``: checkpoint tag used as a unique identifier for checkpoint. If not provided will attempt to load tag in the file named ``latest`` in the checkpoint folder, e.g., ``global_step14``
+ - ``exclude_frozen_parameters``: exclude frozen parameters
+ """
+
+ dtype = _resolve_output_dtype(dtype)
+
+ # Dependency pre-check
+ if safe_serialization:
+ try:
+ from safetensors.torch import save_file
+ except ImportError:
+ print('If you want to use `safe_serialization`, please `pip install safetensors`')
+ raise
+ if max_shard_size is not None:
+ try:
+ from huggingface_hub import split_torch_state_dict_into_shards
+ except ImportError:
+ print('If you want to use `max_shard_size`, please `pip install huggingface_hub`')
+ raise
+
+ # Convert zero checkpoint to state_dict
+ state_dict = get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir,
+ tag,
+ exclude_frozen_parameters,
+ lazy_mode=True)
+
+ # Shard the model if it is too big.
+ weights_name = "model.safetensors" if safe_serialization else "pytorch_model.bin"
+ if max_shard_size is not None:
+ filename_pattern = weights_name.replace(".bin", "{suffix}.bin").replace(".safetensors", "{suffix}.safetensors")
+ # an memory-efficient approach for sharding
+ empty_state_dict = to_torch_tensor(state_dict, return_empty_tensor=True, dtype=dtype)
+ state_dict_split = split_torch_state_dict_into_shards(empty_state_dict,
+ filename_pattern=filename_pattern,
+ max_shard_size=max_shard_size)
+ else:
+ from collections import namedtuple
+ StateDictSplit = namedtuple("StateDictSplit", ["is_sharded", "filename_to_tensors"])
+ state_dict_split = StateDictSplit(is_sharded=False,
+ filename_to_tensors={weights_name: list(state_dict.keys())})
+
+ # Save the model by shard
+ os.makedirs(output_dir, exist_ok=True)
+ filename_to_tensors = state_dict_split.filename_to_tensors.items()
+ for shard_file, tensors in tqdm(filename_to_tensors, desc="Saving checkpoint shards"):
+ shard_state_dict = {tensor_name: state_dict[tensor_name] for tensor_name in tensors}
+ shard_state_dict = to_torch_tensor(shard_state_dict, dtype=dtype)
+ output_path = os.path.join(output_dir, shard_file)
+ if safe_serialization:
+ save_file(shard_state_dict, output_path, metadata={"format": "pt"})
+ else:
+ torch.save(shard_state_dict, output_path)
+ # release the memory of current shard
+ for tensor_name in list(shard_state_dict.keys()):
+ del state_dict[tensor_name]
+ del shard_state_dict[tensor_name]
+ del shard_state_dict
+ gc.collect()
+
+ # Save index if sharded
+ if state_dict_split.is_sharded:
+ index = {
+ "metadata": state_dict_split.metadata,
+ "weight_map": state_dict_split.tensor_to_filename,
+ }
+ save_index_file = "model.safetensors.index.json" if safe_serialization else "pytorch_model.bin.index.json"
+ save_index_file = os.path.join(output_dir, save_index_file)
+ with open(save_index_file, "w", encoding="utf-8") as f:
+ content = json.dumps(index, indent=2, sort_keys=True) + "\n"
+ f.write(content)
+
+
+def convert_zero_checkpoint_to_fp32_state_dict(checkpoint_dir,
+ output_dir,
+ max_shard_size="5GB",
+ safe_serialization=False,
+ tag=None,
+ exclude_frozen_parameters=False):
+ """Backward-compatible fp32 checkpoint conversion."""
+ return convert_zero_checkpoint_to_state_dict(checkpoint_dir,
+ output_dir,
+ dtype=torch.float32,
+ max_shard_size=max_shard_size,
+ safe_serialization=safe_serialization,
+ tag=tag,
+ exclude_frozen_parameters=exclude_frozen_parameters)
+
+
+def load_state_dict_from_zero_checkpoint(model, checkpoint_dir, tag=None):
+ """
+ 1. Put the provided model to cpu
+ 2. Convert ZeRO 2 or 3 checkpoint into a single fp32 consolidated ``state_dict``
+ 3. Load it into the provided model
+
+ Args:
+ - ``model``: the model object to update
+ - ``checkpoint_dir``: path to the desired checkpoint folder. (one that contains the tag-folder, like ``global_step14``)
+ - ``tag``: checkpoint tag used as a unique identifier for checkpoint. If not provided will attempt to load tag in the file named ``latest`` in the checkpoint folder, e.g., ``global_step14``
+
+ Returns:
+ - ``model`: modified model
+
+ Make sure you have plenty of CPU memory available before you call this function. If you don't
+ have enough use the ``zero_to_fp32.py`` utility to do the conversion. You will find it
+ conveniently placed for you in the checkpoint folder.
+
+ A typical usage might be ::
+
+ from deepspeed.utils.zero_to_fp32 import load_state_dict_from_zero_checkpoint
+ model = load_state_dict_from_zero_checkpoint(trainer.model, checkpoint_dir)
+ # submit to model hub or save the model to share with others
+
+ Note, that once this was run, the ``model`` will no longer be usable in the deepspeed context
+ of the same application. i.e. you will need to re-initialize the deepspeed engine, since
+ ``model.load_state_dict(state_dict)`` will remove all the deepspeed magic from it.
+
+ """
+ logger.info("Extracting fp32 weights")
+ state_dict = get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir, tag)
+
+ logger.info("Overwriting model with fp32 weights")
+ model = model.cpu()
+ model.load_state_dict(state_dict, strict=False)
+
+ return model
+
+
+if __name__ == "__main__":
+ parser = argparse.ArgumentParser()
+ parser.add_argument("checkpoint_dir",
+ type=str,
+ help="path to the desired checkpoint folder, e.g., path/checkpoint-12")
+ parser.add_argument("output_dir",
+ type=str,
+ help="directory to the pytorch fp32 state_dict output files"
+ "(e.g. path/checkpoint-12-output/)")
+ parser.add_argument(
+ "--max_shard_size",
+ type=str,
+ default="5GB",
+ help="The maximum size for a checkpoint before being sharded. Checkpoints shard will then be each of size"
+ "lower than this size. If expressed as a string, needs to be digits followed by a unit (like `5MB`"
+ "We default it to 5GB in order for models to be able to run easily on free-tier google colab instances"
+ "without CPU OOM issues.")
+ parser.add_argument(
+ "--safe_serialization",
+ default=False,
+ action='store_true',
+ help="Whether to save the model using `safetensors` or the traditional PyTorch way (that uses `pickle`).")
+ parser.add_argument("-t",
+ "--tag",
+ type=str,
+ default=None,
+ help="checkpoint tag used as a unique identifier for checkpoint. e.g., global_step1")
+ parser.add_argument("--exclude_frozen_parameters", action='store_true', help="exclude frozen parameters")
+ parser.add_argument("-d", "--debug", action='store_true', help="enable debug")
+ args = parser.parse_args()
+
+ debug = args.debug
+
+ convert_zero_checkpoint_to_fp32_state_dict(args.checkpoint_dir,
+ args.output_dir,
+ max_shard_size=args.max_shard_size,
+ safe_serialization=args.safe_serialization,
+ tag=args.tag,
+ exclude_frozen_parameters=args.exclude_frozen_parameters)
diff --git a/qwen3-1.7b-teutonic-5e6/checkpoint-2475/zero_to_torch.py b/qwen3-1.7b-teutonic-5e6/checkpoint-2475/zero_to_torch.py
new file mode 100644
index 0000000000000000000000000000000000000000..81312e252400d77f03cc1a88522f8f18ebe70ee7
--- /dev/null
+++ b/qwen3-1.7b-teutonic-5e6/checkpoint-2475/zero_to_torch.py
@@ -0,0 +1,54 @@
+#!/usr/bin/env python
+
+# SPDX-License-Identifier: Apache-2.0
+# DeepSpeed Team
+
+import argparse
+
+if __package__:
+ from . import zero_to_fp32
+else:
+ import zero_to_fp32
+
+
+def main(args=None):
+ parser = argparse.ArgumentParser(
+ description="Convert a DeepSpeed ZeRO checkpoint to float32, float16, or bfloat16 PyTorch weights.")
+ parser.add_argument("checkpoint_dir",
+ type=str,
+ help="path to the desired checkpoint folder, e.g., path/checkpoint-12")
+ parser.add_argument("output_dir", type=str, help="directory for the converted PyTorch state_dict files")
+ parser.add_argument("--dtype",
+ type=str,
+ choices=sorted(zero_to_fp32.OUTPUT_DTYPE_NAMES),
+ required=True,
+ help="output tensor dtype")
+ parser.add_argument("--max_shard_size",
+ type=str,
+ default="5GB",
+ help="maximum size of each checkpoint shard, such as 5GB or 500MB")
+ parser.add_argument("--safe_serialization",
+ default=False,
+ action='store_true',
+ help="save with safetensors instead of PyTorch pickle serialization")
+ parser.add_argument("-t",
+ "--tag",
+ type=str,
+ default=None,
+ help="checkpoint tag used as a unique identifier, e.g., global_step1")
+ parser.add_argument("--exclude_frozen_parameters", action='store_true', help="exclude frozen parameters")
+ parser.add_argument("-d", "--debug", action='store_true', help="enable debug output")
+ parsed_args = parser.parse_args(args)
+
+ zero_to_fp32.debug = parsed_args.debug
+ zero_to_fp32.convert_zero_checkpoint_to_state_dict(parsed_args.checkpoint_dir,
+ parsed_args.output_dir,
+ dtype=parsed_args.dtype,
+ max_shard_size=parsed_args.max_shard_size,
+ safe_serialization=parsed_args.safe_serialization,
+ tag=parsed_args.tag,
+ exclude_frozen_parameters=parsed_args.exclude_frozen_parameters)
+
+
+if __name__ == "__main__":
+ main()
diff --git a/qwen3-1.7b-teutonic-5e6/checkpoint-500/chat_template.jinja b/qwen3-1.7b-teutonic-5e6/checkpoint-500/chat_template.jinja
new file mode 100644
index 0000000000000000000000000000000000000000..01be9b307daa2d425f7c168c9fb145a286e0afb4
--- /dev/null
+++ b/qwen3-1.7b-teutonic-5e6/checkpoint-500/chat_template.jinja
@@ -0,0 +1,89 @@
+{%- if tools %}
+ {{- '<|im_start|>system\n' }}
+ {%- if messages[0].role == 'system' %}
+ {{- messages[0].content + '\n\n' }}
+ {%- endif %}
+ {{- "# Tools\n\nYou may call one or more functions to assist with the user query.\n\nYou are provided with function signatures within XML tags:\n" }}
+ {%- for tool in tools %}
+ {{- "\n" }}
+ {{- tool | tojson }}
+ {%- endfor %}
+ {{- "\n\n\nFor each function call, return a json object with function name and arguments within XML tags:\n\n{\"name\": , \"arguments\": }\n<|im_end|>\n" }}
+{%- else %}
+ {%- if messages[0].role == 'system' %}
+ {{- '<|im_start|>system\n' + messages[0].content + '<|im_end|>\n' }}
+ {%- endif %}
+{%- endif %}
+{%- set ns = namespace(multi_step_tool=true, last_query_index=messages|length - 1) %}
+{%- for message in messages[::-1] %}
+ {%- set index = (messages|length - 1) - loop.index0 %}
+ {%- if ns.multi_step_tool and message.role == "user" and message.content is string and not(message.content.startswith('') and message.content.endswith('')) %}
+ {%- set ns.multi_step_tool = false %}
+ {%- set ns.last_query_index = index %}
+ {%- endif %}
+{%- endfor %}
+{%- for message in messages %}
+ {%- if message.content is string %}
+ {%- set content = message.content %}
+ {%- else %}
+ {%- set content = '' %}
+ {%- endif %}
+ {%- if (message.role == "user") or (message.role == "system" and not loop.first) %}
+ {{- '<|im_start|>' + message.role + '\n' + content + '<|im_end|>' + '\n' }}
+ {%- elif message.role == "assistant" %}
+ {%- set reasoning_content = '' %}
+ {%- if message.reasoning_content is string %}
+ {%- set reasoning_content = message.reasoning_content %}
+ {%- else %}
+ {%- if '' in content %}
+ {%- set reasoning_content = content.split('')[0].rstrip('\n').split('')[-1].lstrip('\n') %}
+ {%- set content = content.split('')[-1].lstrip('\n') %}
+ {%- endif %}
+ {%- endif %}
+ {%- if loop.index0 > ns.last_query_index %}
+ {%- if loop.last or (not loop.last and reasoning_content) %}
+ {{- '<|im_start|>' + message.role + '\n\n' + reasoning_content.strip('\n') + '\n\n\n' + content.lstrip('\n') }}
+ {%- else %}
+ {{- '<|im_start|>' + message.role + '\n' + content }}
+ {%- endif %}
+ {%- else %}
+ {{- '<|im_start|>' + message.role + '\n' + content }}
+ {%- endif %}
+ {%- if message.tool_calls %}
+ {%- for tool_call in message.tool_calls %}
+ {%- if (loop.first and content) or (not loop.first) %}
+ {{- '\n' }}
+ {%- endif %}
+ {%- if tool_call.function %}
+ {%- set tool_call = tool_call.function %}
+ {%- endif %}
+ {{- '\n{"name": "' }}
+ {{- tool_call.name }}
+ {{- '", "arguments": ' }}
+ {%- if tool_call.arguments is string %}
+ {{- tool_call.arguments }}
+ {%- else %}
+ {{- tool_call.arguments | tojson }}
+ {%- endif %}
+ {{- '}\n' }}
+ {%- endfor %}
+ {%- endif %}
+ {{- '<|im_end|>\n' }}
+ {%- elif message.role == "tool" %}
+ {%- if loop.first or (messages[loop.index0 - 1].role != "tool") %}
+ {{- '<|im_start|>user' }}
+ {%- endif %}
+ {{- '\n\n' }}
+ {{- content }}
+ {{- '\n' }}
+ {%- if loop.last or (messages[loop.index0 + 1].role != "tool") %}
+ {{- '<|im_end|>\n' }}
+ {%- endif %}
+ {%- endif %}
+{%- endfor %}
+{%- if add_generation_prompt %}
+ {{- '<|im_start|>assistant\n' }}
+ {%- if enable_thinking is defined and enable_thinking is false %}
+ {{- '\n\n\n\n' }}
+ {%- endif %}
+{%- endif %}
\ No newline at end of file
diff --git a/qwen3-1.7b-teutonic-5e6/checkpoint-500/config.json b/qwen3-1.7b-teutonic-5e6/checkpoint-500/config.json
new file mode 100644
index 0000000000000000000000000000000000000000..5d9cef07396baadff5390e4508eb33025967a029
--- /dev/null
+++ b/qwen3-1.7b-teutonic-5e6/checkpoint-500/config.json
@@ -0,0 +1,63 @@
+{
+ "architectures": [
+ "Qwen3ForCausalLM"
+ ],
+ "attention_bias": false,
+ "attention_dropout": 0.0,
+ "bos_token_id": null,
+ "dtype": "bfloat16",
+ "eos_token_id": 151645,
+ "head_dim": 128,
+ "hidden_act": "silu",
+ "hidden_size": 2048,
+ "initializer_range": 0.02,
+ "intermediate_size": 6144,
+ "layer_types": [
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention"
+ ],
+ "max_position_embeddings": 40960,
+ "max_window_layers": 28,
+ "model_type": "qwen3",
+ "num_attention_heads": 16,
+ "num_hidden_layers": 28,
+ "num_key_value_heads": 8,
+ "pad_token_id": 151643,
+ "rms_norm_eps": 1e-06,
+ "rope_parameters": {
+ "rope_theta": 1000000,
+ "rope_type": "default"
+ },
+ "sliding_window": null,
+ "tie_word_embeddings": true,
+ "transformers_version": "5.17.0",
+ "use_cache": false,
+ "use_sliding_window": false,
+ "vocab_size": 151936
+}
diff --git a/qwen3-1.7b-teutonic-5e6/checkpoint-500/generation_config.json b/qwen3-1.7b-teutonic-5e6/checkpoint-500/generation_config.json
new file mode 100644
index 0000000000000000000000000000000000000000..e6941fe4b04f26113d6eef6255d005c4d7090bda
--- /dev/null
+++ b/qwen3-1.7b-teutonic-5e6/checkpoint-500/generation_config.json
@@ -0,0 +1,12 @@
+{
+ "do_sample": true,
+ "eos_token_id": [
+ 151645,
+ 151643
+ ],
+ "pad_token_id": 151643,
+ "temperature": 0.6,
+ "top_k": 20,
+ "top_p": 0.95,
+ "transformers_version": "5.17.0"
+}
diff --git a/qwen3-1.7b-teutonic-5e6/checkpoint-500/global_step500/bf16_zero_pp_rank_0_mp_rank_00_optim_states.pt b/qwen3-1.7b-teutonic-5e6/checkpoint-500/global_step500/bf16_zero_pp_rank_0_mp_rank_00_optim_states.pt
new file mode 100644
index 0000000000000000000000000000000000000000..854b672f440410bc3e8f1921b5db6bb30a260bff
--- /dev/null
+++ b/qwen3-1.7b-teutonic-5e6/checkpoint-500/global_step500/bf16_zero_pp_rank_0_mp_rank_00_optim_states.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:8800edf94a3f4750c9e8c47f82dab44c8cc38201965ec29d6346f4d162fd170e
+size 10323466465
diff --git a/qwen3-1.7b-teutonic-5e6/checkpoint-500/global_step500/bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt b/qwen3-1.7b-teutonic-5e6/checkpoint-500/global_step500/bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt
new file mode 100644
index 0000000000000000000000000000000000000000..1a16d5856367942cb4eecc5f78d83ccf6ac6c5a7
--- /dev/null
+++ b/qwen3-1.7b-teutonic-5e6/checkpoint-500/global_step500/bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e136d49fb681060d83ca93ddcef1ea2178e9f0c5c64372db2e92efb55850266f
+size 10323469601
diff --git a/qwen3-1.7b-teutonic-5e6/checkpoint-500/global_step500/mp_rank_00_model_states.pt b/qwen3-1.7b-teutonic-5e6/checkpoint-500/global_step500/mp_rank_00_model_states.pt
new file mode 100644
index 0000000000000000000000000000000000000000..ce49b1c0fe34713c27b1fea1c08d3fdeb87d0d11
--- /dev/null
+++ b/qwen3-1.7b-teutonic-5e6/checkpoint-500/global_step500/mp_rank_00_model_states.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b7412a16595601ac6ba787a41f47b0c59c91caa5b52011b66c0aeb5370017759
+size 3441239653
diff --git a/qwen3-1.7b-teutonic-5e6/checkpoint-500/latest b/qwen3-1.7b-teutonic-5e6/checkpoint-500/latest
new file mode 100644
index 0000000000000000000000000000000000000000..f0b47ce15fff9a01b2a416a473b2148085048a50
--- /dev/null
+++ b/qwen3-1.7b-teutonic-5e6/checkpoint-500/latest
@@ -0,0 +1 @@
+global_step500
\ No newline at end of file
diff --git a/qwen3-1.7b-teutonic-5e6/checkpoint-500/model.safetensors b/qwen3-1.7b-teutonic-5e6/checkpoint-500/model.safetensors
new file mode 100644
index 0000000000000000000000000000000000000000..bf23dac79d2a45730934b7d8aab9289a12cbb5c4
--- /dev/null
+++ b/qwen3-1.7b-teutonic-5e6/checkpoint-500/model.safetensors
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:70efeaa28c9ee409c8711feaa8683fb3fcfacec4f4b27ca7004a1504efe0a937
+size 4063515640
diff --git a/qwen3-1.7b-teutonic-5e6/checkpoint-500/rng_state_0.pth b/qwen3-1.7b-teutonic-5e6/checkpoint-500/rng_state_0.pth
new file mode 100644
index 0000000000000000000000000000000000000000..9bb2f7c587bceefcdb6b3b7d2fac4553d0efcd5c
--- /dev/null
+++ b/qwen3-1.7b-teutonic-5e6/checkpoint-500/rng_state_0.pth
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:089233f905e02a28defe58c22a9508707e22f23429596c648fd56ed174238a34
+size 14917
diff --git a/qwen3-1.7b-teutonic-5e6/checkpoint-500/rng_state_1.pth b/qwen3-1.7b-teutonic-5e6/checkpoint-500/rng_state_1.pth
new file mode 100644
index 0000000000000000000000000000000000000000..b6237e77e3428dfd4402934af3d4a66fe06bd4a4
--- /dev/null
+++ b/qwen3-1.7b-teutonic-5e6/checkpoint-500/rng_state_1.pth
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d9713286962ba6aa18d08540280c05cfd8c3a97d78b57a5169a53bad4ab58862
+size 14917
diff --git a/qwen3-1.7b-teutonic-5e6/checkpoint-500/scheduler.pt b/qwen3-1.7b-teutonic-5e6/checkpoint-500/scheduler.pt
new file mode 100644
index 0000000000000000000000000000000000000000..91206e5c25c136c4f7c581d53f3dc5a254e11b02
--- /dev/null
+++ b/qwen3-1.7b-teutonic-5e6/checkpoint-500/scheduler.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:4380e66fdb61ffd08e35795045b869c90719a46c6838b482b138f11fc30632e3
+size 1465
diff --git a/qwen3-1.7b-teutonic-5e6/checkpoint-500/tokenizer.json b/qwen3-1.7b-teutonic-5e6/checkpoint-500/tokenizer.json
new file mode 100644
index 0000000000000000000000000000000000000000..c7afbed2efcdf019f88ab0572ec29d3bf595dfe2
--- /dev/null
+++ b/qwen3-1.7b-teutonic-5e6/checkpoint-500/tokenizer.json
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:be75606093db2094d7cd20f3c2f385c212750648bd6ea4fb2bf507a6a4c55506
+size 11422650
diff --git a/qwen3-1.7b-teutonic-5e6/checkpoint-500/tokenizer_config.json b/qwen3-1.7b-teutonic-5e6/checkpoint-500/tokenizer_config.json
new file mode 100644
index 0000000000000000000000000000000000000000..770e41d6c92519d525eede4cbcf3ba27f6425311
--- /dev/null
+++ b/qwen3-1.7b-teutonic-5e6/checkpoint-500/tokenizer_config.json
@@ -0,0 +1,30 @@
+{
+ "add_prefix_space": false,
+ "backend": "tokenizers",
+ "bos_token": null,
+ "clean_up_tokenization_spaces": false,
+ "eos_token": "<|im_end|>",
+ "errors": "replace",
+ "extra_special_tokens": [
+ "<|im_start|>",
+ "<|im_end|>",
+ "<|object_ref_start|>",
+ "<|object_ref_end|>",
+ "<|box_start|>",
+ "<|box_end|>",
+ "<|quad_start|>",
+ "<|quad_end|>",
+ "<|vision_start|>",
+ "<|vision_end|>",
+ "<|vision_pad|>",
+ "<|image_pad|>",
+ "<|video_pad|>"
+ ],
+ "is_local": false,
+ "local_files_only": false,
+ "model_max_length": 131072,
+ "pad_token": "<|endoftext|>",
+ "split_special_tokens": false,
+ "tokenizer_class": "Qwen2Tokenizer",
+ "unk_token": null
+}
diff --git a/qwen3-1.7b-teutonic-5e6/checkpoint-500/trainer_state.json b/qwen3-1.7b-teutonic-5e6/checkpoint-500/trainer_state.json
new file mode 100644
index 0000000000000000000000000000000000000000..5ddfd567aeff40337af5c7b88968585cf5e07b83
--- /dev/null
+++ b/qwen3-1.7b-teutonic-5e6/checkpoint-500/trainer_state.json
@@ -0,0 +1,5144 @@
+{
+ "best_global_step": 500,
+ "best_metric": 1.5444872379302979,
+ "best_model_checkpoint": "./checkpoints/qwen3-1.7b-teutonic-5e6/checkpoint-500",
+ "epoch": 1.0101010101010102,
+ "eval_steps": 50,
+ "global_step": 500,
+ "is_hyper_param_search": false,
+ "is_local_process_zero": true,
+ "is_world_process_zero": true,
+ "log_history": [
+ {
+ "entropy": 0.8295683860778809,
+ "epoch": 0.00202020202020202,
+ "grad_norm": 10.886772155761719,
+ "learning_rate": 5e-06,
+ "loss": 1.5185801982879639,
+ "mean_token_accuracy": 0.672874927520752,
+ "num_tokens": 16384.0,
+ "step": 1
+ },
+ {
+ "entropy": 1.1781222820281982,
+ "epoch": 0.00404040404040404,
+ "grad_norm": 13.80163288116455,
+ "learning_rate": 4.997979797979798e-06,
+ "loss": 2.18377685546875,
+ "mean_token_accuracy": 0.5700415372848511,
+ "num_tokens": 32768.0,
+ "step": 2
+ },
+ {
+ "entropy": 0.7972303032875061,
+ "epoch": 0.006060606060606061,
+ "grad_norm": 13.71261215209961,
+ "learning_rate": 4.995959595959596e-06,
+ "loss": 1.5810277462005615,
+ "mean_token_accuracy": 0.6782486438751221,
+ "num_tokens": 49152.0,
+ "step": 3
+ },
+ {
+ "entropy": 0.9709298610687256,
+ "epoch": 0.00808080808080808,
+ "grad_norm": 9.18797779083252,
+ "learning_rate": 4.993939393939394e-06,
+ "loss": 1.6923656463623047,
+ "mean_token_accuracy": 0.6447850465774536,
+ "num_tokens": 65536.0,
+ "step": 4
+ },
+ {
+ "entropy": 1.2161898612976074,
+ "epoch": 0.010101010101010102,
+ "grad_norm": 8.97508430480957,
+ "learning_rate": 4.991919191919192e-06,
+ "loss": 1.9361608028411865,
+ "mean_token_accuracy": 0.5922691822052002,
+ "num_tokens": 81920.0,
+ "step": 5
+ },
+ {
+ "entropy": 0.9950039982795715,
+ "epoch": 0.012121212121212121,
+ "grad_norm": 6.453213214874268,
+ "learning_rate": 4.98989898989899e-06,
+ "loss": 1.5012480020523071,
+ "mean_token_accuracy": 0.6671959161758423,
+ "num_tokens": 98304.0,
+ "step": 6
+ },
+ {
+ "entropy": 1.452682614326477,
+ "epoch": 0.014141414141414142,
+ "grad_norm": 8.50411605834961,
+ "learning_rate": 4.987878787878789e-06,
+ "loss": 2.1297783851623535,
+ "mean_token_accuracy": 0.5807889699935913,
+ "num_tokens": 114688.0,
+ "step": 7
+ },
+ {
+ "entropy": 1.4661014080047607,
+ "epoch": 0.01616161616161616,
+ "grad_norm": 4.290146827697754,
+ "learning_rate": 4.9858585858585865e-06,
+ "loss": 1.7391963005065918,
+ "mean_token_accuracy": 0.611993134021759,
+ "num_tokens": 131072.0,
+ "step": 8
+ },
+ {
+ "entropy": 1.9030903577804565,
+ "epoch": 0.01818181818181818,
+ "grad_norm": 4.503584384918213,
+ "learning_rate": 4.983838383838384e-06,
+ "loss": 2.283596992492676,
+ "mean_token_accuracy": 0.5261968970298767,
+ "num_tokens": 147456.0,
+ "step": 9
+ },
+ {
+ "entropy": 1.2784137725830078,
+ "epoch": 0.020202020202020204,
+ "grad_norm": 3.427410840988159,
+ "learning_rate": 4.981818181818182e-06,
+ "loss": 1.5032392740249634,
+ "mean_token_accuracy": 0.6578529477119446,
+ "num_tokens": 163840.0,
+ "step": 10
+ },
+ {
+ "entropy": 1.7487188577651978,
+ "epoch": 0.022222222222222223,
+ "grad_norm": 3.20536470413208,
+ "learning_rate": 4.97979797979798e-06,
+ "loss": 1.9184643030166626,
+ "mean_token_accuracy": 0.6093673706054688,
+ "num_tokens": 180224.0,
+ "step": 11
+ },
+ {
+ "entropy": 1.6585394144058228,
+ "epoch": 0.024242424242424242,
+ "grad_norm": 3.2463598251342773,
+ "learning_rate": 4.977777777777778e-06,
+ "loss": 1.8364639282226562,
+ "mean_token_accuracy": 0.6014899611473083,
+ "num_tokens": 196608.0,
+ "step": 12
+ },
+ {
+ "entropy": 1.4869519472122192,
+ "epoch": 0.026262626262626262,
+ "grad_norm": 2.8681719303131104,
+ "learning_rate": 4.975757575757576e-06,
+ "loss": 1.6356265544891357,
+ "mean_token_accuracy": 0.6361138224601746,
+ "num_tokens": 212992.0,
+ "step": 13
+ },
+ {
+ "entropy": 1.5179094076156616,
+ "epoch": 0.028282828282828285,
+ "grad_norm": 3.0470707416534424,
+ "learning_rate": 4.973737373737374e-06,
+ "loss": 1.6317660808563232,
+ "mean_token_accuracy": 0.6334269642829895,
+ "num_tokens": 229376.0,
+ "step": 14
+ },
+ {
+ "entropy": 1.7384330034255981,
+ "epoch": 0.030303030303030304,
+ "grad_norm": 2.6149742603302,
+ "learning_rate": 4.971717171717172e-06,
+ "loss": 1.67923903465271,
+ "mean_token_accuracy": 0.6469223499298096,
+ "num_tokens": 245760.0,
+ "step": 15
+ },
+ {
+ "entropy": 2.108799457550049,
+ "epoch": 0.03232323232323232,
+ "grad_norm": 3.694795608520508,
+ "learning_rate": 4.9696969696969696e-06,
+ "loss": 1.9052371978759766,
+ "mean_token_accuracy": 0.5927577018737793,
+ "num_tokens": 262144.0,
+ "step": 16
+ },
+ {
+ "entropy": 1.7532848119735718,
+ "epoch": 0.03434343434343434,
+ "grad_norm": 3.1510567665100098,
+ "learning_rate": 4.9676767676767675e-06,
+ "loss": 1.6382691860198975,
+ "mean_token_accuracy": 0.6311675906181335,
+ "num_tokens": 278528.0,
+ "step": 17
+ },
+ {
+ "entropy": 1.723548412322998,
+ "epoch": 0.03636363636363636,
+ "grad_norm": 3.093825340270996,
+ "learning_rate": 4.965656565656566e-06,
+ "loss": 1.6039624214172363,
+ "mean_token_accuracy": 0.6313507556915283,
+ "num_tokens": 294912.0,
+ "step": 18
+ },
+ {
+ "entropy": 1.4790765047073364,
+ "epoch": 0.03838383838383838,
+ "grad_norm": 2.7150535583496094,
+ "learning_rate": 4.963636363636364e-06,
+ "loss": 1.4173667430877686,
+ "mean_token_accuracy": 0.6627381443977356,
+ "num_tokens": 311296.0,
+ "step": 19
+ },
+ {
+ "entropy": 1.5827536582946777,
+ "epoch": 0.04040404040404041,
+ "grad_norm": 2.610966444015503,
+ "learning_rate": 4.961616161616162e-06,
+ "loss": 1.4604644775390625,
+ "mean_token_accuracy": 0.656509518623352,
+ "num_tokens": 327680.0,
+ "step": 20
+ },
+ {
+ "entropy": 2.054673194885254,
+ "epoch": 0.04242424242424243,
+ "grad_norm": 2.612940788269043,
+ "learning_rate": 4.95959595959596e-06,
+ "loss": 1.962029218673706,
+ "mean_token_accuracy": 0.5719956159591675,
+ "num_tokens": 344064.0,
+ "step": 21
+ },
+ {
+ "entropy": 1.6684198379516602,
+ "epoch": 0.044444444444444446,
+ "grad_norm": 2.5426013469696045,
+ "learning_rate": 4.957575757575758e-06,
+ "loss": 1.591176986694336,
+ "mean_token_accuracy": 0.6519907116889954,
+ "num_tokens": 360448.0,
+ "step": 22
+ },
+ {
+ "entropy": 1.9070855379104614,
+ "epoch": 0.046464646464646465,
+ "grad_norm": 2.5853357315063477,
+ "learning_rate": 4.9555555555555565e-06,
+ "loss": 1.930276870727539,
+ "mean_token_accuracy": 0.5895823240280151,
+ "num_tokens": 376832.0,
+ "step": 23
+ },
+ {
+ "entropy": 1.716689109802246,
+ "epoch": 0.048484848484848485,
+ "grad_norm": 2.7608494758605957,
+ "learning_rate": 4.953535353535354e-06,
+ "loss": 1.739159345626831,
+ "mean_token_accuracy": 0.6028944849967957,
+ "num_tokens": 393216.0,
+ "step": 24
+ },
+ {
+ "entropy": 1.5666695833206177,
+ "epoch": 0.050505050505050504,
+ "grad_norm": 2.5973074436187744,
+ "learning_rate": 4.951515151515152e-06,
+ "loss": 1.6016302108764648,
+ "mean_token_accuracy": 0.626099169254303,
+ "num_tokens": 409600.0,
+ "step": 25
+ },
+ {
+ "entropy": 1.4267652034759521,
+ "epoch": 0.052525252525252523,
+ "grad_norm": 2.163451910018921,
+ "learning_rate": 4.94949494949495e-06,
+ "loss": 1.5093598365783691,
+ "mean_token_accuracy": 0.6530288457870483,
+ "num_tokens": 425984.0,
+ "step": 26
+ },
+ {
+ "entropy": 1.8537150621414185,
+ "epoch": 0.05454545454545454,
+ "grad_norm": 2.6983258724212646,
+ "learning_rate": 4.947474747474748e-06,
+ "loss": 1.8831868171691895,
+ "mean_token_accuracy": 0.5802393555641174,
+ "num_tokens": 442368.0,
+ "step": 27
+ },
+ {
+ "entropy": 1.410015344619751,
+ "epoch": 0.05656565656565657,
+ "grad_norm": 2.4777796268463135,
+ "learning_rate": 4.945454545454546e-06,
+ "loss": 1.4835734367370605,
+ "mean_token_accuracy": 0.6519907116889954,
+ "num_tokens": 458752.0,
+ "step": 28
+ },
+ {
+ "entropy": 1.2399017810821533,
+ "epoch": 0.05858585858585859,
+ "grad_norm": 2.1676478385925293,
+ "learning_rate": 4.943434343434344e-06,
+ "loss": 1.330633282661438,
+ "mean_token_accuracy": 0.6802638173103333,
+ "num_tokens": 475136.0,
+ "step": 29
+ },
+ {
+ "entropy": 1.6478898525238037,
+ "epoch": 0.06060606060606061,
+ "grad_norm": 2.5325536727905273,
+ "learning_rate": 4.941414141414142e-06,
+ "loss": 1.771777868270874,
+ "mean_token_accuracy": 0.6128480434417725,
+ "num_tokens": 491520.0,
+ "step": 30
+ },
+ {
+ "entropy": 1.7915360927581787,
+ "epoch": 0.06262626262626263,
+ "grad_norm": 2.2419557571411133,
+ "learning_rate": 4.93939393939394e-06,
+ "loss": 1.870645523071289,
+ "mean_token_accuracy": 0.6074743270874023,
+ "num_tokens": 507904.0,
+ "step": 31
+ },
+ {
+ "entropy": 1.7638899087905884,
+ "epoch": 0.06464646464646465,
+ "grad_norm": 2.8063058853149414,
+ "learning_rate": 4.937373737373738e-06,
+ "loss": 1.83987557888031,
+ "mean_token_accuracy": 0.5836589932441711,
+ "num_tokens": 524288.0,
+ "step": 32
+ },
+ {
+ "entropy": 1.4097516536712646,
+ "epoch": 0.06666666666666667,
+ "grad_norm": 2.1857714653015137,
+ "learning_rate": 4.935353535353536e-06,
+ "loss": 1.477668285369873,
+ "mean_token_accuracy": 0.6507083773612976,
+ "num_tokens": 540672.0,
+ "step": 33
+ },
+ {
+ "entropy": 1.4447426795959473,
+ "epoch": 0.06868686868686869,
+ "grad_norm": 2.3771870136260986,
+ "learning_rate": 4.933333333333334e-06,
+ "loss": 1.536318063735962,
+ "mean_token_accuracy": 0.6383732557296753,
+ "num_tokens": 557056.0,
+ "step": 34
+ },
+ {
+ "entropy": 1.9658164978027344,
+ "epoch": 0.0707070707070707,
+ "grad_norm": 2.4784936904907227,
+ "learning_rate": 4.931313131313132e-06,
+ "loss": 2.0565035343170166,
+ "mean_token_accuracy": 0.5581949353218079,
+ "num_tokens": 573440.0,
+ "step": 35
+ },
+ {
+ "entropy": 1.6045317649841309,
+ "epoch": 0.07272727272727272,
+ "grad_norm": 2.1128504276275635,
+ "learning_rate": 4.92929292929293e-06,
+ "loss": 1.6234952211380005,
+ "mean_token_accuracy": 0.6293356418609619,
+ "num_tokens": 589824.0,
+ "step": 36
+ },
+ {
+ "entropy": 1.5491269826889038,
+ "epoch": 0.07474747474747474,
+ "grad_norm": 2.1672937870025635,
+ "learning_rate": 4.927272727272728e-06,
+ "loss": 1.5809822082519531,
+ "mean_token_accuracy": 0.6419760584831238,
+ "num_tokens": 606208.0,
+ "step": 37
+ },
+ {
+ "entropy": 1.545534610748291,
+ "epoch": 0.07676767676767676,
+ "grad_norm": 2.2526934146881104,
+ "learning_rate": 4.925252525252526e-06,
+ "loss": 1.5707473754882812,
+ "mean_token_accuracy": 0.6344650983810425,
+ "num_tokens": 622592.0,
+ "step": 38
+ },
+ {
+ "entropy": 1.579596757888794,
+ "epoch": 0.07878787878787878,
+ "grad_norm": 2.3988492488861084,
+ "learning_rate": 4.9232323232323235e-06,
+ "loss": 1.6725983619689941,
+ "mean_token_accuracy": 0.6243282556533813,
+ "num_tokens": 638976.0,
+ "step": 39
+ },
+ {
+ "entropy": 2.0438785552978516,
+ "epoch": 0.08080808080808081,
+ "grad_norm": 2.3314359188079834,
+ "learning_rate": 4.9212121212121214e-06,
+ "loss": 2.0424888134002686,
+ "mean_token_accuracy": 0.570713222026825,
+ "num_tokens": 655360.0,
+ "step": 40
+ },
+ {
+ "entropy": 1.8690773248672485,
+ "epoch": 0.08282828282828283,
+ "grad_norm": 2.130401134490967,
+ "learning_rate": 4.919191919191919e-06,
+ "loss": 1.8796970844268799,
+ "mean_token_accuracy": 0.5882999300956726,
+ "num_tokens": 671744.0,
+ "step": 41
+ },
+ {
+ "entropy": 1.756626009941101,
+ "epoch": 0.08484848484848485,
+ "grad_norm": 2.342318534851074,
+ "learning_rate": 4.917171717171717e-06,
+ "loss": 1.71901535987854,
+ "mean_token_accuracy": 0.6173668503761292,
+ "num_tokens": 688128.0,
+ "step": 42
+ },
+ {
+ "entropy": 1.4642508029937744,
+ "epoch": 0.08686868686868687,
+ "grad_norm": 1.993338704109192,
+ "learning_rate": 4.915151515151516e-06,
+ "loss": 1.4660165309906006,
+ "mean_token_accuracy": 0.6687225103378296,
+ "num_tokens": 704512.0,
+ "step": 43
+ },
+ {
+ "entropy": 1.790807843208313,
+ "epoch": 0.08888888888888889,
+ "grad_norm": 2.3186943531036377,
+ "learning_rate": 4.913131313131314e-06,
+ "loss": 1.7102060317993164,
+ "mean_token_accuracy": 0.6238397359848022,
+ "num_tokens": 720896.0,
+ "step": 44
+ },
+ {
+ "entropy": 1.5970485210418701,
+ "epoch": 0.09090909090909091,
+ "grad_norm": 2.299307346343994,
+ "learning_rate": 4.911111111111112e-06,
+ "loss": 1.5170013904571533,
+ "mean_token_accuracy": 0.652662456035614,
+ "num_tokens": 737280.0,
+ "step": 45
+ },
+ {
+ "entropy": 1.3955466747283936,
+ "epoch": 0.09292929292929293,
+ "grad_norm": 2.171952962875366,
+ "learning_rate": 4.90909090909091e-06,
+ "loss": 1.4059661626815796,
+ "mean_token_accuracy": 0.6654250025749207,
+ "num_tokens": 753664.0,
+ "step": 46
+ },
+ {
+ "entropy": 1.7198575735092163,
+ "epoch": 0.09494949494949495,
+ "grad_norm": 2.385092258453369,
+ "learning_rate": 4.9070707070707075e-06,
+ "loss": 1.730346918106079,
+ "mean_token_accuracy": 0.6191987991333008,
+ "num_tokens": 770048.0,
+ "step": 47
+ },
+ {
+ "entropy": 1.3542555570602417,
+ "epoch": 0.09696969696969697,
+ "grad_norm": 2.1463189125061035,
+ "learning_rate": 4.905050505050505e-06,
+ "loss": 1.346085786819458,
+ "mean_token_accuracy": 0.680446982383728,
+ "num_tokens": 786432.0,
+ "step": 48
+ },
+ {
+ "entropy": 1.8084443807601929,
+ "epoch": 0.09898989898989899,
+ "grad_norm": 2.1505849361419678,
+ "learning_rate": 4.903030303030303e-06,
+ "loss": 1.847151756286621,
+ "mean_token_accuracy": 0.5926355719566345,
+ "num_tokens": 802816.0,
+ "step": 49
+ },
+ {
+ "entropy": 1.751160979270935,
+ "epoch": 0.10101010101010101,
+ "grad_norm": 2.1436259746551514,
+ "learning_rate": 4.901010101010101e-06,
+ "loss": 1.7802404165267944,
+ "mean_token_accuracy": 0.5996580123901367,
+ "num_tokens": 819200.0,
+ "step": 50
+ },
+ {
+ "epoch": 0.10101010101010101,
+ "eval_entropy": 1.6292865045609013,
+ "eval_loss": 1.6725393533706665,
+ "eval_mean_token_accuracy": 0.6230050469598463,
+ "eval_num_tokens": 819200.0,
+ "eval_runtime": 43.9148,
+ "eval_samples_per_second": 22.544,
+ "eval_steps_per_second": 2.824,
+ "step": 50
+ },
+ {
+ "entropy": 1.4428319931030273,
+ "epoch": 0.10303030303030303,
+ "grad_norm": 2.0954954624176025,
+ "learning_rate": 4.898989898989899e-06,
+ "loss": 1.4927232265472412,
+ "mean_token_accuracy": 0.6522349715232849,
+ "num_tokens": 835584.0,
+ "step": 51
+ },
+ {
+ "entropy": 1.7493115663528442,
+ "epoch": 0.10505050505050505,
+ "grad_norm": 2.300030469894409,
+ "learning_rate": 4.896969696969697e-06,
+ "loss": 1.737417221069336,
+ "mean_token_accuracy": 0.6213361024856567,
+ "num_tokens": 851968.0,
+ "step": 52
+ },
+ {
+ "entropy": 1.7801647186279297,
+ "epoch": 0.10707070707070707,
+ "grad_norm": 2.3947081565856934,
+ "learning_rate": 4.894949494949495e-06,
+ "loss": 1.8372564315795898,
+ "mean_token_accuracy": 0.5931240916252136,
+ "num_tokens": 868352.0,
+ "step": 53
+ },
+ {
+ "entropy": 1.6934887170791626,
+ "epoch": 0.10909090909090909,
+ "grad_norm": 2.1981089115142822,
+ "learning_rate": 4.8929292929292936e-06,
+ "loss": 1.734646201133728,
+ "mean_token_accuracy": 0.611932098865509,
+ "num_tokens": 884736.0,
+ "step": 54
+ },
+ {
+ "entropy": 1.3401941061019897,
+ "epoch": 0.1111111111111111,
+ "grad_norm": 2.193511724472046,
+ "learning_rate": 4.8909090909090914e-06,
+ "loss": 1.399888038635254,
+ "mean_token_accuracy": 0.6745847463607788,
+ "num_tokens": 901120.0,
+ "step": 55
+ },
+ {
+ "entropy": 1.8244661092758179,
+ "epoch": 0.11313131313131314,
+ "grad_norm": 2.4358489513397217,
+ "learning_rate": 4.888888888888889e-06,
+ "loss": 1.8716282844543457,
+ "mean_token_accuracy": 0.5867122411727905,
+ "num_tokens": 917504.0,
+ "step": 56
+ },
+ {
+ "entropy": 1.5019619464874268,
+ "epoch": 0.11515151515151516,
+ "grad_norm": 2.1135261058807373,
+ "learning_rate": 4.886868686868687e-06,
+ "loss": 1.523103952407837,
+ "mean_token_accuracy": 0.6414265036582947,
+ "num_tokens": 933888.0,
+ "step": 57
+ },
+ {
+ "entropy": 1.6132855415344238,
+ "epoch": 0.11717171717171718,
+ "grad_norm": 2.026301145553589,
+ "learning_rate": 4.884848484848485e-06,
+ "loss": 1.6233609914779663,
+ "mean_token_accuracy": 0.6279311180114746,
+ "num_tokens": 950272.0,
+ "step": 58
+ },
+ {
+ "entropy": 1.738538384437561,
+ "epoch": 0.1191919191919192,
+ "grad_norm": 2.1539394855499268,
+ "learning_rate": 4.882828282828283e-06,
+ "loss": 1.8147332668304443,
+ "mean_token_accuracy": 0.6124816536903381,
+ "num_tokens": 966656.0,
+ "step": 59
+ },
+ {
+ "entropy": 1.5533764362335205,
+ "epoch": 0.12121212121212122,
+ "grad_norm": 2.2603628635406494,
+ "learning_rate": 4.880808080808081e-06,
+ "loss": 1.562873125076294,
+ "mean_token_accuracy": 0.6398388147354126,
+ "num_tokens": 983040.0,
+ "step": 60
+ },
+ {
+ "entropy": 1.722406268119812,
+ "epoch": 0.12323232323232323,
+ "grad_norm": 2.3630757331848145,
+ "learning_rate": 4.878787878787879e-06,
+ "loss": 1.7461689710617065,
+ "mean_token_accuracy": 0.6023448705673218,
+ "num_tokens": 999424.0,
+ "step": 61
+ },
+ {
+ "entropy": 1.6533517837524414,
+ "epoch": 0.12525252525252525,
+ "grad_norm": 2.2165415287017822,
+ "learning_rate": 4.876767676767677e-06,
+ "loss": 1.676560640335083,
+ "mean_token_accuracy": 0.6437469720840454,
+ "num_tokens": 1015808.0,
+ "step": 62
+ },
+ {
+ "entropy": 1.749995470046997,
+ "epoch": 0.12727272727272726,
+ "grad_norm": 2.1572678089141846,
+ "learning_rate": 4.8747474747474745e-06,
+ "loss": 1.784087896347046,
+ "mean_token_accuracy": 0.5884831547737122,
+ "num_tokens": 1032192.0,
+ "step": 63
+ },
+ {
+ "entropy": 1.442152738571167,
+ "epoch": 0.1292929292929293,
+ "grad_norm": 2.163490056991577,
+ "learning_rate": 4.872727272727273e-06,
+ "loss": 1.4307503700256348,
+ "mean_token_accuracy": 0.6618832349777222,
+ "num_tokens": 1048576.0,
+ "step": 64
+ },
+ {
+ "entropy": 1.2657029628753662,
+ "epoch": 0.13131313131313133,
+ "grad_norm": 2.1225337982177734,
+ "learning_rate": 4.870707070707071e-06,
+ "loss": 1.2731966972351074,
+ "mean_token_accuracy": 0.688568651676178,
+ "num_tokens": 1064960.0,
+ "step": 65
+ },
+ {
+ "entropy": 1.1613880395889282,
+ "epoch": 0.13333333333333333,
+ "grad_norm": 1.9527196884155273,
+ "learning_rate": 4.868686868686869e-06,
+ "loss": 1.13922119140625,
+ "mean_token_accuracy": 0.7389472126960754,
+ "num_tokens": 1081344.0,
+ "step": 66
+ },
+ {
+ "entropy": 1.6936380863189697,
+ "epoch": 0.13535353535353536,
+ "grad_norm": 2.004284620285034,
+ "learning_rate": 4.866666666666667e-06,
+ "loss": 1.6982238292694092,
+ "mean_token_accuracy": 0.6191987991333008,
+ "num_tokens": 1097728.0,
+ "step": 67
+ },
+ {
+ "entropy": 1.750565528869629,
+ "epoch": 0.13737373737373737,
+ "grad_norm": 2.225955009460449,
+ "learning_rate": 4.864646464646466e-06,
+ "loss": 1.796521782875061,
+ "mean_token_accuracy": 0.5934293866157532,
+ "num_tokens": 1114112.0,
+ "step": 68
+ },
+ {
+ "entropy": 1.6608220338821411,
+ "epoch": 0.1393939393939394,
+ "grad_norm": 2.127035617828369,
+ "learning_rate": 4.8626262626262636e-06,
+ "loss": 1.6633095741271973,
+ "mean_token_accuracy": 0.6356253027915955,
+ "num_tokens": 1130496.0,
+ "step": 69
+ },
+ {
+ "entropy": 1.4848661422729492,
+ "epoch": 0.1414141414141414,
+ "grad_norm": 2.0338215827941895,
+ "learning_rate": 4.8606060606060615e-06,
+ "loss": 1.4789674282073975,
+ "mean_token_accuracy": 0.6680508255958557,
+ "num_tokens": 1146880.0,
+ "step": 70
+ },
+ {
+ "entropy": 1.2310466766357422,
+ "epoch": 0.14343434343434344,
+ "grad_norm": 2.105898141860962,
+ "learning_rate": 4.858585858585859e-06,
+ "loss": 1.2301937341690063,
+ "mean_token_accuracy": 0.7040791511535645,
+ "num_tokens": 1163264.0,
+ "step": 71
+ },
+ {
+ "entropy": 1.6310514211654663,
+ "epoch": 0.14545454545454545,
+ "grad_norm": 2.152125358581543,
+ "learning_rate": 4.856565656565657e-06,
+ "loss": 1.643636703491211,
+ "mean_token_accuracy": 0.6221299171447754,
+ "num_tokens": 1179648.0,
+ "step": 72
+ },
+ {
+ "entropy": 1.4747940301895142,
+ "epoch": 0.14747474747474748,
+ "grad_norm": 2.096461296081543,
+ "learning_rate": 4.854545454545455e-06,
+ "loss": 1.42953360080719,
+ "mean_token_accuracy": 0.6651807427406311,
+ "num_tokens": 1196032.0,
+ "step": 73
+ },
+ {
+ "entropy": 1.819259524345398,
+ "epoch": 0.1494949494949495,
+ "grad_norm": 2.2852063179016113,
+ "learning_rate": 4.852525252525253e-06,
+ "loss": 1.845325231552124,
+ "mean_token_accuracy": 0.6036272644996643,
+ "num_tokens": 1212416.0,
+ "step": 74
+ },
+ {
+ "entropy": 1.4233598709106445,
+ "epoch": 0.15151515151515152,
+ "grad_norm": 2.1157381534576416,
+ "learning_rate": 4.850505050505051e-06,
+ "loss": 1.4565438032150269,
+ "mean_token_accuracy": 0.6607230305671692,
+ "num_tokens": 1228800.0,
+ "step": 75
+ },
+ {
+ "entropy": 1.6441459655761719,
+ "epoch": 0.15353535353535352,
+ "grad_norm": 2.2092180252075195,
+ "learning_rate": 4.848484848484849e-06,
+ "loss": 1.6467639207839966,
+ "mean_token_accuracy": 0.6285417675971985,
+ "num_tokens": 1245184.0,
+ "step": 76
+ },
+ {
+ "entropy": 1.6124308109283447,
+ "epoch": 0.15555555555555556,
+ "grad_norm": 2.231876850128174,
+ "learning_rate": 4.846464646464647e-06,
+ "loss": 1.617384672164917,
+ "mean_token_accuracy": 0.6340987086296082,
+ "num_tokens": 1261568.0,
+ "step": 77
+ },
+ {
+ "entropy": 1.828405499458313,
+ "epoch": 0.15757575757575756,
+ "grad_norm": 2.1372313499450684,
+ "learning_rate": 4.8444444444444446e-06,
+ "loss": 1.8333203792572021,
+ "mean_token_accuracy": 0.5979481935501099,
+ "num_tokens": 1277952.0,
+ "step": 78
+ },
+ {
+ "entropy": 1.855564832687378,
+ "epoch": 0.1595959595959596,
+ "grad_norm": 2.1422762870788574,
+ "learning_rate": 4.842424242424243e-06,
+ "loss": 1.9133609533309937,
+ "mean_token_accuracy": 0.570652186870575,
+ "num_tokens": 1294336.0,
+ "step": 79
+ },
+ {
+ "entropy": 1.7908183336257935,
+ "epoch": 0.16161616161616163,
+ "grad_norm": 2.172368049621582,
+ "learning_rate": 4.840404040404041e-06,
+ "loss": 1.8109419345855713,
+ "mean_token_accuracy": 0.60332190990448,
+ "num_tokens": 1310720.0,
+ "step": 80
+ },
+ {
+ "entropy": 1.9537721872329712,
+ "epoch": 0.16363636363636364,
+ "grad_norm": 2.199505090713501,
+ "learning_rate": 4.838383838383839e-06,
+ "loss": 2.0001401901245117,
+ "mean_token_accuracy": 0.5585613250732422,
+ "num_tokens": 1327104.0,
+ "step": 81
+ },
+ {
+ "entropy": 1.5365772247314453,
+ "epoch": 0.16565656565656567,
+ "grad_norm": 2.0778913497924805,
+ "learning_rate": 4.836363636363637e-06,
+ "loss": 1.575089931488037,
+ "mean_token_accuracy": 0.6375183463096619,
+ "num_tokens": 1343488.0,
+ "step": 82
+ },
+ {
+ "entropy": 1.7177143096923828,
+ "epoch": 0.16767676767676767,
+ "grad_norm": 2.1010894775390625,
+ "learning_rate": 4.834343434343435e-06,
+ "loss": 1.7428388595581055,
+ "mean_token_accuracy": 0.6113824844360352,
+ "num_tokens": 1359872.0,
+ "step": 83
+ },
+ {
+ "entropy": 1.5080527067184448,
+ "epoch": 0.1696969696969697,
+ "grad_norm": 2.021969795227051,
+ "learning_rate": 4.832323232323233e-06,
+ "loss": 1.5361201763153076,
+ "mean_token_accuracy": 0.656509518623352,
+ "num_tokens": 1376256.0,
+ "step": 84
+ },
+ {
+ "entropy": 1.5042120218276978,
+ "epoch": 0.1717171717171717,
+ "grad_norm": 2.121314287185669,
+ "learning_rate": 4.830303030303031e-06,
+ "loss": 1.4912033081054688,
+ "mean_token_accuracy": 0.6621274948120117,
+ "num_tokens": 1392640.0,
+ "step": 85
+ },
+ {
+ "entropy": 1.9523948431015015,
+ "epoch": 0.17373737373737375,
+ "grad_norm": 2.1702609062194824,
+ "learning_rate": 4.8282828282828285e-06,
+ "loss": 1.9369449615478516,
+ "mean_token_accuracy": 0.5727283954620361,
+ "num_tokens": 1409024.0,
+ "step": 86
+ },
+ {
+ "entropy": 1.6174770593643188,
+ "epoch": 0.17575757575757575,
+ "grad_norm": 2.222412109375,
+ "learning_rate": 4.826262626262626e-06,
+ "loss": 1.6794973611831665,
+ "mean_token_accuracy": 0.6151685118675232,
+ "num_tokens": 1425408.0,
+ "step": 87
+ },
+ {
+ "entropy": 1.1480307579040527,
+ "epoch": 0.17777777777777778,
+ "grad_norm": 1.98936128616333,
+ "learning_rate": 4.824242424242424e-06,
+ "loss": 1.1324063539505005,
+ "mean_token_accuracy": 0.7193453907966614,
+ "num_tokens": 1441792.0,
+ "step": 88
+ },
+ {
+ "entropy": 1.5290263891220093,
+ "epoch": 0.1797979797979798,
+ "grad_norm": 2.098860263824463,
+ "learning_rate": 4.822222222222222e-06,
+ "loss": 1.556044340133667,
+ "mean_token_accuracy": 0.6415486335754395,
+ "num_tokens": 1458176.0,
+ "step": 89
+ },
+ {
+ "entropy": 1.1366127729415894,
+ "epoch": 0.18181818181818182,
+ "grad_norm": 1.9387420415878296,
+ "learning_rate": 4.820202020202021e-06,
+ "loss": 1.1635141372680664,
+ "mean_token_accuracy": 0.7168416976928711,
+ "num_tokens": 1474560.0,
+ "step": 90
+ },
+ {
+ "entropy": 1.5142875909805298,
+ "epoch": 0.18383838383838383,
+ "grad_norm": 2.259131908416748,
+ "learning_rate": 4.818181818181819e-06,
+ "loss": 1.5329954624176025,
+ "mean_token_accuracy": 0.6471666097640991,
+ "num_tokens": 1490944.0,
+ "step": 91
+ },
+ {
+ "entropy": 1.5197135210037231,
+ "epoch": 0.18585858585858586,
+ "grad_norm": 2.1230807304382324,
+ "learning_rate": 4.816161616161617e-06,
+ "loss": 1.541062593460083,
+ "mean_token_accuracy": 0.6398388147354126,
+ "num_tokens": 1507328.0,
+ "step": 92
+ },
+ {
+ "entropy": 1.5469954013824463,
+ "epoch": 0.18787878787878787,
+ "grad_norm": 2.0568583011627197,
+ "learning_rate": 4.8141414141414146e-06,
+ "loss": 1.5564078092575073,
+ "mean_token_accuracy": 0.6384953856468201,
+ "num_tokens": 1523712.0,
+ "step": 93
+ },
+ {
+ "entropy": 1.5429692268371582,
+ "epoch": 0.1898989898989899,
+ "grad_norm": 2.200144052505493,
+ "learning_rate": 4.8121212121212125e-06,
+ "loss": 1.5292989015579224,
+ "mean_token_accuracy": 0.6373351216316223,
+ "num_tokens": 1540096.0,
+ "step": 94
+ },
+ {
+ "entropy": 1.4556076526641846,
+ "epoch": 0.1919191919191919,
+ "grad_norm": 2.250291585922241,
+ "learning_rate": 4.81010101010101e-06,
+ "loss": 1.426419734954834,
+ "mean_token_accuracy": 0.6604176759719849,
+ "num_tokens": 1556480.0,
+ "step": 95
+ },
+ {
+ "entropy": 1.4021440744400024,
+ "epoch": 0.19393939393939394,
+ "grad_norm": 2.287038803100586,
+ "learning_rate": 4.808080808080808e-06,
+ "loss": 1.4377000331878662,
+ "mean_token_accuracy": 0.6572422981262207,
+ "num_tokens": 1572864.0,
+ "step": 96
+ },
+ {
+ "entropy": 1.658683180809021,
+ "epoch": 0.19595959595959597,
+ "grad_norm": 2.1817963123321533,
+ "learning_rate": 4.806060606060606e-06,
+ "loss": 1.6514620780944824,
+ "mean_token_accuracy": 0.6278700828552246,
+ "num_tokens": 1589248.0,
+ "step": 97
+ },
+ {
+ "entropy": 1.5182844400405884,
+ "epoch": 0.19797979797979798,
+ "grad_norm": 2.144071340560913,
+ "learning_rate": 4.804040404040404e-06,
+ "loss": 1.5554628372192383,
+ "mean_token_accuracy": 0.6311064958572388,
+ "num_tokens": 1605632.0,
+ "step": 98
+ },
+ {
+ "entropy": 1.5997719764709473,
+ "epoch": 0.2,
+ "grad_norm": 2.0921664237976074,
+ "learning_rate": 4.802020202020202e-06,
+ "loss": 1.6185517311096191,
+ "mean_token_accuracy": 0.6286028623580933,
+ "num_tokens": 1622016.0,
+ "step": 99
+ },
+ {
+ "entropy": 1.4011279344558716,
+ "epoch": 0.20202020202020202,
+ "grad_norm": 2.2240700721740723,
+ "learning_rate": 4.800000000000001e-06,
+ "loss": 1.4112927913665771,
+ "mean_token_accuracy": 0.6696995496749878,
+ "num_tokens": 1638400.0,
+ "step": 100
+ },
+ {
+ "epoch": 0.20202020202020202,
+ "eval_entropy": 1.606662715634992,
+ "eval_loss": 1.6283859014511108,
+ "eval_mean_token_accuracy": 0.6293055717983553,
+ "eval_num_tokens": 1638400.0,
+ "eval_runtime": 43.7782,
+ "eval_samples_per_second": 22.614,
+ "eval_steps_per_second": 2.832,
+ "step": 100
+ },
+ {
+ "entropy": 1.6743865013122559,
+ "epoch": 0.20404040404040405,
+ "grad_norm": 2.025153398513794,
+ "learning_rate": 4.7979797979797985e-06,
+ "loss": 1.6404725313186646,
+ "mean_token_accuracy": 0.656509518623352,
+ "num_tokens": 1654784.0,
+ "step": 101
+ },
+ {
+ "entropy": 1.6196308135986328,
+ "epoch": 0.20606060606060606,
+ "grad_norm": 2.037229299545288,
+ "learning_rate": 4.795959595959596e-06,
+ "loss": 1.647303581237793,
+ "mean_token_accuracy": 0.6203590631484985,
+ "num_tokens": 1671168.0,
+ "step": 102
+ },
+ {
+ "entropy": 1.5600125789642334,
+ "epoch": 0.2080808080808081,
+ "grad_norm": 2.144221782684326,
+ "learning_rate": 4.793939393939394e-06,
+ "loss": 1.6199731826782227,
+ "mean_token_accuracy": 0.6249389052391052,
+ "num_tokens": 1687552.0,
+ "step": 103
+ },
+ {
+ "entropy": 2.064497947692871,
+ "epoch": 0.2101010101010101,
+ "grad_norm": 2.3956470489501953,
+ "learning_rate": 4.791919191919192e-06,
+ "loss": 2.048987627029419,
+ "mean_token_accuracy": 0.5507450103759766,
+ "num_tokens": 1703936.0,
+ "step": 104
+ },
+ {
+ "entropy": 1.7934812307357788,
+ "epoch": 0.21212121212121213,
+ "grad_norm": 2.487302780151367,
+ "learning_rate": 4.78989898989899e-06,
+ "loss": 1.8285956382751465,
+ "mean_token_accuracy": 0.5749877691268921,
+ "num_tokens": 1720320.0,
+ "step": 105
+ },
+ {
+ "entropy": 1.365216851234436,
+ "epoch": 0.21414141414141413,
+ "grad_norm": 2.0714964866638184,
+ "learning_rate": 4.787878787878788e-06,
+ "loss": 1.3870220184326172,
+ "mean_token_accuracy": 0.6692721247673035,
+ "num_tokens": 1736704.0,
+ "step": 106
+ },
+ {
+ "entropy": 1.4983874559402466,
+ "epoch": 0.21616161616161617,
+ "grad_norm": 1.9995853900909424,
+ "learning_rate": 4.785858585858586e-06,
+ "loss": 1.4949266910552979,
+ "mean_token_accuracy": 0.6554103493690491,
+ "num_tokens": 1753088.0,
+ "step": 107
+ },
+ {
+ "entropy": 2.0503509044647217,
+ "epoch": 0.21818181818181817,
+ "grad_norm": 2.190884590148926,
+ "learning_rate": 4.783838383838385e-06,
+ "loss": 2.079286813735962,
+ "mean_token_accuracy": 0.5657669901847839,
+ "num_tokens": 1769472.0,
+ "step": 108
+ },
+ {
+ "entropy": 1.5557374954223633,
+ "epoch": 0.2202020202020202,
+ "grad_norm": 2.044100761413574,
+ "learning_rate": 4.7818181818181825e-06,
+ "loss": 1.5735318660736084,
+ "mean_token_accuracy": 0.6359916925430298,
+ "num_tokens": 1785856.0,
+ "step": 109
+ },
+ {
+ "entropy": 1.5567635297775269,
+ "epoch": 0.2222222222222222,
+ "grad_norm": 2.3714160919189453,
+ "learning_rate": 4.77979797979798e-06,
+ "loss": 1.6026921272277832,
+ "mean_token_accuracy": 0.6290302872657776,
+ "num_tokens": 1802240.0,
+ "step": 110
+ },
+ {
+ "entropy": 1.8457536697387695,
+ "epoch": 0.22424242424242424,
+ "grad_norm": 2.202939987182617,
+ "learning_rate": 4.777777777777778e-06,
+ "loss": 1.7986081838607788,
+ "mean_token_accuracy": 0.5952613353729248,
+ "num_tokens": 1818624.0,
+ "step": 111
+ },
+ {
+ "entropy": 1.8822020292282104,
+ "epoch": 0.22626262626262628,
+ "grad_norm": 2.2674992084503174,
+ "learning_rate": 4.775757575757576e-06,
+ "loss": 1.9095954895019531,
+ "mean_token_accuracy": 0.5769418478012085,
+ "num_tokens": 1835008.0,
+ "step": 112
+ },
+ {
+ "entropy": 1.5445998907089233,
+ "epoch": 0.22828282828282828,
+ "grad_norm": 2.2848100662231445,
+ "learning_rate": 4.773737373737374e-06,
+ "loss": 1.5832195281982422,
+ "mean_token_accuracy": 0.6337933540344238,
+ "num_tokens": 1851392.0,
+ "step": 113
+ },
+ {
+ "entropy": 1.934509515762329,
+ "epoch": 0.23030303030303031,
+ "grad_norm": 2.3585174083709717,
+ "learning_rate": 4.771717171717172e-06,
+ "loss": 1.9809319972991943,
+ "mean_token_accuracy": 0.5789570212364197,
+ "num_tokens": 1867776.0,
+ "step": 114
+ },
+ {
+ "entropy": 1.7354214191436768,
+ "epoch": 0.23232323232323232,
+ "grad_norm": 2.13913631439209,
+ "learning_rate": 4.769696969696971e-06,
+ "loss": 1.7527806758880615,
+ "mean_token_accuracy": 0.6061308979988098,
+ "num_tokens": 1884160.0,
+ "step": 115
+ },
+ {
+ "entropy": 1.7714784145355225,
+ "epoch": 0.23434343434343435,
+ "grad_norm": 1.9915403127670288,
+ "learning_rate": 4.7676767676767685e-06,
+ "loss": 1.8065431118011475,
+ "mean_token_accuracy": 0.5986199378967285,
+ "num_tokens": 1900544.0,
+ "step": 116
+ },
+ {
+ "entropy": 1.5272125005722046,
+ "epoch": 0.23636363636363636,
+ "grad_norm": 2.004565715789795,
+ "learning_rate": 4.765656565656566e-06,
+ "loss": 1.5074517726898193,
+ "mean_token_accuracy": 0.6601123809814453,
+ "num_tokens": 1916928.0,
+ "step": 117
+ },
+ {
+ "entropy": 1.5393218994140625,
+ "epoch": 0.2383838383838384,
+ "grad_norm": 2.018089532852173,
+ "learning_rate": 4.763636363636364e-06,
+ "loss": 1.5607573986053467,
+ "mean_token_accuracy": 0.6540058851242065,
+ "num_tokens": 1933312.0,
+ "step": 118
+ },
+ {
+ "entropy": 1.7827534675598145,
+ "epoch": 0.2404040404040404,
+ "grad_norm": 2.062368392944336,
+ "learning_rate": 4.761616161616162e-06,
+ "loss": 1.7844001054763794,
+ "mean_token_accuracy": 0.6187102794647217,
+ "num_tokens": 1949696.0,
+ "step": 119
+ },
+ {
+ "entropy": 1.416417121887207,
+ "epoch": 0.24242424242424243,
+ "grad_norm": 2.0980024337768555,
+ "learning_rate": 4.75959595959596e-06,
+ "loss": 1.4041050672531128,
+ "mean_token_accuracy": 0.6561431288719177,
+ "num_tokens": 1966080.0,
+ "step": 120
+ },
+ {
+ "entropy": 1.3325153589248657,
+ "epoch": 0.24444444444444444,
+ "grad_norm": 1.9985002279281616,
+ "learning_rate": 4.757575757575758e-06,
+ "loss": 1.3197274208068848,
+ "mean_token_accuracy": 0.6806912422180176,
+ "num_tokens": 1982464.0,
+ "step": 121
+ },
+ {
+ "entropy": 1.7306798696517944,
+ "epoch": 0.24646464646464647,
+ "grad_norm": 2.2098441123962402,
+ "learning_rate": 4.755555555555556e-06,
+ "loss": 1.7595295906066895,
+ "mean_token_accuracy": 0.6110160946846008,
+ "num_tokens": 1998848.0,
+ "step": 122
+ },
+ {
+ "entropy": 1.3733264207839966,
+ "epoch": 0.24848484848484848,
+ "grad_norm": 1.9827327728271484,
+ "learning_rate": 4.753535353535354e-06,
+ "loss": 1.4026854038238525,
+ "mean_token_accuracy": 0.65486079454422,
+ "num_tokens": 2015232.0,
+ "step": 123
+ },
+ {
+ "entropy": 1.5910528898239136,
+ "epoch": 0.2505050505050505,
+ "grad_norm": 1.9615319967269897,
+ "learning_rate": 4.751515151515152e-06,
+ "loss": 1.6071114540100098,
+ "mean_token_accuracy": 0.648937463760376,
+ "num_tokens": 2031616.0,
+ "step": 124
+ },
+ {
+ "entropy": 1.3300938606262207,
+ "epoch": 0.25252525252525254,
+ "grad_norm": 1.9878504276275635,
+ "learning_rate": 4.7494949494949495e-06,
+ "loss": 1.3623383045196533,
+ "mean_token_accuracy": 0.6873473525047302,
+ "num_tokens": 2048000.0,
+ "step": 125
+ },
+ {
+ "entropy": 1.6087368726730347,
+ "epoch": 0.2545454545454545,
+ "grad_norm": 2.264647960662842,
+ "learning_rate": 4.747474747474748e-06,
+ "loss": 1.634058952331543,
+ "mean_token_accuracy": 0.6211528778076172,
+ "num_tokens": 2064384.0,
+ "step": 126
+ },
+ {
+ "entropy": 1.536401391029358,
+ "epoch": 0.25656565656565655,
+ "grad_norm": 1.9561539888381958,
+ "learning_rate": 4.745454545454546e-06,
+ "loss": 1.5699501037597656,
+ "mean_token_accuracy": 0.6248167753219604,
+ "num_tokens": 2080768.0,
+ "step": 127
+ },
+ {
+ "entropy": 1.656266212463379,
+ "epoch": 0.2585858585858586,
+ "grad_norm": 2.1124353408813477,
+ "learning_rate": 4.743434343434344e-06,
+ "loss": 1.6612167358398438,
+ "mean_token_accuracy": 0.623900830745697,
+ "num_tokens": 2097152.0,
+ "step": 128
+ },
+ {
+ "entropy": 1.170629620552063,
+ "epoch": 0.2606060606060606,
+ "grad_norm": 1.917840600013733,
+ "learning_rate": 4.741414141414142e-06,
+ "loss": 1.1992300748825073,
+ "mean_token_accuracy": 0.6995603442192078,
+ "num_tokens": 2113536.0,
+ "step": 129
+ },
+ {
+ "entropy": 1.4126694202423096,
+ "epoch": 0.26262626262626265,
+ "grad_norm": 1.9041539430618286,
+ "learning_rate": 4.73939393939394e-06,
+ "loss": 1.4334447383880615,
+ "mean_token_accuracy": 0.6637151837348938,
+ "num_tokens": 2129920.0,
+ "step": 130
+ },
+ {
+ "entropy": 1.8490277528762817,
+ "epoch": 0.26464646464646463,
+ "grad_norm": 2.1440138816833496,
+ "learning_rate": 4.737373737373738e-06,
+ "loss": 1.9226353168487549,
+ "mean_token_accuracy": 0.5805447101593018,
+ "num_tokens": 2146304.0,
+ "step": 131
+ },
+ {
+ "entropy": 1.5000964403152466,
+ "epoch": 0.26666666666666666,
+ "grad_norm": 1.9615508317947388,
+ "learning_rate": 4.735353535353536e-06,
+ "loss": 1.495596170425415,
+ "mean_token_accuracy": 0.6542501449584961,
+ "num_tokens": 2162688.0,
+ "step": 132
+ },
+ {
+ "entropy": 1.9371142387390137,
+ "epoch": 0.2686868686868687,
+ "grad_norm": 2.202200412750244,
+ "learning_rate": 4.7333333333333335e-06,
+ "loss": 1.94151771068573,
+ "mean_token_accuracy": 0.5810332298278809,
+ "num_tokens": 2179072.0,
+ "step": 133
+ },
+ {
+ "entropy": 1.4535126686096191,
+ "epoch": 0.27070707070707073,
+ "grad_norm": 1.9804027080535889,
+ "learning_rate": 4.731313131313131e-06,
+ "loss": 1.4401545524597168,
+ "mean_token_accuracy": 0.6585246920585632,
+ "num_tokens": 2195456.0,
+ "step": 134
+ },
+ {
+ "entropy": 2.0871424674987793,
+ "epoch": 0.2727272727272727,
+ "grad_norm": 2.1417081356048584,
+ "learning_rate": 4.729292929292929e-06,
+ "loss": 2.117375373840332,
+ "mean_token_accuracy": 0.5600268840789795,
+ "num_tokens": 2211840.0,
+ "step": 135
+ },
+ {
+ "entropy": 1.2986469268798828,
+ "epoch": 0.27474747474747474,
+ "grad_norm": 2.0299136638641357,
+ "learning_rate": 4.727272727272728e-06,
+ "loss": 1.3631991147994995,
+ "mean_token_accuracy": 0.6750732660293579,
+ "num_tokens": 2228224.0,
+ "step": 136
+ },
+ {
+ "entropy": 1.163429617881775,
+ "epoch": 0.2767676767676768,
+ "grad_norm": 1.8368210792541504,
+ "learning_rate": 4.725252525252526e-06,
+ "loss": 1.1603795289993286,
+ "mean_token_accuracy": 0.7150708436965942,
+ "num_tokens": 2244608.0,
+ "step": 137
+ },
+ {
+ "entropy": 1.4301519393920898,
+ "epoch": 0.2787878787878788,
+ "grad_norm": 1.996767520904541,
+ "learning_rate": 4.723232323232324e-06,
+ "loss": 1.422170639038086,
+ "mean_token_accuracy": 0.6816682815551758,
+ "num_tokens": 2260992.0,
+ "step": 138
+ },
+ {
+ "entropy": 1.5160106420516968,
+ "epoch": 0.2808080808080808,
+ "grad_norm": 2.004770278930664,
+ "learning_rate": 4.721212121212122e-06,
+ "loss": 1.5385751724243164,
+ "mean_token_accuracy": 0.640693724155426,
+ "num_tokens": 2277376.0,
+ "step": 139
+ },
+ {
+ "entropy": 1.2483867406845093,
+ "epoch": 0.2828282828282828,
+ "grad_norm": 1.8488364219665527,
+ "learning_rate": 4.7191919191919195e-06,
+ "loss": 1.2563843727111816,
+ "mean_token_accuracy": 0.6998046040534973,
+ "num_tokens": 2293760.0,
+ "step": 140
+ },
+ {
+ "entropy": 1.3214129209518433,
+ "epoch": 0.28484848484848485,
+ "grad_norm": 2.1334660053253174,
+ "learning_rate": 4.717171717171717e-06,
+ "loss": 1.3392664194107056,
+ "mean_token_accuracy": 0.6822789311408997,
+ "num_tokens": 2310144.0,
+ "step": 141
+ },
+ {
+ "entropy": 1.611849069595337,
+ "epoch": 0.2868686868686869,
+ "grad_norm": 2.0539803504943848,
+ "learning_rate": 4.715151515151515e-06,
+ "loss": 1.6335396766662598,
+ "mean_token_accuracy": 0.6235954761505127,
+ "num_tokens": 2326528.0,
+ "step": 142
+ },
+ {
+ "entropy": 1.5847896337509155,
+ "epoch": 0.28888888888888886,
+ "grad_norm": 2.1658759117126465,
+ "learning_rate": 4.713131313131313e-06,
+ "loss": 1.603764533996582,
+ "mean_token_accuracy": 0.633671224117279,
+ "num_tokens": 2342912.0,
+ "step": 143
+ },
+ {
+ "entropy": 1.5731406211853027,
+ "epoch": 0.2909090909090909,
+ "grad_norm": 2.0830390453338623,
+ "learning_rate": 4.711111111111111e-06,
+ "loss": 1.558933973312378,
+ "mean_token_accuracy": 0.6392892003059387,
+ "num_tokens": 2359296.0,
+ "step": 144
+ },
+ {
+ "entropy": 1.8901628255844116,
+ "epoch": 0.29292929292929293,
+ "grad_norm": 2.1436922550201416,
+ "learning_rate": 4.709090909090909e-06,
+ "loss": 1.878631353378296,
+ "mean_token_accuracy": 0.57333904504776,
+ "num_tokens": 2375680.0,
+ "step": 145
+ },
+ {
+ "entropy": 1.9344228506088257,
+ "epoch": 0.29494949494949496,
+ "grad_norm": 2.1245176792144775,
+ "learning_rate": 4.707070707070707e-06,
+ "loss": 1.9546704292297363,
+ "mean_token_accuracy": 0.5987420678138733,
+ "num_tokens": 2392064.0,
+ "step": 146
+ },
+ {
+ "entropy": 1.2455096244812012,
+ "epoch": 0.296969696969697,
+ "grad_norm": 1.896581768989563,
+ "learning_rate": 4.705050505050506e-06,
+ "loss": 1.2738728523254395,
+ "mean_token_accuracy": 0.7025524973869324,
+ "num_tokens": 2408448.0,
+ "step": 147
+ },
+ {
+ "entropy": 1.4235948324203491,
+ "epoch": 0.298989898989899,
+ "grad_norm": 2.522636890411377,
+ "learning_rate": 4.7030303030303035e-06,
+ "loss": 1.4524480104446411,
+ "mean_token_accuracy": 0.6441133618354797,
+ "num_tokens": 2424832.0,
+ "step": 148
+ },
+ {
+ "entropy": 1.764552116394043,
+ "epoch": 0.301010101010101,
+ "grad_norm": 2.0232255458831787,
+ "learning_rate": 4.701010101010101e-06,
+ "loss": 1.7811740636825562,
+ "mean_token_accuracy": 0.6229848265647888,
+ "num_tokens": 2441216.0,
+ "step": 149
+ },
+ {
+ "entropy": 1.74843168258667,
+ "epoch": 0.30303030303030304,
+ "grad_norm": 2.1589369773864746,
+ "learning_rate": 4.698989898989899e-06,
+ "loss": 1.7337679862976074,
+ "mean_token_accuracy": 0.5975207686424255,
+ "num_tokens": 2457600.0,
+ "step": 150
+ },
+ {
+ "epoch": 0.30303030303030304,
+ "eval_entropy": 1.6033287221385586,
+ "eval_loss": 1.6036633253097534,
+ "eval_mean_token_accuracy": 0.6329842450157288,
+ "eval_num_tokens": 2457600.0,
+ "eval_runtime": 43.7655,
+ "eval_samples_per_second": 22.621,
+ "eval_steps_per_second": 2.833,
+ "step": 150
+ },
+ {
+ "entropy": 2.0572290420532227,
+ "epoch": 0.30505050505050507,
+ "grad_norm": 2.0511579513549805,
+ "learning_rate": 4.696969696969698e-06,
+ "loss": 2.0196030139923096,
+ "mean_token_accuracy": 0.5519052147865295,
+ "num_tokens": 2473984.0,
+ "step": 151
+ },
+ {
+ "entropy": 1.5792397260665894,
+ "epoch": 0.30707070707070705,
+ "grad_norm": 2.048396587371826,
+ "learning_rate": 4.694949494949496e-06,
+ "loss": 1.575985074043274,
+ "mean_token_accuracy": 0.623961865901947,
+ "num_tokens": 2490368.0,
+ "step": 152
+ },
+ {
+ "entropy": 1.487528681755066,
+ "epoch": 0.3090909090909091,
+ "grad_norm": 2.1903791427612305,
+ "learning_rate": 4.692929292929294e-06,
+ "loss": 1.479973316192627,
+ "mean_token_accuracy": 0.6595017313957214,
+ "num_tokens": 2506752.0,
+ "step": 153
+ },
+ {
+ "entropy": 1.766797423362732,
+ "epoch": 0.3111111111111111,
+ "grad_norm": 2.1843011379241943,
+ "learning_rate": 4.690909090909092e-06,
+ "loss": 1.7993412017822266,
+ "mean_token_accuracy": 0.5965437293052673,
+ "num_tokens": 2523136.0,
+ "step": 154
+ },
+ {
+ "entropy": 1.7058254480361938,
+ "epoch": 0.31313131313131315,
+ "grad_norm": 2.204461097717285,
+ "learning_rate": 4.6888888888888895e-06,
+ "loss": 1.7346007823944092,
+ "mean_token_accuracy": 0.6077185869216919,
+ "num_tokens": 2539520.0,
+ "step": 155
+ },
+ {
+ "entropy": 1.4929591417312622,
+ "epoch": 0.3151515151515151,
+ "grad_norm": 2.1739237308502197,
+ "learning_rate": 4.6868686868686874e-06,
+ "loss": 1.4708621501922607,
+ "mean_token_accuracy": 0.6427088379859924,
+ "num_tokens": 2555904.0,
+ "step": 156
+ },
+ {
+ "entropy": 1.5797587633132935,
+ "epoch": 0.31717171717171716,
+ "grad_norm": 2.150561571121216,
+ "learning_rate": 4.684848484848485e-06,
+ "loss": 1.5921857357025146,
+ "mean_token_accuracy": 0.6262823939323425,
+ "num_tokens": 2572288.0,
+ "step": 157
+ },
+ {
+ "entropy": 1.6198230981826782,
+ "epoch": 0.3191919191919192,
+ "grad_norm": 2.061169385910034,
+ "learning_rate": 4.682828282828283e-06,
+ "loss": 1.6601009368896484,
+ "mean_token_accuracy": 0.619015634059906,
+ "num_tokens": 2588672.0,
+ "step": 158
+ },
+ {
+ "entropy": 1.4026126861572266,
+ "epoch": 0.3212121212121212,
+ "grad_norm": 2.066208839416504,
+ "learning_rate": 4.680808080808081e-06,
+ "loss": 1.4062483310699463,
+ "mean_token_accuracy": 0.6681729555130005,
+ "num_tokens": 2605056.0,
+ "step": 159
+ },
+ {
+ "entropy": 1.4608066082000732,
+ "epoch": 0.32323232323232326,
+ "grad_norm": 2.022627353668213,
+ "learning_rate": 4.678787878787879e-06,
+ "loss": 1.4644970893859863,
+ "mean_token_accuracy": 0.6533341407775879,
+ "num_tokens": 2621440.0,
+ "step": 160
+ },
+ {
+ "entropy": 1.222448468208313,
+ "epoch": 0.32525252525252524,
+ "grad_norm": 2.0399329662323,
+ "learning_rate": 4.676767676767677e-06,
+ "loss": 1.2547852993011475,
+ "mean_token_accuracy": 0.6925378441810608,
+ "num_tokens": 2637824.0,
+ "step": 161
+ },
+ {
+ "entropy": 1.8702255487442017,
+ "epoch": 0.32727272727272727,
+ "grad_norm": 2.177307367324829,
+ "learning_rate": 4.674747474747476e-06,
+ "loss": 1.9041712284088135,
+ "mean_token_accuracy": 0.5754152536392212,
+ "num_tokens": 2654208.0,
+ "step": 162
+ },
+ {
+ "entropy": 1.6861947774887085,
+ "epoch": 0.3292929292929293,
+ "grad_norm": 2.009544610977173,
+ "learning_rate": 4.6727272727272735e-06,
+ "loss": 1.7079355716705322,
+ "mean_token_accuracy": 0.615290641784668,
+ "num_tokens": 2670592.0,
+ "step": 163
+ },
+ {
+ "entropy": 1.3880327939987183,
+ "epoch": 0.33131313131313134,
+ "grad_norm": 2.16359543800354,
+ "learning_rate": 4.670707070707071e-06,
+ "loss": 1.4220571517944336,
+ "mean_token_accuracy": 0.6604787707328796,
+ "num_tokens": 2686976.0,
+ "step": 164
+ },
+ {
+ "entropy": 1.5760643482208252,
+ "epoch": 0.3333333333333333,
+ "grad_norm": 2.09773325920105,
+ "learning_rate": 4.668686868686869e-06,
+ "loss": 1.6036784648895264,
+ "mean_token_accuracy": 0.6267098188400269,
+ "num_tokens": 2703360.0,
+ "step": 165
+ },
+ {
+ "entropy": 1.078303575515747,
+ "epoch": 0.33535353535353535,
+ "grad_norm": 1.7760599851608276,
+ "learning_rate": 4.666666666666667e-06,
+ "loss": 1.0612695217132568,
+ "mean_token_accuracy": 0.7349169254302979,
+ "num_tokens": 2719744.0,
+ "step": 166
+ },
+ {
+ "entropy": 1.3722130060195923,
+ "epoch": 0.3373737373737374,
+ "grad_norm": 2.0816409587860107,
+ "learning_rate": 4.664646464646465e-06,
+ "loss": 1.3693504333496094,
+ "mean_token_accuracy": 0.6682950854301453,
+ "num_tokens": 2736128.0,
+ "step": 167
+ },
+ {
+ "entropy": 1.8752760887145996,
+ "epoch": 0.3393939393939394,
+ "grad_norm": 2.1670358180999756,
+ "learning_rate": 4.662626262626263e-06,
+ "loss": 1.8874578475952148,
+ "mean_token_accuracy": 0.5881778001785278,
+ "num_tokens": 2752512.0,
+ "step": 168
+ },
+ {
+ "entropy": 1.7384581565856934,
+ "epoch": 0.3414141414141414,
+ "grad_norm": 2.0072429180145264,
+ "learning_rate": 4.660606060606061e-06,
+ "loss": 1.7491583824157715,
+ "mean_token_accuracy": 0.6041157841682434,
+ "num_tokens": 2768896.0,
+ "step": 169
+ },
+ {
+ "entropy": 1.6651691198349,
+ "epoch": 0.3434343434343434,
+ "grad_norm": 2.0812578201293945,
+ "learning_rate": 4.658585858585859e-06,
+ "loss": 1.6808438301086426,
+ "mean_token_accuracy": 0.6273204684257507,
+ "num_tokens": 2785280.0,
+ "step": 170
+ },
+ {
+ "entropy": 1.3426616191864014,
+ "epoch": 0.34545454545454546,
+ "grad_norm": 1.9108822345733643,
+ "learning_rate": 4.656565656565657e-06,
+ "loss": 1.3404264450073242,
+ "mean_token_accuracy": 0.6703101992607117,
+ "num_tokens": 2801664.0,
+ "step": 171
+ },
+ {
+ "entropy": 1.631722092628479,
+ "epoch": 0.3474747474747475,
+ "grad_norm": 2.07317852973938,
+ "learning_rate": 4.654545454545455e-06,
+ "loss": 1.6545813083648682,
+ "mean_token_accuracy": 0.6143136024475098,
+ "num_tokens": 2818048.0,
+ "step": 172
+ },
+ {
+ "entropy": 1.3678289651870728,
+ "epoch": 0.34949494949494947,
+ "grad_norm": 1.8517454862594604,
+ "learning_rate": 4.652525252525253e-06,
+ "loss": 1.3759769201278687,
+ "mean_token_accuracy": 0.6693942546844482,
+ "num_tokens": 2834432.0,
+ "step": 173
+ },
+ {
+ "entropy": 1.3927773237228394,
+ "epoch": 0.3515151515151515,
+ "grad_norm": 2.1304867267608643,
+ "learning_rate": 4.650505050505051e-06,
+ "loss": 1.4104211330413818,
+ "mean_token_accuracy": 0.6612725853919983,
+ "num_tokens": 2850816.0,
+ "step": 174
+ },
+ {
+ "entropy": 1.3352863788604736,
+ "epoch": 0.35353535353535354,
+ "grad_norm": 2.2788777351379395,
+ "learning_rate": 4.648484848484849e-06,
+ "loss": 1.3665719032287598,
+ "mean_token_accuracy": 0.6640205383300781,
+ "num_tokens": 2867200.0,
+ "step": 175
+ },
+ {
+ "entropy": 1.6800754070281982,
+ "epoch": 0.35555555555555557,
+ "grad_norm": 2.030787944793701,
+ "learning_rate": 4.646464646464647e-06,
+ "loss": 1.7239181995391846,
+ "mean_token_accuracy": 0.6193209290504456,
+ "num_tokens": 2883584.0,
+ "step": 176
+ },
+ {
+ "entropy": 1.2852731943130493,
+ "epoch": 0.3575757575757576,
+ "grad_norm": 1.9148248434066772,
+ "learning_rate": 4.644444444444445e-06,
+ "loss": 1.2934880256652832,
+ "mean_token_accuracy": 0.6798363327980042,
+ "num_tokens": 2899968.0,
+ "step": 177
+ },
+ {
+ "entropy": 1.6366593837738037,
+ "epoch": 0.3595959595959596,
+ "grad_norm": 2.2264015674591064,
+ "learning_rate": 4.642424242424243e-06,
+ "loss": 1.632002592086792,
+ "mean_token_accuracy": 0.6180385947227478,
+ "num_tokens": 2916352.0,
+ "step": 178
+ },
+ {
+ "entropy": 1.3935565948486328,
+ "epoch": 0.3616161616161616,
+ "grad_norm": 2.0410475730895996,
+ "learning_rate": 4.6404040404040406e-06,
+ "loss": 1.407370686531067,
+ "mean_token_accuracy": 0.6722032427787781,
+ "num_tokens": 2932736.0,
+ "step": 179
+ },
+ {
+ "entropy": 1.0952510833740234,
+ "epoch": 0.36363636363636365,
+ "grad_norm": 1.937270164489746,
+ "learning_rate": 4.6383838383838384e-06,
+ "loss": 1.1101973056793213,
+ "mean_token_accuracy": 0.730947732925415,
+ "num_tokens": 2949120.0,
+ "step": 180
+ },
+ {
+ "entropy": 1.184926152229309,
+ "epoch": 0.3656565656565657,
+ "grad_norm": 1.9867528676986694,
+ "learning_rate": 4.636363636363636e-06,
+ "loss": 1.1932196617126465,
+ "mean_token_accuracy": 0.7085368633270264,
+ "num_tokens": 2965504.0,
+ "step": 181
+ },
+ {
+ "entropy": 1.4552072286605835,
+ "epoch": 0.36767676767676766,
+ "grad_norm": 1.9395766258239746,
+ "learning_rate": 4.634343434343434e-06,
+ "loss": 1.4559956789016724,
+ "mean_token_accuracy": 0.6534562706947327,
+ "num_tokens": 2981888.0,
+ "step": 182
+ },
+ {
+ "entropy": 1.6437443494796753,
+ "epoch": 0.3696969696969697,
+ "grad_norm": 2.1257567405700684,
+ "learning_rate": 4.632323232323233e-06,
+ "loss": 1.6260980367660522,
+ "mean_token_accuracy": 0.625,
+ "num_tokens": 2998272.0,
+ "step": 183
+ },
+ {
+ "entropy": 1.8264633417129517,
+ "epoch": 0.3717171717171717,
+ "grad_norm": 2.187041759490967,
+ "learning_rate": 4.630303030303031e-06,
+ "loss": 1.8431676626205444,
+ "mean_token_accuracy": 0.60332190990448,
+ "num_tokens": 3014656.0,
+ "step": 184
+ },
+ {
+ "entropy": 1.1121351718902588,
+ "epoch": 0.37373737373737376,
+ "grad_norm": 1.8460402488708496,
+ "learning_rate": 4.628282828282829e-06,
+ "loss": 1.1054425239562988,
+ "mean_token_accuracy": 0.7254518866539001,
+ "num_tokens": 3031040.0,
+ "step": 185
+ },
+ {
+ "entropy": 1.3531955480575562,
+ "epoch": 0.37575757575757573,
+ "grad_norm": 2.3463470935821533,
+ "learning_rate": 4.626262626262627e-06,
+ "loss": 1.359381914138794,
+ "mean_token_accuracy": 0.6610894203186035,
+ "num_tokens": 3047424.0,
+ "step": 186
+ },
+ {
+ "entropy": 1.2131577730178833,
+ "epoch": 0.37777777777777777,
+ "grad_norm": 1.9595171213150024,
+ "learning_rate": 4.6242424242424245e-06,
+ "loss": 1.2312005758285522,
+ "mean_token_accuracy": 0.6979115605354309,
+ "num_tokens": 3063808.0,
+ "step": 187
+ },
+ {
+ "entropy": 1.5441125631332397,
+ "epoch": 0.3797979797979798,
+ "grad_norm": 2.2971343994140625,
+ "learning_rate": 4.622222222222222e-06,
+ "loss": 1.5923339128494263,
+ "mean_token_accuracy": 0.6297020316123962,
+ "num_tokens": 3080192.0,
+ "step": 188
+ },
+ {
+ "entropy": 1.7998323440551758,
+ "epoch": 0.38181818181818183,
+ "grad_norm": 2.010552167892456,
+ "learning_rate": 4.62020202020202e-06,
+ "loss": 1.8257204294204712,
+ "mean_token_accuracy": 0.5997191071510315,
+ "num_tokens": 3096576.0,
+ "step": 189
+ },
+ {
+ "entropy": 1.389051914215088,
+ "epoch": 0.3838383838383838,
+ "grad_norm": 2.1183855533599854,
+ "learning_rate": 4.618181818181818e-06,
+ "loss": 1.431575059890747,
+ "mean_token_accuracy": 0.6537005305290222,
+ "num_tokens": 3112960.0,
+ "step": 190
+ },
+ {
+ "entropy": 2.0066142082214355,
+ "epoch": 0.38585858585858585,
+ "grad_norm": 2.061399221420288,
+ "learning_rate": 4.616161616161616e-06,
+ "loss": 2.0477406978607178,
+ "mean_token_accuracy": 0.5454934239387512,
+ "num_tokens": 3129344.0,
+ "step": 191
+ },
+ {
+ "entropy": 1.3063198328018188,
+ "epoch": 0.3878787878787879,
+ "grad_norm": 1.9490301609039307,
+ "learning_rate": 4.614141414141414e-06,
+ "loss": 1.3293564319610596,
+ "mean_token_accuracy": 0.6797142028808594,
+ "num_tokens": 3145728.0,
+ "step": 192
+ },
+ {
+ "entropy": 1.3636538982391357,
+ "epoch": 0.3898989898989899,
+ "grad_norm": 2.025458574295044,
+ "learning_rate": 4.612121212121212e-06,
+ "loss": 1.3736209869384766,
+ "mean_token_accuracy": 0.679286777973175,
+ "num_tokens": 3162112.0,
+ "step": 193
+ },
+ {
+ "entropy": 1.5669925212860107,
+ "epoch": 0.39191919191919194,
+ "grad_norm": 2.064033269882202,
+ "learning_rate": 4.6101010101010106e-06,
+ "loss": 1.5587083101272583,
+ "mean_token_accuracy": 0.6267709136009216,
+ "num_tokens": 3178496.0,
+ "step": 194
+ },
+ {
+ "entropy": 1.4489716291427612,
+ "epoch": 0.3939393939393939,
+ "grad_norm": 2.0174880027770996,
+ "learning_rate": 4.6080808080808085e-06,
+ "loss": 1.4331417083740234,
+ "mean_token_accuracy": 0.6571812629699707,
+ "num_tokens": 3194880.0,
+ "step": 195
+ },
+ {
+ "entropy": 2.026416778564453,
+ "epoch": 0.39595959595959596,
+ "grad_norm": 2.1820878982543945,
+ "learning_rate": 4.606060606060606e-06,
+ "loss": 2.029003143310547,
+ "mean_token_accuracy": 0.563568651676178,
+ "num_tokens": 3211264.0,
+ "step": 196
+ },
+ {
+ "entropy": 2.1844022274017334,
+ "epoch": 0.397979797979798,
+ "grad_norm": 2.175349473953247,
+ "learning_rate": 4.604040404040404e-06,
+ "loss": 2.2053825855255127,
+ "mean_token_accuracy": 0.5553248524665833,
+ "num_tokens": 3227648.0,
+ "step": 197
+ },
+ {
+ "entropy": 1.6750906705856323,
+ "epoch": 0.4,
+ "grad_norm": 1.8300689458847046,
+ "learning_rate": 4.602020202020203e-06,
+ "loss": 1.692289113998413,
+ "mean_token_accuracy": 0.6345261335372925,
+ "num_tokens": 3244032.0,
+ "step": 198
+ },
+ {
+ "entropy": 1.6192772388458252,
+ "epoch": 0.402020202020202,
+ "grad_norm": 1.9788341522216797,
+ "learning_rate": 4.600000000000001e-06,
+ "loss": 1.628842830657959,
+ "mean_token_accuracy": 0.615229606628418,
+ "num_tokens": 3260416.0,
+ "step": 199
+ },
+ {
+ "entropy": 1.444870948791504,
+ "epoch": 0.40404040404040403,
+ "grad_norm": 2.05141544342041,
+ "learning_rate": 4.597979797979799e-06,
+ "loss": 1.4365839958190918,
+ "mean_token_accuracy": 0.652723491191864,
+ "num_tokens": 3276800.0,
+ "step": 200
+ },
+ {
+ "epoch": 0.40404040404040403,
+ "eval_entropy": 1.5766179907706477,
+ "eval_loss": 1.5878442525863647,
+ "eval_mean_token_accuracy": 0.6355829551335304,
+ "eval_num_tokens": 3276800.0,
+ "eval_runtime": 43.7272,
+ "eval_samples_per_second": 22.64,
+ "eval_steps_per_second": 2.836,
+ "step": 200
+ },
+ {
+ "entropy": 1.3354029655456543,
+ "epoch": 0.40606060606060607,
+ "grad_norm": 2.012895345687866,
+ "learning_rate": 4.595959595959597e-06,
+ "loss": 1.3289990425109863,
+ "mean_token_accuracy": 0.685271143913269,
+ "num_tokens": 3293184.0,
+ "step": 201
+ },
+ {
+ "entropy": 1.1990011930465698,
+ "epoch": 0.4080808080808081,
+ "grad_norm": 1.9462409019470215,
+ "learning_rate": 4.5939393939393945e-06,
+ "loss": 1.224869728088379,
+ "mean_token_accuracy": 0.6976673007011414,
+ "num_tokens": 3309568.0,
+ "step": 202
+ },
+ {
+ "entropy": 1.6442710161209106,
+ "epoch": 0.4101010101010101,
+ "grad_norm": 2.064532995223999,
+ "learning_rate": 4.591919191919192e-06,
+ "loss": 1.6800963878631592,
+ "mean_token_accuracy": 0.6111993193626404,
+ "num_tokens": 3325952.0,
+ "step": 203
+ },
+ {
+ "entropy": 1.8336306810379028,
+ "epoch": 0.4121212121212121,
+ "grad_norm": 2.478731393814087,
+ "learning_rate": 4.58989898989899e-06,
+ "loss": 1.8518702983856201,
+ "mean_token_accuracy": 0.6021006107330322,
+ "num_tokens": 3342336.0,
+ "step": 204
+ },
+ {
+ "entropy": 1.6767220497131348,
+ "epoch": 0.41414141414141414,
+ "grad_norm": 2.0676512718200684,
+ "learning_rate": 4.587878787878788e-06,
+ "loss": 1.674522042274475,
+ "mean_token_accuracy": 0.6178553700447083,
+ "num_tokens": 3358720.0,
+ "step": 205
+ },
+ {
+ "entropy": 1.7236497402191162,
+ "epoch": 0.4161616161616162,
+ "grad_norm": 2.057074785232544,
+ "learning_rate": 4.585858585858586e-06,
+ "loss": 1.725832462310791,
+ "mean_token_accuracy": 0.620114803314209,
+ "num_tokens": 3375104.0,
+ "step": 206
+ },
+ {
+ "entropy": 1.6493065357208252,
+ "epoch": 0.41818181818181815,
+ "grad_norm": 2.0946099758148193,
+ "learning_rate": 4.583838383838384e-06,
+ "loss": 1.6826295852661133,
+ "mean_token_accuracy": 0.6251221299171448,
+ "num_tokens": 3391488.0,
+ "step": 207
+ },
+ {
+ "entropy": 1.3565926551818848,
+ "epoch": 0.4202020202020202,
+ "grad_norm": 1.9611284732818604,
+ "learning_rate": 4.581818181818183e-06,
+ "loss": 1.346787452697754,
+ "mean_token_accuracy": 0.685332179069519,
+ "num_tokens": 3407872.0,
+ "step": 208
+ },
+ {
+ "entropy": 1.4401211738586426,
+ "epoch": 0.4222222222222222,
+ "grad_norm": 1.8614766597747803,
+ "learning_rate": 4.579797979797981e-06,
+ "loss": 1.4717891216278076,
+ "mean_token_accuracy": 0.650036633014679,
+ "num_tokens": 3424256.0,
+ "step": 209
+ },
+ {
+ "entropy": 1.194276213645935,
+ "epoch": 0.42424242424242425,
+ "grad_norm": 2.0625810623168945,
+ "learning_rate": 4.5777777777777785e-06,
+ "loss": 1.2106804847717285,
+ "mean_token_accuracy": 0.700537383556366,
+ "num_tokens": 3440640.0,
+ "step": 210
+ },
+ {
+ "entropy": 1.7622313499450684,
+ "epoch": 0.4262626262626263,
+ "grad_norm": 2.330610752105713,
+ "learning_rate": 4.575757575757576e-06,
+ "loss": 1.7872710227966309,
+ "mean_token_accuracy": 0.598436713218689,
+ "num_tokens": 3457024.0,
+ "step": 211
+ },
+ {
+ "entropy": 1.409427523612976,
+ "epoch": 0.42828282828282827,
+ "grad_norm": 2.0988259315490723,
+ "learning_rate": 4.573737373737374e-06,
+ "loss": 1.416553020477295,
+ "mean_token_accuracy": 0.6682950854301453,
+ "num_tokens": 3473408.0,
+ "step": 212
+ },
+ {
+ "entropy": 1.6856107711791992,
+ "epoch": 0.4303030303030303,
+ "grad_norm": 2.341475009918213,
+ "learning_rate": 4.571717171717172e-06,
+ "loss": 1.692287802696228,
+ "mean_token_accuracy": 0.6024059653282166,
+ "num_tokens": 3489792.0,
+ "step": 213
+ },
+ {
+ "entropy": 1.997343897819519,
+ "epoch": 0.43232323232323233,
+ "grad_norm": 2.200498104095459,
+ "learning_rate": 4.56969696969697e-06,
+ "loss": 2.026289939880371,
+ "mean_token_accuracy": 0.5781631469726562,
+ "num_tokens": 3506176.0,
+ "step": 214
+ },
+ {
+ "entropy": 1.5175039768218994,
+ "epoch": 0.43434343434343436,
+ "grad_norm": 2.105257272720337,
+ "learning_rate": 4.567676767676768e-06,
+ "loss": 1.521841049194336,
+ "mean_token_accuracy": 0.6404494643211365,
+ "num_tokens": 3522560.0,
+ "step": 215
+ },
+ {
+ "entropy": 1.3616528511047363,
+ "epoch": 0.43636363636363634,
+ "grad_norm": 2.0035297870635986,
+ "learning_rate": 4.565656565656566e-06,
+ "loss": 1.3674360513687134,
+ "mean_token_accuracy": 0.6771494746208191,
+ "num_tokens": 3538944.0,
+ "step": 216
+ },
+ {
+ "entropy": 1.42499840259552,
+ "epoch": 0.4383838383838384,
+ "grad_norm": 1.9115629196166992,
+ "learning_rate": 4.563636363636364e-06,
+ "loss": 1.4129266738891602,
+ "mean_token_accuracy": 0.6686614751815796,
+ "num_tokens": 3555328.0,
+ "step": 217
+ },
+ {
+ "entropy": 1.2844709157943726,
+ "epoch": 0.4404040404040404,
+ "grad_norm": 2.3313121795654297,
+ "learning_rate": 4.5616161616161616e-06,
+ "loss": 1.315006971359253,
+ "mean_token_accuracy": 0.681546151638031,
+ "num_tokens": 3571712.0,
+ "step": 218
+ },
+ {
+ "entropy": 1.5227075815200806,
+ "epoch": 0.44242424242424244,
+ "grad_norm": 2.1849124431610107,
+ "learning_rate": 4.55959595959596e-06,
+ "loss": 1.517989158630371,
+ "mean_token_accuracy": 0.6466169953346252,
+ "num_tokens": 3588096.0,
+ "step": 219
+ },
+ {
+ "entropy": 1.494642734527588,
+ "epoch": 0.4444444444444444,
+ "grad_norm": 2.3109116554260254,
+ "learning_rate": 4.557575757575758e-06,
+ "loss": 1.525421142578125,
+ "mean_token_accuracy": 0.6392281651496887,
+ "num_tokens": 3604480.0,
+ "step": 220
+ },
+ {
+ "entropy": 1.6428948640823364,
+ "epoch": 0.44646464646464645,
+ "grad_norm": 2.1182680130004883,
+ "learning_rate": 4.555555555555556e-06,
+ "loss": 1.6634926795959473,
+ "mean_token_accuracy": 0.6207864880561829,
+ "num_tokens": 3620864.0,
+ "step": 221
+ },
+ {
+ "entropy": 1.7323675155639648,
+ "epoch": 0.4484848484848485,
+ "grad_norm": 2.2620837688446045,
+ "learning_rate": 4.553535353535354e-06,
+ "loss": 1.7312631607055664,
+ "mean_token_accuracy": 0.6138250827789307,
+ "num_tokens": 3637248.0,
+ "step": 222
+ },
+ {
+ "entropy": 1.7863894701004028,
+ "epoch": 0.4505050505050505,
+ "grad_norm": 2.1416971683502197,
+ "learning_rate": 4.551515151515152e-06,
+ "loss": 1.799318552017212,
+ "mean_token_accuracy": 0.5906203985214233,
+ "num_tokens": 3653632.0,
+ "step": 223
+ },
+ {
+ "entropy": 1.7404330968856812,
+ "epoch": 0.45252525252525255,
+ "grad_norm": 2.0814504623413086,
+ "learning_rate": 4.54949494949495e-06,
+ "loss": 1.742197036743164,
+ "mean_token_accuracy": 0.6050317287445068,
+ "num_tokens": 3670016.0,
+ "step": 224
+ },
+ {
+ "entropy": 1.4387869834899902,
+ "epoch": 0.45454545454545453,
+ "grad_norm": 2.1386022567749023,
+ "learning_rate": 4.547474747474748e-06,
+ "loss": 1.4602317810058594,
+ "mean_token_accuracy": 0.6502198576927185,
+ "num_tokens": 3686400.0,
+ "step": 225
+ },
+ {
+ "entropy": 1.2733348608016968,
+ "epoch": 0.45656565656565656,
+ "grad_norm": 2.02687668800354,
+ "learning_rate": 4.5454545454545455e-06,
+ "loss": 1.2683714628219604,
+ "mean_token_accuracy": 0.6966902613639832,
+ "num_tokens": 3702784.0,
+ "step": 226
+ },
+ {
+ "entropy": 1.4554595947265625,
+ "epoch": 0.4585858585858586,
+ "grad_norm": 2.169268846511841,
+ "learning_rate": 4.543434343434343e-06,
+ "loss": 1.4541833400726318,
+ "mean_token_accuracy": 0.6522960662841797,
+ "num_tokens": 3719168.0,
+ "step": 227
+ },
+ {
+ "entropy": 1.487573266029358,
+ "epoch": 0.46060606060606063,
+ "grad_norm": 1.9726165533065796,
+ "learning_rate": 4.541414141414141e-06,
+ "loss": 1.5024409294128418,
+ "mean_token_accuracy": 0.6509526371955872,
+ "num_tokens": 3735552.0,
+ "step": 228
+ },
+ {
+ "entropy": 1.3991138935089111,
+ "epoch": 0.4626262626262626,
+ "grad_norm": 2.0992283821105957,
+ "learning_rate": 4.539393939393939e-06,
+ "loss": 1.4016860723495483,
+ "mean_token_accuracy": 0.6639594435691833,
+ "num_tokens": 3751936.0,
+ "step": 229
+ },
+ {
+ "entropy": 1.6564134359359741,
+ "epoch": 0.46464646464646464,
+ "grad_norm": 2.047656297683716,
+ "learning_rate": 4.537373737373738e-06,
+ "loss": 1.6598721742630005,
+ "mean_token_accuracy": 0.6166951656341553,
+ "num_tokens": 3768320.0,
+ "step": 230
+ },
+ {
+ "entropy": 1.5165014266967773,
+ "epoch": 0.4666666666666667,
+ "grad_norm": 2.079996109008789,
+ "learning_rate": 4.535353535353536e-06,
+ "loss": 1.4980010986328125,
+ "mean_token_accuracy": 0.6370908617973328,
+ "num_tokens": 3784704.0,
+ "step": 231
+ },
+ {
+ "entropy": 1.5409225225448608,
+ "epoch": 0.4686868686868687,
+ "grad_norm": 2.0996923446655273,
+ "learning_rate": 4.533333333333334e-06,
+ "loss": 1.5504257678985596,
+ "mean_token_accuracy": 0.6465559601783752,
+ "num_tokens": 3801088.0,
+ "step": 232
+ },
+ {
+ "entropy": 1.3590043783187866,
+ "epoch": 0.4707070707070707,
+ "grad_norm": 2.14617919921875,
+ "learning_rate": 4.531313131313132e-06,
+ "loss": 1.3581812381744385,
+ "mean_token_accuracy": 0.6683561205863953,
+ "num_tokens": 3817472.0,
+ "step": 233
+ },
+ {
+ "entropy": 1.641785740852356,
+ "epoch": 0.4727272727272727,
+ "grad_norm": 1.9499926567077637,
+ "learning_rate": 4.5292929292929295e-06,
+ "loss": 1.6653470993041992,
+ "mean_token_accuracy": 0.6319614052772522,
+ "num_tokens": 3833856.0,
+ "step": 234
+ },
+ {
+ "entropy": 1.4700758457183838,
+ "epoch": 0.47474747474747475,
+ "grad_norm": 1.9411437511444092,
+ "learning_rate": 4.527272727272727e-06,
+ "loss": 1.4762463569641113,
+ "mean_token_accuracy": 0.6631656289100647,
+ "num_tokens": 3850240.0,
+ "step": 235
+ },
+ {
+ "entropy": 1.5418941974639893,
+ "epoch": 0.4767676767676768,
+ "grad_norm": 2.3191940784454346,
+ "learning_rate": 4.525252525252526e-06,
+ "loss": 1.5151214599609375,
+ "mean_token_accuracy": 0.6326331496238708,
+ "num_tokens": 3866624.0,
+ "step": 236
+ },
+ {
+ "entropy": 2.1102776527404785,
+ "epoch": 0.47878787878787876,
+ "grad_norm": 2.2287707328796387,
+ "learning_rate": 4.523232323232324e-06,
+ "loss": 2.106567859649658,
+ "mean_token_accuracy": 0.5506839156150818,
+ "num_tokens": 3883008.0,
+ "step": 237
+ },
+ {
+ "entropy": 1.7448827028274536,
+ "epoch": 0.4808080808080808,
+ "grad_norm": 2.2352893352508545,
+ "learning_rate": 4.521212121212122e-06,
+ "loss": 1.7495017051696777,
+ "mean_token_accuracy": 0.60326087474823,
+ "num_tokens": 3899392.0,
+ "step": 238
+ },
+ {
+ "entropy": 1.315240740776062,
+ "epoch": 0.48282828282828283,
+ "grad_norm": 1.9933631420135498,
+ "learning_rate": 4.51919191919192e-06,
+ "loss": 1.315302848815918,
+ "mean_token_accuracy": 0.686431348323822,
+ "num_tokens": 3915776.0,
+ "step": 239
+ },
+ {
+ "entropy": 1.5851637125015259,
+ "epoch": 0.48484848484848486,
+ "grad_norm": 2.153303623199463,
+ "learning_rate": 4.517171717171718e-06,
+ "loss": 1.5947662591934204,
+ "mean_token_accuracy": 0.6297020316123962,
+ "num_tokens": 3932160.0,
+ "step": 240
+ },
+ {
+ "entropy": 1.8050944805145264,
+ "epoch": 0.4868686868686869,
+ "grad_norm": 2.025022506713867,
+ "learning_rate": 4.5151515151515155e-06,
+ "loss": 1.8051010370254517,
+ "mean_token_accuracy": 0.6055202484130859,
+ "num_tokens": 3948544.0,
+ "step": 241
+ },
+ {
+ "entropy": 1.518296718597412,
+ "epoch": 0.4888888888888889,
+ "grad_norm": 2.1021833419799805,
+ "learning_rate": 4.513131313131313e-06,
+ "loss": 1.5396809577941895,
+ "mean_token_accuracy": 0.6359916925430298,
+ "num_tokens": 3964928.0,
+ "step": 242
+ },
+ {
+ "entropy": 1.518903136253357,
+ "epoch": 0.4909090909090909,
+ "grad_norm": 2.0960140228271484,
+ "learning_rate": 4.511111111111111e-06,
+ "loss": 1.5528055429458618,
+ "mean_token_accuracy": 0.6224963068962097,
+ "num_tokens": 3981312.0,
+ "step": 243
+ },
+ {
+ "entropy": 1.6960705518722534,
+ "epoch": 0.49292929292929294,
+ "grad_norm": 1.8580718040466309,
+ "learning_rate": 4.50909090909091e-06,
+ "loss": 1.7192862033843994,
+ "mean_token_accuracy": 0.6259770393371582,
+ "num_tokens": 3997696.0,
+ "step": 244
+ },
+ {
+ "entropy": 1.4633033275604248,
+ "epoch": 0.494949494949495,
+ "grad_norm": 1.8983049392700195,
+ "learning_rate": 4.507070707070708e-06,
+ "loss": 1.451701283454895,
+ "mean_token_accuracy": 0.6669516563415527,
+ "num_tokens": 4014080.0,
+ "step": 245
+ },
+ {
+ "entropy": 1.6508095264434814,
+ "epoch": 0.49696969696969695,
+ "grad_norm": 2.298679828643799,
+ "learning_rate": 4.505050505050506e-06,
+ "loss": 1.6596897840499878,
+ "mean_token_accuracy": 0.6115046143531799,
+ "num_tokens": 4030464.0,
+ "step": 246
+ },
+ {
+ "entropy": 1.313779354095459,
+ "epoch": 0.498989898989899,
+ "grad_norm": 1.9894335269927979,
+ "learning_rate": 4.503030303030304e-06,
+ "loss": 1.3285409212112427,
+ "mean_token_accuracy": 0.6802027225494385,
+ "num_tokens": 4046848.0,
+ "step": 247
+ },
+ {
+ "entropy": 1.5139521360397339,
+ "epoch": 0.501010101010101,
+ "grad_norm": 2.028320789337158,
+ "learning_rate": 4.501010101010102e-06,
+ "loss": 1.5366487503051758,
+ "mean_token_accuracy": 0.6416707634925842,
+ "num_tokens": 4063232.0,
+ "step": 248
+ },
+ {
+ "entropy": 1.5277045965194702,
+ "epoch": 0.503030303030303,
+ "grad_norm": 2.3160979747772217,
+ "learning_rate": 4.4989898989898995e-06,
+ "loss": 1.5571036338806152,
+ "mean_token_accuracy": 0.6436858773231506,
+ "num_tokens": 4079616.0,
+ "step": 249
+ },
+ {
+ "entropy": 1.6084188222885132,
+ "epoch": 0.5050505050505051,
+ "grad_norm": 2.20550274848938,
+ "learning_rate": 4.496969696969697e-06,
+ "loss": 1.622127652168274,
+ "mean_token_accuracy": 0.6284196376800537,
+ "num_tokens": 4096000.0,
+ "step": 250
+ },
+ {
+ "epoch": 0.5050505050505051,
+ "eval_entropy": 1.5547234262189558,
+ "eval_loss": 1.5764786005020142,
+ "eval_mean_token_accuracy": 0.6375306306346771,
+ "eval_num_tokens": 4096000.0,
+ "eval_runtime": 43.7607,
+ "eval_samples_per_second": 22.623,
+ "eval_steps_per_second": 2.834,
+ "step": 250
+ },
+ {
+ "entropy": 1.4309135675430298,
+ "epoch": 0.5070707070707071,
+ "grad_norm": 2.080864906311035,
+ "learning_rate": 4.494949494949495e-06,
+ "loss": 1.4481091499328613,
+ "mean_token_accuracy": 0.6694552898406982,
+ "num_tokens": 4112384.0,
+ "step": 251
+ },
+ {
+ "entropy": 1.4067270755767822,
+ "epoch": 0.509090909090909,
+ "grad_norm": 2.036475419998169,
+ "learning_rate": 4.492929292929293e-06,
+ "loss": 1.4035298824310303,
+ "mean_token_accuracy": 0.6547996997833252,
+ "num_tokens": 4128768.0,
+ "step": 252
+ },
+ {
+ "entropy": 2.1751608848571777,
+ "epoch": 0.5111111111111111,
+ "grad_norm": 1.953995943069458,
+ "learning_rate": 4.490909090909091e-06,
+ "loss": 2.1616692543029785,
+ "mean_token_accuracy": 0.5592941045761108,
+ "num_tokens": 4145152.0,
+ "step": 253
+ },
+ {
+ "entropy": 1.45353102684021,
+ "epoch": 0.5131313131313131,
+ "grad_norm": 2.084437370300293,
+ "learning_rate": 4.488888888888889e-06,
+ "loss": 1.4633586406707764,
+ "mean_token_accuracy": 0.6620664596557617,
+ "num_tokens": 4161536.0,
+ "step": 254
+ },
+ {
+ "entropy": 1.64310884475708,
+ "epoch": 0.5151515151515151,
+ "grad_norm": 2.0700111389160156,
+ "learning_rate": 4.486868686868688e-06,
+ "loss": 1.6868078708648682,
+ "mean_token_accuracy": 0.616328775882721,
+ "num_tokens": 4177920.0,
+ "step": 255
+ },
+ {
+ "entropy": 1.5113472938537598,
+ "epoch": 0.5171717171717172,
+ "grad_norm": 2.102180242538452,
+ "learning_rate": 4.4848484848484855e-06,
+ "loss": 1.5240120887756348,
+ "mean_token_accuracy": 0.638067901134491,
+ "num_tokens": 4194304.0,
+ "step": 256
+ },
+ {
+ "entropy": 1.5752851963043213,
+ "epoch": 0.5191919191919192,
+ "grad_norm": 2.026737689971924,
+ "learning_rate": 4.4828282828282834e-06,
+ "loss": 1.6041791439056396,
+ "mean_token_accuracy": 0.6275647282600403,
+ "num_tokens": 4210688.0,
+ "step": 257
+ },
+ {
+ "entropy": 1.548423171043396,
+ "epoch": 0.5212121212121212,
+ "grad_norm": 2.0578935146331787,
+ "learning_rate": 4.480808080808081e-06,
+ "loss": 1.563529372215271,
+ "mean_token_accuracy": 0.6348925232887268,
+ "num_tokens": 4227072.0,
+ "step": 258
+ },
+ {
+ "entropy": 1.5324457883834839,
+ "epoch": 0.5232323232323233,
+ "grad_norm": 2.216235637664795,
+ "learning_rate": 4.478787878787879e-06,
+ "loss": 1.5559678077697754,
+ "mean_token_accuracy": 0.6300073266029358,
+ "num_tokens": 4243456.0,
+ "step": 259
+ },
+ {
+ "entropy": 1.5275540351867676,
+ "epoch": 0.5252525252525253,
+ "grad_norm": 2.037306070327759,
+ "learning_rate": 4.476767676767677e-06,
+ "loss": 1.541567087173462,
+ "mean_token_accuracy": 0.6424035429954529,
+ "num_tokens": 4259840.0,
+ "step": 260
+ },
+ {
+ "entropy": 1.6842185258865356,
+ "epoch": 0.5272727272727272,
+ "grad_norm": 2.0241005420684814,
+ "learning_rate": 4.474747474747475e-06,
+ "loss": 1.6819056272506714,
+ "mean_token_accuracy": 0.6102222800254822,
+ "num_tokens": 4276224.0,
+ "step": 261
+ },
+ {
+ "entropy": 1.5656248331069946,
+ "epoch": 0.5292929292929293,
+ "grad_norm": 1.993054986000061,
+ "learning_rate": 4.472727272727273e-06,
+ "loss": 1.6132277250289917,
+ "mean_token_accuracy": 0.634709358215332,
+ "num_tokens": 4292608.0,
+ "step": 262
+ },
+ {
+ "entropy": 1.6354466676712036,
+ "epoch": 0.5313131313131313,
+ "grad_norm": 2.06508731842041,
+ "learning_rate": 4.470707070707071e-06,
+ "loss": 1.6475149393081665,
+ "mean_token_accuracy": 0.6121763586997986,
+ "num_tokens": 4308992.0,
+ "step": 263
+ },
+ {
+ "entropy": 1.5267232656478882,
+ "epoch": 0.5333333333333333,
+ "grad_norm": 2.1227569580078125,
+ "learning_rate": 4.468686868686869e-06,
+ "loss": 1.5468671321868896,
+ "mean_token_accuracy": 0.6365413069725037,
+ "num_tokens": 4325376.0,
+ "step": 264
+ },
+ {
+ "entropy": 1.2252761125564575,
+ "epoch": 0.5353535353535354,
+ "grad_norm": 2.0596134662628174,
+ "learning_rate": 4.4666666666666665e-06,
+ "loss": 1.2242389917373657,
+ "mean_token_accuracy": 0.6973620057106018,
+ "num_tokens": 4341760.0,
+ "step": 265
+ },
+ {
+ "entropy": 1.6007431745529175,
+ "epoch": 0.5373737373737374,
+ "grad_norm": 1.9341918230056763,
+ "learning_rate": 4.464646464646465e-06,
+ "loss": 1.5989094972610474,
+ "mean_token_accuracy": 0.6284196376800537,
+ "num_tokens": 4358144.0,
+ "step": 266
+ },
+ {
+ "entropy": 1.2708780765533447,
+ "epoch": 0.5393939393939394,
+ "grad_norm": 4.594091415405273,
+ "learning_rate": 4.462626262626263e-06,
+ "loss": 1.3484312295913696,
+ "mean_token_accuracy": 0.6790425181388855,
+ "num_tokens": 4374528.0,
+ "step": 267
+ },
+ {
+ "entropy": 1.5423723459243774,
+ "epoch": 0.5414141414141415,
+ "grad_norm": 1.933602213859558,
+ "learning_rate": 4.460606060606061e-06,
+ "loss": 1.594527244567871,
+ "mean_token_accuracy": 0.6386785507202148,
+ "num_tokens": 4390912.0,
+ "step": 268
+ },
+ {
+ "entropy": 1.7213952541351318,
+ "epoch": 0.5434343434343434,
+ "grad_norm": 2.195904016494751,
+ "learning_rate": 4.458585858585859e-06,
+ "loss": 1.724353551864624,
+ "mean_token_accuracy": 0.6008182764053345,
+ "num_tokens": 4407296.0,
+ "step": 269
+ },
+ {
+ "entropy": 1.4772557020187378,
+ "epoch": 0.5454545454545454,
+ "grad_norm": 2.0990796089172363,
+ "learning_rate": 4.456565656565657e-06,
+ "loss": 1.486825942993164,
+ "mean_token_accuracy": 0.6507083773612976,
+ "num_tokens": 4423680.0,
+ "step": 270
+ },
+ {
+ "entropy": 1.6876367330551147,
+ "epoch": 0.5474747474747474,
+ "grad_norm": 2.1944479942321777,
+ "learning_rate": 4.454545454545455e-06,
+ "loss": 1.71107816696167,
+ "mean_token_accuracy": 0.6027112603187561,
+ "num_tokens": 4440064.0,
+ "step": 271
+ },
+ {
+ "entropy": 1.5480726957321167,
+ "epoch": 0.5494949494949495,
+ "grad_norm": 2.1579341888427734,
+ "learning_rate": 4.452525252525253e-06,
+ "loss": 1.544647216796875,
+ "mean_token_accuracy": 0.6345261335372925,
+ "num_tokens": 4456448.0,
+ "step": 272
+ },
+ {
+ "entropy": 1.6161645650863647,
+ "epoch": 0.5515151515151515,
+ "grad_norm": 1.981154203414917,
+ "learning_rate": 4.4505050505050505e-06,
+ "loss": 1.6109068393707275,
+ "mean_token_accuracy": 0.6318392753601074,
+ "num_tokens": 4472832.0,
+ "step": 273
+ },
+ {
+ "entropy": 1.2576326131820679,
+ "epoch": 0.5535353535353535,
+ "grad_norm": 1.95668625831604,
+ "learning_rate": 4.448484848484848e-06,
+ "loss": 1.2614656686782837,
+ "mean_token_accuracy": 0.7025524973869324,
+ "num_tokens": 4489216.0,
+ "step": 274
+ },
+ {
+ "entropy": 1.405206561088562,
+ "epoch": 0.5555555555555556,
+ "grad_norm": 2.0302884578704834,
+ "learning_rate": 4.446464646464646e-06,
+ "loss": 1.416546106338501,
+ "mean_token_accuracy": 0.6753786206245422,
+ "num_tokens": 4505600.0,
+ "step": 275
+ },
+ {
+ "entropy": 1.9038625955581665,
+ "epoch": 0.5575757575757576,
+ "grad_norm": 2.06475567817688,
+ "learning_rate": 4.444444444444444e-06,
+ "loss": 1.9273614883422852,
+ "mean_token_accuracy": 0.5813996195793152,
+ "num_tokens": 4521984.0,
+ "step": 276
+ },
+ {
+ "entropy": 1.7208465337753296,
+ "epoch": 0.5595959595959596,
+ "grad_norm": 2.0136189460754395,
+ "learning_rate": 4.442424242424243e-06,
+ "loss": 1.7260158061981201,
+ "mean_token_accuracy": 0.6006350517272949,
+ "num_tokens": 4538368.0,
+ "step": 277
+ },
+ {
+ "entropy": 1.7570570707321167,
+ "epoch": 0.5616161616161616,
+ "grad_norm": 2.148594379425049,
+ "learning_rate": 4.440404040404041e-06,
+ "loss": 1.7799286842346191,
+ "mean_token_accuracy": 0.5943453907966614,
+ "num_tokens": 4554752.0,
+ "step": 278
+ },
+ {
+ "entropy": 1.6388157606124878,
+ "epoch": 0.5636363636363636,
+ "grad_norm": 2.1301987171173096,
+ "learning_rate": 4.438383838383839e-06,
+ "loss": 1.63419508934021,
+ "mean_token_accuracy": 0.6405715942382812,
+ "num_tokens": 4571136.0,
+ "step": 279
+ },
+ {
+ "entropy": 1.5902295112609863,
+ "epoch": 0.5656565656565656,
+ "grad_norm": 2.0526790618896484,
+ "learning_rate": 4.436363636363637e-06,
+ "loss": 1.6278276443481445,
+ "mean_token_accuracy": 0.615229606628418,
+ "num_tokens": 4587520.0,
+ "step": 280
+ },
+ {
+ "entropy": 1.5494027137756348,
+ "epoch": 0.5676767676767677,
+ "grad_norm": 1.9863859415054321,
+ "learning_rate": 4.434343434343435e-06,
+ "loss": 1.5622975826263428,
+ "mean_token_accuracy": 0.6261602640151978,
+ "num_tokens": 4603904.0,
+ "step": 281
+ },
+ {
+ "entropy": 1.2140685319900513,
+ "epoch": 0.5696969696969697,
+ "grad_norm": 2.4206902980804443,
+ "learning_rate": 4.432323232323233e-06,
+ "loss": 1.2389612197875977,
+ "mean_token_accuracy": 0.6943697929382324,
+ "num_tokens": 4620288.0,
+ "step": 282
+ },
+ {
+ "entropy": 1.4347270727157593,
+ "epoch": 0.5717171717171717,
+ "grad_norm": 2.0198237895965576,
+ "learning_rate": 4.430303030303031e-06,
+ "loss": 1.4648659229278564,
+ "mean_token_accuracy": 0.6573033928871155,
+ "num_tokens": 4636672.0,
+ "step": 283
+ },
+ {
+ "entropy": 1.3239331245422363,
+ "epoch": 0.5737373737373738,
+ "grad_norm": 1.9862704277038574,
+ "learning_rate": 4.428282828282829e-06,
+ "loss": 1.3520365953445435,
+ "mean_token_accuracy": 0.6852100491523743,
+ "num_tokens": 4653056.0,
+ "step": 284
+ },
+ {
+ "entropy": 1.7534631490707397,
+ "epoch": 0.5757575757575758,
+ "grad_norm": 2.0964176654815674,
+ "learning_rate": 4.426262626262627e-06,
+ "loss": 1.7659757137298584,
+ "mean_token_accuracy": 0.5975207686424255,
+ "num_tokens": 4669440.0,
+ "step": 285
+ },
+ {
+ "entropy": 1.1573433876037598,
+ "epoch": 0.5777777777777777,
+ "grad_norm": 2.0671567916870117,
+ "learning_rate": 4.424242424242425e-06,
+ "loss": 1.1479461193084717,
+ "mean_token_accuracy": 0.7143380641937256,
+ "num_tokens": 4685824.0,
+ "step": 286
+ },
+ {
+ "entropy": 1.2439504861831665,
+ "epoch": 0.5797979797979798,
+ "grad_norm": 2.0317327976226807,
+ "learning_rate": 4.422222222222223e-06,
+ "loss": 1.255782961845398,
+ "mean_token_accuracy": 0.691255509853363,
+ "num_tokens": 4702208.0,
+ "step": 287
+ },
+ {
+ "entropy": 1.5569473505020142,
+ "epoch": 0.5818181818181818,
+ "grad_norm": 2.0759670734405518,
+ "learning_rate": 4.4202020202020205e-06,
+ "loss": 1.5564974546432495,
+ "mean_token_accuracy": 0.6284807324409485,
+ "num_tokens": 4718592.0,
+ "step": 288
+ },
+ {
+ "entropy": 1.478676199913025,
+ "epoch": 0.5838383838383838,
+ "grad_norm": 1.9528205394744873,
+ "learning_rate": 4.418181818181818e-06,
+ "loss": 1.491654396057129,
+ "mean_token_accuracy": 0.6520518064498901,
+ "num_tokens": 4734976.0,
+ "step": 289
+ },
+ {
+ "entropy": 1.2454503774642944,
+ "epoch": 0.5858585858585859,
+ "grad_norm": 1.9008079767227173,
+ "learning_rate": 4.416161616161616e-06,
+ "loss": 1.253904938697815,
+ "mean_token_accuracy": 0.6971787810325623,
+ "num_tokens": 4751360.0,
+ "step": 290
+ },
+ {
+ "entropy": 1.1740810871124268,
+ "epoch": 0.5878787878787879,
+ "grad_norm": 1.8888787031173706,
+ "learning_rate": 4.414141414141415e-06,
+ "loss": 1.195070505142212,
+ "mean_token_accuracy": 0.7048119306564331,
+ "num_tokens": 4767744.0,
+ "step": 291
+ },
+ {
+ "entropy": 1.6064486503601074,
+ "epoch": 0.5898989898989899,
+ "grad_norm": 2.073559284210205,
+ "learning_rate": 4.412121212121213e-06,
+ "loss": 1.6180689334869385,
+ "mean_token_accuracy": 0.6242061257362366,
+ "num_tokens": 4784128.0,
+ "step": 292
+ },
+ {
+ "entropy": 1.6119383573532104,
+ "epoch": 0.591919191919192,
+ "grad_norm": 1.8773425817489624,
+ "learning_rate": 4.410101010101011e-06,
+ "loss": 1.5875662565231323,
+ "mean_token_accuracy": 0.647838294506073,
+ "num_tokens": 4800512.0,
+ "step": 293
+ },
+ {
+ "entropy": 1.392905592918396,
+ "epoch": 0.593939393939394,
+ "grad_norm": 2.1699368953704834,
+ "learning_rate": 4.408080808080809e-06,
+ "loss": 1.3949127197265625,
+ "mean_token_accuracy": 0.6538837552070618,
+ "num_tokens": 4816896.0,
+ "step": 294
+ },
+ {
+ "entropy": 1.5057002305984497,
+ "epoch": 0.5959595959595959,
+ "grad_norm": 2.2730712890625,
+ "learning_rate": 4.4060606060606066e-06,
+ "loss": 1.4755051136016846,
+ "mean_token_accuracy": 0.6498534679412842,
+ "num_tokens": 4833280.0,
+ "step": 295
+ },
+ {
+ "entropy": 1.5468902587890625,
+ "epoch": 0.597979797979798,
+ "grad_norm": 1.9911075830459595,
+ "learning_rate": 4.4040404040404044e-06,
+ "loss": 1.539963722229004,
+ "mean_token_accuracy": 0.6520518064498901,
+ "num_tokens": 4849664.0,
+ "step": 296
+ },
+ {
+ "entropy": 1.5196901559829712,
+ "epoch": 0.6,
+ "grad_norm": 1.9859540462493896,
+ "learning_rate": 4.402020202020202e-06,
+ "loss": 1.523442029953003,
+ "mean_token_accuracy": 0.6417317986488342,
+ "num_tokens": 4866048.0,
+ "step": 297
+ },
+ {
+ "entropy": 1.8251866102218628,
+ "epoch": 0.602020202020202,
+ "grad_norm": 2.2566516399383545,
+ "learning_rate": 4.4e-06,
+ "loss": 1.8667771816253662,
+ "mean_token_accuracy": 0.5872007608413696,
+ "num_tokens": 4882432.0,
+ "step": 298
+ },
+ {
+ "entropy": 1.6816744804382324,
+ "epoch": 0.604040404040404,
+ "grad_norm": 2.0107715129852295,
+ "learning_rate": 4.397979797979798e-06,
+ "loss": 1.6907548904418945,
+ "mean_token_accuracy": 0.6176722049713135,
+ "num_tokens": 4898816.0,
+ "step": 299
+ },
+ {
+ "entropy": 1.4992223978042603,
+ "epoch": 0.6060606060606061,
+ "grad_norm": 2.1236329078674316,
+ "learning_rate": 4.395959595959596e-06,
+ "loss": 1.5015790462493896,
+ "mean_token_accuracy": 0.6372129917144775,
+ "num_tokens": 4915200.0,
+ "step": 300
+ },
+ {
+ "epoch": 0.6060606060606061,
+ "eval_entropy": 1.5544315297757425,
+ "eval_loss": 1.5676765441894531,
+ "eval_mean_token_accuracy": 0.6389380799185845,
+ "eval_num_tokens": 4915200.0,
+ "eval_runtime": 43.7857,
+ "eval_samples_per_second": 22.61,
+ "eval_steps_per_second": 2.832,
+ "step": 300
+ },
+ {
+ "entropy": 1.518556833267212,
+ "epoch": 0.6080808080808081,
+ "grad_norm": 2.88371205329895,
+ "learning_rate": 4.393939393939394e-06,
+ "loss": 1.5310916900634766,
+ "mean_token_accuracy": 0.6551660895347595,
+ "num_tokens": 4931584.0,
+ "step": 301
+ },
+ {
+ "entropy": 1.4650386571884155,
+ "epoch": 0.6101010101010101,
+ "grad_norm": 2.1849780082702637,
+ "learning_rate": 4.391919191919193e-06,
+ "loss": 1.4405598640441895,
+ "mean_token_accuracy": 0.6531509757041931,
+ "num_tokens": 4947968.0,
+ "step": 302
+ },
+ {
+ "entropy": 1.5209505558013916,
+ "epoch": 0.6121212121212121,
+ "grad_norm": 2.1135010719299316,
+ "learning_rate": 4.3898989898989905e-06,
+ "loss": 1.5324647426605225,
+ "mean_token_accuracy": 0.6561431288719177,
+ "num_tokens": 4964352.0,
+ "step": 303
+ },
+ {
+ "entropy": 1.448391318321228,
+ "epoch": 0.6141414141414141,
+ "grad_norm": 1.9959306716918945,
+ "learning_rate": 4.387878787878788e-06,
+ "loss": 1.4721965789794922,
+ "mean_token_accuracy": 0.6578529477119446,
+ "num_tokens": 4980736.0,
+ "step": 304
+ },
+ {
+ "entropy": 1.4609358310699463,
+ "epoch": 0.6161616161616161,
+ "grad_norm": 2.1308915615081787,
+ "learning_rate": 4.385858585858586e-06,
+ "loss": 1.4707520008087158,
+ "mean_token_accuracy": 0.6482046842575073,
+ "num_tokens": 4997120.0,
+ "step": 305
+ },
+ {
+ "entropy": 1.7125155925750732,
+ "epoch": 0.6181818181818182,
+ "grad_norm": 2.2170841693878174,
+ "learning_rate": 4.383838383838384e-06,
+ "loss": 1.7630269527435303,
+ "mean_token_accuracy": 0.5999022722244263,
+ "num_tokens": 5013504.0,
+ "step": 306
+ },
+ {
+ "entropy": 1.6740268468856812,
+ "epoch": 0.6202020202020202,
+ "grad_norm": 1.983644723892212,
+ "learning_rate": 4.381818181818182e-06,
+ "loss": 1.6849563121795654,
+ "mean_token_accuracy": 0.6237176060676575,
+ "num_tokens": 5029888.0,
+ "step": 307
+ },
+ {
+ "entropy": 1.3754971027374268,
+ "epoch": 0.6222222222222222,
+ "grad_norm": 2.005277395248413,
+ "learning_rate": 4.37979797979798e-06,
+ "loss": 1.385213851928711,
+ "mean_token_accuracy": 0.6638984084129333,
+ "num_tokens": 5046272.0,
+ "step": 308
+ },
+ {
+ "entropy": 1.5476733446121216,
+ "epoch": 0.6242424242424243,
+ "grad_norm": 2.4317610263824463,
+ "learning_rate": 4.377777777777778e-06,
+ "loss": 1.5417041778564453,
+ "mean_token_accuracy": 0.6323888897895813,
+ "num_tokens": 5062656.0,
+ "step": 309
+ },
+ {
+ "entropy": 1.4437031745910645,
+ "epoch": 0.6262626262626263,
+ "grad_norm": 2.1464109420776367,
+ "learning_rate": 4.375757575757576e-06,
+ "loss": 1.4704627990722656,
+ "mean_token_accuracy": 0.6772105693817139,
+ "num_tokens": 5079040.0,
+ "step": 310
+ },
+ {
+ "entropy": 1.4859641790390015,
+ "epoch": 0.6282828282828283,
+ "grad_norm": 2.0800492763519287,
+ "learning_rate": 4.373737373737374e-06,
+ "loss": 1.5072834491729736,
+ "mean_token_accuracy": 0.658707857131958,
+ "num_tokens": 5095424.0,
+ "step": 311
+ },
+ {
+ "entropy": 1.8733419179916382,
+ "epoch": 0.6303030303030303,
+ "grad_norm": 2.120965003967285,
+ "learning_rate": 4.3717171717171715e-06,
+ "loss": 1.8567254543304443,
+ "mean_token_accuracy": 0.5847581624984741,
+ "num_tokens": 5111808.0,
+ "step": 312
+ },
+ {
+ "entropy": 1.4578243494033813,
+ "epoch": 0.6323232323232323,
+ "grad_norm": 1.9577823877334595,
+ "learning_rate": 4.36969696969697e-06,
+ "loss": 1.4758052825927734,
+ "mean_token_accuracy": 0.6590742468833923,
+ "num_tokens": 5128192.0,
+ "step": 313
+ },
+ {
+ "entropy": 1.8680084943771362,
+ "epoch": 0.6343434343434343,
+ "grad_norm": 2.170994520187378,
+ "learning_rate": 4.367676767676768e-06,
+ "loss": 1.904400110244751,
+ "mean_token_accuracy": 0.5857962965965271,
+ "num_tokens": 5144576.0,
+ "step": 314
+ },
+ {
+ "entropy": 1.6488304138183594,
+ "epoch": 0.6363636363636364,
+ "grad_norm": 1.959490418434143,
+ "learning_rate": 4.365656565656566e-06,
+ "loss": 1.6570477485656738,
+ "mean_token_accuracy": 0.6257327795028687,
+ "num_tokens": 5160960.0,
+ "step": 315
+ },
+ {
+ "entropy": 1.5038025379180908,
+ "epoch": 0.6383838383838384,
+ "grad_norm": 2.072697401046753,
+ "learning_rate": 4.363636363636364e-06,
+ "loss": 1.5234860181808472,
+ "mean_token_accuracy": 0.6494259834289551,
+ "num_tokens": 5177344.0,
+ "step": 316
+ },
+ {
+ "entropy": 1.4154654741287231,
+ "epoch": 0.6404040404040404,
+ "grad_norm": 1.894587755203247,
+ "learning_rate": 4.361616161616162e-06,
+ "loss": 1.4131592512130737,
+ "mean_token_accuracy": 0.6667073965072632,
+ "num_tokens": 5193728.0,
+ "step": 317
+ },
+ {
+ "entropy": 1.5813250541687012,
+ "epoch": 0.6424242424242425,
+ "grad_norm": 2.4088737964630127,
+ "learning_rate": 4.35959595959596e-06,
+ "loss": 1.6236622333526611,
+ "mean_token_accuracy": 0.6313507556915283,
+ "num_tokens": 5210112.0,
+ "step": 318
+ },
+ {
+ "entropy": 1.4140045642852783,
+ "epoch": 0.6444444444444445,
+ "grad_norm": 1.991557240486145,
+ "learning_rate": 4.3575757575757576e-06,
+ "loss": 1.4242517948150635,
+ "mean_token_accuracy": 0.6569980382919312,
+ "num_tokens": 5226496.0,
+ "step": 319
+ },
+ {
+ "entropy": 1.615628957748413,
+ "epoch": 0.6464646464646465,
+ "grad_norm": 1.8819077014923096,
+ "learning_rate": 4.3555555555555555e-06,
+ "loss": 1.6331532001495361,
+ "mean_token_accuracy": 0.6299462914466858,
+ "num_tokens": 5242880.0,
+ "step": 320
+ },
+ {
+ "entropy": 1.3118394613265991,
+ "epoch": 0.6484848484848484,
+ "grad_norm": 2.0589284896850586,
+ "learning_rate": 4.353535353535353e-06,
+ "loss": 1.2880034446716309,
+ "mean_token_accuracy": 0.6842941045761108,
+ "num_tokens": 5259264.0,
+ "step": 321
+ },
+ {
+ "entropy": 1.2786612510681152,
+ "epoch": 0.6505050505050505,
+ "grad_norm": 2.033839225769043,
+ "learning_rate": 4.351515151515152e-06,
+ "loss": 1.2829055786132812,
+ "mean_token_accuracy": 0.6910112500190735,
+ "num_tokens": 5275648.0,
+ "step": 322
+ },
+ {
+ "entropy": 1.5632988214492798,
+ "epoch": 0.6525252525252525,
+ "grad_norm": 2.1970419883728027,
+ "learning_rate": 4.34949494949495e-06,
+ "loss": 1.5865097045898438,
+ "mean_token_accuracy": 0.642892062664032,
+ "num_tokens": 5292032.0,
+ "step": 323
+ },
+ {
+ "entropy": 1.1542725563049316,
+ "epoch": 0.6545454545454545,
+ "grad_norm": 1.9750654697418213,
+ "learning_rate": 4.347474747474748e-06,
+ "loss": 1.1401035785675049,
+ "mean_token_accuracy": 0.7061553597450256,
+ "num_tokens": 5308416.0,
+ "step": 324
+ },
+ {
+ "entropy": 1.6879942417144775,
+ "epoch": 0.6565656565656566,
+ "grad_norm": 2.0022354125976562,
+ "learning_rate": 4.345454545454546e-06,
+ "loss": 1.70950448513031,
+ "mean_token_accuracy": 0.6089398860931396,
+ "num_tokens": 5324800.0,
+ "step": 325
+ },
+ {
+ "entropy": 1.7589699029922485,
+ "epoch": 0.6585858585858586,
+ "grad_norm": 1.925520420074463,
+ "learning_rate": 4.343434343434344e-06,
+ "loss": 1.7749860286712646,
+ "mean_token_accuracy": 0.6015510559082031,
+ "num_tokens": 5341184.0,
+ "step": 326
+ },
+ {
+ "entropy": 1.4720325469970703,
+ "epoch": 0.6606060606060606,
+ "grad_norm": 1.9487124681472778,
+ "learning_rate": 4.341414141414142e-06,
+ "loss": 1.4983797073364258,
+ "mean_token_accuracy": 0.6537005305290222,
+ "num_tokens": 5357568.0,
+ "step": 327
+ },
+ {
+ "entropy": 1.676164150238037,
+ "epoch": 0.6626262626262627,
+ "grad_norm": 2.13053035736084,
+ "learning_rate": 4.33939393939394e-06,
+ "loss": 1.699425458908081,
+ "mean_token_accuracy": 0.6100390553474426,
+ "num_tokens": 5373952.0,
+ "step": 328
+ },
+ {
+ "entropy": 1.3424437046051025,
+ "epoch": 0.6646464646464646,
+ "grad_norm": 2.0245954990386963,
+ "learning_rate": 4.337373737373738e-06,
+ "loss": 1.3393046855926514,
+ "mean_token_accuracy": 0.677760124206543,
+ "num_tokens": 5390336.0,
+ "step": 329
+ },
+ {
+ "entropy": 1.4926583766937256,
+ "epoch": 0.6666666666666666,
+ "grad_norm": 1.893343448638916,
+ "learning_rate": 4.335353535353536e-06,
+ "loss": 1.4779269695281982,
+ "mean_token_accuracy": 0.6713483333587646,
+ "num_tokens": 5406720.0,
+ "step": 330
+ },
+ {
+ "entropy": 1.5753449201583862,
+ "epoch": 0.6686868686868687,
+ "grad_norm": 2.051647424697876,
+ "learning_rate": 4.333333333333334e-06,
+ "loss": 1.622597336769104,
+ "mean_token_accuracy": 0.6436248421669006,
+ "num_tokens": 5423104.0,
+ "step": 331
+ },
+ {
+ "entropy": 1.4573988914489746,
+ "epoch": 0.6707070707070707,
+ "grad_norm": 1.8709567785263062,
+ "learning_rate": 4.331313131313132e-06,
+ "loss": 1.4902422428131104,
+ "mean_token_accuracy": 0.6682950854301453,
+ "num_tokens": 5439488.0,
+ "step": 332
+ },
+ {
+ "entropy": 1.378867745399475,
+ "epoch": 0.6727272727272727,
+ "grad_norm": 2.235928773880005,
+ "learning_rate": 4.32929292929293e-06,
+ "loss": 1.4067103862762451,
+ "mean_token_accuracy": 0.662432849407196,
+ "num_tokens": 5455872.0,
+ "step": 333
+ },
+ {
+ "entropy": 1.7553350925445557,
+ "epoch": 0.6747474747474748,
+ "grad_norm": 2.0335066318511963,
+ "learning_rate": 4.327272727272728e-06,
+ "loss": 1.772943139076233,
+ "mean_token_accuracy": 0.5953834652900696,
+ "num_tokens": 5472256.0,
+ "step": 334
+ },
+ {
+ "entropy": 1.1587097644805908,
+ "epoch": 0.6767676767676768,
+ "grad_norm": 1.8764615058898926,
+ "learning_rate": 4.3252525252525255e-06,
+ "loss": 1.1396210193634033,
+ "mean_token_accuracy": 0.7197117805480957,
+ "num_tokens": 5488640.0,
+ "step": 335
+ },
+ {
+ "entropy": 1.4819872379302979,
+ "epoch": 0.6787878787878788,
+ "grad_norm": 2.0907511711120605,
+ "learning_rate": 4.323232323232323e-06,
+ "loss": 1.460550308227539,
+ "mean_token_accuracy": 0.6520518064498901,
+ "num_tokens": 5505024.0,
+ "step": 336
+ },
+ {
+ "entropy": 1.0744472742080688,
+ "epoch": 0.6808080808080809,
+ "grad_norm": 2.227606773376465,
+ "learning_rate": 4.321212121212121e-06,
+ "loss": 1.0909301042556763,
+ "mean_token_accuracy": 0.7258182764053345,
+ "num_tokens": 5521408.0,
+ "step": 337
+ },
+ {
+ "entropy": 1.507999300956726,
+ "epoch": 0.6828282828282828,
+ "grad_norm": 2.0332415103912354,
+ "learning_rate": 4.31919191919192e-06,
+ "loss": 1.5173046588897705,
+ "mean_token_accuracy": 0.6374572515487671,
+ "num_tokens": 5537792.0,
+ "step": 338
+ },
+ {
+ "entropy": 1.6792720556259155,
+ "epoch": 0.6848484848484848,
+ "grad_norm": 2.0194997787475586,
+ "learning_rate": 4.317171717171718e-06,
+ "loss": 1.679445743560791,
+ "mean_token_accuracy": 0.6138250827789307,
+ "num_tokens": 5554176.0,
+ "step": 339
+ },
+ {
+ "entropy": 1.6470589637756348,
+ "epoch": 0.6868686868686869,
+ "grad_norm": 2.09116268157959,
+ "learning_rate": 4.315151515151516e-06,
+ "loss": 1.6386632919311523,
+ "mean_token_accuracy": 0.6146799921989441,
+ "num_tokens": 5570560.0,
+ "step": 340
+ },
+ {
+ "entropy": 1.3612488508224487,
+ "epoch": 0.6888888888888889,
+ "grad_norm": 2.1586217880249023,
+ "learning_rate": 4.313131313131314e-06,
+ "loss": 1.3698725700378418,
+ "mean_token_accuracy": 0.6696995496749878,
+ "num_tokens": 5586944.0,
+ "step": 341
+ },
+ {
+ "entropy": 1.400327205657959,
+ "epoch": 0.6909090909090909,
+ "grad_norm": 2.082094192504883,
+ "learning_rate": 4.3111111111111115e-06,
+ "loss": 1.396535873413086,
+ "mean_token_accuracy": 0.6780654788017273,
+ "num_tokens": 5603328.0,
+ "step": 342
+ },
+ {
+ "entropy": 1.389290690422058,
+ "epoch": 0.692929292929293,
+ "grad_norm": 2.0780303478240967,
+ "learning_rate": 4.309090909090909e-06,
+ "loss": 1.4275782108306885,
+ "mean_token_accuracy": 0.6656082272529602,
+ "num_tokens": 5619712.0,
+ "step": 343
+ },
+ {
+ "entropy": 1.5133870840072632,
+ "epoch": 0.694949494949495,
+ "grad_norm": 1.9819531440734863,
+ "learning_rate": 4.307070707070707e-06,
+ "loss": 1.5309596061706543,
+ "mean_token_accuracy": 0.6373351216316223,
+ "num_tokens": 5636096.0,
+ "step": 344
+ },
+ {
+ "entropy": 1.483219861984253,
+ "epoch": 0.696969696969697,
+ "grad_norm": 1.9794325828552246,
+ "learning_rate": 4.305050505050505e-06,
+ "loss": 1.4693087339401245,
+ "mean_token_accuracy": 0.6550439596176147,
+ "num_tokens": 5652480.0,
+ "step": 345
+ },
+ {
+ "entropy": 1.381489634513855,
+ "epoch": 0.6989898989898989,
+ "grad_norm": 2.0637686252593994,
+ "learning_rate": 4.303030303030303e-06,
+ "loss": 1.3862476348876953,
+ "mean_token_accuracy": 0.6631045341491699,
+ "num_tokens": 5668864.0,
+ "step": 346
+ },
+ {
+ "entropy": 1.9860024452209473,
+ "epoch": 0.701010101010101,
+ "grad_norm": 2.1626803874969482,
+ "learning_rate": 4.301010101010101e-06,
+ "loss": 1.9815950393676758,
+ "mean_token_accuracy": 0.5698583126068115,
+ "num_tokens": 5685248.0,
+ "step": 347
+ },
+ {
+ "entropy": 1.5044004917144775,
+ "epoch": 0.703030303030303,
+ "grad_norm": 2.060976266860962,
+ "learning_rate": 4.298989898989899e-06,
+ "loss": 1.4937127828598022,
+ "mean_token_accuracy": 0.6524792313575745,
+ "num_tokens": 5701632.0,
+ "step": 348
+ },
+ {
+ "entropy": 1.3397772312164307,
+ "epoch": 0.705050505050505,
+ "grad_norm": 2.0162901878356934,
+ "learning_rate": 4.296969696969698e-06,
+ "loss": 1.3358572721481323,
+ "mean_token_accuracy": 0.6949804425239563,
+ "num_tokens": 5718016.0,
+ "step": 349
+ },
+ {
+ "entropy": 1.145772099494934,
+ "epoch": 0.7070707070707071,
+ "grad_norm": 2.097634792327881,
+ "learning_rate": 4.2949494949494955e-06,
+ "loss": 1.1833416223526,
+ "mean_token_accuracy": 0.7104909420013428,
+ "num_tokens": 5734400.0,
+ "step": 350
+ },
+ {
+ "epoch": 0.7070707070707071,
+ "eval_entropy": 1.5476290602837839,
+ "eval_loss": 1.5603480339050293,
+ "eval_mean_token_accuracy": 0.640111118555069,
+ "eval_num_tokens": 5734400.0,
+ "eval_runtime": 43.7844,
+ "eval_samples_per_second": 22.611,
+ "eval_steps_per_second": 2.832,
+ "step": 350
+ },
+ {
+ "entropy": 1.4311926364898682,
+ "epoch": 0.7090909090909091,
+ "grad_norm": 2.002321720123291,
+ "learning_rate": 4.292929292929293e-06,
+ "loss": 1.4534825086593628,
+ "mean_token_accuracy": 0.6614558100700378,
+ "num_tokens": 5750784.0,
+ "step": 351
+ },
+ {
+ "entropy": 1.3983922004699707,
+ "epoch": 0.7111111111111111,
+ "grad_norm": 2.1724867820739746,
+ "learning_rate": 4.290909090909091e-06,
+ "loss": 1.3969402313232422,
+ "mean_token_accuracy": 0.6652418375015259,
+ "num_tokens": 5767168.0,
+ "step": 352
+ },
+ {
+ "entropy": 1.8218920230865479,
+ "epoch": 0.7131313131313132,
+ "grad_norm": 2.1629281044006348,
+ "learning_rate": 4.288888888888889e-06,
+ "loss": 1.8196980953216553,
+ "mean_token_accuracy": 0.6027112603187561,
+ "num_tokens": 5783552.0,
+ "step": 353
+ },
+ {
+ "entropy": 1.5322320461273193,
+ "epoch": 0.7151515151515152,
+ "grad_norm": 1.8486647605895996,
+ "learning_rate": 4.286868686868687e-06,
+ "loss": 1.5504610538482666,
+ "mean_token_accuracy": 0.6533341407775879,
+ "num_tokens": 5799936.0,
+ "step": 354
+ },
+ {
+ "entropy": 1.935966968536377,
+ "epoch": 0.7171717171717171,
+ "grad_norm": 2.252814292907715,
+ "learning_rate": 4.284848484848485e-06,
+ "loss": 1.9300384521484375,
+ "mean_token_accuracy": 0.5656448602676392,
+ "num_tokens": 5816320.0,
+ "step": 355
+ },
+ {
+ "entropy": 1.6711398363113403,
+ "epoch": 0.7191919191919192,
+ "grad_norm": 2.023379325866699,
+ "learning_rate": 4.282828282828283e-06,
+ "loss": 1.6584455966949463,
+ "mean_token_accuracy": 0.6235954761505127,
+ "num_tokens": 5832704.0,
+ "step": 356
+ },
+ {
+ "entropy": 1.3922803401947021,
+ "epoch": 0.7212121212121212,
+ "grad_norm": 2.0487611293792725,
+ "learning_rate": 4.280808080808081e-06,
+ "loss": 1.4220250844955444,
+ "mean_token_accuracy": 0.6591963768005371,
+ "num_tokens": 5849088.0,
+ "step": 357
+ },
+ {
+ "entropy": 1.59521484375,
+ "epoch": 0.7232323232323232,
+ "grad_norm": 1.8598614931106567,
+ "learning_rate": 4.278787878787879e-06,
+ "loss": 1.5979329347610474,
+ "mean_token_accuracy": 0.6237176060676575,
+ "num_tokens": 5865472.0,
+ "step": 358
+ },
+ {
+ "entropy": 1.5810115337371826,
+ "epoch": 0.7252525252525253,
+ "grad_norm": 2.140115737915039,
+ "learning_rate": 4.276767676767677e-06,
+ "loss": 1.5774705410003662,
+ "mean_token_accuracy": 0.6274425983428955,
+ "num_tokens": 5881856.0,
+ "step": 359
+ },
+ {
+ "entropy": 1.5839109420776367,
+ "epoch": 0.7272727272727273,
+ "grad_norm": 2.0947749614715576,
+ "learning_rate": 4.274747474747475e-06,
+ "loss": 1.601511001586914,
+ "mean_token_accuracy": 0.630984365940094,
+ "num_tokens": 5898240.0,
+ "step": 360
+ },
+ {
+ "entropy": 1.195770263671875,
+ "epoch": 0.7292929292929293,
+ "grad_norm": 1.8740363121032715,
+ "learning_rate": 4.272727272727273e-06,
+ "loss": 1.195652961730957,
+ "mean_token_accuracy": 0.7077430486679077,
+ "num_tokens": 5914624.0,
+ "step": 361
+ },
+ {
+ "entropy": 1.375698208808899,
+ "epoch": 0.7313131313131314,
+ "grad_norm": 1.9580703973770142,
+ "learning_rate": 4.270707070707071e-06,
+ "loss": 1.3746864795684814,
+ "mean_token_accuracy": 0.6775158643722534,
+ "num_tokens": 5931008.0,
+ "step": 362
+ },
+ {
+ "entropy": 1.7259451150894165,
+ "epoch": 0.7333333333333333,
+ "grad_norm": 2.3127365112304688,
+ "learning_rate": 4.268686868686869e-06,
+ "loss": 1.745998501777649,
+ "mean_token_accuracy": 0.6040546894073486,
+ "num_tokens": 5947392.0,
+ "step": 363
+ },
+ {
+ "entropy": 1.209228754043579,
+ "epoch": 0.7353535353535353,
+ "grad_norm": 1.9004793167114258,
+ "learning_rate": 4.266666666666668e-06,
+ "loss": 1.232427716255188,
+ "mean_token_accuracy": 0.6925989389419556,
+ "num_tokens": 5963776.0,
+ "step": 364
+ },
+ {
+ "entropy": 1.2811263799667358,
+ "epoch": 0.7373737373737373,
+ "grad_norm": 1.9568246603012085,
+ "learning_rate": 4.2646464646464655e-06,
+ "loss": 1.291853427886963,
+ "mean_token_accuracy": 0.6884465217590332,
+ "num_tokens": 5980160.0,
+ "step": 365
+ },
+ {
+ "entropy": 1.843044638633728,
+ "epoch": 0.7393939393939394,
+ "grad_norm": 2.132735252380371,
+ "learning_rate": 4.262626262626263e-06,
+ "loss": 1.8818857669830322,
+ "mean_token_accuracy": 0.5785295367240906,
+ "num_tokens": 5996544.0,
+ "step": 366
+ },
+ {
+ "entropy": 1.3353440761566162,
+ "epoch": 0.7414141414141414,
+ "grad_norm": 1.8893013000488281,
+ "learning_rate": 4.260606060606061e-06,
+ "loss": 1.354011058807373,
+ "mean_token_accuracy": 0.6797752976417542,
+ "num_tokens": 6012928.0,
+ "step": 367
+ },
+ {
+ "entropy": 1.3889728784561157,
+ "epoch": 0.7434343434343434,
+ "grad_norm": 1.980757236480713,
+ "learning_rate": 4.258585858585859e-06,
+ "loss": 1.3671875,
+ "mean_token_accuracy": 0.6627381443977356,
+ "num_tokens": 6029312.0,
+ "step": 368
+ },
+ {
+ "entropy": 1.4605348110198975,
+ "epoch": 0.7454545454545455,
+ "grad_norm": 2.1033780574798584,
+ "learning_rate": 4.256565656565657e-06,
+ "loss": 1.455024003982544,
+ "mean_token_accuracy": 0.6451514363288879,
+ "num_tokens": 6045696.0,
+ "step": 369
+ },
+ {
+ "entropy": 1.4382058382034302,
+ "epoch": 0.7474747474747475,
+ "grad_norm": 2.1068074703216553,
+ "learning_rate": 4.254545454545455e-06,
+ "loss": 1.4422768354415894,
+ "mean_token_accuracy": 0.6630434989929199,
+ "num_tokens": 6062080.0,
+ "step": 370
+ },
+ {
+ "entropy": 1.8046759366989136,
+ "epoch": 0.7494949494949495,
+ "grad_norm": 2.214466094970703,
+ "learning_rate": 4.252525252525253e-06,
+ "loss": 1.8247106075286865,
+ "mean_token_accuracy": 0.6003297567367554,
+ "num_tokens": 6078464.0,
+ "step": 371
+ },
+ {
+ "entropy": 1.671690583229065,
+ "epoch": 0.7515151515151515,
+ "grad_norm": 1.9790785312652588,
+ "learning_rate": 4.250505050505051e-06,
+ "loss": 1.6907753944396973,
+ "mean_token_accuracy": 0.6232290863990784,
+ "num_tokens": 6094848.0,
+ "step": 372
+ },
+ {
+ "entropy": 1.6881897449493408,
+ "epoch": 0.7535353535353535,
+ "grad_norm": 1.8638463020324707,
+ "learning_rate": 4.248484848484849e-06,
+ "loss": 1.7132716178894043,
+ "mean_token_accuracy": 0.6357474327087402,
+ "num_tokens": 6111232.0,
+ "step": 373
+ },
+ {
+ "entropy": 1.2555122375488281,
+ "epoch": 0.7555555555555555,
+ "grad_norm": 2.082378387451172,
+ "learning_rate": 4.246464646464647e-06,
+ "loss": 1.240688681602478,
+ "mean_token_accuracy": 0.6965070962905884,
+ "num_tokens": 6127616.0,
+ "step": 374
+ },
+ {
+ "entropy": 1.847135305404663,
+ "epoch": 0.7575757575757576,
+ "grad_norm": 2.142962694168091,
+ "learning_rate": 4.244444444444445e-06,
+ "loss": 1.8840911388397217,
+ "mean_token_accuracy": 0.5822545289993286,
+ "num_tokens": 6144000.0,
+ "step": 375
+ },
+ {
+ "entropy": 1.239328384399414,
+ "epoch": 0.7595959595959596,
+ "grad_norm": 2.0589468479156494,
+ "learning_rate": 4.242424242424243e-06,
+ "loss": 1.256324052810669,
+ "mean_token_accuracy": 0.6845383644104004,
+ "num_tokens": 6160384.0,
+ "step": 376
+ },
+ {
+ "entropy": 1.5047202110290527,
+ "epoch": 0.7616161616161616,
+ "grad_norm": 1.7951308488845825,
+ "learning_rate": 4.240404040404041e-06,
+ "loss": 1.5063304901123047,
+ "mean_token_accuracy": 0.6583414673805237,
+ "num_tokens": 6176768.0,
+ "step": 377
+ },
+ {
+ "entropy": 1.4072566032409668,
+ "epoch": 0.7636363636363637,
+ "grad_norm": 2.1670594215393066,
+ "learning_rate": 4.238383838383839e-06,
+ "loss": 1.4205389022827148,
+ "mean_token_accuracy": 0.658707857131958,
+ "num_tokens": 6193152.0,
+ "step": 378
+ },
+ {
+ "entropy": 1.3207885026931763,
+ "epoch": 0.7656565656565657,
+ "grad_norm": 1.9017083644866943,
+ "learning_rate": 4.236363636363637e-06,
+ "loss": 1.3169896602630615,
+ "mean_token_accuracy": 0.6954079270362854,
+ "num_tokens": 6209536.0,
+ "step": 379
+ },
+ {
+ "entropy": 1.4762436151504517,
+ "epoch": 0.7676767676767676,
+ "grad_norm": 2.0023069381713867,
+ "learning_rate": 4.234343434343435e-06,
+ "loss": 1.4725041389465332,
+ "mean_token_accuracy": 0.6620053648948669,
+ "num_tokens": 6225920.0,
+ "step": 380
+ },
+ {
+ "entropy": 1.4895304441452026,
+ "epoch": 0.7696969696969697,
+ "grad_norm": 1.9528017044067383,
+ "learning_rate": 4.2323232323232325e-06,
+ "loss": 1.49650239944458,
+ "mean_token_accuracy": 0.6508915424346924,
+ "num_tokens": 6242304.0,
+ "step": 381
+ },
+ {
+ "entropy": 1.3024516105651855,
+ "epoch": 0.7717171717171717,
+ "grad_norm": 1.9855588674545288,
+ "learning_rate": 4.2303030303030304e-06,
+ "loss": 1.34218168258667,
+ "mean_token_accuracy": 0.6838055849075317,
+ "num_tokens": 6258688.0,
+ "step": 382
+ },
+ {
+ "entropy": 1.6005626916885376,
+ "epoch": 0.7737373737373737,
+ "grad_norm": 2.028286933898926,
+ "learning_rate": 4.228282828282828e-06,
+ "loss": 1.6299283504486084,
+ "mean_token_accuracy": 0.6315950155258179,
+ "num_tokens": 6275072.0,
+ "step": 383
+ },
+ {
+ "entropy": 1.7050484418869019,
+ "epoch": 0.7757575757575758,
+ "grad_norm": 2.474047899246216,
+ "learning_rate": 4.226262626262626e-06,
+ "loss": 1.777151346206665,
+ "mean_token_accuracy": 0.5919027924537659,
+ "num_tokens": 6291456.0,
+ "step": 384
+ },
+ {
+ "entropy": 1.5014543533325195,
+ "epoch": 0.7777777777777778,
+ "grad_norm": 1.9866594076156616,
+ "learning_rate": 4.224242424242425e-06,
+ "loss": 1.5308949947357178,
+ "mean_token_accuracy": 0.639106035232544,
+ "num_tokens": 6307840.0,
+ "step": 385
+ },
+ {
+ "entropy": 1.6957359313964844,
+ "epoch": 0.7797979797979798,
+ "grad_norm": 2.229820966720581,
+ "learning_rate": 4.222222222222223e-06,
+ "loss": 1.728761911392212,
+ "mean_token_accuracy": 0.6242061257362366,
+ "num_tokens": 6324224.0,
+ "step": 386
+ },
+ {
+ "entropy": 1.6823521852493286,
+ "epoch": 0.7818181818181819,
+ "grad_norm": 2.033383369445801,
+ "learning_rate": 4.220202020202021e-06,
+ "loss": 1.7310154438018799,
+ "mean_token_accuracy": 0.6257938742637634,
+ "num_tokens": 6340608.0,
+ "step": 387
+ },
+ {
+ "entropy": 1.710307240486145,
+ "epoch": 0.7838383838383839,
+ "grad_norm": 2.061861038208008,
+ "learning_rate": 4.218181818181819e-06,
+ "loss": 1.7066943645477295,
+ "mean_token_accuracy": 0.6123595237731934,
+ "num_tokens": 6356992.0,
+ "step": 388
+ },
+ {
+ "entropy": 1.241638422012329,
+ "epoch": 0.7858585858585858,
+ "grad_norm": 1.9743852615356445,
+ "learning_rate": 4.2161616161616165e-06,
+ "loss": 1.2520272731781006,
+ "mean_token_accuracy": 0.692415714263916,
+ "num_tokens": 6373376.0,
+ "step": 389
+ },
+ {
+ "entropy": 1.5918776988983154,
+ "epoch": 0.7878787878787878,
+ "grad_norm": 2.156675100326538,
+ "learning_rate": 4.214141414141414e-06,
+ "loss": 1.6060255765914917,
+ "mean_token_accuracy": 0.6178553700447083,
+ "num_tokens": 6389760.0,
+ "step": 390
+ },
+ {
+ "entropy": 1.7307862043380737,
+ "epoch": 0.7898989898989899,
+ "grad_norm": 2.171247959136963,
+ "learning_rate": 4.212121212121212e-06,
+ "loss": 1.7327581644058228,
+ "mean_token_accuracy": 0.6502808928489685,
+ "num_tokens": 6406144.0,
+ "step": 391
+ },
+ {
+ "entropy": 1.6725553274154663,
+ "epoch": 0.7919191919191919,
+ "grad_norm": 2.020228624343872,
+ "learning_rate": 4.21010101010101e-06,
+ "loss": 1.6803646087646484,
+ "mean_token_accuracy": 0.6383732557296753,
+ "num_tokens": 6422528.0,
+ "step": 392
+ },
+ {
+ "entropy": 1.4120928049087524,
+ "epoch": 0.793939393939394,
+ "grad_norm": 1.9896639585494995,
+ "learning_rate": 4.208080808080808e-06,
+ "loss": 1.3974279165267944,
+ "mean_token_accuracy": 0.6661577820777893,
+ "num_tokens": 6438912.0,
+ "step": 393
+ },
+ {
+ "entropy": 0.933545708656311,
+ "epoch": 0.795959595959596,
+ "grad_norm": 1.6884223222732544,
+ "learning_rate": 4.206060606060606e-06,
+ "loss": 0.9287986755371094,
+ "mean_token_accuracy": 0.7636175155639648,
+ "num_tokens": 6455296.0,
+ "step": 394
+ },
+ {
+ "entropy": 1.419004201889038,
+ "epoch": 0.797979797979798,
+ "grad_norm": 2.2590551376342773,
+ "learning_rate": 4.204040404040405e-06,
+ "loss": 1.4652538299560547,
+ "mean_token_accuracy": 0.6589521169662476,
+ "num_tokens": 6471680.0,
+ "step": 395
+ },
+ {
+ "entropy": 1.750221610069275,
+ "epoch": 0.8,
+ "grad_norm": 2.195030450820923,
+ "learning_rate": 4.2020202020202026e-06,
+ "loss": 1.7328863143920898,
+ "mean_token_accuracy": 0.6008182764053345,
+ "num_tokens": 6488064.0,
+ "step": 396
+ },
+ {
+ "entropy": 1.7440015077590942,
+ "epoch": 0.802020202020202,
+ "grad_norm": 1.9833927154541016,
+ "learning_rate": 4.2000000000000004e-06,
+ "loss": 1.7617835998535156,
+ "mean_token_accuracy": 0.5992916226387024,
+ "num_tokens": 6504448.0,
+ "step": 397
+ },
+ {
+ "entropy": 1.1636452674865723,
+ "epoch": 0.804040404040404,
+ "grad_norm": 1.9506802558898926,
+ "learning_rate": 4.197979797979798e-06,
+ "loss": 1.1344032287597656,
+ "mean_token_accuracy": 0.7122618556022644,
+ "num_tokens": 6520832.0,
+ "step": 398
+ },
+ {
+ "entropy": 1.6173542737960815,
+ "epoch": 0.806060606060606,
+ "grad_norm": 2.0997231006622314,
+ "learning_rate": 4.195959595959596e-06,
+ "loss": 1.6303956508636475,
+ "mean_token_accuracy": 0.6215192675590515,
+ "num_tokens": 6537216.0,
+ "step": 399
+ },
+ {
+ "entropy": 1.4391542673110962,
+ "epoch": 0.8080808080808081,
+ "grad_norm": 2.356828212738037,
+ "learning_rate": 4.193939393939394e-06,
+ "loss": 1.4465923309326172,
+ "mean_token_accuracy": 0.6594406366348267,
+ "num_tokens": 6553600.0,
+ "step": 400
+ },
+ {
+ "epoch": 0.8080808080808081,
+ "eval_entropy": 1.5504949785047961,
+ "eval_loss": 1.5544542074203491,
+ "eval_mean_token_accuracy": 0.6409837569921247,
+ "eval_num_tokens": 6553600.0,
+ "eval_runtime": 43.7986,
+ "eval_samples_per_second": 22.603,
+ "eval_steps_per_second": 2.831,
+ "step": 400
+ },
+ {
+ "entropy": 1.60548996925354,
+ "epoch": 0.8101010101010101,
+ "grad_norm": 2.1894404888153076,
+ "learning_rate": 4.191919191919192e-06,
+ "loss": 1.6116085052490234,
+ "mean_token_accuracy": 0.6276868581771851,
+ "num_tokens": 6569984.0,
+ "step": 401
+ },
+ {
+ "entropy": 1.436041235923767,
+ "epoch": 0.8121212121212121,
+ "grad_norm": 2.1180264949798584,
+ "learning_rate": 4.18989898989899e-06,
+ "loss": 1.4203698635101318,
+ "mean_token_accuracy": 0.6554714441299438,
+ "num_tokens": 6586368.0,
+ "step": 402
+ },
+ {
+ "entropy": 1.1348106861114502,
+ "epoch": 0.8141414141414142,
+ "grad_norm": 2.0420851707458496,
+ "learning_rate": 4.187878787878788e-06,
+ "loss": 1.1424527168273926,
+ "mean_token_accuracy": 0.7154372334480286,
+ "num_tokens": 6602752.0,
+ "step": 403
+ },
+ {
+ "entropy": 1.4039727449417114,
+ "epoch": 0.8161616161616162,
+ "grad_norm": 2.148340940475464,
+ "learning_rate": 4.185858585858586e-06,
+ "loss": 1.4246132373809814,
+ "mean_token_accuracy": 0.6723864078521729,
+ "num_tokens": 6619136.0,
+ "step": 404
+ },
+ {
+ "entropy": 1.4222626686096191,
+ "epoch": 0.8181818181818182,
+ "grad_norm": 1.9436827898025513,
+ "learning_rate": 4.1838383838383835e-06,
+ "loss": 1.4235990047454834,
+ "mean_token_accuracy": 0.6759281754493713,
+ "num_tokens": 6635520.0,
+ "step": 405
+ },
+ {
+ "entropy": 1.4771310091018677,
+ "epoch": 0.8202020202020202,
+ "grad_norm": 2.0953726768493652,
+ "learning_rate": 4.181818181818182e-06,
+ "loss": 1.501934289932251,
+ "mean_token_accuracy": 0.6535173654556274,
+ "num_tokens": 6651904.0,
+ "step": 406
+ },
+ {
+ "entropy": 1.526256799697876,
+ "epoch": 0.8222222222222222,
+ "grad_norm": 2.245994806289673,
+ "learning_rate": 4.17979797979798e-06,
+ "loss": 1.539383888244629,
+ "mean_token_accuracy": 0.6417317986488342,
+ "num_tokens": 6668288.0,
+ "step": 407
+ },
+ {
+ "entropy": 1.1716097593307495,
+ "epoch": 0.8242424242424242,
+ "grad_norm": 1.8283652067184448,
+ "learning_rate": 4.177777777777778e-06,
+ "loss": 1.1798359155654907,
+ "mean_token_accuracy": 0.7123839855194092,
+ "num_tokens": 6684672.0,
+ "step": 408
+ },
+ {
+ "entropy": 1.4581540822982788,
+ "epoch": 0.8262626262626263,
+ "grad_norm": 1.9325168132781982,
+ "learning_rate": 4.175757575757576e-06,
+ "loss": 1.4526585340499878,
+ "mean_token_accuracy": 0.662432849407196,
+ "num_tokens": 6701056.0,
+ "step": 409
+ },
+ {
+ "entropy": 1.4073102474212646,
+ "epoch": 0.8282828282828283,
+ "grad_norm": 2.1543467044830322,
+ "learning_rate": 4.173737373737375e-06,
+ "loss": 1.437713861465454,
+ "mean_token_accuracy": 0.6725696325302124,
+ "num_tokens": 6717440.0,
+ "step": 410
+ },
+ {
+ "entropy": 1.5970062017440796,
+ "epoch": 0.8303030303030303,
+ "grad_norm": 2.1714792251586914,
+ "learning_rate": 4.1717171717171726e-06,
+ "loss": 1.613295078277588,
+ "mean_token_accuracy": 0.6229848265647888,
+ "num_tokens": 6733824.0,
+ "step": 411
+ },
+ {
+ "entropy": 1.449837327003479,
+ "epoch": 0.8323232323232324,
+ "grad_norm": 2.2499871253967285,
+ "learning_rate": 4.1696969696969705e-06,
+ "loss": 1.489758014678955,
+ "mean_token_accuracy": 0.645639955997467,
+ "num_tokens": 6750208.0,
+ "step": 412
+ },
+ {
+ "entropy": 1.7696173191070557,
+ "epoch": 0.8343434343434344,
+ "grad_norm": 2.3637073040008545,
+ "learning_rate": 4.167676767676768e-06,
+ "loss": 1.8147599697113037,
+ "mean_token_accuracy": 0.5821323990821838,
+ "num_tokens": 6766592.0,
+ "step": 413
+ },
+ {
+ "entropy": 1.99376380443573,
+ "epoch": 0.8363636363636363,
+ "grad_norm": 2.265683174133301,
+ "learning_rate": 4.165656565656566e-06,
+ "loss": 2.009024143218994,
+ "mean_token_accuracy": 0.5591108798980713,
+ "num_tokens": 6782976.0,
+ "step": 414
+ },
+ {
+ "entropy": 1.7276980876922607,
+ "epoch": 0.8383838383838383,
+ "grad_norm": 1.9407477378845215,
+ "learning_rate": 4.163636363636364e-06,
+ "loss": 1.7130229473114014,
+ "mean_token_accuracy": 0.6202979683876038,
+ "num_tokens": 6799360.0,
+ "step": 415
+ },
+ {
+ "entropy": 1.5418223142623901,
+ "epoch": 0.8404040404040404,
+ "grad_norm": 1.9765743017196655,
+ "learning_rate": 4.161616161616162e-06,
+ "loss": 1.5627632141113281,
+ "mean_token_accuracy": 0.6433194875717163,
+ "num_tokens": 6815744.0,
+ "step": 416
+ },
+ {
+ "entropy": 1.6040345430374146,
+ "epoch": 0.8424242424242424,
+ "grad_norm": 2.1942877769470215,
+ "learning_rate": 4.15959595959596e-06,
+ "loss": 1.6044622659683228,
+ "mean_token_accuracy": 0.6278700828552246,
+ "num_tokens": 6832128.0,
+ "step": 417
+ },
+ {
+ "entropy": 1.8461577892303467,
+ "epoch": 0.8444444444444444,
+ "grad_norm": 2.289196729660034,
+ "learning_rate": 4.157575757575758e-06,
+ "loss": 1.8679372072219849,
+ "mean_token_accuracy": 0.5887884497642517,
+ "num_tokens": 6848512.0,
+ "step": 418
+ },
+ {
+ "entropy": 1.4079350233078003,
+ "epoch": 0.8464646464646465,
+ "grad_norm": 1.9526047706604004,
+ "learning_rate": 4.155555555555556e-06,
+ "loss": 1.4145114421844482,
+ "mean_token_accuracy": 0.6731802821159363,
+ "num_tokens": 6864896.0,
+ "step": 419
+ },
+ {
+ "entropy": 1.1973973512649536,
+ "epoch": 0.8484848484848485,
+ "grad_norm": 2.0716679096221924,
+ "learning_rate": 4.1535353535353536e-06,
+ "loss": 1.260810136795044,
+ "mean_token_accuracy": 0.701697587966919,
+ "num_tokens": 6881280.0,
+ "step": 420
+ },
+ {
+ "entropy": 1.507702350616455,
+ "epoch": 0.8505050505050505,
+ "grad_norm": 2.0233314037323,
+ "learning_rate": 4.151515151515152e-06,
+ "loss": 1.513720154762268,
+ "mean_token_accuracy": 0.6497313380241394,
+ "num_tokens": 6897664.0,
+ "step": 421
+ },
+ {
+ "entropy": 1.54402756690979,
+ "epoch": 0.8525252525252526,
+ "grad_norm": 2.23366379737854,
+ "learning_rate": 4.14949494949495e-06,
+ "loss": 1.5434964895248413,
+ "mean_token_accuracy": 0.6502198576927185,
+ "num_tokens": 6914048.0,
+ "step": 422
+ },
+ {
+ "entropy": 1.820296049118042,
+ "epoch": 0.8545454545454545,
+ "grad_norm": 2.066905975341797,
+ "learning_rate": 4.147474747474748e-06,
+ "loss": 1.8356924057006836,
+ "mean_token_accuracy": 0.5886663198471069,
+ "num_tokens": 6930432.0,
+ "step": 423
+ },
+ {
+ "entropy": 1.4747720956802368,
+ "epoch": 0.8565656565656565,
+ "grad_norm": 2.040022850036621,
+ "learning_rate": 4.145454545454546e-06,
+ "loss": 1.4495999813079834,
+ "mean_token_accuracy": 0.6532731056213379,
+ "num_tokens": 6946816.0,
+ "step": 424
+ },
+ {
+ "entropy": 1.7536696195602417,
+ "epoch": 0.8585858585858586,
+ "grad_norm": 2.0884320735931396,
+ "learning_rate": 4.143434343434344e-06,
+ "loss": 1.7520158290863037,
+ "mean_token_accuracy": 0.6083903312683105,
+ "num_tokens": 6963200.0,
+ "step": 425
+ },
+ {
+ "entropy": 1.715582251548767,
+ "epoch": 0.8606060606060606,
+ "grad_norm": 2.2991514205932617,
+ "learning_rate": 4.141414141414142e-06,
+ "loss": 1.7138090133666992,
+ "mean_token_accuracy": 0.607107937335968,
+ "num_tokens": 6979584.0,
+ "step": 426
+ },
+ {
+ "entropy": 1.8116382360458374,
+ "epoch": 0.8626262626262626,
+ "grad_norm": 2.289909839630127,
+ "learning_rate": 4.13939393939394e-06,
+ "loss": 1.7755894660949707,
+ "mean_token_accuracy": 0.5776746273040771,
+ "num_tokens": 6995968.0,
+ "step": 427
+ },
+ {
+ "entropy": 1.487213373184204,
+ "epoch": 0.8646464646464647,
+ "grad_norm": 1.8834803104400635,
+ "learning_rate": 4.1373737373737375e-06,
+ "loss": 1.511157751083374,
+ "mean_token_accuracy": 0.6546165347099304,
+ "num_tokens": 7012352.0,
+ "step": 428
+ },
+ {
+ "entropy": 1.2769891023635864,
+ "epoch": 0.8666666666666667,
+ "grad_norm": 1.892616629600525,
+ "learning_rate": 4.135353535353535e-06,
+ "loss": 1.2772598266601562,
+ "mean_token_accuracy": 0.6984611749649048,
+ "num_tokens": 7028736.0,
+ "step": 429
+ },
+ {
+ "entropy": 1.579519271850586,
+ "epoch": 0.8686868686868687,
+ "grad_norm": 1.9801563024520874,
+ "learning_rate": 4.133333333333333e-06,
+ "loss": 1.5647528171539307,
+ "mean_token_accuracy": 0.6482046842575073,
+ "num_tokens": 7045120.0,
+ "step": 430
+ },
+ {
+ "entropy": 1.407600998878479,
+ "epoch": 0.8707070707070707,
+ "grad_norm": 2.1737446784973145,
+ "learning_rate": 4.131313131313132e-06,
+ "loss": 1.4184494018554688,
+ "mean_token_accuracy": 0.6640815734863281,
+ "num_tokens": 7061504.0,
+ "step": 431
+ },
+ {
+ "entropy": 1.9118441343307495,
+ "epoch": 0.8727272727272727,
+ "grad_norm": 1.9541205167770386,
+ "learning_rate": 4.12929292929293e-06,
+ "loss": 1.9581422805786133,
+ "mean_token_accuracy": 0.564667820930481,
+ "num_tokens": 7077888.0,
+ "step": 432
+ },
+ {
+ "entropy": 1.925604224205017,
+ "epoch": 0.8747474747474747,
+ "grad_norm": 1.9696223735809326,
+ "learning_rate": 4.127272727272728e-06,
+ "loss": 1.9226163625717163,
+ "mean_token_accuracy": 0.5870786309242249,
+ "num_tokens": 7094272.0,
+ "step": 433
+ },
+ {
+ "entropy": 1.3994735479354858,
+ "epoch": 0.8767676767676768,
+ "grad_norm": 2.003532886505127,
+ "learning_rate": 4.125252525252526e-06,
+ "loss": 1.4291752576828003,
+ "mean_token_accuracy": 0.669516384601593,
+ "num_tokens": 7110656.0,
+ "step": 434
+ },
+ {
+ "entropy": 1.5550929307937622,
+ "epoch": 0.8787878787878788,
+ "grad_norm": 2.2658586502075195,
+ "learning_rate": 4.1232323232323236e-06,
+ "loss": 1.5533335208892822,
+ "mean_token_accuracy": 0.6242672204971313,
+ "num_tokens": 7127040.0,
+ "step": 435
+ },
+ {
+ "entropy": 1.6622785329818726,
+ "epoch": 0.8808080808080808,
+ "grad_norm": 2.0748393535614014,
+ "learning_rate": 4.1212121212121215e-06,
+ "loss": 1.700000524520874,
+ "mean_token_accuracy": 0.6221299171447754,
+ "num_tokens": 7143424.0,
+ "step": 436
+ },
+ {
+ "entropy": 1.1223996877670288,
+ "epoch": 0.8828282828282829,
+ "grad_norm": 2.0348756313323975,
+ "learning_rate": 4.119191919191919e-06,
+ "loss": 1.143293857574463,
+ "mean_token_accuracy": 0.7045676708221436,
+ "num_tokens": 7159808.0,
+ "step": 437
+ },
+ {
+ "entropy": 1.7156380414962769,
+ "epoch": 0.8848484848484849,
+ "grad_norm": 2.306549549102783,
+ "learning_rate": 4.117171717171717e-06,
+ "loss": 1.7625794410705566,
+ "mean_token_accuracy": 0.6049706935882568,
+ "num_tokens": 7176192.0,
+ "step": 438
+ },
+ {
+ "entropy": 1.8507202863693237,
+ "epoch": 0.8868686868686869,
+ "grad_norm": 2.2955853939056396,
+ "learning_rate": 4.115151515151515e-06,
+ "loss": 1.8923052549362183,
+ "mean_token_accuracy": 0.5938568711280823,
+ "num_tokens": 7192576.0,
+ "step": 439
+ },
+ {
+ "entropy": 1.813754916191101,
+ "epoch": 0.8888888888888888,
+ "grad_norm": 2.095700263977051,
+ "learning_rate": 4.113131313131313e-06,
+ "loss": 1.8375307321548462,
+ "mean_token_accuracy": 0.5848192572593689,
+ "num_tokens": 7208960.0,
+ "step": 440
+ },
+ {
+ "entropy": 1.4185007810592651,
+ "epoch": 0.8909090909090909,
+ "grad_norm": 2.118213176727295,
+ "learning_rate": 4.111111111111111e-06,
+ "loss": 1.4548171758651733,
+ "mean_token_accuracy": 0.657608687877655,
+ "num_tokens": 7225344.0,
+ "step": 441
+ },
+ {
+ "entropy": 1.1041690111160278,
+ "epoch": 0.8929292929292929,
+ "grad_norm": 1.9638988971710205,
+ "learning_rate": 4.10909090909091e-06,
+ "loss": 1.1232322454452515,
+ "mean_token_accuracy": 0.7068881392478943,
+ "num_tokens": 7241728.0,
+ "step": 442
+ },
+ {
+ "entropy": 1.6423180103302002,
+ "epoch": 0.8949494949494949,
+ "grad_norm": 2.32987117767334,
+ "learning_rate": 4.1070707070707075e-06,
+ "loss": 1.665462613105774,
+ "mean_token_accuracy": 0.6133365631103516,
+ "num_tokens": 7258112.0,
+ "step": 443
+ },
+ {
+ "entropy": 1.4778310060501099,
+ "epoch": 0.896969696969697,
+ "grad_norm": 2.1247661113739014,
+ "learning_rate": 4.105050505050505e-06,
+ "loss": 1.4554922580718994,
+ "mean_token_accuracy": 0.6474108695983887,
+ "num_tokens": 7274496.0,
+ "step": 444
+ },
+ {
+ "entropy": 1.3777755498886108,
+ "epoch": 0.898989898989899,
+ "grad_norm": 1.9243263006210327,
+ "learning_rate": 4.103030303030303e-06,
+ "loss": 1.3942883014678955,
+ "mean_token_accuracy": 0.6656692624092102,
+ "num_tokens": 7290880.0,
+ "step": 445
+ },
+ {
+ "entropy": 1.1036415100097656,
+ "epoch": 0.901010101010101,
+ "grad_norm": 1.9711178541183472,
+ "learning_rate": 4.101010101010101e-06,
+ "loss": 1.1070351600646973,
+ "mean_token_accuracy": 0.7213605046272278,
+ "num_tokens": 7307264.0,
+ "step": 446
+ },
+ {
+ "entropy": 1.695487380027771,
+ "epoch": 0.9030303030303031,
+ "grad_norm": 2.1113150119781494,
+ "learning_rate": 4.098989898989899e-06,
+ "loss": 1.6993653774261475,
+ "mean_token_accuracy": 0.5992305874824524,
+ "num_tokens": 7323648.0,
+ "step": 447
+ },
+ {
+ "entropy": 1.496254801750183,
+ "epoch": 0.9050505050505051,
+ "grad_norm": 2.072986602783203,
+ "learning_rate": 4.096969696969697e-06,
+ "loss": 1.5046567916870117,
+ "mean_token_accuracy": 0.6558378338813782,
+ "num_tokens": 7340032.0,
+ "step": 448
+ },
+ {
+ "entropy": 2.1845033168792725,
+ "epoch": 0.907070707070707,
+ "grad_norm": 2.3246262073516846,
+ "learning_rate": 4.094949494949495e-06,
+ "loss": 2.1508708000183105,
+ "mean_token_accuracy": 0.5376160144805908,
+ "num_tokens": 7356416.0,
+ "step": 449
+ },
+ {
+ "entropy": 1.6329164505004883,
+ "epoch": 0.9090909090909091,
+ "grad_norm": 1.997310757637024,
+ "learning_rate": 4.092929292929294e-06,
+ "loss": 1.6422264575958252,
+ "mean_token_accuracy": 0.6232290863990784,
+ "num_tokens": 7372800.0,
+ "step": 450
+ },
+ {
+ "epoch": 0.9090909090909091,
+ "eval_entropy": 1.5515337480652718,
+ "eval_loss": 1.5485161542892456,
+ "eval_mean_token_accuracy": 0.6417716929028111,
+ "eval_num_tokens": 7372800.0,
+ "eval_runtime": 43.7901,
+ "eval_samples_per_second": 22.608,
+ "eval_steps_per_second": 2.832,
+ "step": 450
+ },
+ {
+ "entropy": 1.7854291200637817,
+ "epoch": 0.9111111111111111,
+ "grad_norm": 2.0212173461914062,
+ "learning_rate": 4.0909090909090915e-06,
+ "loss": 1.7920666933059692,
+ "mean_token_accuracy": 0.5950170755386353,
+ "num_tokens": 7389184.0,
+ "step": 451
+ },
+ {
+ "entropy": 1.216304898262024,
+ "epoch": 0.9131313131313131,
+ "grad_norm": 1.8800705671310425,
+ "learning_rate": 4.088888888888889e-06,
+ "loss": 1.2014267444610596,
+ "mean_token_accuracy": 0.7076819539070129,
+ "num_tokens": 7405568.0,
+ "step": 452
+ },
+ {
+ "entropy": 1.5593703985214233,
+ "epoch": 0.9151515151515152,
+ "grad_norm": 2.101381301879883,
+ "learning_rate": 4.086868686868687e-06,
+ "loss": 1.559610366821289,
+ "mean_token_accuracy": 0.6337933540344238,
+ "num_tokens": 7421952.0,
+ "step": 453
+ },
+ {
+ "entropy": 1.4199841022491455,
+ "epoch": 0.9171717171717172,
+ "grad_norm": 2.0038692951202393,
+ "learning_rate": 4.084848484848485e-06,
+ "loss": 1.4040653705596924,
+ "mean_token_accuracy": 0.660845160484314,
+ "num_tokens": 7438336.0,
+ "step": 454
+ },
+ {
+ "entropy": 1.2010453939437866,
+ "epoch": 0.9191919191919192,
+ "grad_norm": 2.004110336303711,
+ "learning_rate": 4.082828282828283e-06,
+ "loss": 1.194115400314331,
+ "mean_token_accuracy": 0.6955910921096802,
+ "num_tokens": 7454720.0,
+ "step": 455
+ },
+ {
+ "entropy": 1.2295804023742676,
+ "epoch": 0.9212121212121213,
+ "grad_norm": 2.130387544631958,
+ "learning_rate": 4.080808080808081e-06,
+ "loss": 1.2125787734985352,
+ "mean_token_accuracy": 0.6993771195411682,
+ "num_tokens": 7471104.0,
+ "step": 456
+ },
+ {
+ "entropy": 1.3706485033035278,
+ "epoch": 0.9232323232323232,
+ "grad_norm": 2.1667706966400146,
+ "learning_rate": 4.07878787878788e-06,
+ "loss": 1.3886957168579102,
+ "mean_token_accuracy": 0.6686614751815796,
+ "num_tokens": 7487488.0,
+ "step": 457
+ },
+ {
+ "entropy": 1.714059591293335,
+ "epoch": 0.9252525252525252,
+ "grad_norm": 2.0084264278411865,
+ "learning_rate": 4.0767676767676775e-06,
+ "loss": 1.724153757095337,
+ "mean_token_accuracy": 0.602161705493927,
+ "num_tokens": 7503872.0,
+ "step": 458
+ },
+ {
+ "entropy": 1.3170617818832397,
+ "epoch": 0.9272727272727272,
+ "grad_norm": 1.9923994541168213,
+ "learning_rate": 4.0747474747474754e-06,
+ "loss": 1.3518096208572388,
+ "mean_token_accuracy": 0.6801416873931885,
+ "num_tokens": 7520256.0,
+ "step": 459
+ },
+ {
+ "entropy": 1.7946380376815796,
+ "epoch": 0.9292929292929293,
+ "grad_norm": 2.3475260734558105,
+ "learning_rate": 4.072727272727273e-06,
+ "loss": 1.824514389038086,
+ "mean_token_accuracy": 0.6005740165710449,
+ "num_tokens": 7536640.0,
+ "step": 460
+ },
+ {
+ "entropy": 1.3695449829101562,
+ "epoch": 0.9313131313131313,
+ "grad_norm": 2.0983943939208984,
+ "learning_rate": 4.070707070707071e-06,
+ "loss": 1.3934273719787598,
+ "mean_token_accuracy": 0.6723253726959229,
+ "num_tokens": 7553024.0,
+ "step": 461
+ },
+ {
+ "entropy": 1.5890663862228394,
+ "epoch": 0.9333333333333333,
+ "grad_norm": 2.1465413570404053,
+ "learning_rate": 4.068686868686869e-06,
+ "loss": 1.611652135848999,
+ "mean_token_accuracy": 0.6408158540725708,
+ "num_tokens": 7569408.0,
+ "step": 462
+ },
+ {
+ "entropy": 1.2593727111816406,
+ "epoch": 0.9353535353535354,
+ "grad_norm": 1.7885241508483887,
+ "learning_rate": 4.066666666666667e-06,
+ "loss": 1.2781705856323242,
+ "mean_token_accuracy": 0.6946751475334167,
+ "num_tokens": 7585792.0,
+ "step": 463
+ },
+ {
+ "entropy": 1.5680429935455322,
+ "epoch": 0.9373737373737374,
+ "grad_norm": 2.1466636657714844,
+ "learning_rate": 4.064646464646465e-06,
+ "loss": 1.60218346118927,
+ "mean_token_accuracy": 0.6334269642829895,
+ "num_tokens": 7602176.0,
+ "step": 464
+ },
+ {
+ "entropy": 1.681670069694519,
+ "epoch": 0.9393939393939394,
+ "grad_norm": 2.2211875915527344,
+ "learning_rate": 4.062626262626263e-06,
+ "loss": 1.6819908618927002,
+ "mean_token_accuracy": 0.6177943348884583,
+ "num_tokens": 7618560.0,
+ "step": 465
+ },
+ {
+ "entropy": 1.2239924669265747,
+ "epoch": 0.9414141414141414,
+ "grad_norm": 1.9997022151947021,
+ "learning_rate": 4.060606060606061e-06,
+ "loss": 1.2383348941802979,
+ "mean_token_accuracy": 0.6922325491905212,
+ "num_tokens": 7634944.0,
+ "step": 466
+ },
+ {
+ "entropy": 1.291772723197937,
+ "epoch": 0.9434343434343434,
+ "grad_norm": 2.0868117809295654,
+ "learning_rate": 4.058585858585859e-06,
+ "loss": 1.2879221439361572,
+ "mean_token_accuracy": 0.6883854269981384,
+ "num_tokens": 7651328.0,
+ "step": 467
+ },
+ {
+ "entropy": 1.2860186100006104,
+ "epoch": 0.9454545454545454,
+ "grad_norm": 1.8586393594741821,
+ "learning_rate": 4.056565656565657e-06,
+ "loss": 1.2820203304290771,
+ "mean_token_accuracy": 0.6802638173103333,
+ "num_tokens": 7667712.0,
+ "step": 468
+ },
+ {
+ "entropy": 1.860795259475708,
+ "epoch": 0.9474747474747475,
+ "grad_norm": 2.364405393600464,
+ "learning_rate": 4.054545454545455e-06,
+ "loss": 1.8880727291107178,
+ "mean_token_accuracy": 0.5821323990821838,
+ "num_tokens": 7684096.0,
+ "step": 469
+ },
+ {
+ "entropy": 1.6017589569091797,
+ "epoch": 0.9494949494949495,
+ "grad_norm": 2.023054599761963,
+ "learning_rate": 4.052525252525253e-06,
+ "loss": 1.5993852615356445,
+ "mean_token_accuracy": 0.6366634368896484,
+ "num_tokens": 7700480.0,
+ "step": 470
+ },
+ {
+ "entropy": 1.8498003482818604,
+ "epoch": 0.9515151515151515,
+ "grad_norm": 2.4474003314971924,
+ "learning_rate": 4.050505050505051e-06,
+ "loss": 1.868700623512268,
+ "mean_token_accuracy": 0.5826209187507629,
+ "num_tokens": 7716864.0,
+ "step": 471
+ },
+ {
+ "entropy": 1.5625540018081665,
+ "epoch": 0.9535353535353536,
+ "grad_norm": 2.1018362045288086,
+ "learning_rate": 4.048484848484849e-06,
+ "loss": 1.571077585220337,
+ "mean_token_accuracy": 0.6433805823326111,
+ "num_tokens": 7733248.0,
+ "step": 472
+ },
+ {
+ "entropy": 1.8534579277038574,
+ "epoch": 0.9555555555555556,
+ "grad_norm": 2.407588243484497,
+ "learning_rate": 4.046464646464647e-06,
+ "loss": 1.8789153099060059,
+ "mean_token_accuracy": 0.5859794616699219,
+ "num_tokens": 7749632.0,
+ "step": 473
+ },
+ {
+ "entropy": 1.3264559507369995,
+ "epoch": 0.9575757575757575,
+ "grad_norm": 2.007199764251709,
+ "learning_rate": 4.044444444444445e-06,
+ "loss": 1.3489338159561157,
+ "mean_token_accuracy": 0.6650586128234863,
+ "num_tokens": 7766016.0,
+ "step": 474
+ },
+ {
+ "entropy": 1.1361761093139648,
+ "epoch": 0.9595959595959596,
+ "grad_norm": 1.9226998090744019,
+ "learning_rate": 4.0424242424242425e-06,
+ "loss": 1.156738519668579,
+ "mean_token_accuracy": 0.7226428985595703,
+ "num_tokens": 7782400.0,
+ "step": 475
+ },
+ {
+ "entropy": 1.550872802734375,
+ "epoch": 0.9616161616161616,
+ "grad_norm": 2.0082473754882812,
+ "learning_rate": 4.04040404040404e-06,
+ "loss": 1.5657379627227783,
+ "mean_token_accuracy": 0.642953097820282,
+ "num_tokens": 7798784.0,
+ "step": 476
+ },
+ {
+ "entropy": 1.0122721195220947,
+ "epoch": 0.9636363636363636,
+ "grad_norm": 1.9946776628494263,
+ "learning_rate": 4.038383838383838e-06,
+ "loss": 1.0301119089126587,
+ "mean_token_accuracy": 0.7361993193626404,
+ "num_tokens": 7815168.0,
+ "step": 477
+ },
+ {
+ "entropy": 1.61614990234375,
+ "epoch": 0.9656565656565657,
+ "grad_norm": 2.5009427070617676,
+ "learning_rate": 4.036363636363637e-06,
+ "loss": 1.6194994449615479,
+ "mean_token_accuracy": 0.620175838470459,
+ "num_tokens": 7831552.0,
+ "step": 478
+ },
+ {
+ "entropy": 1.5723443031311035,
+ "epoch": 0.9676767676767677,
+ "grad_norm": 1.9101917743682861,
+ "learning_rate": 4.034343434343435e-06,
+ "loss": 1.5586073398590088,
+ "mean_token_accuracy": 0.6223741769790649,
+ "num_tokens": 7847936.0,
+ "step": 479
+ },
+ {
+ "entropy": 1.271834373474121,
+ "epoch": 0.9696969696969697,
+ "grad_norm": 1.768438458442688,
+ "learning_rate": 4.032323232323233e-06,
+ "loss": 1.2466087341308594,
+ "mean_token_accuracy": 0.692354679107666,
+ "num_tokens": 7864320.0,
+ "step": 480
+ },
+ {
+ "entropy": 1.4550020694732666,
+ "epoch": 0.9717171717171718,
+ "grad_norm": 1.9753917455673218,
+ "learning_rate": 4.030303030303031e-06,
+ "loss": 1.4689980745315552,
+ "mean_token_accuracy": 0.6530288457870483,
+ "num_tokens": 7880704.0,
+ "step": 481
+ },
+ {
+ "entropy": 2.023750066757202,
+ "epoch": 0.9737373737373738,
+ "grad_norm": 2.3014674186706543,
+ "learning_rate": 4.0282828282828285e-06,
+ "loss": 2.0601019859313965,
+ "mean_token_accuracy": 0.5528212189674377,
+ "num_tokens": 7897088.0,
+ "step": 482
+ },
+ {
+ "entropy": 1.3225218057632446,
+ "epoch": 0.9757575757575757,
+ "grad_norm": 1.9937688112258911,
+ "learning_rate": 4.0262626262626264e-06,
+ "loss": 1.3186583518981934,
+ "mean_token_accuracy": 0.6751343607902527,
+ "num_tokens": 7913472.0,
+ "step": 483
+ },
+ {
+ "entropy": 1.4357911348342896,
+ "epoch": 0.9777777777777777,
+ "grad_norm": 1.9591492414474487,
+ "learning_rate": 4.024242424242424e-06,
+ "loss": 1.4157384634017944,
+ "mean_token_accuracy": 0.6618832349777222,
+ "num_tokens": 7929856.0,
+ "step": 484
+ },
+ {
+ "entropy": 1.470274567604065,
+ "epoch": 0.9797979797979798,
+ "grad_norm": 1.9318779706954956,
+ "learning_rate": 4.022222222222222e-06,
+ "loss": 1.4754853248596191,
+ "mean_token_accuracy": 0.6497923731803894,
+ "num_tokens": 7946240.0,
+ "step": 485
+ },
+ {
+ "entropy": 1.345750331878662,
+ "epoch": 0.9818181818181818,
+ "grad_norm": 1.9079619646072388,
+ "learning_rate": 4.02020202020202e-06,
+ "loss": 1.336526870727539,
+ "mean_token_accuracy": 0.6800805926322937,
+ "num_tokens": 7962624.0,
+ "step": 486
+ },
+ {
+ "entropy": 1.4641839265823364,
+ "epoch": 0.9838383838383838,
+ "grad_norm": 1.8413265943527222,
+ "learning_rate": 4.018181818181818e-06,
+ "loss": 1.4556326866149902,
+ "mean_token_accuracy": 0.6591963768005371,
+ "num_tokens": 7979008.0,
+ "step": 487
+ },
+ {
+ "entropy": 1.7966911792755127,
+ "epoch": 0.9858585858585859,
+ "grad_norm": 2.138899564743042,
+ "learning_rate": 4.016161616161616e-06,
+ "loss": 1.822898507118225,
+ "mean_token_accuracy": 0.5959941148757935,
+ "num_tokens": 7995392.0,
+ "step": 488
+ },
+ {
+ "entropy": 1.4334503412246704,
+ "epoch": 0.9878787878787879,
+ "grad_norm": 2.206122636795044,
+ "learning_rate": 4.014141414141415e-06,
+ "loss": 1.4127681255340576,
+ "mean_token_accuracy": 0.6609672904014587,
+ "num_tokens": 8011776.0,
+ "step": 489
+ },
+ {
+ "entropy": 1.684112548828125,
+ "epoch": 0.98989898989899,
+ "grad_norm": 2.0637400150299072,
+ "learning_rate": 4.0121212121212125e-06,
+ "loss": 1.6790317296981812,
+ "mean_token_accuracy": 0.6159012913703918,
+ "num_tokens": 8028160.0,
+ "step": 490
+ },
+ {
+ "entropy": 2.0716898441314697,
+ "epoch": 0.9919191919191919,
+ "grad_norm": 3.3912627696990967,
+ "learning_rate": 4.01010101010101e-06,
+ "loss": 2.0814826488494873,
+ "mean_token_accuracy": 0.5522105693817139,
+ "num_tokens": 8044544.0,
+ "step": 491
+ },
+ {
+ "entropy": 1.8764609098434448,
+ "epoch": 0.9939393939393939,
+ "grad_norm": 2.00569748878479,
+ "learning_rate": 4.008080808080808e-06,
+ "loss": 1.9150068759918213,
+ "mean_token_accuracy": 0.5823766589164734,
+ "num_tokens": 8060928.0,
+ "step": 492
+ },
+ {
+ "entropy": 1.630242943763733,
+ "epoch": 0.9959595959595959,
+ "grad_norm": 2.0648858547210693,
+ "learning_rate": 4.006060606060607e-06,
+ "loss": 1.6354784965515137,
+ "mean_token_accuracy": 0.621213972568512,
+ "num_tokens": 8077312.0,
+ "step": 493
+ },
+ {
+ "entropy": 1.7174078226089478,
+ "epoch": 0.997979797979798,
+ "grad_norm": 2.1376583576202393,
+ "learning_rate": 4.004040404040405e-06,
+ "loss": 1.7092773914337158,
+ "mean_token_accuracy": 0.6129701733589172,
+ "num_tokens": 8093696.0,
+ "step": 494
+ },
+ {
+ "entropy": 1.2112717628479004,
+ "epoch": 1.0,
+ "grad_norm": 1.9164371490478516,
+ "learning_rate": 4.002020202020203e-06,
+ "loss": 1.2106354236602783,
+ "mean_token_accuracy": 0.7040180563926697,
+ "num_tokens": 8110080.0,
+ "step": 495
+ },
+ {
+ "entropy": 1.2168891429901123,
+ "epoch": 1.002020202020202,
+ "grad_norm": 1.9845075607299805,
+ "learning_rate": 4.000000000000001e-06,
+ "loss": 1.1398870944976807,
+ "mean_token_accuracy": 0.7053004503250122,
+ "num_tokens": 8126464.0,
+ "step": 496
+ },
+ {
+ "entropy": 1.4670923948287964,
+ "epoch": 1.004040404040404,
+ "grad_norm": 2.0864014625549316,
+ "learning_rate": 3.9979797979797986e-06,
+ "loss": 1.4241242408752441,
+ "mean_token_accuracy": 0.6605398058891296,
+ "num_tokens": 8142848.0,
+ "step": 497
+ },
+ {
+ "entropy": 1.560593843460083,
+ "epoch": 1.006060606060606,
+ "grad_norm": 1.8882137537002563,
+ "learning_rate": 3.9959595959595964e-06,
+ "loss": 1.5164835453033447,
+ "mean_token_accuracy": 0.6610894203186035,
+ "num_tokens": 8159232.0,
+ "step": 498
+ },
+ {
+ "entropy": 1.2573609352111816,
+ "epoch": 1.0080808080808081,
+ "grad_norm": 2.011486530303955,
+ "learning_rate": 3.993939393939394e-06,
+ "loss": 1.2119636535644531,
+ "mean_token_accuracy": 0.6832559704780579,
+ "num_tokens": 8175616.0,
+ "step": 499
+ },
+ {
+ "entropy": 1.6119122505187988,
+ "epoch": 1.0101010101010102,
+ "grad_norm": 2.0089032649993896,
+ "learning_rate": 3.991919191919192e-06,
+ "loss": 1.5869622230529785,
+ "mean_token_accuracy": 0.6301905512809753,
+ "num_tokens": 8192000.0,
+ "step": 500
+ },
+ {
+ "epoch": 1.0101010101010102,
+ "eval_entropy": 1.4907484121860997,
+ "eval_loss": 1.5444872379302979,
+ "eval_mean_token_accuracy": 0.6426531960887294,
+ "eval_num_tokens": 8192000.0,
+ "eval_runtime": 43.729,
+ "eval_samples_per_second": 22.639,
+ "eval_steps_per_second": 2.836,
+ "step": 500
+ }
+ ],
+ "logging_steps": 1,
+ "max_steps": 2475,
+ "num_input_tokens_seen": 0,
+ "num_train_epochs": 5,
+ "save_steps": 500,
+ "stateful_callbacks": {
+ "TrainerControl": {
+ "args": {
+ "should_epoch_stop": false,
+ "should_evaluate": false,
+ "should_log": false,
+ "should_save": true,
+ "should_training_stop": false
+ },
+ "attributes": {}
+ }
+ },
+ "total_flos": 6.9275322679296e+16,
+ "train_batch_size": 4,
+ "trial_name": null,
+ "trial_params": null
+}
diff --git a/qwen3-1.7b-teutonic-5e6/checkpoint-500/training_args.bin b/qwen3-1.7b-teutonic-5e6/checkpoint-500/training_args.bin
new file mode 100644
index 0000000000000000000000000000000000000000..37649d4bae96a1df88666daf20b4e5b6e092d976
--- /dev/null
+++ b/qwen3-1.7b-teutonic-5e6/checkpoint-500/training_args.bin
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:58bbcabfe99a1cf24f2e76aab66b507c2b720d9271dc9f17c8208d741a9c3a65
+size 7121
diff --git a/qwen3-1.7b-teutonic-5e6/checkpoint-500/zero_to_fp32.py b/qwen3-1.7b-teutonic-5e6/checkpoint-500/zero_to_fp32.py
new file mode 100644
index 0000000000000000000000000000000000000000..3970548c400fd4361bd923b763db90e963ddaf8c
--- /dev/null
+++ b/qwen3-1.7b-teutonic-5e6/checkpoint-500/zero_to_fp32.py
@@ -0,0 +1,854 @@
+#!/usr/bin/env python
+
+# Copyright (c) Microsoft Corporation.
+# SPDX-License-Identifier: Apache-2.0
+
+# DeepSpeed Team
+
+# This script extracts fp32 consolidated weights from a zero 1, 2 and 3 DeepSpeed checkpoints. It gets
+# copied into the top level checkpoint dir, so the user can easily do the conversion at any point in
+# the future. Once extracted, the weights don't require DeepSpeed and can be used in any
+# application.
+#
+# example:
+# python zero_to_fp32.py . output_dir/
+# or
+# python zero_to_fp32.py . output_dir/ --safe_serialization
+
+import argparse
+import torch
+import glob
+import math
+import os
+import re
+import gc
+import json
+import numpy as np
+from tqdm import tqdm
+from collections import OrderedDict
+from dataclasses import dataclass
+
+# while this script doesn't use deepspeed to recover data, since the checkpoints are pickled with
+# DeepSpeed data structures it has to be available in the current python environment.
+from deepspeed.utils import logger
+from deepspeed.checkpoint.constants import (DS_VERSION, OPTIMIZER_STATE_DICT, SINGLE_PARTITION_OF_FP32_GROUPS,
+ FP32_FLAT_GROUPS, ZERO_STAGE, PARTITION_COUNT, PARAM_SHAPES, BUFFER_NAMES,
+ FROZEN_PARAM_SHAPES, FROZEN_PARAM_FRAGMENTS, AUTOEP_LAYERS_KEY,
+ AUTOEP_LAYERS_KEY_LEGACY, AUTOEP_ZERO3_EXPERT_STATE_FORMAT_KEY,
+ AUTOEP_ZERO3_PARTITIONED_EXPERT_STATE_FORMAT, PARAM_ALIGNMENT_PADDINGS)
+
+
+@dataclass
+class zero_model_state:
+ buffers: dict()
+ param_shapes: dict()
+ shared_params: list
+ ds_version: int
+ frozen_param_shapes: dict()
+ frozen_param_fragments: dict()
+
+
+debug = 0
+
+# load to cpu
+device = torch.device('cpu')
+
+OUTPUT_DTYPE_NAMES = {
+ 'float32': torch.float32,
+ 'fp32': torch.float32,
+ 'float16': torch.float16,
+ 'fp16': torch.float16,
+ 'bfloat16': torch.bfloat16,
+ 'bf16': torch.bfloat16,
+}
+
+
+def _resolve_output_dtype(dtype):
+ requested_dtype = dtype
+ if isinstance(dtype, str):
+ dtype_name = dtype[6:] if dtype.startswith('torch.') else dtype
+ dtype = OUTPUT_DTYPE_NAMES.get(dtype_name.lower())
+ if dtype not in set(OUTPUT_DTYPE_NAMES.values()):
+ supported = ', '.join(sorted(OUTPUT_DTYPE_NAMES))
+ raise ValueError(f"Unsupported output dtype {requested_dtype!r}. Choose one of: {supported}")
+ return dtype
+
+
+def atoi(text):
+ return int(text) if text.isdigit() else text
+
+
+def natural_keys(text):
+ '''
+ alist.sort(key=natural_keys) sorts in human order
+ http://nedbatchelder.com/blog/200712/human_sorting.html
+ (See Toothy's implementation in the comments)
+ '''
+ return [atoi(c) for c in re.split(r'(\d+)', text)]
+
+
+def get_model_state_file(checkpoint_dir, zero_stage):
+ if not os.path.isdir(checkpoint_dir):
+ raise FileNotFoundError(f"Directory '{checkpoint_dir}' doesn't exist")
+
+ # there should be only one file
+ if zero_stage <= 2:
+ file = os.path.join(checkpoint_dir, "mp_rank_00_model_states.pt")
+ elif zero_stage == 3:
+ file = os.path.join(checkpoint_dir, "zero_pp_rank_0_mp_rank_00_model_states.pt")
+
+ if not os.path.exists(file):
+ raise FileNotFoundError(f"can't find model states file at '{file}'")
+
+ return file
+
+
+def get_checkpoint_files(checkpoint_dir, glob_pattern):
+ # XXX: need to test that this simple glob rule works for multi-node setup too
+ ckpt_files = sorted(glob.glob(os.path.join(checkpoint_dir, glob_pattern)), key=natural_keys)
+
+ if len(ckpt_files) == 0:
+ raise FileNotFoundError(f"can't find {glob_pattern} files in directory '{checkpoint_dir}'")
+
+ return ckpt_files
+
+
+def get_optim_files(checkpoint_dir):
+ return get_checkpoint_files(checkpoint_dir, "*_optim_states.pt")
+
+
+def get_model_state_files(checkpoint_dir):
+ return get_checkpoint_files(checkpoint_dir, "*_model_states.pt")
+
+
+def _has_autoep_zero3_partitioned_metadata(state_dict):
+ autoep_layers = state_dict.get(AUTOEP_LAYERS_KEY)
+ if autoep_layers is None:
+ autoep_layers = state_dict.get(AUTOEP_LAYERS_KEY_LEGACY)
+ if not isinstance(autoep_layers, list):
+ return False
+ return any(
+ isinstance(entry, dict)
+ and entry.get(AUTOEP_ZERO3_EXPERT_STATE_FORMAT_KEY) == AUTOEP_ZERO3_PARTITIONED_EXPERT_STATE_FORMAT
+ for entry in autoep_layers)
+
+
+def _raise_if_autoep_zero3_partitioned_state(state_dict):
+ if _has_autoep_zero3_partitioned_metadata(state_dict):
+ raise NotImplementedError("zero_to_fp32 does not support AutoEP ZeRO-3 partition-native checkpoints. "
+ "AutoEP expert parameters are partitioned over expert replica groups, so "
+ "global data-parallel consolidation would produce incomplete expert tensors. "
+ "Use ds_to_universal.py for expert-aware conversion.")
+
+
+def _raise_if_autoep_zero3_partitioned_checkpoint(model_files):
+ for file in model_files:
+ state_dict = torch.load(file, map_location=device, weights_only=False)
+ _raise_if_autoep_zero3_partitioned_state(state_dict)
+
+
+def parse_model_states(files):
+ zero_model_states = []
+ for file in files:
+ state_dict = torch.load(file, map_location=device, weights_only=False)
+ _raise_if_autoep_zero3_partitioned_state(state_dict)
+
+ if BUFFER_NAMES not in state_dict:
+ raise ValueError(f"{file} is not a model state checkpoint")
+ buffer_names = state_dict[BUFFER_NAMES]
+ if debug:
+ print("Found buffers:", buffer_names)
+
+ # recover just the buffers while restoring them to fp32 if they were saved in fp16
+ buffers = {k: v.float() for k, v in state_dict["module"].items() if k in buffer_names}
+ param_shapes = state_dict[PARAM_SHAPES]
+
+ # collect parameters that are included in param_shapes
+ param_names = []
+ for s in param_shapes:
+ for name in s.keys():
+ param_names.append(name)
+
+ # update with frozen parameters
+ frozen_param_shapes = state_dict.get(FROZEN_PARAM_SHAPES, None)
+ if frozen_param_shapes is not None:
+ if debug:
+ print(f"Found frozen_param_shapes: {frozen_param_shapes}")
+ param_names += list(frozen_param_shapes.keys())
+
+ # handle shared params
+ shared_params = [[k, v] for k, v in state_dict["shared_params"].items()]
+
+ ds_version = state_dict.get(DS_VERSION, None)
+
+ frozen_param_fragments = state_dict.get(FROZEN_PARAM_FRAGMENTS, None)
+
+ z_model_state = zero_model_state(buffers=buffers,
+ param_shapes=param_shapes,
+ shared_params=shared_params,
+ ds_version=ds_version,
+ frozen_param_shapes=frozen_param_shapes,
+ frozen_param_fragments=frozen_param_fragments)
+ zero_model_states.append(z_model_state)
+
+ return zero_model_states
+
+
+def parse_optim_states(files, ds_checkpoint_dir):
+ total_files = len(files)
+ state_dicts = []
+ for f in tqdm(files, desc='Loading checkpoint shards'):
+ state_dict = torch.load(f, map_location=device, mmap=True, weights_only=False)
+ # immediately discard the potentially huge 2 optimizer states as we only care for fp32 master weights
+ # and also handle the case where it was already removed by another helper script
+ state_dict["optimizer_state_dict"].pop("optimizer_state_dict", None)
+ state_dicts.append(state_dict)
+
+ if ZERO_STAGE not in state_dicts[0][OPTIMIZER_STATE_DICT]:
+ raise ValueError(f"{files[0]} is not a zero checkpoint")
+ zero_stage = state_dicts[0][OPTIMIZER_STATE_DICT][ZERO_STAGE]
+ world_size = state_dicts[0][OPTIMIZER_STATE_DICT][PARTITION_COUNT]
+
+ # For ZeRO-2 each param group can have different partition_count as data parallelism for expert
+ # parameters can be different from data parallelism for non-expert parameters. So we can just
+ # use the max of the partition_count to get the dp world_size.
+
+ if type(world_size) is list:
+ world_size = max(world_size)
+
+ if world_size != total_files:
+ raise ValueError(
+ f"Expected {world_size} of '*_optim_states.pt' under '{ds_checkpoint_dir}' but found {total_files} files. "
+ "Possibly due to an overwrite of an old checkpoint, or a checkpoint didn't get saved by one or more processes."
+ )
+
+ # the groups are named differently in each stage
+ if zero_stage <= 2:
+ fp32_groups_key = SINGLE_PARTITION_OF_FP32_GROUPS
+ elif zero_stage == 3:
+ fp32_groups_key = FP32_FLAT_GROUPS
+ else:
+ raise ValueError(f"unknown zero stage {zero_stage}")
+
+ fp32_flat_groups = [state_dicts[i][OPTIMIZER_STATE_DICT][fp32_groups_key] for i in range(len(state_dicts))]
+ param_alignment_paddings = state_dicts[0][OPTIMIZER_STATE_DICT].get(PARAM_ALIGNMENT_PADDINGS)
+ return zero_stage, world_size, fp32_flat_groups, param_alignment_paddings
+
+
+def _get_fp32_state_dict_from_zero_checkpoint(ds_checkpoint_dir, exclude_frozen_parameters):
+ """
+ Returns fp32 state_dict reconstructed from ds checkpoint
+
+ Args:
+ - ``ds_checkpoint_dir``: path to the deepspeed checkpoint folder (where the optimizer files are)
+
+ """
+ print(f"Processing zero checkpoint '{ds_checkpoint_dir}'")
+
+ # parse_model_states rejects AutoEP ZeRO-3 partition-native checkpoints
+ # before the expensive optimizer-shard load below.
+ model_files = get_model_state_files(ds_checkpoint_dir)
+ zero_model_states = parse_model_states(model_files)
+ print(f'Parsing checkpoint created by deepspeed=={zero_model_states[0].ds_version}')
+
+ optim_files = get_optim_files(ds_checkpoint_dir)
+ zero_stage, world_size, fp32_flat_groups, param_alignment_paddings = parse_optim_states(
+ optim_files, ds_checkpoint_dir)
+ print(f"Detected checkpoint of type zero stage {zero_stage}, world_size: {world_size}")
+
+ if zero_stage <= 2:
+ return _get_fp32_state_dict_from_zero2_checkpoint(world_size, fp32_flat_groups, zero_model_states,
+ exclude_frozen_parameters, param_alignment_paddings)
+ elif zero_stage == 3:
+ return _get_fp32_state_dict_from_zero3_checkpoint(world_size, fp32_flat_groups, zero_model_states,
+ exclude_frozen_parameters)
+
+
+def _zero2_merge_frozen_params(state_dict, zero_model_states):
+ if zero_model_states[0].frozen_param_shapes is None or len(zero_model_states[0].frozen_param_shapes) == 0:
+ return
+
+ frozen_param_shapes = zero_model_states[0].frozen_param_shapes
+ frozen_param_fragments = zero_model_states[0].frozen_param_fragments
+
+ if debug:
+ num_elem = sum(s.numel() for s in frozen_param_shapes.values())
+ print(f'rank 0: {FROZEN_PARAM_SHAPES}.numel = {num_elem}')
+
+ wanted_params = len(frozen_param_shapes)
+ wanted_numel = sum(s.numel() for s in frozen_param_shapes.values())
+ avail_numel = sum([p.numel() for p in frozen_param_fragments.values()])
+ print(f'Frozen params: Have {avail_numel} numels to process.')
+ print(f'Frozen params: Need {wanted_numel} numels in {wanted_params} params')
+
+ total_params = 0
+ total_numel = 0
+ for name, shape in frozen_param_shapes.items():
+ total_params += 1
+ unpartitioned_numel = shape.numel()
+ total_numel += unpartitioned_numel
+
+ state_dict[name] = frozen_param_fragments[name]
+
+ if debug:
+ print(f"{name} full shape: {shape} unpartitioned numel {unpartitioned_numel} ")
+
+ print(f"Reconstructed Frozen fp32 state dict with {total_params} params {total_numel} elements")
+
+
+def _has_callable(obj, fn):
+ attr = getattr(obj, fn, None)
+ return callable(attr)
+
+
+def _zero2_merge_trainable_params(state_dict,
+ world_size,
+ fp32_flat_groups,
+ zero_model_states,
+ param_alignment_paddings=None):
+ param_shapes = zero_model_states[0].param_shapes
+
+ # Reconstruction protocol:
+ #
+ # XXX: document this
+
+ if debug:
+ for i in range(world_size):
+ for j in range(len(fp32_flat_groups[0])):
+ print(f"{FP32_FLAT_GROUPS}[{i}][{j}].shape={fp32_flat_groups[i][j].shape}")
+
+ # XXX: memory usage doubles here (zero2)
+ num_param_groups = len(fp32_flat_groups[0])
+ merged_single_partition_of_fp32_groups = []
+ for i in range(num_param_groups):
+ merged_partitions = [sd[i] for sd in fp32_flat_groups]
+ full_single_fp32_vector = torch.cat(merged_partitions, 0)
+ merged_single_partition_of_fp32_groups.append(full_single_fp32_vector)
+ avail_numel = sum(
+ [full_single_fp32_vector.numel() for full_single_fp32_vector in merged_single_partition_of_fp32_groups])
+
+ if debug:
+ wanted_params = sum([len(shapes) for shapes in param_shapes])
+ wanted_numel = sum([sum(shape.numel() for shape in shapes.values()) for shapes in param_shapes])
+ # not asserting if there is a mismatch due to possible padding
+ print(f"Have {avail_numel} numels to process.")
+ print(f"Need {wanted_numel} numels in {wanted_params} params.")
+
+ # params
+ # XXX: for huge models that can't fit into the host's RAM we will have to recode this to support
+ # out-of-core computing solution
+ total_numel = 0
+ total_params = 0
+ for group_idx, (shapes,
+ full_single_fp32_vector) in enumerate(zip(param_shapes, merged_single_partition_of_fp32_groups)):
+ offset = 0
+ avail_numel = full_single_fp32_vector.numel()
+ group_alignment_paddings = None
+ if param_alignment_paddings is not None:
+ group_alignment_paddings = param_alignment_paddings[group_idx]
+ if len(group_alignment_paddings) != len(shapes):
+ raise ValueError(f"Expected {len(shapes)} parameter alignment paddings for group {group_idx}, "
+ f"but found {len(group_alignment_paddings)}")
+
+ for param_idx, (name, shape) in enumerate(shapes.items()):
+
+ unpartitioned_numel = shape.numel() if _has_callable(shape, 'numel') else math.prod(shape)
+ total_numel += unpartitioned_numel
+ total_params += 1
+
+ if debug:
+ print(f"{name} full shape: {shape} unpartitioned numel {unpartitioned_numel} ")
+ state_dict[name] = full_single_fp32_vector.narrow(0, offset, unpartitioned_numel).view(shape)
+ offset += unpartitioned_numel
+ if group_alignment_paddings is not None:
+ offset += group_alignment_paddings[param_idx]
+
+ # Z2 started to align to 2*world_size to improve nccl performance. Therefore both offset and
+ # avail_numel can differ by anywhere between 0..2*world_size. Due to two unrelated complex
+ # paddings performed in the code it's almost impossible to predict the exact numbers w/o the
+ # live optimizer object, so we are checking that the numbers are within the right range
+ align_to = 2 * world_size
+
+ def zero2_align(x):
+ return align_to * math.ceil(x / align_to)
+
+ if debug:
+ print(f"original offset={offset}, avail_numel={avail_numel}")
+
+ offset = zero2_align(offset)
+ avail_numel = zero2_align(avail_numel)
+
+ if debug:
+ print(f"aligned offset={offset}, avail_numel={avail_numel}")
+
+ # Sanity check
+ if offset != avail_numel:
+ raise ValueError(f"consumed {offset} numels out of {avail_numel} - something is wrong")
+
+ print(f"Reconstructed fp32 state dict with {total_params} params {total_numel} elements")
+
+
+def _get_fp32_state_dict_from_zero2_checkpoint(world_size,
+ fp32_flat_groups,
+ zero_model_states,
+ exclude_frozen_parameters,
+ param_alignment_paddings=None):
+ state_dict = OrderedDict()
+
+ # buffers
+ buffers = zero_model_states[0].buffers
+ state_dict.update(buffers)
+ if debug:
+ print(f"added {len(buffers)} buffers")
+
+ if not exclude_frozen_parameters:
+ _zero2_merge_frozen_params(state_dict, zero_model_states)
+
+ _zero2_merge_trainable_params(state_dict, world_size, fp32_flat_groups, zero_model_states,
+ param_alignment_paddings)
+
+ # recover shared parameters
+ for pair in zero_model_states[0].shared_params:
+ if pair[1] in state_dict:
+ state_dict[pair[0]] = state_dict[pair[1]]
+
+ return state_dict
+
+
+def zero3_partitioned_param_info(unpartitioned_numel, world_size):
+ remainder = unpartitioned_numel % world_size
+ padding_numel = (world_size - remainder) if remainder else 0
+ partitioned_numel = math.ceil(unpartitioned_numel / world_size)
+ return partitioned_numel, padding_numel
+
+
+def _zero3_merge_frozen_params(state_dict, world_size, zero_model_states):
+ if zero_model_states[0].frozen_param_shapes is None or len(zero_model_states[0].frozen_param_shapes) == 0:
+ return
+
+ if debug:
+ for i in range(world_size):
+ num_elem = sum(s.numel() for s in zero_model_states[i].frozen_param_fragments.values())
+ print(f'rank {i}: {FROZEN_PARAM_SHAPES}.numel = {num_elem}')
+
+ frozen_param_shapes = zero_model_states[0].frozen_param_shapes
+ wanted_params = len(frozen_param_shapes)
+ wanted_numel = sum(s.numel() for s in frozen_param_shapes.values())
+ avail_numel = sum([p.numel() for p in zero_model_states[0].frozen_param_fragments.values()]) * world_size
+ print(f'Frozen params: Have {avail_numel} numels to process.')
+ print(f'Frozen params: Need {wanted_numel} numels in {wanted_params} params')
+
+ total_params = 0
+ total_numel = 0
+ for name, shape in zero_model_states[0].frozen_param_shapes.items():
+ total_params += 1
+ unpartitioned_numel = shape.numel()
+ total_numel += unpartitioned_numel
+
+ param_frags = tuple(model_state.frozen_param_fragments[name] for model_state in zero_model_states)
+ state_dict[name] = torch.cat(param_frags, 0).narrow(0, 0, unpartitioned_numel).view(shape)
+
+ partitioned_numel, partitioned_padding_numel = zero3_partitioned_param_info(unpartitioned_numel, world_size)
+
+ if debug:
+ print(
+ f"Frozen params: {total_params} {name} full shape: {shape} partition0 numel={partitioned_numel} partitioned_padding_numel={partitioned_padding_numel}"
+ )
+
+ print(f"Reconstructed Frozen fp32 state dict with {total_params} params {total_numel} elements")
+
+
+class GatheredTensor:
+ """
+ A pseudo tensor that collects partitioned weights.
+ It is more memory efficient when there are multiple groups.
+ """
+
+ def __init__(self, flat_groups, flat_groups_offset, offset, partitioned_numel, shape):
+ self.flat_groups = flat_groups
+ self.flat_groups_offset = flat_groups_offset
+ self.offset = offset
+ self.partitioned_numel = partitioned_numel
+ self.shape = shape
+ self.dtype = self.flat_groups[0][0].dtype
+
+ def contiguous(self):
+ """
+ Merge partitioned weights from flat_groups into a single tensor.
+ """
+ end_idx = self.offset + self.partitioned_numel
+ world_size = len(self.flat_groups)
+ pad_flat_param_chunks = []
+
+ for rank_i in range(world_size):
+ # for each rank, we need to collect weights from related group/groups
+ flat_groups_at_rank_i = self.flat_groups[rank_i]
+ start_group_id = None
+ end_group_id = None
+ for group_id in range(len(self.flat_groups_offset)):
+ if self.flat_groups_offset[group_id] <= self.offset < self.flat_groups_offset[group_id + 1]:
+ start_group_id = group_id
+ if self.flat_groups_offset[group_id] < end_idx <= self.flat_groups_offset[group_id + 1]:
+ end_group_id = group_id
+ break
+ # collect weights from related group/groups
+ for group_id in range(start_group_id, end_group_id + 1):
+ flat_tensor = flat_groups_at_rank_i[group_id]
+ start_offset = self.offset - self.flat_groups_offset[group_id]
+ end_offset = min(end_idx, self.flat_groups_offset[group_id + 1]) - self.flat_groups_offset[group_id]
+ pad_flat_param_chunks.append(flat_tensor[start_offset:end_offset])
+
+ # collect weights from all ranks
+ pad_flat_param = torch.cat(pad_flat_param_chunks, dim=0)
+ param = pad_flat_param[:self.shape.numel()].view(self.shape).contiguous()
+ return param
+
+
+def _zero3_merge_trainable_params(state_dict, world_size, fp32_flat_groups, zero_model_states):
+ param_shapes = zero_model_states[0].param_shapes
+ avail_numel = sum([flat_group.numel() for flat_group in fp32_flat_groups[0]]) * world_size
+
+ # Reconstruction protocol: For zero3 we need to zip the partitions together at boundary of each
+ # param, re-consolidating each param, while dealing with padding if any
+
+ # merge list of dicts, preserving order
+ param_shapes = {k: v for d in param_shapes for k, v in d.items()}
+
+ if debug:
+ for i in range(world_size):
+ print(f"{FP32_FLAT_GROUPS}[{i}].shape={fp32_flat_groups[i].shape}")
+
+ wanted_params = len(param_shapes)
+ wanted_numel = sum(shape.numel() for shape in param_shapes.values())
+ # not asserting if there is a mismatch due to possible padding
+ avail_numel = fp32_flat_groups[0].numel() * world_size
+ print(f"Trainable params: Have {avail_numel} numels to process.")
+ print(f"Trainable params: Need {wanted_numel} numels in {wanted_params} params.")
+
+ # params
+ # XXX: for huge models that can't fit into the host's RAM we will have to recode this to support
+ # out-of-core computing solution
+ offset = 0
+ total_numel = 0
+ total_params = 0
+ flat_groups_offset = [0] + list(np.cumsum([flat_tensor.numel() for flat_tensor in fp32_flat_groups[0]]))
+ for name, shape in tqdm(param_shapes.items(), desc='Gathering sharded weights'):
+ unpartitioned_numel = shape.numel()
+ total_numel += unpartitioned_numel
+ total_params += 1
+ partitioned_numel, partitioned_padding_numel = zero3_partitioned_param_info(unpartitioned_numel, world_size)
+
+ if debug:
+ print(
+ f"Trainable params: {total_params} {name} full shape: {shape} partition0 numel={partitioned_numel} partitioned_padding_numel={partitioned_padding_numel}"
+ )
+
+ # memory efficient tensor
+ tensor = GatheredTensor(fp32_flat_groups, flat_groups_offset, offset, partitioned_numel, shape)
+ state_dict[name] = tensor
+ offset += partitioned_numel
+
+ offset *= world_size
+
+ # Sanity check
+ if offset != avail_numel:
+ raise ValueError(f"consumed {offset} numels out of {avail_numel} - something is wrong")
+
+ print(f"Reconstructed Trainable fp32 state dict with {total_params} params {total_numel} elements")
+
+
+def _get_fp32_state_dict_from_zero3_checkpoint(world_size, fp32_flat_groups, zero_model_states,
+ exclude_frozen_parameters):
+ state_dict = OrderedDict()
+
+ # buffers
+ buffers = zero_model_states[0].buffers
+ state_dict.update(buffers)
+ if debug:
+ print(f"added {len(buffers)} buffers")
+
+ if not exclude_frozen_parameters:
+ _zero3_merge_frozen_params(state_dict, world_size, zero_model_states)
+
+ _zero3_merge_trainable_params(state_dict, world_size, fp32_flat_groups, zero_model_states)
+
+ # recover shared parameters
+ for pair in zero_model_states[0].shared_params:
+ if pair[1] in state_dict:
+ state_dict[pair[0]] = state_dict[pair[1]]
+
+ return state_dict
+
+
+def to_torch_tensor(state_dict, return_empty_tensor=False, dtype=None):
+ """
+ Convert state_dict of GatheredTensor to torch tensor
+ """
+ if dtype is not None:
+ dtype = _resolve_output_dtype(dtype)
+
+ torch_state_dict = {}
+ converted_tensors = {}
+ for name, tensor in state_dict.items():
+ tensor_id = id(tensor)
+ if tensor_id in converted_tensors: # shared tensors
+ shared_tensor = torch_state_dict[converted_tensors[tensor_id]]
+ torch_state_dict[name] = shared_tensor
+ else:
+ converted_tensors[tensor_id] = name
+ if return_empty_tensor:
+ torch_state_dict[name] = torch.empty(tensor.shape, dtype=dtype or tensor.dtype)
+ else:
+ contiguous_tensor = tensor.contiguous()
+ torch_state_dict[name] = contiguous_tensor.to(dtype=dtype) if dtype else contiguous_tensor
+ return torch_state_dict
+
+
+def get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir,
+ tag=None,
+ exclude_frozen_parameters=False,
+ lazy_mode=False):
+ """
+ Convert ZeRO 2 or 3 checkpoint into a single fp32 consolidated state_dict that can be loaded with
+ ``load_state_dict()`` and used for training without DeepSpeed or shared with others, for example
+ via a model hub.
+
+ Args:
+ - ``checkpoint_dir``: path to the desired checkpoint folder
+ - ``tag``: checkpoint tag used as a unique identifier for checkpoint. If not provided will attempt to load tag in 'latest' file. e.g., ``global_step14``
+ - ``exclude_frozen_parameters``: exclude frozen parameters
+ - ``lazy_mode``: get state_dict in lazy mode. It returns a dict of pesduo tensor instead of torch tensor, which is more memory efficient.
+ Convert the pesduo tensor to torch tensor by ``.contiguous()``
+
+ Returns:
+ - pytorch ``state_dict``
+
+ A typical usage might be ::
+
+ from deepspeed.utils.zero_to_fp32 import get_fp32_state_dict_from_zero_checkpoint
+ # do the training and checkpoint saving
+ state_dict = get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir) # already on cpu
+ model = model.cpu() # move to cpu
+ model.load_state_dict(state_dict)
+ # submit to model hub or save the model to share with others
+
+ In this example the ``model`` will no longer be usable in the deepspeed context of the same
+ application. i.e. you will need to re-initialize the deepspeed engine, since
+ ``model.load_state_dict(state_dict)`` will remove all the deepspeed magic from it.
+
+ If you want it all done for you, use ``load_state_dict_from_zero_checkpoint`` instead.
+
+ Note: the above usage may not work if your application doesn't have sufficient free CPU memory.
+ You may need to use the offline approach using the ``zero_to_fp32.py`` script that is saved with
+ the checkpoint. Or you can load state_dict in lazy mode ::
+
+ from deepspeed.utils.zero_to_fp32 import get_fp32_state_dict_from_zero_checkpoint
+ state_dict = get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir, lazy_mode=True) # not on cpu
+ for name, lazy_tensor in state_dict.item():
+ tensor = lazy_tensor.contiguous() # to cpu
+ print(name, tensor)
+ # del tensor to release memory if it no longer in use
+ """
+ if tag is None:
+ latest_path = os.path.join(checkpoint_dir, 'latest')
+ if os.path.isfile(latest_path):
+ with open(latest_path, 'r') as fd:
+ tag = fd.read().strip()
+ else:
+ raise ValueError(f"Unable to find 'latest' file at {latest_path}")
+
+ ds_checkpoint_dir = os.path.join(checkpoint_dir, tag)
+
+ if not os.path.isdir(ds_checkpoint_dir):
+ raise FileNotFoundError(f"Directory '{ds_checkpoint_dir}' doesn't exist")
+
+ state_dict = _get_fp32_state_dict_from_zero_checkpoint(ds_checkpoint_dir, exclude_frozen_parameters)
+ if lazy_mode:
+ return state_dict
+ else:
+ return to_torch_tensor(state_dict)
+
+
+def convert_zero_checkpoint_to_state_dict(checkpoint_dir,
+ output_dir,
+ dtype=torch.float32,
+ max_shard_size="5GB",
+ safe_serialization=False,
+ tag=None,
+ exclude_frozen_parameters=False):
+ """
+ Convert ZeRO 2 or 3 checkpoint into a consolidated ``state_dict`` file that can be
+ loaded with ``torch.load(file)`` + ``load_state_dict()`` and used for training without DeepSpeed.
+
+ Args:
+ - ``checkpoint_dir``: path to the desired checkpoint folder. (one that contains the tag-folder, like ``global_step14``)
+ - ``output_dir``: directory for the PyTorch state_dict output files
+ - ``dtype``: output tensor dtype. Supports float32, float16, and bfloat16 as strings or torch dtypes.
+ - ``max_shard_size``: the maximum size for a checkpoint before being sharded, default value is 5GB
+ - ``safe_serialization``: whether to save the model using `safetensors` or the traditional PyTorch way (that uses `pickle`).
+ - ``tag``: checkpoint tag used as a unique identifier for checkpoint. If not provided will attempt to load tag in the file named ``latest`` in the checkpoint folder, e.g., ``global_step14``
+ - ``exclude_frozen_parameters``: exclude frozen parameters
+ """
+
+ dtype = _resolve_output_dtype(dtype)
+
+ # Dependency pre-check
+ if safe_serialization:
+ try:
+ from safetensors.torch import save_file
+ except ImportError:
+ print('If you want to use `safe_serialization`, please `pip install safetensors`')
+ raise
+ if max_shard_size is not None:
+ try:
+ from huggingface_hub import split_torch_state_dict_into_shards
+ except ImportError:
+ print('If you want to use `max_shard_size`, please `pip install huggingface_hub`')
+ raise
+
+ # Convert zero checkpoint to state_dict
+ state_dict = get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir,
+ tag,
+ exclude_frozen_parameters,
+ lazy_mode=True)
+
+ # Shard the model if it is too big.
+ weights_name = "model.safetensors" if safe_serialization else "pytorch_model.bin"
+ if max_shard_size is not None:
+ filename_pattern = weights_name.replace(".bin", "{suffix}.bin").replace(".safetensors", "{suffix}.safetensors")
+ # an memory-efficient approach for sharding
+ empty_state_dict = to_torch_tensor(state_dict, return_empty_tensor=True, dtype=dtype)
+ state_dict_split = split_torch_state_dict_into_shards(empty_state_dict,
+ filename_pattern=filename_pattern,
+ max_shard_size=max_shard_size)
+ else:
+ from collections import namedtuple
+ StateDictSplit = namedtuple("StateDictSplit", ["is_sharded", "filename_to_tensors"])
+ state_dict_split = StateDictSplit(is_sharded=False,
+ filename_to_tensors={weights_name: list(state_dict.keys())})
+
+ # Save the model by shard
+ os.makedirs(output_dir, exist_ok=True)
+ filename_to_tensors = state_dict_split.filename_to_tensors.items()
+ for shard_file, tensors in tqdm(filename_to_tensors, desc="Saving checkpoint shards"):
+ shard_state_dict = {tensor_name: state_dict[tensor_name] for tensor_name in tensors}
+ shard_state_dict = to_torch_tensor(shard_state_dict, dtype=dtype)
+ output_path = os.path.join(output_dir, shard_file)
+ if safe_serialization:
+ save_file(shard_state_dict, output_path, metadata={"format": "pt"})
+ else:
+ torch.save(shard_state_dict, output_path)
+ # release the memory of current shard
+ for tensor_name in list(shard_state_dict.keys()):
+ del state_dict[tensor_name]
+ del shard_state_dict[tensor_name]
+ del shard_state_dict
+ gc.collect()
+
+ # Save index if sharded
+ if state_dict_split.is_sharded:
+ index = {
+ "metadata": state_dict_split.metadata,
+ "weight_map": state_dict_split.tensor_to_filename,
+ }
+ save_index_file = "model.safetensors.index.json" if safe_serialization else "pytorch_model.bin.index.json"
+ save_index_file = os.path.join(output_dir, save_index_file)
+ with open(save_index_file, "w", encoding="utf-8") as f:
+ content = json.dumps(index, indent=2, sort_keys=True) + "\n"
+ f.write(content)
+
+
+def convert_zero_checkpoint_to_fp32_state_dict(checkpoint_dir,
+ output_dir,
+ max_shard_size="5GB",
+ safe_serialization=False,
+ tag=None,
+ exclude_frozen_parameters=False):
+ """Backward-compatible fp32 checkpoint conversion."""
+ return convert_zero_checkpoint_to_state_dict(checkpoint_dir,
+ output_dir,
+ dtype=torch.float32,
+ max_shard_size=max_shard_size,
+ safe_serialization=safe_serialization,
+ tag=tag,
+ exclude_frozen_parameters=exclude_frozen_parameters)
+
+
+def load_state_dict_from_zero_checkpoint(model, checkpoint_dir, tag=None):
+ """
+ 1. Put the provided model to cpu
+ 2. Convert ZeRO 2 or 3 checkpoint into a single fp32 consolidated ``state_dict``
+ 3. Load it into the provided model
+
+ Args:
+ - ``model``: the model object to update
+ - ``checkpoint_dir``: path to the desired checkpoint folder. (one that contains the tag-folder, like ``global_step14``)
+ - ``tag``: checkpoint tag used as a unique identifier for checkpoint. If not provided will attempt to load tag in the file named ``latest`` in the checkpoint folder, e.g., ``global_step14``
+
+ Returns:
+ - ``model`: modified model
+
+ Make sure you have plenty of CPU memory available before you call this function. If you don't
+ have enough use the ``zero_to_fp32.py`` utility to do the conversion. You will find it
+ conveniently placed for you in the checkpoint folder.
+
+ A typical usage might be ::
+
+ from deepspeed.utils.zero_to_fp32 import load_state_dict_from_zero_checkpoint
+ model = load_state_dict_from_zero_checkpoint(trainer.model, checkpoint_dir)
+ # submit to model hub or save the model to share with others
+
+ Note, that once this was run, the ``model`` will no longer be usable in the deepspeed context
+ of the same application. i.e. you will need to re-initialize the deepspeed engine, since
+ ``model.load_state_dict(state_dict)`` will remove all the deepspeed magic from it.
+
+ """
+ logger.info("Extracting fp32 weights")
+ state_dict = get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir, tag)
+
+ logger.info("Overwriting model with fp32 weights")
+ model = model.cpu()
+ model.load_state_dict(state_dict, strict=False)
+
+ return model
+
+
+if __name__ == "__main__":
+ parser = argparse.ArgumentParser()
+ parser.add_argument("checkpoint_dir",
+ type=str,
+ help="path to the desired checkpoint folder, e.g., path/checkpoint-12")
+ parser.add_argument("output_dir",
+ type=str,
+ help="directory to the pytorch fp32 state_dict output files"
+ "(e.g. path/checkpoint-12-output/)")
+ parser.add_argument(
+ "--max_shard_size",
+ type=str,
+ default="5GB",
+ help="The maximum size for a checkpoint before being sharded. Checkpoints shard will then be each of size"
+ "lower than this size. If expressed as a string, needs to be digits followed by a unit (like `5MB`"
+ "We default it to 5GB in order for models to be able to run easily on free-tier google colab instances"
+ "without CPU OOM issues.")
+ parser.add_argument(
+ "--safe_serialization",
+ default=False,
+ action='store_true',
+ help="Whether to save the model using `safetensors` or the traditional PyTorch way (that uses `pickle`).")
+ parser.add_argument("-t",
+ "--tag",
+ type=str,
+ default=None,
+ help="checkpoint tag used as a unique identifier for checkpoint. e.g., global_step1")
+ parser.add_argument("--exclude_frozen_parameters", action='store_true', help="exclude frozen parameters")
+ parser.add_argument("-d", "--debug", action='store_true', help="enable debug")
+ args = parser.parse_args()
+
+ debug = args.debug
+
+ convert_zero_checkpoint_to_fp32_state_dict(args.checkpoint_dir,
+ args.output_dir,
+ max_shard_size=args.max_shard_size,
+ safe_serialization=args.safe_serialization,
+ tag=args.tag,
+ exclude_frozen_parameters=args.exclude_frozen_parameters)
diff --git a/qwen3-1.7b-teutonic-5e6/checkpoint-500/zero_to_torch.py b/qwen3-1.7b-teutonic-5e6/checkpoint-500/zero_to_torch.py
new file mode 100644
index 0000000000000000000000000000000000000000..81312e252400d77f03cc1a88522f8f18ebe70ee7
--- /dev/null
+++ b/qwen3-1.7b-teutonic-5e6/checkpoint-500/zero_to_torch.py
@@ -0,0 +1,54 @@
+#!/usr/bin/env python
+
+# SPDX-License-Identifier: Apache-2.0
+# DeepSpeed Team
+
+import argparse
+
+if __package__:
+ from . import zero_to_fp32
+else:
+ import zero_to_fp32
+
+
+def main(args=None):
+ parser = argparse.ArgumentParser(
+ description="Convert a DeepSpeed ZeRO checkpoint to float32, float16, or bfloat16 PyTorch weights.")
+ parser.add_argument("checkpoint_dir",
+ type=str,
+ help="path to the desired checkpoint folder, e.g., path/checkpoint-12")
+ parser.add_argument("output_dir", type=str, help="directory for the converted PyTorch state_dict files")
+ parser.add_argument("--dtype",
+ type=str,
+ choices=sorted(zero_to_fp32.OUTPUT_DTYPE_NAMES),
+ required=True,
+ help="output tensor dtype")
+ parser.add_argument("--max_shard_size",
+ type=str,
+ default="5GB",
+ help="maximum size of each checkpoint shard, such as 5GB or 500MB")
+ parser.add_argument("--safe_serialization",
+ default=False,
+ action='store_true',
+ help="save with safetensors instead of PyTorch pickle serialization")
+ parser.add_argument("-t",
+ "--tag",
+ type=str,
+ default=None,
+ help="checkpoint tag used as a unique identifier, e.g., global_step1")
+ parser.add_argument("--exclude_frozen_parameters", action='store_true', help="exclude frozen parameters")
+ parser.add_argument("-d", "--debug", action='store_true', help="enable debug output")
+ parsed_args = parser.parse_args(args)
+
+ zero_to_fp32.debug = parsed_args.debug
+ zero_to_fp32.convert_zero_checkpoint_to_state_dict(parsed_args.checkpoint_dir,
+ parsed_args.output_dir,
+ dtype=parsed_args.dtype,
+ max_shard_size=parsed_args.max_shard_size,
+ safe_serialization=parsed_args.safe_serialization,
+ tag=parsed_args.tag,
+ exclude_frozen_parameters=parsed_args.exclude_frozen_parameters)
+
+
+if __name__ == "__main__":
+ main()