Upload folder using huggingface_hub

.gitattributes

@@ -55,3 +55,4 @@ g1_jupyter_notebook__edit/250M/tokenizer.json filter=lfs diff=lfs merge=lfs -tex
 g1_jupyter_notebook__edit/500M/tokenizer.json filter=lfs diff=lfs merge=lfs -text
 g1_jupyter_notebook__edit/1000M/tokenizer.json filter=lfs diff=lfs merge=lfs -text
 g2_1_oldcode_cm_newcode/250M/tokenizer.json filter=lfs diff=lfs merge=lfs -text
+g2_1_oldcode_cm_newcode/500M/tokenizer.json filter=lfs diff=lfs merge=lfs -text

g2_1_oldcode_cm_newcode/500M/README.md

@@ -0,0 +1,60 @@
+---
+library_name: transformers
+license: other
+base_model: Qwen/Qwen3-8B-Base
+tags:
+- llama-factory
+- full
+- generated_from_trainer
+model-index:
+- name: 500M
+  results: []
+---
+
+<!-- This model card has been generated automatically according to the information the Trainer had access to. You
+should probably proofread and complete it, then remove this comment. -->
+
+# 500M
+
+This model is a fine-tuned version of [/lus/flare/projects/MultiModalLLM/model-weights/Qwen/Qwen3-8B-Base](https://huggingface.co//lus/flare/projects/MultiModalLLM/model-weights/Qwen/Qwen3-8B-Base) on the rv5_g2_1_oldcode_cm_newcode and the instruct_replay datasets.
+
+## Model description
+
+More information needed
+
+## Intended uses & limitations
+
+More information needed
+
+## Training and evaluation data
+
+More information needed
+
+## Training procedure
+
+### Training hyperparameters
+
+The following hyperparameters were used during training:
+- learning_rate: 2e-05
+- train_batch_size: 1
+- eval_batch_size: 8
+- seed: 42
+- distributed_type: multi-GPU
+- num_devices: 12
+- total_train_batch_size: 12
+- total_eval_batch_size: 96
+- optimizer: Use OptimizerNames.ADAMW_TORCH_FUSED with betas=(0.9,0.95) and epsilon=1e-08 and optimizer_args=No additional optimizer arguments
+- lr_scheduler_type: warmup_stable_decay
+- lr_scheduler_warmup_steps: 178
+- training_steps: 5597
+
+### Training results
+
+
+
+### Framework versions
+
+- Transformers 4.57.6
+- Pytorch 2.10.0a0+git449b176
+- Datasets 4.5.0
+- Tokenizers 0.22.2

g2_1_oldcode_cm_newcode/500M/added_tokens.json

@@ -0,0 +1,28 @@
+{
+  "</think>": 151668,
+  "</tool_call>": 151658,
+  "</tool_response>": 151666,
+  "<think>": 151667,
+  "<tool_call>": 151657,
+  "<tool_response>": 151665,
+  "<|box_end|>": 151649,
+  "<|box_start|>": 151648,
+  "<|endoftext|>": 151643,
+  "<|file_sep|>": 151664,
+  "<|fim_middle|>": 151660,
+  "<|fim_pad|>": 151662,
+  "<|fim_prefix|>": 151659,
+  "<|fim_suffix|>": 151661,
+  "<|im_end|>": 151645,
+  "<|im_start|>": 151644,
+  "<|image_pad|>": 151655,
+  "<|object_ref_end|>": 151647,
+  "<|object_ref_start|>": 151646,
+  "<|quad_end|>": 151651,
+  "<|quad_start|>": 151650,
+  "<|repo_name|>": 151663,
+  "<|video_pad|>": 151656,
+  "<|vision_end|>": 151653,
+  "<|vision_pad|>": 151654,
+  "<|vision_start|>": 151652
+}

g2_1_oldcode_cm_newcode/500M/all_results.json

@@ -0,0 +1,8 @@
+{
+    "epoch": 0.4919142204253823,
+    "total_flos": 2.4985159616803373e+19,
+    "train_loss": 0.048519859191283855,
+    "train_runtime": 5685.2195,
+    "train_samples_per_second": 11.814,
+    "train_steps_per_second": 0.984
+}

g2_1_oldcode_cm_newcode/500M/chat_template.jinja

@@ -0,0 +1,85 @@
+{%- if tools %}
+    {{- '<|im_start|>system\n' }}
+    {%- if messages[0].role == 'system' %}
+        {{- messages[0].content + '\n\n' }}
+    {%- endif %}
+    {{- "# Tools\n\nYou may call one or more functions to assist with the user query.\n\nYou are provided with function signatures within <tools></tools> XML tags:\n<tools>" }}
+    {%- for tool in tools %}
+        {{- "\n" }}
+        {{- tool | tojson }}
+    {%- endfor %}
+    {{- "\n</tools>\n\nFor each function call, return a json object with function name and arguments within <tool_call></tool_call> XML tags:\n<tool_call>\n{\"name\": <function-name>, \"arguments\": <args-json-object>}\n</tool_call><|im_end|>\n" }}
+{%- else %}
+    {%- if messages[0].role == 'system' %}
+        {{- '<|im_start|>system\n' + messages[0].content + '<|im_end|>\n' }}
+    {%- endif %}
+{%- endif %}
+{%- set ns = namespace(multi_step_tool=true, last_query_index=messages|length - 1) %}
+{%- for message in messages[::-1] %}
+    {%- set index = (messages|length - 1) - loop.index0 %}
+    {%- if ns.multi_step_tool and message.role == "user" and not(message.content.startswith('<tool_response>') and message.content.endswith('</tool_response>')) %}
+        {%- set ns.multi_step_tool = false %}
+        {%- set ns.last_query_index = index %}
+    {%- endif %}
+{%- endfor %}
+{%- for message in messages %}
+    {%- if (message.role == "user") or (message.role == "system" and not loop.first) %}
+        {{- '<|im_start|>' + message.role + '\n' + message.content + '<|im_end|>' + '\n' }}
+    {%- elif message.role == "assistant" %}
+        {%- set content = message.content %}
+        {%- set reasoning_content = '' %}
+        {%- if message.reasoning_content is defined and message.reasoning_content is not none %}
+            {%- set reasoning_content = message.reasoning_content %}
+        {%- else %}
+            {%- if '</think>' in message.content %}
+                {%- set content = message.content.split('</think>')[-1].lstrip('\n') %}
+                {%- set reasoning_content = message.content.split('</think>')[0].rstrip('\n').split('<think>')[-1].lstrip('\n') %}
+            {%- endif %}
+        {%- endif %}
+        {%- if loop.index0 > ns.last_query_index %}
+            {%- if loop.last or (not loop.last and reasoning_content) %}
+                {{- '<|im_start|>' + message.role + '\n<think>\n' + reasoning_content.strip('\n') + '\n</think>\n\n' + content.lstrip('\n') }}
+            {%- else %}
+                {{- '<|im_start|>' + message.role + '\n' + content }}
+            {%- endif %}
+        {%- else %}
+            {{- '<|im_start|>' + message.role + '\n' + content }}
+        {%- endif %}
+        {%- if message.tool_calls %}
+            {%- for tool_call in message.tool_calls %}
+                {%- if (loop.first and content) or (not loop.first) %}
+                    {{- '\n' }}
+                {%- endif %}
+                {%- if tool_call.function %}
+                    {%- set tool_call = tool_call.function %}
+                {%- endif %}
+                {{- '<tool_call>\n{"name": "' }}
+                {{- tool_call.name }}
+                {{- '", "arguments": ' }}
+                {%- if tool_call.arguments is string %}
+                    {{- tool_call.arguments }}
+                {%- else %}
+                    {{- tool_call.arguments | tojson }}
+                {%- endif %}
+                {{- '}\n</tool_call>' }}
+            {%- endfor %}
+        {%- endif %}
+        {{- '<|im_end|>\n' }}
+    {%- elif message.role == "tool" %}
+        {%- if loop.first or (messages[loop.index0 - 1].role != "tool") %}
+            {{- '<|im_start|>user' }}
+        {%- endif %}
+        {{- '\n<tool_response>\n' }}
+        {{- message.content }}
+        {{- '\n</tool_response>' }}
+        {%- if loop.last or (messages[loop.index0 + 1].role != "tool") %}
+            {{- '<|im_end|>\n' }}
+        {%- endif %}
+    {%- endif %}
+{%- endfor %}
+{%- if add_generation_prompt %}
+    {{- '<|im_start|>assistant\n' }}
+    {%- if enable_thinking is defined and enable_thinking is false %}
+        {{- '<think>\n\n</think>\n\n' }}
+    {%- endif %}
+{%- endif %}

g2_1_oldcode_cm_newcode/500M/config.json

@@ -0,0 +1,68 @@
+{
+  "architectures": [
+    "Qwen3ForCausalLM"
+  ],
+  "attention_bias": false,
+  "attention_dropout": 0.0,
+  "dtype": "bfloat16",
+  "eos_token_id": 151643,
+  "head_dim": 128,
+  "hidden_act": "silu",
+  "hidden_size": 4096,
+  "initializer_range": 0.02,
+  "intermediate_size": 12288,
+  "layer_types": [
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention"
+  ],
+  "max_position_embeddings": 32768,
+  "max_window_layers": 36,
+  "model_type": "qwen3",
+  "num_attention_heads": 32,
+  "num_hidden_layers": 36,
+  "num_key_value_heads": 8,
+  "pad_token_id": 151643,
+  "rms_norm_eps": 1e-06,
+  "rope_scaling": null,
+  "rope_theta": 1000000,
+  "sliding_window": null,
+  "tie_word_embeddings": false,
+  "transformers_version": "4.57.6",
+  "use_cache": false,
+  "use_sliding_window": false,
+  "vocab_size": 151936
+}

g2_1_oldcode_cm_newcode/500M/generation_config.json

@@ -0,0 +1,8 @@
+{
+  "eos_token_id": [
+    151643
+  ],
+  "max_new_tokens": 2048,
+  "pad_token_id": 151643,
+  "transformers_version": "4.57.6"
+}

g2_1_oldcode_cm_newcode/500M/model-00001-of-00004.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:7ddcad18e535b8d78bfa56e017acd9bd03c012a3677a904f9587bcc7b73c2e05
+size 4902257696

g2_1_oldcode_cm_newcode/500M/model-00002-of-00004.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a1adbd08fe672ead141299f1a6deb4f85d35c79e292be597f32ac85589c7a7fa
+size 4915960368

g2_1_oldcode_cm_newcode/500M/model-00003-of-00004.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:945aa6ae272d5e8a7516410d441be5c2535dabd46186521bc683bffd807a6c22
+size 4983068496

g2_1_oldcode_cm_newcode/500M/model-00004-of-00004.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:7696ed7bd6a6cd0cf6c89b04d68e950b53e9b9bcfe1edd05949e929352ceb982
+size 1580230264

g2_1_oldcode_cm_newcode/500M/model.safetensors.index.json

@@ -0,0 +1,407 @@
+{
+  "metadata": {
+    "total_parameters": 8190735360,
+    "total_size": 16381470720
+  },
+  "weight_map": {
+    "lm_head.weight": "model-00004-of-00004.safetensors",
+    "model.embed_tokens.weight": "model-00001-of-00004.safetensors",
+    "model.layers.0.input_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.0.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.0.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.0.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.0.post_attention_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.0.self_attn.k_norm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.0.self_attn.k_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.0.self_attn.o_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.0.self_attn.q_norm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.0.self_attn.q_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.0.self_attn.v_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.1.input_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.1.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.1.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.1.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.1.post_attention_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.1.self_attn.k_norm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.1.self_attn.k_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.1.self_attn.o_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.1.self_attn.q_norm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.1.self_attn.q_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.1.self_attn.v_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.10.input_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.10.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.10.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.10.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.10.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.10.self_attn.k_norm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.10.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.10.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.10.self_attn.q_norm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.10.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.10.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.11.input_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.11.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.11.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.11.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.11.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.11.self_attn.k_norm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.11.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.11.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.11.self_attn.q_norm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.11.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.11.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.12.input_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.12.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.12.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.12.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.12.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.12.self_attn.k_norm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.12.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.12.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.12.self_attn.q_norm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.12.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.12.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.13.input_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.13.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.13.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.13.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.13.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.13.self_attn.k_norm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.13.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.13.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.13.self_attn.q_norm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.13.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.13.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.14.input_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.14.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.14.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.14.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.14.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.14.self_attn.k_norm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.14.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.14.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.14.self_attn.q_norm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.14.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.14.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.15.input_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.15.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.15.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.15.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.15.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.15.self_attn.k_norm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.15.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.15.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.15.self_attn.q_norm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.15.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.15.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.16.input_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.16.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.16.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.16.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.16.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.16.self_attn.k_norm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.16.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.16.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.16.self_attn.q_norm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.16.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.16.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.17.input_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.17.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.17.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.17.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.17.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.17.self_attn.k_norm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.17.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.17.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.17.self_attn.q_norm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.17.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.17.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.18.input_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.18.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.18.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.18.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.18.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.18.self_attn.k_norm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.18.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.18.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.18.self_attn.q_norm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.18.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.18.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.19.input_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.19.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.19.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.19.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.19.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.19.self_attn.k_norm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.19.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.19.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.19.self_attn.q_norm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.19.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.19.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.2.input_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.2.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.2.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.2.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.2.post_attention_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.2.self_attn.k_norm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.2.self_attn.k_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.2.self_attn.o_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.2.self_attn.q_norm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.2.self_attn.q_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.2.self_attn.v_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.20.input_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.20.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.20.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.20.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.20.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.20.self_attn.k_norm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.20.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.20.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.20.self_attn.q_norm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.20.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.20.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.21.input_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.21.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.21.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.21.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.21.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.21.self_attn.k_norm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.21.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.21.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.21.self_attn.q_norm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.21.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.21.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.22.input_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.22.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.22.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.22.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.22.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.22.self_attn.k_norm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.22.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.22.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.22.self_attn.q_norm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.22.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.22.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.23.input_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.23.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.23.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.23.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.23.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.23.self_attn.k_norm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.23.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.23.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.23.self_attn.q_norm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.23.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.23.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.24.input_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.24.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.24.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.24.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.24.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.24.self_attn.k_norm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.24.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.24.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.24.self_attn.q_norm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.24.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.24.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.25.input_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.25.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.25.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.25.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.25.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.25.self_attn.k_norm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.25.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.25.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.25.self_attn.q_norm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.25.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.25.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.26.input_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.26.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.26.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.26.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.26.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.26.self_attn.k_norm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.26.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.26.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.26.self_attn.q_norm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.26.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.26.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.27.input_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.27.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.27.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.27.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.27.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.27.self_attn.k_norm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.27.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.27.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.27.self_attn.q_norm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.27.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.27.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.28.input_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.28.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.28.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.28.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.28.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.28.self_attn.k_norm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.28.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.28.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.28.self_attn.q_norm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.28.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.28.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.29.input_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.29.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.29.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.29.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.29.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.29.self_attn.k_norm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.29.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.29.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.29.self_attn.q_norm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.29.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.29.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.3.input_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.3.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.3.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.3.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.3.post_attention_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.3.self_attn.k_norm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.3.self_attn.k_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.3.self_attn.o_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.3.self_attn.q_norm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.3.self_attn.q_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.3.self_attn.v_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.30.input_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.30.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.30.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.30.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.30.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.30.self_attn.k_norm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.30.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.30.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.30.self_attn.q_norm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.30.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.30.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.31.input_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.31.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.31.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.31.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.31.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.31.self_attn.k_norm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.31.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.31.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.31.self_attn.q_norm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.31.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.31.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.32.input_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.32.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.32.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.32.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.32.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.32.self_attn.k_norm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.32.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.32.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.32.self_attn.q_norm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.32.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.32.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.33.input_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.33.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.33.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.33.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.33.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.33.self_attn.k_norm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.33.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.33.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.33.self_attn.q_norm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.33.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.33.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.34.input_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.34.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.34.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.34.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.34.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.34.self_attn.k_norm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.34.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.34.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.34.self_attn.q_norm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.34.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.34.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.35.input_layernorm.weight": "model-00004-of-00004.safetensors",
+    "model.layers.35.mlp.down_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.35.mlp.gate_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.35.mlp.up_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.35.post_attention_layernorm.weight": "model-00004-of-00004.safetensors",
+    "model.layers.35.self_attn.k_norm.weight": "model-00004-of-00004.safetensors",
+    "model.layers.35.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.35.self_attn.o_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.35.self_attn.q_norm.weight": "model-00004-of-00004.safetensors",
+    "model.layers.35.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.35.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.4.input_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.4.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.4.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.4.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.4.post_attention_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.4.self_attn.k_norm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.4.self_attn.k_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.4.self_attn.o_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.4.self_attn.q_norm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.4.self_attn.q_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.4.self_attn.v_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.5.input_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.5.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.5.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.5.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.5.post_attention_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.5.self_attn.k_norm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.5.self_attn.k_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.5.self_attn.o_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.5.self_attn.q_norm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.5.self_attn.q_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.5.self_attn.v_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.6.input_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.6.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.6.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.6.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.6.post_attention_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.6.self_attn.k_norm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.6.self_attn.k_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.6.self_attn.o_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.6.self_attn.q_norm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.6.self_attn.q_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.6.self_attn.v_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.7.input_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.7.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.7.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.7.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.7.post_attention_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.7.self_attn.k_norm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.7.self_attn.k_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.7.self_attn.o_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.7.self_attn.q_norm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.7.self_attn.q_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.7.self_attn.v_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.8.input_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.8.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.8.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.8.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.8.post_attention_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.8.self_attn.k_norm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.8.self_attn.k_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.8.self_attn.o_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.8.self_attn.q_norm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.8.self_attn.q_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.8.self_attn.v_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.9.input_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.9.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.9.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.9.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.9.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.9.self_attn.k_norm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.9.self_attn.k_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.9.self_attn.o_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.9.self_attn.q_norm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.9.self_attn.q_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.9.self_attn.v_proj.weight": "model-00001-of-00004.safetensors",
+    "model.norm.weight": "model-00004-of-00004.safetensors"
+  }
+}

g2_1_oldcode_cm_newcode/500M/special_tokens_map.json

@@ -0,0 +1,31 @@
+{
+  "additional_special_tokens": [
+    "<|im_start|>",
+    "<|im_end|>",
+    "<|object_ref_start|>",
+    "<|object_ref_end|>",
+    "<|box_start|>",
+    "<|box_end|>",
+    "<|quad_start|>",
+    "<|quad_end|>",
+    "<|vision_start|>",
+    "<|vision_end|>",
+    "<|vision_pad|>",
+    "<|image_pad|>",
+    "<|video_pad|>"
+  ],
+  "eos_token": {
+    "content": "<|endoftext|>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "pad_token": {
+    "content": "<|endoftext|>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  }
+}

g2_1_oldcode_cm_newcode/500M/tokenizer.json

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:aeb13307a71acd8fe81861d94ad54ab689df773318809eed3cbe794b4492dae4
+size 11422654

g2_1_oldcode_cm_newcode/500M/tokenizer_config.json

@@ -0,0 +1,240 @@
+{
+  "add_bos_token": false,
+  "add_prefix_space": false,
+  "added_tokens_decoder": {
+    "151643": {
+      "content": "<|endoftext|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151644": {
+      "content": "<|im_start|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151645": {
+      "content": "<|im_end|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151646": {
+      "content": "<|object_ref_start|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151647": {
+      "content": "<|object_ref_end|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151648": {
+      "content": "<|box_start|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151649": {
+      "content": "<|box_end|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151650": {
+      "content": "<|quad_start|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151651": {
+      "content": "<|quad_end|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151652": {
+      "content": "<|vision_start|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151653": {
+      "content": "<|vision_end|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151654": {
+      "content": "<|vision_pad|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151655": {
+      "content": "<|image_pad|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151656": {
+      "content": "<|video_pad|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151657": {
+      "content": "<tool_call>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151658": {
+      "content": "</tool_call>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151659": {
+      "content": "<|fim_prefix|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151660": {
+      "content": "<|fim_middle|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151661": {
+      "content": "<|fim_suffix|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151662": {
+      "content": "<|fim_pad|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151663": {
+      "content": "<|repo_name|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151664": {
+      "content": "<|file_sep|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151665": {
+      "content": "<tool_response>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151666": {
+      "content": "</tool_response>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151667": {
+      "content": "<think>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151668": {
+      "content": "</think>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    }
+  },
+  "additional_special_tokens": [
+    "<|im_start|>",
+    "<|im_end|>",
+    "<|object_ref_start|>",
+    "<|object_ref_end|>",
+    "<|box_start|>",
+    "<|box_end|>",
+    "<|quad_start|>",
+    "<|quad_end|>",
+    "<|vision_start|>",
+    "<|vision_end|>",
+    "<|vision_pad|>",
+    "<|image_pad|>",
+    "<|video_pad|>"
+  ],
+  "bos_token": null,
+  "clean_up_tokenization_spaces": false,
+  "eos_token": "<|endoftext|>",
+  "errors": "replace",
+  "extra_special_tokens": {},
+  "model_max_length": 131072,
+  "pad_token": "<|endoftext|>",
+  "padding_side": "right",
+  "split_special_tokens": false,
+  "tokenizer_class": "Qwen2Tokenizer",
+  "unk_token": null
+}

g2_1_oldcode_cm_newcode/500M/train_results.json

@@ -0,0 +1,8 @@
+{
+    "epoch": 0.4919142204253823,
+    "total_flos": 2.4985159616803373e+19,
+    "train_loss": 0.048519859191283855,
+    "train_runtime": 5685.2195,
+    "train_samples_per_second": 11.814,
+    "train_steps_per_second": 0.984
+}

g2_1_oldcode_cm_newcode/500M/trainer_log.jsonl

@@ -0,0 +1,52 @@
+{"current_steps": 5090, "total_steps": 5597, "loss": 0.589, "lr": 1.9999810479633703e-05, "epoch": 0.4473545438565653, "percentage": 90.94, "elapsed_time": "0:00:26", "remaining_time": "0:00:02"}
+{"current_steps": 5100, "total_steps": 5597, "loss": 0.5323, "lr": 1.997707677020786e-05, "epoch": 0.4482334329407629, "percentage": 91.12, "elapsed_time": "0:02:17", "remaining_time": "0:00:13"}
+{"current_steps": 5110, "total_steps": 5597, "loss": 0.5446, "lr": 1.9916538196124637e-05, "epoch": 0.44911232202496043, "percentage": 91.3, "elapsed_time": "0:04:08", "remaining_time": "0:00:23"}
+{"current_steps": 5120, "total_steps": 5597, "loss": 0.517, "lr": 1.9818425303863402e-05, "epoch": 0.449991211109158, "percentage": 91.48, "elapsed_time": "0:06:00", "remaining_time": "0:00:33"}
+{"current_steps": 5130, "total_steps": 5597, "loss": 0.5261, "lr": 1.9683111732582887e-05, "epoch": 0.4508701001933556, "percentage": 91.66, "elapsed_time": "0:07:51", "remaining_time": "0:00:42"}
+{"current_steps": 5140, "total_steps": 5597, "loss": 0.5404, "lr": 1.951111279120706e-05, "epoch": 0.4517489892775532, "percentage": 91.83, "elapsed_time": "0:09:42", "remaining_time": "0:00:51"}
+{"current_steps": 5150, "total_steps": 5597, "loss": 0.5269, "lr": 1.9303083495996028e-05, "epoch": 0.4526278783617507, "percentage": 92.01, "elapsed_time": "0:11:34", "remaining_time": "0:01:00"}
+{"current_steps": 5160, "total_steps": 5597, "loss": 0.5414, "lr": 1.9059816076075482e-05, "epoch": 0.4535067674459483, "percentage": 92.19, "elapsed_time": "0:13:25", "remaining_time": "0:01:08"}
+{"current_steps": 5170, "total_steps": 5597, "loss": 0.5404, "lr": 1.8782236956424142e-05, "epoch": 0.4543856565301459, "percentage": 92.37, "elapsed_time": "0:15:16", "remaining_time": "0:01:15"}
+{"current_steps": 5180, "total_steps": 5597, "loss": 0.5278, "lr": 1.8471403229808897e-05, "epoch": 0.4552645456143435, "percentage": 92.55, "elapsed_time": "0:17:08", "remaining_time": "0:01:22"}
+{"current_steps": 5190, "total_steps": 5597, "loss": 0.5479, "lr": 1.8128498631103333e-05, "epoch": 0.456143434698541, "percentage": 92.73, "elapsed_time": "0:18:59", "remaining_time": "0:01:29"}
+{"current_steps": 5200, "total_steps": 5597, "loss": 0.515, "lr": 1.7754829029320507e-05, "epoch": 0.4570223237827386, "percentage": 92.91, "elapsed_time": "0:20:51", "remaining_time": "0:01:35"}
+{"current_steps": 5210, "total_steps": 5597, "loss": 0.5267, "lr": 1.7351817454527473e-05, "epoch": 0.4579012128669362, "percentage": 93.09, "elapsed_time": "0:22:42", "remaining_time": "0:01:41"}
+{"current_steps": 5220, "total_steps": 5597, "loss": 0.5165, "lr": 1.6920998678580345e-05, "epoch": 0.4587801019511338, "percentage": 93.26, "elapsed_time": "0:24:33", "remaining_time": "0:01:46"}
+{"current_steps": 5230, "total_steps": 5597, "loss": 0.5316, "lr": 1.6464013370317854e-05, "epoch": 0.45965899103533137, "percentage": 93.44, "elapsed_time": "0:26:25", "remaining_time": "0:01:51"}
+{"current_steps": 5240, "total_steps": 5597, "loss": 0.535, "lr": 1.5982601847471915e-05, "epoch": 0.4605378801195289, "percentage": 93.62, "elapsed_time": "0:28:16", "remaining_time": "0:01:55"}
+{"current_steps": 5250, "total_steps": 5597, "loss": 0.5459, "lr": 1.5478597449089553e-05, "epoch": 0.4614167692037265, "percentage": 93.8, "elapsed_time": "0:30:07", "remaining_time": "0:01:59"}
+{"current_steps": 5260, "total_steps": 5597, "loss": 0.5432, "lr": 1.4953919553705725e-05, "epoch": 0.46229565828792407, "percentage": 93.98, "elapsed_time": "0:31:59", "remaining_time": "0:02:02"}
+{"current_steps": 5270, "total_steps": 5597, "loss": 0.5714, "lr": 1.441056626985559e-05, "epoch": 0.46317454737212166, "percentage": 94.16, "elapsed_time": "0:33:50", "remaining_time": "0:02:05"}
+{"current_steps": 5280, "total_steps": 5597, "loss": 0.5473, "lr": 1.3850606826762668e-05, "epoch": 0.4640534364563192, "percentage": 94.34, "elapsed_time": "0:35:41", "remaining_time": "0:02:08"}
+{"current_steps": 5290, "total_steps": 5597, "loss": 0.5238, "lr": 1.3276173694181128e-05, "epoch": 0.4649323255405168, "percentage": 94.51, "elapsed_time": "0:37:33", "remaining_time": "0:02:10"}
+{"current_steps": 5300, "total_steps": 5597, "loss": 0.5291, "lr": 1.2689454461401822e-05, "epoch": 0.46581121462471436, "percentage": 94.69, "elapsed_time": "0:39:24", "remaining_time": "0:02:12"}
+{"current_steps": 5310, "total_steps": 5597, "loss": 0.5181, "lr": 1.2092683506348939e-05, "epoch": 0.46669010370891195, "percentage": 94.87, "elapsed_time": "0:41:15", "remaining_time": "0:02:13"}
+{"current_steps": 5320, "total_steps": 5597, "loss": 0.5234, "lr": 1.14881334864935e-05, "epoch": 0.4675689927931095, "percentage": 95.05, "elapsed_time": "0:43:06", "remaining_time": "0:02:14"}
+{"current_steps": 5330, "total_steps": 5597, "loss": 0.5611, "lr": 1.0878106683988481e-05, "epoch": 0.46844788187730707, "percentage": 95.23, "elapsed_time": "0:44:57", "remaining_time": "0:02:15"}
+{"current_steps": 5340, "total_steps": 5597, "loss": 0.5543, "lr": 1.026492623798551e-05, "epoch": 0.46932677096150466, "percentage": 95.41, "elapsed_time": "0:46:49", "remaining_time": "0:02:15"}
+{"current_steps": 5350, "total_steps": 5597, "loss": 0.5379, "lr": 9.650927297522743e-06, "epoch": 0.47020566004570225, "percentage": 95.59, "elapsed_time": "0:48:40", "remaining_time": "0:02:14"}
+{"current_steps": 5360, "total_steps": 5597, "loss": 0.545, "lr": 9.038448128676012e-06, "epoch": 0.47108454912989983, "percentage": 95.77, "elapsed_time": "0:50:31", "remaining_time": "0:02:14"}
+{"current_steps": 5370, "total_steps": 5597, "loss": 0.531, "lr": 8.42982120983949e-06, "epoch": 0.47196343821409736, "percentage": 95.94, "elapsed_time": "0:52:22", "remaining_time": "0:02:12"}
+{"current_steps": 5380, "total_steps": 5597, "loss": 0.5297, "lr": 7.827364349047303e-06, "epoch": 0.47284232729829495, "percentage": 96.12, "elapsed_time": "0:54:13", "remaining_time": "0:02:11"}
+{"current_steps": 5390, "total_steps": 5597, "loss": 0.5202, "lr": 7.233371857163706e-06, "epoch": 0.47372121638249254, "percentage": 96.3, "elapsed_time": "0:56:04", "remaining_time": "0:02:09"}
+{"current_steps": 5400, "total_steps": 5597, "loss": 0.5233, "lr": 6.650105810556492e-06, "epoch": 0.4746001054666901, "percentage": 96.48, "elapsed_time": "0:57:56", "remaining_time": "0:02:06"}
+{"current_steps": 5410, "total_steps": 5597, "loss": 0.5213, "lr": 6.079787436527681e-06, "epoch": 0.47547899455088766, "percentage": 96.66, "elapsed_time": "0:59:47", "remaining_time": "0:02:03"}
+{"current_steps": 5420, "total_steps": 5597, "loss": 0.5345, "lr": 5.524588654307961e-06, "epoch": 0.47635788363508524, "percentage": 96.84, "elapsed_time": "1:01:38", "remaining_time": "0:02:00"}
+{"current_steps": 5430, "total_steps": 5597, "loss": 0.5417, "lr": 4.986623803828985e-06, "epoch": 0.47723677271928283, "percentage": 97.02, "elapsed_time": "1:03:29", "remaining_time": "0:01:57"}
+{"current_steps": 5440, "total_steps": 5597, "loss": 0.5196, "lr": 4.467941593772423e-06, "epoch": 0.4781156618034804, "percentage": 97.19, "elapsed_time": "1:05:20", "remaining_time": "0:01:53"}
+{"current_steps": 5450, "total_steps": 5597, "loss": 0.5365, "lr": 3.9705172995597606e-06, "epoch": 0.47899455088767795, "percentage": 97.37, "elapsed_time": "1:07:11", "remaining_time": "0:01:48"}
+{"current_steps": 5460, "total_steps": 5597, "loss": 0.5399, "lr": 3.4962452409948346e-06, "epoch": 0.47987343997187554, "percentage": 97.55, "elapsed_time": "1:09:02", "remaining_time": "0:01:43"}
+{"current_steps": 5470, "total_steps": 5597, "loss": 0.5548, "lr": 3.046931568206211e-06, "epoch": 0.4807523290560731, "percentage": 97.73, "elapsed_time": "1:10:53", "remaining_time": "0:01:38"}
+{"current_steps": 5480, "total_steps": 5597, "loss": 0.5186, "lr": 2.624287383362404e-06, "epoch": 0.4816312181402707, "percentage": 97.91, "elapsed_time": "1:12:45", "remaining_time": "0:01:33"}
+{"current_steps": 5490, "total_steps": 5597, "loss": 0.5369, "lr": 2.2299222243541845e-06, "epoch": 0.4825101072244683, "percentage": 98.09, "elapsed_time": "1:14:36", "remaining_time": "0:01:27"}
+{"current_steps": 5500, "total_steps": 5597, "loss": 0.524, "lr": 1.8653379352598246e-06, "epoch": 0.48338899630866583, "percentage": 98.27, "elapsed_time": "1:16:27", "remaining_time": "0:01:20"}
+{"current_steps": 5510, "total_steps": 5597, "loss": 0.5411, "lr": 1.5319229469361075e-06, "epoch": 0.4842678853928634, "percentage": 98.45, "elapsed_time": "1:18:18", "remaining_time": "0:01:14"}
+{"current_steps": 5520, "total_steps": 5597, "loss": 0.5421, "lr": 1.2309469895160917e-06, "epoch": 0.485146774477061, "percentage": 98.62, "elapsed_time": "1:20:09", "remaining_time": "0:01:07"}
+{"current_steps": 5530, "total_steps": 5597, "loss": 0.5364, "lr": 9.635562569497974e-07, "epoch": 0.4860256635612586, "percentage": 98.8, "elapsed_time": "1:22:01", "remaining_time": "0:00:59"}
+{"current_steps": 5540, "total_steps": 5597, "loss": 0.5218, "lr": 7.307690420024867e-07, "epoch": 0.4869045526454561, "percentage": 98.98, "elapsed_time": "1:23:52", "remaining_time": "0:00:51"}
+{"current_steps": 5550, "total_steps": 5597, "loss": 0.5479, "lr": 5.334718583335942e-07, "epoch": 0.4877834417296537, "percentage": 99.16, "elapsed_time": "1:25:43", "remaining_time": "0:00:43"}
+{"current_steps": 5560, "total_steps": 5597, "loss": 0.5463, "lr": 3.7241606442442763e-07, "epoch": 0.4886623308138513, "percentage": 99.34, "elapsed_time": "1:27:35", "remaining_time": "0:00:34"}
+{"current_steps": 5570, "total_steps": 5597, "loss": 0.5216, "lr": 2.482150022115879e-07, "epoch": 0.4895412198980489, "percentage": 99.52, "elapsed_time": "1:29:26", "remaining_time": "0:00:26"}
+{"current_steps": 5580, "total_steps": 5597, "loss": 0.528, "lr": 1.6134166132293174e-07, "epoch": 0.4904201089822464, "percentage": 99.7, "elapsed_time": "1:31:17", "remaining_time": "0:00:16"}
+{"current_steps": 5590, "total_steps": 5597, "loss": 0.5224, "lr": 1.1212687781127357e-07, "epoch": 0.491298998066444, "percentage": 99.87, "elapsed_time": "1:33:08", "remaining_time": "0:00:06"}
+{"current_steps": 5597, "total_steps": 5597, "epoch": 0.4919142204253823, "percentage": 100.0, "elapsed_time": "1:34:45", "remaining_time": "0:00:00"}

g2_1_oldcode_cm_newcode/500M/trainer_state.json

@@ -0,0 +1,3956 @@
+{
+  "best_global_step": null,
+  "best_metric": null,
+  "best_model_checkpoint": null,
+  "epoch": 0.4919142204253823,
+  "eval_steps": 500,
+  "global_step": 5597,
+  "is_hyper_param_search": false,
+  "is_local_process_zero": true,
+  "is_world_process_zero": true,
+  "log_history": [
+    {
+      "epoch": 0.0008788890841975743,
+      "grad_norm": 0.7299243807792664,
+      "learning_rate": 1.1061797752808989e-06,
+      "loss": 0.5786,
+      "step": 10
+    },
+    {
+      "epoch": 0.0017577781683951485,
+      "grad_norm": 0.7039077281951904,
+      "learning_rate": 2.2241573033707866e-06,
+      "loss": 0.5844,
+      "step": 20
+    },
+    {
+      "epoch": 0.002636667252592723,
+      "grad_norm": 0.594024121761322,
+      "learning_rate": 3.342134831460675e-06,
+      "loss": 0.5971,
+      "step": 30
+    },
+    {
+      "epoch": 0.003515556336790297,
+      "grad_norm": 0.7417164444923401,
+      "learning_rate": 4.4601123595505624e-06,
+      "loss": 0.5748,
+      "step": 40
+    },
+    {
+      "epoch": 0.004394445420987872,
+      "grad_norm": 0.9814750552177429,
+      "learning_rate": 5.5780898876404495e-06,
+      "loss": 0.5991,
+      "step": 50
+    },
+    {
+      "epoch": 0.005273334505185446,
+      "grad_norm": 0.6245082020759583,
+      "learning_rate": 6.6960674157303374e-06,
+      "loss": 0.5847,
+      "step": 60
+    },
+    {
+      "epoch": 0.00615222358938302,
+      "grad_norm": 0.5894784331321716,
+      "learning_rate": 7.814044943820226e-06,
+      "loss": 0.5843,
+      "step": 70
+    },
+    {
+      "epoch": 0.007031112673580594,
+      "grad_norm": 0.6612409353256226,
+      "learning_rate": 8.932022471910112e-06,
+      "loss": 0.572,
+      "step": 80
+    },
+    {
+      "epoch": 0.007910001757778168,
+      "grad_norm": 0.614031195640564,
+      "learning_rate": 1.005e-05,
+      "loss": 0.5755,
+      "step": 90
+    },
+    {
+      "epoch": 0.008788890841975743,
+      "grad_norm": 0.6163526773452759,
+      "learning_rate": 1.1167977528089888e-05,
+      "loss": 0.5672,
+      "step": 100
+    },
+    {
+      "epoch": 0.009667779926173316,
+      "grad_norm": 0.6364962458610535,
+      "learning_rate": 1.2285955056179776e-05,
+      "loss": 0.5639,
+      "step": 110
+    },
+    {
+      "epoch": 0.010546669010370891,
+      "grad_norm": 0.6238141655921936,
+      "learning_rate": 1.3403932584269664e-05,
+      "loss": 0.5805,
+      "step": 120
+    },
+    {
+      "epoch": 0.011425558094568465,
+      "grad_norm": 0.7033671736717224,
+      "learning_rate": 1.4521910112359552e-05,
+      "loss": 0.5794,
+      "step": 130
+    },
+    {
+      "epoch": 0.01230444717876604,
+      "grad_norm": 0.5810057520866394,
+      "learning_rate": 1.563988764044944e-05,
+      "loss": 0.5318,
+      "step": 140
+    },
+    {
+      "epoch": 0.013183336262963615,
+      "grad_norm": 0.6278232336044312,
+      "learning_rate": 1.6757865168539328e-05,
+      "loss": 0.5694,
+      "step": 150
+    },
+    {
+      "epoch": 0.014062225347161188,
+      "grad_norm": 0.5975486040115356,
+      "learning_rate": 1.7875842696629214e-05,
+      "loss": 0.5692,
+      "step": 160
+    },
+    {
+      "epoch": 0.014941114431358763,
+      "grad_norm": 0.582416296005249,
+      "learning_rate": 1.89938202247191e-05,
+      "loss": 0.5773,
+      "step": 170
+    },
+    {
+      "epoch": 0.015820003515556336,
+      "grad_norm": 0.6315813660621643,
+      "learning_rate": 2e-05,
+      "loss": 0.5881,
+      "step": 180
+    },
+    {
+      "epoch": 0.01669889259975391,
+      "grad_norm": 0.5048695802688599,
+      "learning_rate": 2e-05,
+      "loss": 0.6048,
+      "step": 190
+    },
+    {
+      "epoch": 0.017577781683951486,
+      "grad_norm": 0.5755158066749573,
+      "learning_rate": 2e-05,
+      "loss": 0.5961,
+      "step": 200
+    },
+    {
+      "epoch": 0.01845667076814906,
+      "grad_norm": 0.46752920746803284,
+      "learning_rate": 2e-05,
+      "loss": 0.5767,
+      "step": 210
+    },
+    {
+      "epoch": 0.019335559852346633,
+      "grad_norm": 0.5730629563331604,
+      "learning_rate": 2e-05,
+      "loss": 0.593,
+      "step": 220
+    },
+    {
+      "epoch": 0.02021444893654421,
+      "grad_norm": 0.4156428575515747,
+      "learning_rate": 2e-05,
+      "loss": 0.5804,
+      "step": 230
+    },
+    {
+      "epoch": 0.021093338020741783,
+      "grad_norm": 0.4448103606700897,
+      "learning_rate": 2e-05,
+      "loss": 0.5891,
+      "step": 240
+    },
+    {
+      "epoch": 0.021972227104939356,
+      "grad_norm": 0.446362167596817,
+      "learning_rate": 2e-05,
+      "loss": 0.5669,
+      "step": 250
+    },
+    {
+      "epoch": 0.02285111618913693,
+      "grad_norm": 0.42595115303993225,
+      "learning_rate": 2e-05,
+      "loss": 0.5885,
+      "step": 260
+    },
+    {
+      "epoch": 0.023730005273334506,
+      "grad_norm": 0.45942461490631104,
+      "learning_rate": 2e-05,
+      "loss": 0.5812,
+      "step": 270
+    },
+    {
+      "epoch": 0.02460889435753208,
+      "grad_norm": 0.4314509332180023,
+      "learning_rate": 2e-05,
+      "loss": 0.5984,
+      "step": 280
+    },
+    {
+      "epoch": 0.025487783441729653,
+      "grad_norm": 0.4355176091194153,
+      "learning_rate": 2e-05,
+      "loss": 0.5995,
+      "step": 290
+    },
+    {
+      "epoch": 0.02636667252592723,
+      "grad_norm": 0.4335140287876129,
+      "learning_rate": 2e-05,
+      "loss": 0.5725,
+      "step": 300
+    },
+    {
+      "epoch": 0.027245561610124803,
+      "grad_norm": 0.41898155212402344,
+      "learning_rate": 2e-05,
+      "loss": 0.5947,
+      "step": 310
+    },
+    {
+      "epoch": 0.028124450694322376,
+      "grad_norm": 0.35632404685020447,
+      "learning_rate": 2e-05,
+      "loss": 0.5575,
+      "step": 320
+    },
+    {
+      "epoch": 0.02900333977851995,
+      "grad_norm": 0.4908845126628876,
+      "learning_rate": 2e-05,
+      "loss": 0.5874,
+      "step": 330
+    },
+    {
+      "epoch": 0.029882228862717526,
+      "grad_norm": 0.39167770743370056,
+      "learning_rate": 2e-05,
+      "loss": 0.6033,
+      "step": 340
+    },
+    {
+      "epoch": 0.0307611179469151,
+      "grad_norm": 0.41944095492362976,
+      "learning_rate": 2e-05,
+      "loss": 0.578,
+      "step": 350
+    },
+    {
+      "epoch": 0.03164000703111267,
+      "grad_norm": 0.4073144197463989,
+      "learning_rate": 2e-05,
+      "loss": 0.5851,
+      "step": 360
+    },
+    {
+      "epoch": 0.032518896115310246,
+      "grad_norm": 0.3751962184906006,
+      "learning_rate": 2e-05,
+      "loss": 0.5799,
+      "step": 370
+    },
+    {
+      "epoch": 0.03339778519950782,
+      "grad_norm": 0.39586400985717773,
+      "learning_rate": 2e-05,
+      "loss": 0.5798,
+      "step": 380
+    },
+    {
+      "epoch": 0.0342766742837054,
+      "grad_norm": 0.4123347997665405,
+      "learning_rate": 2e-05,
+      "loss": 0.574,
+      "step": 390
+    },
+    {
+      "epoch": 0.03515556336790297,
+      "grad_norm": 0.4077567160129547,
+      "learning_rate": 2e-05,
+      "loss": 0.6083,
+      "step": 400
+    },
+    {
+      "epoch": 0.036034452452100546,
+      "grad_norm": 0.36184021830558777,
+      "learning_rate": 2e-05,
+      "loss": 0.5791,
+      "step": 410
+    },
+    {
+      "epoch": 0.03691334153629812,
+      "grad_norm": 0.39715006947517395,
+      "learning_rate": 2e-05,
+      "loss": 0.5809,
+      "step": 420
+    },
+    {
+      "epoch": 0.03779223062049569,
+      "grad_norm": 0.34982648491859436,
+      "learning_rate": 2e-05,
+      "loss": 0.5651,
+      "step": 430
+    },
+    {
+      "epoch": 0.038671119704693266,
+      "grad_norm": 0.35893237590789795,
+      "learning_rate": 2e-05,
+      "loss": 0.581,
+      "step": 440
+    },
+    {
+      "epoch": 0.03955000878889084,
+      "grad_norm": 0.38406050205230713,
+      "learning_rate": 2e-05,
+      "loss": 0.5961,
+      "step": 450
+    },
+    {
+      "epoch": 0.04042889787308842,
+      "grad_norm": 0.4047548174858093,
+      "learning_rate": 2e-05,
+      "loss": 0.561,
+      "step": 460
+    },
+    {
+      "epoch": 0.04130778695728599,
+      "grad_norm": 0.3789917826652527,
+      "learning_rate": 2e-05,
+      "loss": 0.5899,
+      "step": 470
+    },
+    {
+      "epoch": 0.042186676041483566,
+      "grad_norm": 0.40871259570121765,
+      "learning_rate": 2e-05,
+      "loss": 0.5848,
+      "step": 480
+    },
+    {
+      "epoch": 0.04306556512568114,
+      "grad_norm": 0.3634383976459503,
+      "learning_rate": 2e-05,
+      "loss": 0.5942,
+      "step": 490
+    },
+    {
+      "epoch": 0.04394445420987871,
+      "grad_norm": 0.3622599244117737,
+      "learning_rate": 2e-05,
+      "loss": 0.5632,
+      "step": 500
+    },
+    {
+      "epoch": 0.044823343294076286,
+      "grad_norm": 0.3681706488132477,
+      "learning_rate": 2e-05,
+      "loss": 0.5918,
+      "step": 510
+    },
+    {
+      "epoch": 0.04570223237827386,
+      "grad_norm": 0.4069708287715912,
+      "learning_rate": 2e-05,
+      "loss": 0.5745,
+      "step": 520
+    },
+    {
+      "epoch": 0.04658112146247144,
+      "grad_norm": 0.40182244777679443,
+      "learning_rate": 2e-05,
+      "loss": 0.575,
+      "step": 530
+    },
+    {
+      "epoch": 0.04746001054666901,
+      "grad_norm": 0.39383378624916077,
+      "learning_rate": 2e-05,
+      "loss": 0.5431,
+      "step": 540
+    },
+    {
+      "epoch": 0.048338899630866586,
+      "grad_norm": 0.35547786951065063,
+      "learning_rate": 2e-05,
+      "loss": 0.5653,
+      "step": 550
+    },
+    {
+      "epoch": 0.04921778871506416,
+      "grad_norm": 0.35708558559417725,
+      "learning_rate": 2e-05,
+      "loss": 0.5726,
+      "step": 560
+    },
+    {
+      "epoch": 0.05009667779926173,
+      "grad_norm": 0.41257601976394653,
+      "learning_rate": 2e-05,
+      "loss": 0.5546,
+      "step": 570
+    },
+    {
+      "epoch": 0.050975566883459306,
+      "grad_norm": 0.37748679518699646,
+      "learning_rate": 2e-05,
+      "loss": 0.5796,
+      "step": 580
+    },
+    {
+      "epoch": 0.05185445596765688,
+      "grad_norm": 0.3628121614456177,
+      "learning_rate": 2e-05,
+      "loss": 0.5542,
+      "step": 590
+    },
+    {
+      "epoch": 0.05273334505185446,
+      "grad_norm": 0.31530919671058655,
+      "learning_rate": 2e-05,
+      "loss": 0.5643,
+      "step": 600
+    },
+    {
+      "epoch": 0.05361223413605203,
+      "grad_norm": 0.3966090679168701,
+      "learning_rate": 2e-05,
+      "loss": 0.588,
+      "step": 610
+    },
+    {
+      "epoch": 0.054491123220249606,
+      "grad_norm": 0.41289761662483215,
+      "learning_rate": 2e-05,
+      "loss": 0.5868,
+      "step": 620
+    },
+    {
+      "epoch": 0.05537001230444718,
+      "grad_norm": 0.37477827072143555,
+      "learning_rate": 2e-05,
+      "loss": 0.5764,
+      "step": 630
+    },
+    {
+      "epoch": 0.05624890138864475,
+      "grad_norm": 0.4318097233772278,
+      "learning_rate": 2e-05,
+      "loss": 0.581,
+      "step": 640
+    },
+    {
+      "epoch": 0.057127790472842326,
+      "grad_norm": 0.36890605092048645,
+      "learning_rate": 2e-05,
+      "loss": 0.5623,
+      "step": 650
+    },
+    {
+      "epoch": 0.0580066795570399,
+      "grad_norm": 0.409492164850235,
+      "learning_rate": 2e-05,
+      "loss": 0.5848,
+      "step": 660
+    },
+    {
+      "epoch": 0.05888556864123748,
+      "grad_norm": 0.3958004415035248,
+      "learning_rate": 2e-05,
+      "loss": 0.5851,
+      "step": 670
+    },
+    {
+      "epoch": 0.05976445772543505,
+      "grad_norm": 0.3854322135448456,
+      "learning_rate": 2e-05,
+      "loss": 0.5867,
+      "step": 680
+    },
+    {
+      "epoch": 0.060643346809632626,
+      "grad_norm": 0.3919129967689514,
+      "learning_rate": 2e-05,
+      "loss": 0.5859,
+      "step": 690
+    },
+    {
+      "epoch": 0.0615222358938302,
+      "grad_norm": 0.34421873092651367,
+      "learning_rate": 2e-05,
+      "loss": 0.5695,
+      "step": 700
+    },
+    {
+      "epoch": 0.06240112497802777,
+      "grad_norm": 0.36081117391586304,
+      "learning_rate": 2e-05,
+      "loss": 0.561,
+      "step": 710
+    },
+    {
+      "epoch": 0.06328001406222535,
+      "grad_norm": 0.3740138113498688,
+      "learning_rate": 2e-05,
+      "loss": 0.5705,
+      "step": 720
+    },
+    {
+      "epoch": 0.06415890314642292,
+      "grad_norm": 0.35506170988082886,
+      "learning_rate": 2e-05,
+      "loss": 0.5534,
+      "step": 730
+    },
+    {
+      "epoch": 0.06503779223062049,
+      "grad_norm": 0.380117267370224,
+      "learning_rate": 2e-05,
+      "loss": 0.5559,
+      "step": 740
+    },
+    {
+      "epoch": 0.06591668131481807,
+      "grad_norm": 0.36159446835517883,
+      "learning_rate": 2e-05,
+      "loss": 0.5886,
+      "step": 750
+    },
+    {
+      "epoch": 0.06679557039901564,
+      "grad_norm": 0.3657621145248413,
+      "learning_rate": 2e-05,
+      "loss": 0.553,
+      "step": 760
+    },
+    {
+      "epoch": 0.06767445948321321,
+      "grad_norm": 0.3414924740791321,
+      "learning_rate": 2e-05,
+      "loss": 0.5935,
+      "step": 770
+    },
+    {
+      "epoch": 0.0685533485674108,
+      "grad_norm": 0.32405906915664673,
+      "learning_rate": 2e-05,
+      "loss": 0.5272,
+      "step": 780
+    },
+    {
+      "epoch": 0.06943223765160837,
+      "grad_norm": 0.3730144500732422,
+      "learning_rate": 2e-05,
+      "loss": 0.5562,
+      "step": 790
+    },
+    {
+      "epoch": 0.07031112673580595,
+      "grad_norm": 0.4821453392505646,
+      "learning_rate": 2e-05,
+      "loss": 0.575,
+      "step": 800
+    },
+    {
+      "epoch": 0.07119001582000352,
+      "grad_norm": 0.48737847805023193,
+      "learning_rate": 2e-05,
+      "loss": 0.6033,
+      "step": 810
+    },
+    {
+      "epoch": 0.07206890490420109,
+      "grad_norm": 0.5840171575546265,
+      "learning_rate": 2e-05,
+      "loss": 0.5745,
+      "step": 820
+    },
+    {
+      "epoch": 0.07294779398839867,
+      "grad_norm": 0.3971763551235199,
+      "learning_rate": 2e-05,
+      "loss": 0.5696,
+      "step": 830
+    },
+    {
+      "epoch": 0.07382668307259624,
+      "grad_norm": 0.3556879460811615,
+      "learning_rate": 2e-05,
+      "loss": 0.5993,
+      "step": 840
+    },
+    {
+      "epoch": 0.07470557215679381,
+      "grad_norm": 0.3166077733039856,
+      "learning_rate": 2e-05,
+      "loss": 0.5711,
+      "step": 850
+    },
+    {
+      "epoch": 0.07558446124099139,
+      "grad_norm": 0.37802746891975403,
+      "learning_rate": 2e-05,
+      "loss": 0.5806,
+      "step": 860
+    },
+    {
+      "epoch": 0.07646335032518896,
+      "grad_norm": 0.33548033237457275,
+      "learning_rate": 2e-05,
+      "loss": 0.5619,
+      "step": 870
+    },
+    {
+      "epoch": 0.07734223940938653,
+      "grad_norm": 0.3750935196876526,
+      "learning_rate": 2e-05,
+      "loss": 0.5636,
+      "step": 880
+    },
+    {
+      "epoch": 0.0782211284935841,
+      "grad_norm": 0.359465628862381,
+      "learning_rate": 2e-05,
+      "loss": 0.5978,
+      "step": 890
+    },
+    {
+      "epoch": 0.07910001757778168,
+      "grad_norm": 0.39016494154930115,
+      "learning_rate": 2e-05,
+      "loss": 0.5784,
+      "step": 900
+    },
+    {
+      "epoch": 0.07997890666197925,
+      "grad_norm": 0.32137224078178406,
+      "learning_rate": 2e-05,
+      "loss": 0.5958,
+      "step": 910
+    },
+    {
+      "epoch": 0.08085779574617684,
+      "grad_norm": 0.3473421037197113,
+      "learning_rate": 2e-05,
+      "loss": 0.5707,
+      "step": 920
+    },
+    {
+      "epoch": 0.08173668483037441,
+      "grad_norm": 0.35025525093078613,
+      "learning_rate": 2e-05,
+      "loss": 0.5776,
+      "step": 930
+    },
+    {
+      "epoch": 0.08261557391457199,
+      "grad_norm": 0.4126445949077606,
+      "learning_rate": 2e-05,
+      "loss": 0.5645,
+      "step": 940
+    },
+    {
+      "epoch": 0.08349446299876956,
+      "grad_norm": 0.3999921679496765,
+      "learning_rate": 2e-05,
+      "loss": 0.5806,
+      "step": 950
+    },
+    {
+      "epoch": 0.08437335208296713,
+      "grad_norm": 0.38202181458473206,
+      "learning_rate": 2e-05,
+      "loss": 0.5775,
+      "step": 960
+    },
+    {
+      "epoch": 0.0852522411671647,
+      "grad_norm": 0.3595036566257477,
+      "learning_rate": 2e-05,
+      "loss": 0.5797,
+      "step": 970
+    },
+    {
+      "epoch": 0.08613113025136228,
+      "grad_norm": 0.3591906726360321,
+      "learning_rate": 2e-05,
+      "loss": 0.5875,
+      "step": 980
+    },
+    {
+      "epoch": 0.08701001933555985,
+      "grad_norm": 0.3604753613471985,
+      "learning_rate": 2e-05,
+      "loss": 0.576,
+      "step": 990
+    },
+    {
+      "epoch": 0.08788890841975742,
+      "grad_norm": 0.35856151580810547,
+      "learning_rate": 2e-05,
+      "loss": 0.5704,
+      "step": 1000
+    },
+    {
+      "epoch": 0.088767797503955,
+      "grad_norm": 0.3498944938182831,
+      "learning_rate": 2e-05,
+      "loss": 0.6101,
+      "step": 1010
+    },
+    {
+      "epoch": 0.08964668658815257,
+      "grad_norm": 0.34088367223739624,
+      "learning_rate": 2e-05,
+      "loss": 0.5809,
+      "step": 1020
+    },
+    {
+      "epoch": 0.09052557567235014,
+      "grad_norm": 0.37535375356674194,
+      "learning_rate": 2e-05,
+      "loss": 0.569,
+      "step": 1030
+    },
+    {
+      "epoch": 0.09140446475654772,
+      "grad_norm": 0.34113436937332153,
+      "learning_rate": 2e-05,
+      "loss": 0.5821,
+      "step": 1040
+    },
+    {
+      "epoch": 0.09228335384074529,
+      "grad_norm": 0.3853510618209839,
+      "learning_rate": 2e-05,
+      "loss": 0.6217,
+      "step": 1050
+    },
+    {
+      "epoch": 0.09316224292494288,
+      "grad_norm": 0.39107996225357056,
+      "learning_rate": 2e-05,
+      "loss": 0.5619,
+      "step": 1060
+    },
+    {
+      "epoch": 0.09404113200914045,
+      "grad_norm": 0.3326006531715393,
+      "learning_rate": 2e-05,
+      "loss": 0.5712,
+      "step": 1070
+    },
+    {
+      "epoch": 0.09492002109333803,
+      "grad_norm": 0.5044306516647339,
+      "learning_rate": 2e-05,
+      "loss": 0.5799,
+      "step": 1080
+    },
+    {
+      "epoch": 0.0957989101775356,
+      "grad_norm": 0.4056992530822754,
+      "learning_rate": 2e-05,
+      "loss": 0.5592,
+      "step": 1090
+    },
+    {
+      "epoch": 0.09667779926173317,
+      "grad_norm": 0.42873504757881165,
+      "learning_rate": 2e-05,
+      "loss": 0.5932,
+      "step": 1100
+    },
+    {
+      "epoch": 0.09755668834593074,
+      "grad_norm": 0.3402392566204071,
+      "learning_rate": 2e-05,
+      "loss": 0.5675,
+      "step": 1110
+    },
+    {
+      "epoch": 0.09843557743012832,
+      "grad_norm": 0.36879539489746094,
+      "learning_rate": 2e-05,
+      "loss": 0.5393,
+      "step": 1120
+    },
+    {
+      "epoch": 0.09931446651432589,
+      "grad_norm": 0.3497093617916107,
+      "learning_rate": 2e-05,
+      "loss": 0.6054,
+      "step": 1130
+    },
+    {
+      "epoch": 0.10019335559852346,
+      "grad_norm": 0.3965706527233124,
+      "learning_rate": 2e-05,
+      "loss": 0.5851,
+      "step": 1140
+    },
+    {
+      "epoch": 0.10107224468272104,
+      "grad_norm": 0.35941794514656067,
+      "learning_rate": 2e-05,
+      "loss": 0.5703,
+      "step": 1150
+    },
+    {
+      "epoch": 0.10195113376691861,
+      "grad_norm": 0.3452138900756836,
+      "learning_rate": 2e-05,
+      "loss": 0.5854,
+      "step": 1160
+    },
+    {
+      "epoch": 0.10283002285111618,
+      "grad_norm": 0.38294243812561035,
+      "learning_rate": 2e-05,
+      "loss": 0.5553,
+      "step": 1170
+    },
+    {
+      "epoch": 0.10370891193531376,
+      "grad_norm": 0.3560450077056885,
+      "learning_rate": 2e-05,
+      "loss": 0.5847,
+      "step": 1180
+    },
+    {
+      "epoch": 0.10458780101951133,
+      "grad_norm": 0.34364673495292664,
+      "learning_rate": 2e-05,
+      "loss": 0.5627,
+      "step": 1190
+    },
+    {
+      "epoch": 0.10546669010370892,
+      "grad_norm": 0.3834216892719269,
+      "learning_rate": 2e-05,
+      "loss": 0.5617,
+      "step": 1200
+    },
+    {
+      "epoch": 0.10634557918790649,
+      "grad_norm": 0.34551456570625305,
+      "learning_rate": 2e-05,
+      "loss": 0.555,
+      "step": 1210
+    },
+    {
+      "epoch": 0.10722446827210406,
+      "grad_norm": 0.3550628423690796,
+      "learning_rate": 2e-05,
+      "loss": 0.569,
+      "step": 1220
+    },
+    {
+      "epoch": 0.10810335735630164,
+      "grad_norm": 0.39383211731910706,
+      "learning_rate": 2e-05,
+      "loss": 0.5475,
+      "step": 1230
+    },
+    {
+      "epoch": 0.10898224644049921,
+      "grad_norm": 0.45007845759391785,
+      "learning_rate": 2e-05,
+      "loss": 0.5749,
+      "step": 1240
+    },
+    {
+      "epoch": 0.10986113552469678,
+      "grad_norm": 0.33808740973472595,
+      "learning_rate": 2e-05,
+      "loss": 0.5613,
+      "step": 1250
+    },
+    {
+      "epoch": 0.11074002460889436,
+      "grad_norm": 0.3664572238922119,
+      "learning_rate": 2e-05,
+      "loss": 0.5525,
+      "step": 1260
+    },
+    {
+      "epoch": 0.11161891369309193,
+      "grad_norm": 0.3647216260433197,
+      "learning_rate": 2e-05,
+      "loss": 0.5793,
+      "step": 1270
+    },
+    {
+      "epoch": 0.1124978027772895,
+      "grad_norm": 0.3669637441635132,
+      "learning_rate": 2e-05,
+      "loss": 0.5572,
+      "step": 1280
+    },
+    {
+      "epoch": 0.11337669186148708,
+      "grad_norm": 0.3178112506866455,
+      "learning_rate": 2e-05,
+      "loss": 0.5509,
+      "step": 1290
+    },
+    {
+      "epoch": 0.11425558094568465,
+      "grad_norm": 0.360220342874527,
+      "learning_rate": 2e-05,
+      "loss": 0.5589,
+      "step": 1300
+    },
+    {
+      "epoch": 0.11513447002988222,
+      "grad_norm": 0.36686965823173523,
+      "learning_rate": 2e-05,
+      "loss": 0.5752,
+      "step": 1310
+    },
+    {
+      "epoch": 0.1160133591140798,
+      "grad_norm": 0.357736200094223,
+      "learning_rate": 2e-05,
+      "loss": 0.5901,
+      "step": 1320
+    },
+    {
+      "epoch": 0.11689224819827737,
+      "grad_norm": 0.3455224633216858,
+      "learning_rate": 2e-05,
+      "loss": 0.5614,
+      "step": 1330
+    },
+    {
+      "epoch": 0.11777113728247496,
+      "grad_norm": 0.34142857789993286,
+      "learning_rate": 2e-05,
+      "loss": 0.5563,
+      "step": 1340
+    },
+    {
+      "epoch": 0.11865002636667253,
+      "grad_norm": 0.33547478914260864,
+      "learning_rate": 2e-05,
+      "loss": 0.5347,
+      "step": 1350
+    },
+    {
+      "epoch": 0.1195289154508701,
+      "grad_norm": 0.3595579266548157,
+      "learning_rate": 2e-05,
+      "loss": 0.5747,
+      "step": 1360
+    },
+    {
+      "epoch": 0.12040780453506768,
+      "grad_norm": 0.4105367362499237,
+      "learning_rate": 2e-05,
+      "loss": 0.5848,
+      "step": 1370
+    },
+    {
+      "epoch": 0.12128669361926525,
+      "grad_norm": 0.45205193758010864,
+      "learning_rate": 2e-05,
+      "loss": 0.5863,
+      "step": 1380
+    },
+    {
+      "epoch": 0.12216558270346282,
+      "grad_norm": 0.3702628016471863,
+      "learning_rate": 2e-05,
+      "loss": 0.5727,
+      "step": 1390
+    },
+    {
+      "epoch": 0.1230444717876604,
+      "grad_norm": 0.3627513647079468,
+      "learning_rate": 2e-05,
+      "loss": 0.554,
+      "step": 1400
+    },
+    {
+      "epoch": 0.12392336087185797,
+      "grad_norm": 0.37213680148124695,
+      "learning_rate": 2e-05,
+      "loss": 0.5441,
+      "step": 1410
+    },
+    {
+      "epoch": 0.12480224995605554,
+      "grad_norm": 0.33166980743408203,
+      "learning_rate": 2e-05,
+      "loss": 0.5596,
+      "step": 1420
+    },
+    {
+      "epoch": 0.12568113904025313,
+      "grad_norm": 0.36080440878868103,
+      "learning_rate": 2e-05,
+      "loss": 0.605,
+      "step": 1430
+    },
+    {
+      "epoch": 0.1265600281244507,
+      "grad_norm": 0.39948782324790955,
+      "learning_rate": 2e-05,
+      "loss": 0.5642,
+      "step": 1440
+    },
+    {
+      "epoch": 0.12743891720864828,
+      "grad_norm": 0.3613622784614563,
+      "learning_rate": 2e-05,
+      "loss": 0.5624,
+      "step": 1450
+    },
+    {
+      "epoch": 0.12831780629284584,
+      "grad_norm": 0.41446202993392944,
+      "learning_rate": 2e-05,
+      "loss": 0.5966,
+      "step": 1460
+    },
+    {
+      "epoch": 0.12919669537704342,
+      "grad_norm": 0.3632044196128845,
+      "learning_rate": 2e-05,
+      "loss": 0.558,
+      "step": 1470
+    },
+    {
+      "epoch": 0.13007558446124098,
+      "grad_norm": 0.3540528118610382,
+      "learning_rate": 2e-05,
+      "loss": 0.5239,
+      "step": 1480
+    },
+    {
+      "epoch": 0.13095447354543857,
+      "grad_norm": 0.43492934107780457,
+      "learning_rate": 2e-05,
+      "loss": 0.5639,
+      "step": 1490
+    },
+    {
+      "epoch": 0.13183336262963613,
+      "grad_norm": 0.3482060432434082,
+      "learning_rate": 2e-05,
+      "loss": 0.5946,
+      "step": 1500
+    },
+    {
+      "epoch": 0.13271225171383372,
+      "grad_norm": 0.34748104214668274,
+      "learning_rate": 2e-05,
+      "loss": 0.5628,
+      "step": 1510
+    },
+    {
+      "epoch": 0.13359114079803128,
+      "grad_norm": 0.3222218453884125,
+      "learning_rate": 2e-05,
+      "loss": 0.5933,
+      "step": 1520
+    },
+    {
+      "epoch": 0.13447002988222886,
+      "grad_norm": 0.3792349398136139,
+      "learning_rate": 2e-05,
+      "loss": 0.562,
+      "step": 1530
+    },
+    {
+      "epoch": 0.13534891896642642,
+      "grad_norm": 0.37555038928985596,
+      "learning_rate": 2e-05,
+      "loss": 0.567,
+      "step": 1540
+    },
+    {
+      "epoch": 0.136227808050624,
+      "grad_norm": 0.3451753854751587,
+      "learning_rate": 2e-05,
+      "loss": 0.5585,
+      "step": 1550
+    },
+    {
+      "epoch": 0.1371066971348216,
+      "grad_norm": 0.32497790455818176,
+      "learning_rate": 2e-05,
+      "loss": 0.5659,
+      "step": 1560
+    },
+    {
+      "epoch": 0.13798558621901916,
+      "grad_norm": 0.35301199555397034,
+      "learning_rate": 2e-05,
+      "loss": 0.6012,
+      "step": 1570
+    },
+    {
+      "epoch": 0.13886447530321674,
+      "grad_norm": 0.3340965807437897,
+      "learning_rate": 2e-05,
+      "loss": 0.6125,
+      "step": 1580
+    },
+    {
+      "epoch": 0.1397433643874143,
+      "grad_norm": 0.3181551396846771,
+      "learning_rate": 2e-05,
+      "loss": 0.5529,
+      "step": 1590
+    },
+    {
+      "epoch": 0.1406222534716119,
+      "grad_norm": 0.3974955677986145,
+      "learning_rate": 2e-05,
+      "loss": 0.585,
+      "step": 1600
+    },
+    {
+      "epoch": 0.14150114255580945,
+      "grad_norm": 0.347336083650589,
+      "learning_rate": 2e-05,
+      "loss": 0.5606,
+      "step": 1610
+    },
+    {
+      "epoch": 0.14238003164000704,
+      "grad_norm": 0.3495638072490692,
+      "learning_rate": 2e-05,
+      "loss": 0.5906,
+      "step": 1620
+    },
+    {
+      "epoch": 0.1432589207242046,
+      "grad_norm": 0.3735026717185974,
+      "learning_rate": 2e-05,
+      "loss": 0.5591,
+      "step": 1630
+    },
+    {
+      "epoch": 0.14413780980840218,
+      "grad_norm": 0.3471795618534088,
+      "learning_rate": 2e-05,
+      "loss": 0.5705,
+      "step": 1640
+    },
+    {
+      "epoch": 0.14501669889259974,
+      "grad_norm": 0.36941561102867126,
+      "learning_rate": 2e-05,
+      "loss": 0.6011,
+      "step": 1650
+    },
+    {
+      "epoch": 0.14589558797679733,
+      "grad_norm": 0.3396647870540619,
+      "learning_rate": 2e-05,
+      "loss": 0.5717,
+      "step": 1660
+    },
+    {
+      "epoch": 0.1467744770609949,
+      "grad_norm": 0.31963610649108887,
+      "learning_rate": 2e-05,
+      "loss": 0.5655,
+      "step": 1670
+    },
+    {
+      "epoch": 0.14765336614519248,
+      "grad_norm": 0.44198107719421387,
+      "learning_rate": 2e-05,
+      "loss": 0.5869,
+      "step": 1680
+    },
+    {
+      "epoch": 0.14853225522939006,
+      "grad_norm": 0.33693188428878784,
+      "learning_rate": 2e-05,
+      "loss": 0.5459,
+      "step": 1690
+    },
+    {
+      "epoch": 0.14941114431358762,
+      "grad_norm": 0.3307777941226959,
+      "learning_rate": 2e-05,
+      "loss": 0.5901,
+      "step": 1700
+    },
+    {
+      "epoch": 0.1502900333977852,
+      "grad_norm": 0.3152621388435364,
+      "learning_rate": 2e-05,
+      "loss": 0.5596,
+      "step": 1710
+    },
+    {
+      "epoch": 0.15116892248198277,
+      "grad_norm": 0.5023934841156006,
+      "learning_rate": 2e-05,
+      "loss": 0.5634,
+      "step": 1720
+    },
+    {
+      "epoch": 0.15204781156618036,
+      "grad_norm": 0.3895277678966522,
+      "learning_rate": 2e-05,
+      "loss": 0.5763,
+      "step": 1730
+    },
+    {
+      "epoch": 0.15292670065037792,
+      "grad_norm": 0.3810598850250244,
+      "learning_rate": 2e-05,
+      "loss": 0.576,
+      "step": 1740
+    },
+    {
+      "epoch": 0.1538055897345755,
+      "grad_norm": 0.3689657151699066,
+      "learning_rate": 2e-05,
+      "loss": 0.5474,
+      "step": 1750
+    },
+    {
+      "epoch": 0.15468447881877306,
+      "grad_norm": 0.3715059459209442,
+      "learning_rate": 2e-05,
+      "loss": 0.5491,
+      "step": 1760
+    },
+    {
+      "epoch": 0.15556336790297065,
+      "grad_norm": 0.33421632647514343,
+      "learning_rate": 2e-05,
+      "loss": 0.5425,
+      "step": 1770
+    },
+    {
+      "epoch": 0.1564422569871682,
+      "grad_norm": 0.5304133296012878,
+      "learning_rate": 2e-05,
+      "loss": 0.5716,
+      "step": 1780
+    },
+    {
+      "epoch": 0.1573211460713658,
+      "grad_norm": 0.3863163888454437,
+      "learning_rate": 2e-05,
+      "loss": 0.5795,
+      "step": 1790
+    },
+    {
+      "epoch": 0.15820003515556336,
+      "grad_norm": 0.5527595281600952,
+      "learning_rate": 2e-05,
+      "loss": 0.5561,
+      "step": 1800
+    },
+    {
+      "epoch": 0.15907892423976094,
+      "grad_norm": 0.4207919239997864,
+      "learning_rate": 2e-05,
+      "loss": 0.5837,
+      "step": 1810
+    },
+    {
+      "epoch": 0.1599578133239585,
+      "grad_norm": 0.29921954870224,
+      "learning_rate": 2e-05,
+      "loss": 0.5299,
+      "step": 1820
+    },
+    {
+      "epoch": 0.1608367024081561,
+      "grad_norm": 0.34096500277519226,
+      "learning_rate": 2e-05,
+      "loss": 0.5767,
+      "step": 1830
+    },
+    {
+      "epoch": 0.16171559149235368,
+      "grad_norm": 0.37093764543533325,
+      "learning_rate": 2e-05,
+      "loss": 0.5597,
+      "step": 1840
+    },
+    {
+      "epoch": 0.16259448057655124,
+      "grad_norm": 0.392698734998703,
+      "learning_rate": 2e-05,
+      "loss": 0.5546,
+      "step": 1850
+    },
+    {
+      "epoch": 0.16347336966074882,
+      "grad_norm": 0.37084656953811646,
+      "learning_rate": 2e-05,
+      "loss": 0.564,
+      "step": 1860
+    },
+    {
+      "epoch": 0.16435225874494638,
+      "grad_norm": 0.5617843866348267,
+      "learning_rate": 2e-05,
+      "loss": 0.5669,
+      "step": 1870
+    },
+    {
+      "epoch": 0.16523114782914397,
+      "grad_norm": 0.3692229390144348,
+      "learning_rate": 2e-05,
+      "loss": 0.5628,
+      "step": 1880
+    },
+    {
+      "epoch": 0.16611003691334153,
+      "grad_norm": 0.3360963761806488,
+      "learning_rate": 2e-05,
+      "loss": 0.551,
+      "step": 1890
+    },
+    {
+      "epoch": 0.16698892599753912,
+      "grad_norm": 0.3678745925426483,
+      "learning_rate": 2e-05,
+      "loss": 0.5376,
+      "step": 1900
+    },
+    {
+      "epoch": 0.16786781508173668,
+      "grad_norm": 0.3097802996635437,
+      "learning_rate": 2e-05,
+      "loss": 0.5924,
+      "step": 1910
+    },
+    {
+      "epoch": 0.16874670416593426,
+      "grad_norm": 0.3742968738079071,
+      "learning_rate": 2e-05,
+      "loss": 0.5538,
+      "step": 1920
+    },
+    {
+      "epoch": 0.16962559325013182,
+      "grad_norm": 0.3674496114253998,
+      "learning_rate": 2e-05,
+      "loss": 0.5837,
+      "step": 1930
+    },
+    {
+      "epoch": 0.1705044823343294,
+      "grad_norm": 0.41291990876197815,
+      "learning_rate": 2e-05,
+      "loss": 0.5453,
+      "step": 1940
+    },
+    {
+      "epoch": 0.17138337141852697,
+      "grad_norm": 0.35975903272628784,
+      "learning_rate": 2e-05,
+      "loss": 0.5478,
+      "step": 1950
+    },
+    {
+      "epoch": 0.17226226050272456,
+      "grad_norm": 0.31317245960235596,
+      "learning_rate": 2e-05,
+      "loss": 0.5891,
+      "step": 1960
+    },
+    {
+      "epoch": 0.17314114958692214,
+      "grad_norm": 0.3947499394416809,
+      "learning_rate": 2e-05,
+      "loss": 0.5555,
+      "step": 1970
+    },
+    {
+      "epoch": 0.1740200386711197,
+      "grad_norm": 0.41339772939682007,
+      "learning_rate": 2e-05,
+      "loss": 0.5683,
+      "step": 1980
+    },
+    {
+      "epoch": 0.1748989277553173,
+      "grad_norm": 0.3648429811000824,
+      "learning_rate": 2e-05,
+      "loss": 0.5639,
+      "step": 1990
+    },
+    {
+      "epoch": 0.17577781683951485,
+      "grad_norm": 0.31032198667526245,
+      "learning_rate": 2e-05,
+      "loss": 0.5613,
+      "step": 2000
+    },
+    {
+      "epoch": 0.17665670592371244,
+      "grad_norm": 0.3212377429008484,
+      "learning_rate": 2e-05,
+      "loss": 0.5501,
+      "step": 2010
+    },
+    {
+      "epoch": 0.17753559500791,
+      "grad_norm": 0.34276482462882996,
+      "learning_rate": 2e-05,
+      "loss": 0.5801,
+      "step": 2020
+    },
+    {
+      "epoch": 0.17841448409210758,
+      "grad_norm": 0.32408374547958374,
+      "learning_rate": 2e-05,
+      "loss": 0.574,
+      "step": 2030
+    },
+    {
+      "epoch": 0.17929337317630514,
+      "grad_norm": 0.31184566020965576,
+      "learning_rate": 2e-05,
+      "loss": 0.5566,
+      "step": 2040
+    },
+    {
+      "epoch": 0.18017226226050273,
+      "grad_norm": 0.32635489106178284,
+      "learning_rate": 2e-05,
+      "loss": 0.5475,
+      "step": 2050
+    },
+    {
+      "epoch": 0.1810511513447003,
+      "grad_norm": 0.3519839644432068,
+      "learning_rate": 2e-05,
+      "loss": 0.5652,
+      "step": 2060
+    },
+    {
+      "epoch": 0.18193004042889788,
+      "grad_norm": 0.37629371881484985,
+      "learning_rate": 2e-05,
+      "loss": 0.5521,
+      "step": 2070
+    },
+    {
+      "epoch": 0.18280892951309544,
+      "grad_norm": 0.3756372034549713,
+      "learning_rate": 2e-05,
+      "loss": 0.5621,
+      "step": 2080
+    },
+    {
+      "epoch": 0.18368781859729302,
+      "grad_norm": 0.3752441704273224,
+      "learning_rate": 2e-05,
+      "loss": 0.5802,
+      "step": 2090
+    },
+    {
+      "epoch": 0.18456670768149058,
+      "grad_norm": 0.3569568693637848,
+      "learning_rate": 2e-05,
+      "loss": 0.5828,
+      "step": 2100
+    },
+    {
+      "epoch": 0.18544559676568817,
+      "grad_norm": 0.32334834337234497,
+      "learning_rate": 2e-05,
+      "loss": 0.5479,
+      "step": 2110
+    },
+    {
+      "epoch": 0.18632448584988576,
+      "grad_norm": 0.35802629590034485,
+      "learning_rate": 2e-05,
+      "loss": 0.566,
+      "step": 2120
+    },
+    {
+      "epoch": 0.18720337493408332,
+      "grad_norm": 0.5078518986701965,
+      "learning_rate": 2e-05,
+      "loss": 0.5586,
+      "step": 2130
+    },
+    {
+      "epoch": 0.1880822640182809,
+      "grad_norm": 0.36526328325271606,
+      "learning_rate": 2e-05,
+      "loss": 0.5616,
+      "step": 2140
+    },
+    {
+      "epoch": 0.18896115310247846,
+      "grad_norm": 0.33778101205825806,
+      "learning_rate": 2e-05,
+      "loss": 0.5666,
+      "step": 2150
+    },
+    {
+      "epoch": 0.18984004218667605,
+      "grad_norm": 0.38717159628868103,
+      "learning_rate": 2e-05,
+      "loss": 0.5605,
+      "step": 2160
+    },
+    {
+      "epoch": 0.1907189312708736,
+      "grad_norm": 0.325623095035553,
+      "learning_rate": 2e-05,
+      "loss": 0.5419,
+      "step": 2170
+    },
+    {
+      "epoch": 0.1915978203550712,
+      "grad_norm": 0.4406554400920868,
+      "learning_rate": 2e-05,
+      "loss": 0.5856,
+      "step": 2180
+    },
+    {
+      "epoch": 0.19247670943926876,
+      "grad_norm": 0.3608305752277374,
+      "learning_rate": 2e-05,
+      "loss": 0.5397,
+      "step": 2190
+    },
+    {
+      "epoch": 0.19335559852346634,
+      "grad_norm": 0.4066288471221924,
+      "learning_rate": 2e-05,
+      "loss": 0.5884,
+      "step": 2200
+    },
+    {
+      "epoch": 0.1942344876076639,
+      "grad_norm": 0.37264975905418396,
+      "learning_rate": 2e-05,
+      "loss": 0.6049,
+      "step": 2210
+    },
+    {
+      "epoch": 0.1951133766918615,
+      "grad_norm": 0.3412500321865082,
+      "learning_rate": 2e-05,
+      "loss": 0.5679,
+      "step": 2220
+    },
+    {
+      "epoch": 0.19599226577605905,
+      "grad_norm": 0.36977285146713257,
+      "learning_rate": 2e-05,
+      "loss": 0.5589,
+      "step": 2230
+    },
+    {
+      "epoch": 0.19687115486025664,
+      "grad_norm": 0.320394903421402,
+      "learning_rate": 2e-05,
+      "loss": 0.5536,
+      "step": 2240
+    },
+    {
+      "epoch": 0.19775004394445422,
+      "grad_norm": 0.4130312502384186,
+      "learning_rate": 2e-05,
+      "loss": 0.5678,
+      "step": 2250
+    },
+    {
+      "epoch": 0.19862893302865178,
+      "grad_norm": 0.3542708158493042,
+      "learning_rate": 2e-05,
+      "loss": 0.5871,
+      "step": 2260
+    },
+    {
+      "epoch": 0.19950782211284937,
+      "grad_norm": 0.36599817872047424,
+      "learning_rate": 2e-05,
+      "loss": 0.5481,
+      "step": 2270
+    },
+    {
+      "epoch": 0.20038671119704693,
+      "grad_norm": 0.42361220717430115,
+      "learning_rate": 2e-05,
+      "loss": 0.546,
+      "step": 2280
+    },
+    {
+      "epoch": 0.20126560028124452,
+      "grad_norm": 0.3402586579322815,
+      "learning_rate": 2e-05,
+      "loss": 0.58,
+      "step": 2290
+    },
+    {
+      "epoch": 0.20214448936544208,
+      "grad_norm": 0.3884638845920563,
+      "learning_rate": 2e-05,
+      "loss": 0.5393,
+      "step": 2300
+    },
+    {
+      "epoch": 0.20302337844963966,
+      "grad_norm": 0.366931676864624,
+      "learning_rate": 2e-05,
+      "loss": 0.596,
+      "step": 2310
+    },
+    {
+      "epoch": 0.20390226753383722,
+      "grad_norm": 0.32991132140159607,
+      "learning_rate": 2e-05,
+      "loss": 0.5522,
+      "step": 2320
+    },
+    {
+      "epoch": 0.2047811566180348,
+      "grad_norm": 0.39930519461631775,
+      "learning_rate": 2e-05,
+      "loss": 0.5836,
+      "step": 2330
+    },
+    {
+      "epoch": 0.20566004570223237,
+      "grad_norm": 0.37963399291038513,
+      "learning_rate": 2e-05,
+      "loss": 0.548,
+      "step": 2340
+    },
+    {
+      "epoch": 0.20653893478642996,
+      "grad_norm": 0.365708589553833,
+      "learning_rate": 2e-05,
+      "loss": 0.5646,
+      "step": 2350
+    },
+    {
+      "epoch": 0.20741782387062752,
+      "grad_norm": 0.38639870285987854,
+      "learning_rate": 2e-05,
+      "loss": 0.5348,
+      "step": 2360
+    },
+    {
+      "epoch": 0.2082967129548251,
+      "grad_norm": 0.31267666816711426,
+      "learning_rate": 2e-05,
+      "loss": 0.541,
+      "step": 2370
+    },
+    {
+      "epoch": 0.20917560203902266,
+      "grad_norm": 0.42928454279899597,
+      "learning_rate": 2e-05,
+      "loss": 0.5665,
+      "step": 2380
+    },
+    {
+      "epoch": 0.21005449112322025,
+      "grad_norm": 0.3244638442993164,
+      "learning_rate": 2e-05,
+      "loss": 0.5904,
+      "step": 2390
+    },
+    {
+      "epoch": 0.21093338020741784,
+      "grad_norm": 0.30141741037368774,
+      "learning_rate": 2e-05,
+      "loss": 0.5446,
+      "step": 2400
+    },
+    {
+      "epoch": 0.2118122692916154,
+      "grad_norm": 0.3784339427947998,
+      "learning_rate": 2e-05,
+      "loss": 0.5936,
+      "step": 2410
+    },
+    {
+      "epoch": 0.21269115837581298,
+      "grad_norm": 0.40315935015678406,
+      "learning_rate": 2e-05,
+      "loss": 0.5707,
+      "step": 2420
+    },
+    {
+      "epoch": 0.21357004746001054,
+      "grad_norm": 0.35042673349380493,
+      "learning_rate": 2e-05,
+      "loss": 0.5684,
+      "step": 2430
+    },
+    {
+      "epoch": 0.21444893654420813,
+      "grad_norm": 0.37262678146362305,
+      "learning_rate": 2e-05,
+      "loss": 0.5531,
+      "step": 2440
+    },
+    {
+      "epoch": 0.2153278256284057,
+      "grad_norm": 0.3180401027202606,
+      "learning_rate": 2e-05,
+      "loss": 0.5727,
+      "step": 2450
+    },
+    {
+      "epoch": 0.21620671471260328,
+      "grad_norm": 0.3344835937023163,
+      "learning_rate": 2e-05,
+      "loss": 0.5707,
+      "step": 2460
+    },
+    {
+      "epoch": 0.21708560379680084,
+      "grad_norm": 0.3284481167793274,
+      "learning_rate": 2e-05,
+      "loss": 0.5497,
+      "step": 2470
+    },
+    {
+      "epoch": 0.21796449288099842,
+      "grad_norm": 0.37181684374809265,
+      "learning_rate": 2e-05,
+      "loss": 0.5904,
+      "step": 2480
+    },
+    {
+      "epoch": 0.21884338196519598,
+      "grad_norm": 0.3416779637336731,
+      "learning_rate": 2e-05,
+      "loss": 0.5507,
+      "step": 2490
+    },
+    {
+      "epoch": 0.21972227104939357,
+      "grad_norm": 0.347848504781723,
+      "learning_rate": 2e-05,
+      "loss": 0.5684,
+      "step": 2500
+    },
+    {
+      "epoch": 0.22060116013359113,
+      "grad_norm": 0.3230196237564087,
+      "learning_rate": 2e-05,
+      "loss": 0.5456,
+      "step": 2510
+    },
+    {
+      "epoch": 0.22148004921778872,
+      "grad_norm": 0.38809800148010254,
+      "learning_rate": 2e-05,
+      "loss": 0.5517,
+      "step": 2520
+    },
+    {
+      "epoch": 0.2223589383019863,
+      "grad_norm": 0.3716602027416229,
+      "learning_rate": 2e-05,
+      "loss": 0.5606,
+      "step": 2530
+    },
+    {
+      "epoch": 0.22323782738618386,
+      "grad_norm": 0.342180460691452,
+      "learning_rate": 2e-05,
+      "loss": 0.55,
+      "step": 2540
+    },
+    {
+      "epoch": 0.22411671647038145,
+      "grad_norm": 0.40020951628685,
+      "learning_rate": 2e-05,
+      "loss": 0.5326,
+      "step": 2550
+    },
+    {
+      "epoch": 0.224995605554579,
+      "grad_norm": 0.37780147790908813,
+      "learning_rate": 2e-05,
+      "loss": 0.5473,
+      "step": 2560
+    },
+    {
+      "epoch": 0.2258744946387766,
+      "grad_norm": 0.3578275740146637,
+      "learning_rate": 2e-05,
+      "loss": 0.5474,
+      "step": 2570
+    },
+    {
+      "epoch": 0.22675338372297416,
+      "grad_norm": 0.33693841099739075,
+      "learning_rate": 2e-05,
+      "loss": 0.5686,
+      "step": 2580
+    },
+    {
+      "epoch": 0.22763227280717174,
+      "grad_norm": 0.41994157433509827,
+      "learning_rate": 2e-05,
+      "loss": 0.5629,
+      "step": 2590
+    },
+    {
+      "epoch": 0.2285111618913693,
+      "grad_norm": 0.3703048527240753,
+      "learning_rate": 2e-05,
+      "loss": 0.5535,
+      "step": 2600
+    },
+    {
+      "epoch": 0.2293900509755669,
+      "grad_norm": 0.3215305805206299,
+      "learning_rate": 2e-05,
+      "loss": 0.5304,
+      "step": 2610
+    },
+    {
+      "epoch": 0.23026894005976445,
+      "grad_norm": 0.3829116225242615,
+      "learning_rate": 2e-05,
+      "loss": 0.5717,
+      "step": 2620
+    },
+    {
+      "epoch": 0.23114782914396204,
+      "grad_norm": 0.31442129611968994,
+      "learning_rate": 2e-05,
+      "loss": 0.5543,
+      "step": 2630
+    },
+    {
+      "epoch": 0.2320267182281596,
+      "grad_norm": 0.46074509620666504,
+      "learning_rate": 2e-05,
+      "loss": 0.5811,
+      "step": 2640
+    },
+    {
+      "epoch": 0.23290560731235718,
+      "grad_norm": 0.37541741132736206,
+      "learning_rate": 2e-05,
+      "loss": 0.5561,
+      "step": 2650
+    },
+    {
+      "epoch": 0.23378449639655474,
+      "grad_norm": 0.32522743940353394,
+      "learning_rate": 2e-05,
+      "loss": 0.5649,
+      "step": 2660
+    },
+    {
+      "epoch": 0.23466338548075233,
+      "grad_norm": 0.3570707142353058,
+      "learning_rate": 2e-05,
+      "loss": 0.5615,
+      "step": 2670
+    },
+    {
+      "epoch": 0.23554227456494992,
+      "grad_norm": 0.33403363823890686,
+      "learning_rate": 2e-05,
+      "loss": 0.5733,
+      "step": 2680
+    },
+    {
+      "epoch": 0.23642116364914748,
+      "grad_norm": 0.3274398744106293,
+      "learning_rate": 2e-05,
+      "loss": 0.5624,
+      "step": 2690
+    },
+    {
+      "epoch": 0.23730005273334506,
+      "grad_norm": 0.35692623257637024,
+      "learning_rate": 2e-05,
+      "loss": 0.5733,
+      "step": 2700
+    },
+    {
+      "epoch": 0.23817894181754262,
+      "grad_norm": 0.3499510586261749,
+      "learning_rate": 2e-05,
+      "loss": 0.581,
+      "step": 2710
+    },
+    {
+      "epoch": 0.2390578309017402,
+      "grad_norm": 0.3738914430141449,
+      "learning_rate": 2e-05,
+      "loss": 0.5481,
+      "step": 2720
+    },
+    {
+      "epoch": 0.23993671998593777,
+      "grad_norm": 0.4006647765636444,
+      "learning_rate": 2e-05,
+      "loss": 0.5337,
+      "step": 2730
+    },
+    {
+      "epoch": 0.24081560907013536,
+      "grad_norm": 0.5295825600624084,
+      "learning_rate": 2e-05,
+      "loss": 0.561,
+      "step": 2740
+    },
+    {
+      "epoch": 0.24169449815433292,
+      "grad_norm": 0.34328094124794006,
+      "learning_rate": 2e-05,
+      "loss": 0.5517,
+      "step": 2750
+    },
+    {
+      "epoch": 0.2425733872385305,
+      "grad_norm": 0.36072251200675964,
+      "learning_rate": 2e-05,
+      "loss": 0.5486,
+      "step": 2760
+    },
+    {
+      "epoch": 0.24345227632272806,
+      "grad_norm": 0.32866302132606506,
+      "learning_rate": 2e-05,
+      "loss": 0.5764,
+      "step": 2770
+    },
+    {
+      "epoch": 0.24433116540692565,
+      "grad_norm": 0.3333951532840729,
+      "learning_rate": 2e-05,
+      "loss": 0.5775,
+      "step": 2780
+    },
+    {
+      "epoch": 0.2452100544911232,
+      "grad_norm": 0.41300830245018005,
+      "learning_rate": 2e-05,
+      "loss": 0.5609,
+      "step": 2790
+    },
+    {
+      "epoch": 0.2460889435753208,
+      "grad_norm": 0.3127140700817108,
+      "learning_rate": 2e-05,
+      "loss": 0.5711,
+      "step": 2800
+    },
+    {
+      "epoch": 0.24696783265951838,
+      "grad_norm": 0.33722373843193054,
+      "learning_rate": 2e-05,
+      "loss": 0.5421,
+      "step": 2810
+    },
+    {
+      "epoch": 0.24784672174371594,
+      "grad_norm": 0.3545294404029846,
+      "learning_rate": 2e-05,
+      "loss": 0.568,
+      "step": 2820
+    },
+    {
+      "epoch": 0.24872561082791353,
+      "grad_norm": 0.345797598361969,
+      "learning_rate": 2e-05,
+      "loss": 0.5306,
+      "step": 2830
+    },
+    {
+      "epoch": 0.2496044999121111,
+      "grad_norm": 0.40061184763908386,
+      "learning_rate": 2e-05,
+      "loss": 0.5467,
+      "step": 2840
+    },
+    {
+      "epoch": 0.2504833889963087,
+      "grad_norm": 0.32904642820358276,
+      "learning_rate": 2e-05,
+      "loss": 0.5294,
+      "step": 2850
+    },
+    {
+      "epoch": 0.25136227808050626,
+      "grad_norm": 0.3608121871948242,
+      "learning_rate": 2e-05,
+      "loss": 0.5564,
+      "step": 2860
+    },
+    {
+      "epoch": 0.2522411671647038,
+      "grad_norm": 0.4847877025604248,
+      "learning_rate": 2e-05,
+      "loss": 0.5627,
+      "step": 2870
+    },
+    {
+      "epoch": 0.2531200562489014,
+      "grad_norm": 0.3074108362197876,
+      "learning_rate": 2e-05,
+      "loss": 0.5427,
+      "step": 2880
+    },
+    {
+      "epoch": 0.25399894533309897,
+      "grad_norm": 0.3400770127773285,
+      "learning_rate": 2e-05,
+      "loss": 0.5395,
+      "step": 2890
+    },
+    {
+      "epoch": 0.25487783441729656,
+      "grad_norm": 0.3570431172847748,
+      "learning_rate": 2e-05,
+      "loss": 0.5627,
+      "step": 2900
+    },
+    {
+      "epoch": 0.2557567235014941,
+      "grad_norm": 0.3542000651359558,
+      "learning_rate": 2e-05,
+      "loss": 0.5667,
+      "step": 2910
+    },
+    {
+      "epoch": 0.2566356125856917,
+      "grad_norm": 0.3159409761428833,
+      "learning_rate": 2e-05,
+      "loss": 0.5561,
+      "step": 2920
+    },
+    {
+      "epoch": 0.25751450166988926,
+      "grad_norm": 0.3355763554573059,
+      "learning_rate": 2e-05,
+      "loss": 0.553,
+      "step": 2930
+    },
+    {
+      "epoch": 0.25839339075408685,
+      "grad_norm": 0.3804934024810791,
+      "learning_rate": 2e-05,
+      "loss": 0.5515,
+      "step": 2940
+    },
+    {
+      "epoch": 0.25927227983828444,
+      "grad_norm": 0.3341805636882782,
+      "learning_rate": 2e-05,
+      "loss": 0.5695,
+      "step": 2950
+    },
+    {
+      "epoch": 0.26015116892248197,
+      "grad_norm": 0.32842281460762024,
+      "learning_rate": 2e-05,
+      "loss": 0.5395,
+      "step": 2960
+    },
+    {
+      "epoch": 0.26103005800667956,
+      "grad_norm": 0.3404848575592041,
+      "learning_rate": 2e-05,
+      "loss": 0.5427,
+      "step": 2970
+    },
+    {
+      "epoch": 0.26190894709087714,
+      "grad_norm": 0.38430777192115784,
+      "learning_rate": 2e-05,
+      "loss": 0.5637,
+      "step": 2980
+    },
+    {
+      "epoch": 0.26278783617507473,
+      "grad_norm": 0.30766552686691284,
+      "learning_rate": 2e-05,
+      "loss": 0.5596,
+      "step": 2990
+    },
+    {
+      "epoch": 0.26366672525927226,
+      "grad_norm": 0.351011723279953,
+      "learning_rate": 2e-05,
+      "loss": 0.5807,
+      "step": 3000
+    },
+    {
+      "epoch": 0.26454561434346985,
+      "grad_norm": 0.35301610827445984,
+      "learning_rate": 2e-05,
+      "loss": 0.5583,
+      "step": 3010
+    },
+    {
+      "epoch": 0.26542450342766744,
+      "grad_norm": 0.350064218044281,
+      "learning_rate": 2e-05,
+      "loss": 0.5608,
+      "step": 3020
+    },
+    {
+      "epoch": 0.266303392511865,
+      "grad_norm": 0.3541603982448578,
+      "learning_rate": 2e-05,
+      "loss": 0.5427,
+      "step": 3030
+    },
+    {
+      "epoch": 0.26718228159606255,
+      "grad_norm": 0.3428955674171448,
+      "learning_rate": 2e-05,
+      "loss": 0.5658,
+      "step": 3040
+    },
+    {
+      "epoch": 0.26806117068026014,
+      "grad_norm": 0.34655940532684326,
+      "learning_rate": 2e-05,
+      "loss": 0.5748,
+      "step": 3050
+    },
+    {
+      "epoch": 0.26894005976445773,
+      "grad_norm": 0.3420259356498718,
+      "learning_rate": 2e-05,
+      "loss": 0.5447,
+      "step": 3060
+    },
+    {
+      "epoch": 0.2698189488486553,
+      "grad_norm": 0.3394719958305359,
+      "learning_rate": 2e-05,
+      "loss": 0.5539,
+      "step": 3070
+    },
+    {
+      "epoch": 0.27069783793285285,
+      "grad_norm": 0.33060818910598755,
+      "learning_rate": 2e-05,
+      "loss": 0.576,
+      "step": 3080
+    },
+    {
+      "epoch": 0.27157672701705043,
+      "grad_norm": 0.34807607531547546,
+      "learning_rate": 2e-05,
+      "loss": 0.5524,
+      "step": 3090
+    },
+    {
+      "epoch": 0.272455616101248,
+      "grad_norm": 0.33848682045936584,
+      "learning_rate": 2e-05,
+      "loss": 0.5283,
+      "step": 3100
+    },
+    {
+      "epoch": 0.2733345051854456,
+      "grad_norm": 0.31702762842178345,
+      "learning_rate": 2e-05,
+      "loss": 0.5731,
+      "step": 3110
+    },
+    {
+      "epoch": 0.2742133942696432,
+      "grad_norm": 0.30397921800613403,
+      "learning_rate": 2e-05,
+      "loss": 0.5403,
+      "step": 3120
+    },
+    {
+      "epoch": 0.2750922833538407,
+      "grad_norm": 0.37567466497421265,
+      "learning_rate": 2e-05,
+      "loss": 0.5754,
+      "step": 3130
+    },
+    {
+      "epoch": 0.2759711724380383,
+      "grad_norm": 0.3388182520866394,
+      "learning_rate": 2e-05,
+      "loss": 0.5512,
+      "step": 3140
+    },
+    {
+      "epoch": 0.2768500615222359,
+      "grad_norm": 0.36109867691993713,
+      "learning_rate": 2e-05,
+      "loss": 0.5507,
+      "step": 3150
+    },
+    {
+      "epoch": 0.2777289506064335,
+      "grad_norm": 0.3950864374637604,
+      "learning_rate": 2e-05,
+      "loss": 0.5427,
+      "step": 3160
+    },
+    {
+      "epoch": 0.278607839690631,
+      "grad_norm": 0.32862675189971924,
+      "learning_rate": 2e-05,
+      "loss": 0.5705,
+      "step": 3170
+    },
+    {
+      "epoch": 0.2794867287748286,
+      "grad_norm": 0.3726232945919037,
+      "learning_rate": 2e-05,
+      "loss": 0.5456,
+      "step": 3180
+    },
+    {
+      "epoch": 0.2803656178590262,
+      "grad_norm": 0.3371773958206177,
+      "learning_rate": 2e-05,
+      "loss": 0.5813,
+      "step": 3190
+    },
+    {
+      "epoch": 0.2812445069432238,
+      "grad_norm": 0.3559187650680542,
+      "learning_rate": 2e-05,
+      "loss": 0.5193,
+      "step": 3200
+    },
+    {
+      "epoch": 0.2821233960274213,
+      "grad_norm": 0.3417247235774994,
+      "learning_rate": 2e-05,
+      "loss": 0.5369,
+      "step": 3210
+    },
+    {
+      "epoch": 0.2830022851116189,
+      "grad_norm": 0.3516131341457367,
+      "learning_rate": 2e-05,
+      "loss": 0.539,
+      "step": 3220
+    },
+    {
+      "epoch": 0.2838811741958165,
+      "grad_norm": 0.29232025146484375,
+      "learning_rate": 2e-05,
+      "loss": 0.5746,
+      "step": 3230
+    },
+    {
+      "epoch": 0.2847600632800141,
+      "grad_norm": 0.3936523199081421,
+      "learning_rate": 2e-05,
+      "loss": 0.5592,
+      "step": 3240
+    },
+    {
+      "epoch": 0.28563895236421166,
+      "grad_norm": 0.3399224579334259,
+      "learning_rate": 2e-05,
+      "loss": 0.5602,
+      "step": 3250
+    },
+    {
+      "epoch": 0.2865178414484092,
+      "grad_norm": 0.36498552560806274,
+      "learning_rate": 2e-05,
+      "loss": 0.5346,
+      "step": 3260
+    },
+    {
+      "epoch": 0.2873967305326068,
+      "grad_norm": 0.3364720344543457,
+      "learning_rate": 2e-05,
+      "loss": 0.5615,
+      "step": 3270
+    },
+    {
+      "epoch": 0.28827561961680437,
+      "grad_norm": 0.3698938190937042,
+      "learning_rate": 2e-05,
+      "loss": 0.5675,
+      "step": 3280
+    },
+    {
+      "epoch": 0.28915450870100196,
+      "grad_norm": 0.3517701327800751,
+      "learning_rate": 2e-05,
+      "loss": 0.5494,
+      "step": 3290
+    },
+    {
+      "epoch": 0.2900333977851995,
+      "grad_norm": 0.5226931571960449,
+      "learning_rate": 2e-05,
+      "loss": 0.5624,
+      "step": 3300
+    },
+    {
+      "epoch": 0.2909122868693971,
+      "grad_norm": 0.36056846380233765,
+      "learning_rate": 2e-05,
+      "loss": 0.5549,
+      "step": 3310
+    },
+    {
+      "epoch": 0.29179117595359466,
+      "grad_norm": 0.36480969190597534,
+      "learning_rate": 2e-05,
+      "loss": 0.555,
+      "step": 3320
+    },
+    {
+      "epoch": 0.29267006503779225,
+      "grad_norm": 0.4041381776332855,
+      "learning_rate": 2e-05,
+      "loss": 0.5707,
+      "step": 3330
+    },
+    {
+      "epoch": 0.2935489541219898,
+      "grad_norm": 0.3795798420906067,
+      "learning_rate": 2e-05,
+      "loss": 0.5224,
+      "step": 3340
+    },
+    {
+      "epoch": 0.29442784320618737,
+      "grad_norm": 0.35352370142936707,
+      "learning_rate": 2e-05,
+      "loss": 0.5392,
+      "step": 3350
+    },
+    {
+      "epoch": 0.29530673229038495,
+      "grad_norm": 0.4036523997783661,
+      "learning_rate": 2e-05,
+      "loss": 0.557,
+      "step": 3360
+    },
+    {
+      "epoch": 0.29618562137458254,
+      "grad_norm": 0.3892285227775574,
+      "learning_rate": 2e-05,
+      "loss": 0.5704,
+      "step": 3370
+    },
+    {
+      "epoch": 0.29706451045878013,
+      "grad_norm": 0.3575426936149597,
+      "learning_rate": 2e-05,
+      "loss": 0.5564,
+      "step": 3380
+    },
+    {
+      "epoch": 0.29794339954297766,
+      "grad_norm": 0.35260751843452454,
+      "learning_rate": 2e-05,
+      "loss": 0.5423,
+      "step": 3390
+    },
+    {
+      "epoch": 0.29882228862717525,
+      "grad_norm": 0.3310370445251465,
+      "learning_rate": 2e-05,
+      "loss": 0.5423,
+      "step": 3400
+    },
+    {
+      "epoch": 0.29970117771137283,
+      "grad_norm": 0.35556313395500183,
+      "learning_rate": 2e-05,
+      "loss": 0.5498,
+      "step": 3410
+    },
+    {
+      "epoch": 0.3005800667955704,
+      "grad_norm": 0.3494598865509033,
+      "learning_rate": 2e-05,
+      "loss": 0.532,
+      "step": 3420
+    },
+    {
+      "epoch": 0.30145895587976795,
+      "grad_norm": 0.32750406861305237,
+      "learning_rate": 2e-05,
+      "loss": 0.5501,
+      "step": 3430
+    },
+    {
+      "epoch": 0.30233784496396554,
+      "grad_norm": 0.34658825397491455,
+      "learning_rate": 2e-05,
+      "loss": 0.5513,
+      "step": 3440
+    },
+    {
+      "epoch": 0.30321673404816313,
+      "grad_norm": 0.3852780759334564,
+      "learning_rate": 2e-05,
+      "loss": 0.5597,
+      "step": 3450
+    },
+    {
+      "epoch": 0.3040956231323607,
+      "grad_norm": 0.3383525013923645,
+      "learning_rate": 2e-05,
+      "loss": 0.5396,
+      "step": 3460
+    },
+    {
+      "epoch": 0.30497451221655825,
+      "grad_norm": 0.32666006684303284,
+      "learning_rate": 2e-05,
+      "loss": 0.5472,
+      "step": 3470
+    },
+    {
+      "epoch": 0.30585340130075583,
+      "grad_norm": 0.3554231524467468,
+      "learning_rate": 2e-05,
+      "loss": 0.5587,
+      "step": 3480
+    },
+    {
+      "epoch": 0.3067322903849534,
+      "grad_norm": 0.3399517834186554,
+      "learning_rate": 2e-05,
+      "loss": 0.5218,
+      "step": 3490
+    },
+    {
+      "epoch": 0.307611179469151,
+      "grad_norm": 0.36986055970191956,
+      "learning_rate": 2e-05,
+      "loss": 0.5445,
+      "step": 3500
+    },
+    {
+      "epoch": 0.30849006855334854,
+      "grad_norm": 0.3844973146915436,
+      "learning_rate": 2e-05,
+      "loss": 0.548,
+      "step": 3510
+    },
+    {
+      "epoch": 0.3093689576375461,
+      "grad_norm": 0.3885951042175293,
+      "learning_rate": 2e-05,
+      "loss": 0.5599,
+      "step": 3520
+    },
+    {
+      "epoch": 0.3102478467217437,
+      "grad_norm": 0.3187924325466156,
+      "learning_rate": 2e-05,
+      "loss": 0.5647,
+      "step": 3530
+    },
+    {
+      "epoch": 0.3111267358059413,
+      "grad_norm": 0.3450549244880676,
+      "learning_rate": 2e-05,
+      "loss": 0.5657,
+      "step": 3540
+    },
+    {
+      "epoch": 0.3120056248901389,
+      "grad_norm": 0.37729015946388245,
+      "learning_rate": 2e-05,
+      "loss": 0.5515,
+      "step": 3550
+    },
+    {
+      "epoch": 0.3128845139743364,
+      "grad_norm": 0.3477257490158081,
+      "learning_rate": 2e-05,
+      "loss": 0.529,
+      "step": 3560
+    },
+    {
+      "epoch": 0.313763403058534,
+      "grad_norm": 0.4619958698749542,
+      "learning_rate": 2e-05,
+      "loss": 0.5473,
+      "step": 3570
+    },
+    {
+      "epoch": 0.3146422921427316,
+      "grad_norm": 0.4123147130012512,
+      "learning_rate": 2e-05,
+      "loss": 0.5699,
+      "step": 3580
+    },
+    {
+      "epoch": 0.3155211812269292,
+      "grad_norm": 0.331781268119812,
+      "learning_rate": 2e-05,
+      "loss": 0.5477,
+      "step": 3590
+    },
+    {
+      "epoch": 0.3164000703111267,
+      "grad_norm": 0.4143794775009155,
+      "learning_rate": 2e-05,
+      "loss": 0.5437,
+      "step": 3600
+    },
+    {
+      "epoch": 0.3172789593953243,
+      "grad_norm": 0.37911897897720337,
+      "learning_rate": 2e-05,
+      "loss": 0.5698,
+      "step": 3610
+    },
+    {
+      "epoch": 0.3181578484795219,
+      "grad_norm": 0.37786126136779785,
+      "learning_rate": 2e-05,
+      "loss": 0.5721,
+      "step": 3620
+    },
+    {
+      "epoch": 0.3190367375637195,
+      "grad_norm": 0.36114272475242615,
+      "learning_rate": 2e-05,
+      "loss": 0.559,
+      "step": 3630
+    },
+    {
+      "epoch": 0.319915626647917,
+      "grad_norm": 0.3579804599285126,
+      "learning_rate": 2e-05,
+      "loss": 0.5228,
+      "step": 3640
+    },
+    {
+      "epoch": 0.3207945157321146,
+      "grad_norm": 0.36827799677848816,
+      "learning_rate": 2e-05,
+      "loss": 0.5861,
+      "step": 3650
+    },
+    {
+      "epoch": 0.3216734048163122,
+      "grad_norm": 0.37947359681129456,
+      "learning_rate": 2e-05,
+      "loss": 0.5403,
+      "step": 3660
+    },
+    {
+      "epoch": 0.32255229390050977,
+      "grad_norm": 0.34253841638565063,
+      "learning_rate": 2e-05,
+      "loss": 0.5205,
+      "step": 3670
+    },
+    {
+      "epoch": 0.32343118298470735,
+      "grad_norm": 0.37916991114616394,
+      "learning_rate": 2e-05,
+      "loss": 0.5573,
+      "step": 3680
+    },
+    {
+      "epoch": 0.3243100720689049,
+      "grad_norm": 0.3785051703453064,
+      "learning_rate": 2e-05,
+      "loss": 0.5497,
+      "step": 3690
+    },
+    {
+      "epoch": 0.3251889611531025,
+      "grad_norm": 0.36646854877471924,
+      "learning_rate": 2e-05,
+      "loss": 0.5681,
+      "step": 3700
+    },
+    {
+      "epoch": 0.32606785023730006,
+      "grad_norm": 0.3919835090637207,
+      "learning_rate": 2e-05,
+      "loss": 0.5499,
+      "step": 3710
+    },
+    {
+      "epoch": 0.32694673932149765,
+      "grad_norm": 0.36537960171699524,
+      "learning_rate": 2e-05,
+      "loss": 0.5377,
+      "step": 3720
+    },
+    {
+      "epoch": 0.3278256284056952,
+      "grad_norm": 0.3292663097381592,
+      "learning_rate": 2e-05,
+      "loss": 0.5147,
+      "step": 3730
+    },
+    {
+      "epoch": 0.32870451748989277,
+      "grad_norm": 0.5153776407241821,
+      "learning_rate": 2e-05,
+      "loss": 0.5659,
+      "step": 3740
+    },
+    {
+      "epoch": 0.32958340657409035,
+      "grad_norm": 0.3422275483608246,
+      "learning_rate": 2e-05,
+      "loss": 0.534,
+      "step": 3750
+    },
+    {
+      "epoch": 0.33046229565828794,
+      "grad_norm": 0.3704967498779297,
+      "learning_rate": 2e-05,
+      "loss": 0.5554,
+      "step": 3760
+    },
+    {
+      "epoch": 0.3313411847424855,
+      "grad_norm": 0.3404252827167511,
+      "learning_rate": 2e-05,
+      "loss": 0.5522,
+      "step": 3770
+    },
+    {
+      "epoch": 0.33222007382668306,
+      "grad_norm": 0.3768712878227234,
+      "learning_rate": 2e-05,
+      "loss": 0.5656,
+      "step": 3780
+    },
+    {
+      "epoch": 0.33309896291088065,
+      "grad_norm": 0.3620288670063019,
+      "learning_rate": 2e-05,
+      "loss": 0.5426,
+      "step": 3790
+    },
+    {
+      "epoch": 0.33397785199507823,
+      "grad_norm": 0.3485812246799469,
+      "learning_rate": 2e-05,
+      "loss": 0.5649,
+      "step": 3800
+    },
+    {
+      "epoch": 0.3348567410792758,
+      "grad_norm": 0.3773175776004791,
+      "learning_rate": 2e-05,
+      "loss": 0.5354,
+      "step": 3810
+    },
+    {
+      "epoch": 0.33573563016347335,
+      "grad_norm": 0.3337115943431854,
+      "learning_rate": 2e-05,
+      "loss": 0.5237,
+      "step": 3820
+    },
+    {
+      "epoch": 0.33661451924767094,
+      "grad_norm": 0.3432636559009552,
+      "learning_rate": 2e-05,
+      "loss": 0.5472,
+      "step": 3830
+    },
+    {
+      "epoch": 0.3374934083318685,
+      "grad_norm": 0.34719574451446533,
+      "learning_rate": 2e-05,
+      "loss": 0.5549,
+      "step": 3840
+    },
+    {
+      "epoch": 0.3383722974160661,
+      "grad_norm": 0.4003327786922455,
+      "learning_rate": 2e-05,
+      "loss": 0.5552,
+      "step": 3850
+    },
+    {
+      "epoch": 0.33925118650026365,
+      "grad_norm": 0.33857205510139465,
+      "learning_rate": 2e-05,
+      "loss": 0.5514,
+      "step": 3860
+    },
+    {
+      "epoch": 0.34013007558446123,
+      "grad_norm": 0.39958328008651733,
+      "learning_rate": 2e-05,
+      "loss": 0.5478,
+      "step": 3870
+    },
+    {
+      "epoch": 0.3410089646686588,
+      "grad_norm": 0.3632514178752899,
+      "learning_rate": 2e-05,
+      "loss": 0.5475,
+      "step": 3880
+    },
+    {
+      "epoch": 0.3418878537528564,
+      "grad_norm": 0.3496982157230377,
+      "learning_rate": 2e-05,
+      "loss": 0.5664,
+      "step": 3890
+    },
+    {
+      "epoch": 0.34276674283705394,
+      "grad_norm": 0.36692944169044495,
+      "learning_rate": 2e-05,
+      "loss": 0.5593,
+      "step": 3900
+    },
+    {
+      "epoch": 0.3436456319212515,
+      "grad_norm": 0.36224329471588135,
+      "learning_rate": 2e-05,
+      "loss": 0.5545,
+      "step": 3910
+    },
+    {
+      "epoch": 0.3445245210054491,
+      "grad_norm": 0.3247584402561188,
+      "learning_rate": 2e-05,
+      "loss": 0.5464,
+      "step": 3920
+    },
+    {
+      "epoch": 0.3454034100896467,
+      "grad_norm": 0.3404799997806549,
+      "learning_rate": 2e-05,
+      "loss": 0.5406,
+      "step": 3930
+    },
+    {
+      "epoch": 0.3462822991738443,
+      "grad_norm": 0.3565092384815216,
+      "learning_rate": 2e-05,
+      "loss": 0.5262,
+      "step": 3940
+    },
+    {
+      "epoch": 0.3471611882580418,
+      "grad_norm": 0.379717081785202,
+      "learning_rate": 2e-05,
+      "loss": 0.5366,
+      "step": 3950
+    },
+    {
+      "epoch": 0.3480400773422394,
+      "grad_norm": 0.3445590138435364,
+      "learning_rate": 2e-05,
+      "loss": 0.5446,
+      "step": 3960
+    },
+    {
+      "epoch": 0.348918966426437,
+      "grad_norm": 0.38638412952423096,
+      "learning_rate": 2e-05,
+      "loss": 0.5126,
+      "step": 3970
+    },
+    {
+      "epoch": 0.3497978555106346,
+      "grad_norm": 0.3413693308830261,
+      "learning_rate": 2e-05,
+      "loss": 0.529,
+      "step": 3980
+    },
+    {
+      "epoch": 0.3506767445948321,
+      "grad_norm": 0.35638323426246643,
+      "learning_rate": 2e-05,
+      "loss": 0.5663,
+      "step": 3990
+    },
+    {
+      "epoch": 0.3515556336790297,
+      "grad_norm": 0.34048929810523987,
+      "learning_rate": 2e-05,
+      "loss": 0.524,
+      "step": 4000
+    },
+    {
+      "epoch": 0.3524345227632273,
+      "grad_norm": 0.40266934037208557,
+      "learning_rate": 2e-05,
+      "loss": 0.5393,
+      "step": 4010
+    },
+    {
+      "epoch": 0.3533134118474249,
+      "grad_norm": 0.3910423815250397,
+      "learning_rate": 2e-05,
+      "loss": 0.5617,
+      "step": 4020
+    },
+    {
+      "epoch": 0.3541923009316224,
+      "grad_norm": 0.39282137155532837,
+      "learning_rate": 2e-05,
+      "loss": 0.5575,
+      "step": 4030
+    },
+    {
+      "epoch": 0.35507119001582,
+      "grad_norm": 0.36524611711502075,
+      "learning_rate": 2e-05,
+      "loss": 0.559,
+      "step": 4040
+    },
+    {
+      "epoch": 0.3559500791000176,
+      "grad_norm": 0.3243862986564636,
+      "learning_rate": 2e-05,
+      "loss": 0.5457,
+      "step": 4050
+    },
+    {
+      "epoch": 0.35682896818421517,
+      "grad_norm": 0.31815096735954285,
+      "learning_rate": 2e-05,
+      "loss": 0.5532,
+      "step": 4060
+    },
+    {
+      "epoch": 0.3577078572684127,
+      "grad_norm": 0.3698352873325348,
+      "learning_rate": 2e-05,
+      "loss": 0.5708,
+      "step": 4070
+    },
+    {
+      "epoch": 0.3585867463526103,
+      "grad_norm": 0.4162037968635559,
+      "learning_rate": 2e-05,
+      "loss": 0.5794,
+      "step": 4080
+    },
+    {
+      "epoch": 0.3594656354368079,
+      "grad_norm": 0.41902992129325867,
+      "learning_rate": 2e-05,
+      "loss": 0.565,
+      "step": 4090
+    },
+    {
+      "epoch": 0.36034452452100546,
+      "grad_norm": 0.36418578028678894,
+      "learning_rate": 2e-05,
+      "loss": 0.5609,
+      "step": 4100
+    },
+    {
+      "epoch": 0.36122341360520305,
+      "grad_norm": 0.4600087106227875,
+      "learning_rate": 2e-05,
+      "loss": 0.5629,
+      "step": 4110
+    },
+    {
+      "epoch": 0.3621023026894006,
+      "grad_norm": 0.41888490319252014,
+      "learning_rate": 2e-05,
+      "loss": 0.5504,
+      "step": 4120
+    },
+    {
+      "epoch": 0.36298119177359817,
+      "grad_norm": 0.36782190203666687,
+      "learning_rate": 2e-05,
+      "loss": 0.5252,
+      "step": 4130
+    },
+    {
+      "epoch": 0.36386008085779575,
+      "grad_norm": 0.3366917073726654,
+      "learning_rate": 2e-05,
+      "loss": 0.5605,
+      "step": 4140
+    },
+    {
+      "epoch": 0.36473896994199334,
+      "grad_norm": 0.3785319924354553,
+      "learning_rate": 2e-05,
+      "loss": 0.5632,
+      "step": 4150
+    },
+    {
+      "epoch": 0.3656178590261909,
+      "grad_norm": 0.32834818959236145,
+      "learning_rate": 2e-05,
+      "loss": 0.5234,
+      "step": 4160
+    },
+    {
+      "epoch": 0.36649674811038846,
+      "grad_norm": 0.3661729395389557,
+      "learning_rate": 2e-05,
+      "loss": 0.5687,
+      "step": 4170
+    },
+    {
+      "epoch": 0.36737563719458605,
+      "grad_norm": 0.370278924703598,
+      "learning_rate": 2e-05,
+      "loss": 0.5517,
+      "step": 4180
+    },
+    {
+      "epoch": 0.36825452627878363,
+      "grad_norm": 0.3554653227329254,
+      "learning_rate": 2e-05,
+      "loss": 0.5337,
+      "step": 4190
+    },
+    {
+      "epoch": 0.36913341536298117,
+      "grad_norm": 0.35374918580055237,
+      "learning_rate": 2e-05,
+      "loss": 0.5384,
+      "step": 4200
+    },
+    {
+      "epoch": 0.37001230444717875,
+      "grad_norm": 0.3210890591144562,
+      "learning_rate": 2e-05,
+      "loss": 0.5435,
+      "step": 4210
+    },
+    {
+      "epoch": 0.37089119353137634,
+      "grad_norm": 0.3437519967556,
+      "learning_rate": 2e-05,
+      "loss": 0.5242,
+      "step": 4220
+    },
+    {
+      "epoch": 0.3717700826155739,
+      "grad_norm": 0.35392579436302185,
+      "learning_rate": 2e-05,
+      "loss": 0.5472,
+      "step": 4230
+    },
+    {
+      "epoch": 0.3726489716997715,
+      "grad_norm": 0.3219797611236572,
+      "learning_rate": 2e-05,
+      "loss": 0.5364,
+      "step": 4240
+    },
+    {
+      "epoch": 0.37352786078396905,
+      "grad_norm": 0.36878517270088196,
+      "learning_rate": 2e-05,
+      "loss": 0.5483,
+      "step": 4250
+    },
+    {
+      "epoch": 0.37440674986816663,
+      "grad_norm": 0.3591812551021576,
+      "learning_rate": 2e-05,
+      "loss": 0.5786,
+      "step": 4260
+    },
+    {
+      "epoch": 0.3752856389523642,
+      "grad_norm": 0.337785542011261,
+      "learning_rate": 2e-05,
+      "loss": 0.5284,
+      "step": 4270
+    },
+    {
+      "epoch": 0.3761645280365618,
+      "grad_norm": 0.38380974531173706,
+      "learning_rate": 2e-05,
+      "loss": 0.5532,
+      "step": 4280
+    },
+    {
+      "epoch": 0.37704341712075934,
+      "grad_norm": 0.42100170254707336,
+      "learning_rate": 2e-05,
+      "loss": 0.5508,
+      "step": 4290
+    },
+    {
+      "epoch": 0.3779223062049569,
+      "grad_norm": 0.335602343082428,
+      "learning_rate": 2e-05,
+      "loss": 0.5445,
+      "step": 4300
+    },
+    {
+      "epoch": 0.3788011952891545,
+      "grad_norm": 0.341927170753479,
+      "learning_rate": 2e-05,
+      "loss": 0.5527,
+      "step": 4310
+    },
+    {
+      "epoch": 0.3796800843733521,
+      "grad_norm": 0.3622886538505554,
+      "learning_rate": 2e-05,
+      "loss": 0.5381,
+      "step": 4320
+    },
+    {
+      "epoch": 0.38055897345754963,
+      "grad_norm": 0.3521440029144287,
+      "learning_rate": 2e-05,
+      "loss": 0.5686,
+      "step": 4330
+    },
+    {
+      "epoch": 0.3814378625417472,
+      "grad_norm": 0.34552690386772156,
+      "learning_rate": 2e-05,
+      "loss": 0.5529,
+      "step": 4340
+    },
+    {
+      "epoch": 0.3823167516259448,
+      "grad_norm": 0.3347887098789215,
+      "learning_rate": 2e-05,
+      "loss": 0.5459,
+      "step": 4350
+    },
+    {
+      "epoch": 0.3831956407101424,
+      "grad_norm": 0.341581791639328,
+      "learning_rate": 2e-05,
+      "loss": 0.5374,
+      "step": 4360
+    },
+    {
+      "epoch": 0.38407452979434,
+      "grad_norm": 0.4854869544506073,
+      "learning_rate": 2e-05,
+      "loss": 0.5352,
+      "step": 4370
+    },
+    {
+      "epoch": 0.3849534188785375,
+      "grad_norm": 0.3230288028717041,
+      "learning_rate": 2e-05,
+      "loss": 0.5492,
+      "step": 4380
+    },
+    {
+      "epoch": 0.3858323079627351,
+      "grad_norm": 0.36125731468200684,
+      "learning_rate": 2e-05,
+      "loss": 0.5641,
+      "step": 4390
+    },
+    {
+      "epoch": 0.3867111970469327,
+      "grad_norm": 0.30713826417922974,
+      "learning_rate": 2e-05,
+      "loss": 0.5484,
+      "step": 4400
+    },
+    {
+      "epoch": 0.3875900861311303,
+      "grad_norm": 0.31878671050071716,
+      "learning_rate": 2e-05,
+      "loss": 0.5438,
+      "step": 4410
+    },
+    {
+      "epoch": 0.3884689752153278,
+      "grad_norm": 0.38744089007377625,
+      "learning_rate": 2e-05,
+      "loss": 0.5309,
+      "step": 4420
+    },
+    {
+      "epoch": 0.3893478642995254,
+      "grad_norm": 0.39923980832099915,
+      "learning_rate": 2e-05,
+      "loss": 0.5086,
+      "step": 4430
+    },
+    {
+      "epoch": 0.390226753383723,
+      "grad_norm": 0.44628486037254333,
+      "learning_rate": 2e-05,
+      "loss": 0.5359,
+      "step": 4440
+    },
+    {
+      "epoch": 0.39110564246792057,
+      "grad_norm": 0.3254205286502838,
+      "learning_rate": 2e-05,
+      "loss": 0.5181,
+      "step": 4450
+    },
+    {
+      "epoch": 0.3919845315521181,
+      "grad_norm": 0.32878732681274414,
+      "learning_rate": 2e-05,
+      "loss": 0.5055,
+      "step": 4460
+    },
+    {
+      "epoch": 0.3928634206363157,
+      "grad_norm": 0.35472363233566284,
+      "learning_rate": 2e-05,
+      "loss": 0.5707,
+      "step": 4470
+    },
+    {
+      "epoch": 0.3937423097205133,
+      "grad_norm": 0.3601676821708679,
+      "learning_rate": 2e-05,
+      "loss": 0.5646,
+      "step": 4480
+    },
+    {
+      "epoch": 0.39462119880471086,
+      "grad_norm": 0.3487088084220886,
+      "learning_rate": 2e-05,
+      "loss": 0.5271,
+      "step": 4490
+    },
+    {
+      "epoch": 0.39550008788890845,
+      "grad_norm": 0.3668665885925293,
+      "learning_rate": 2e-05,
+      "loss": 0.5511,
+      "step": 4500
+    },
+    {
+      "epoch": 0.396378976973106,
+      "grad_norm": 0.3314940333366394,
+      "learning_rate": 2e-05,
+      "loss": 0.5465,
+      "step": 4510
+    },
+    {
+      "epoch": 0.39725786605730357,
+      "grad_norm": 0.3193863332271576,
+      "learning_rate": 2e-05,
+      "loss": 0.5265,
+      "step": 4520
+    },
+    {
+      "epoch": 0.39813675514150115,
+      "grad_norm": 0.40486985445022583,
+      "learning_rate": 2e-05,
+      "loss": 0.5503,
+      "step": 4530
+    },
+    {
+      "epoch": 0.39901564422569874,
+      "grad_norm": 0.44396767020225525,
+      "learning_rate": 2e-05,
+      "loss": 0.5451,
+      "step": 4540
+    },
+    {
+      "epoch": 0.39989453330989627,
+      "grad_norm": 0.3681359589099884,
+      "learning_rate": 2e-05,
+      "loss": 0.5136,
+      "step": 4550
+    },
+    {
+      "epoch": 0.40077342239409386,
+      "grad_norm": 0.35657134652137756,
+      "learning_rate": 2e-05,
+      "loss": 0.5721,
+      "step": 4560
+    },
+    {
+      "epoch": 0.40165231147829145,
+      "grad_norm": 0.32666659355163574,
+      "learning_rate": 2e-05,
+      "loss": 0.5694,
+      "step": 4570
+    },
+    {
+      "epoch": 0.40253120056248903,
+      "grad_norm": 0.3304166793823242,
+      "learning_rate": 2e-05,
+      "loss": 0.5437,
+      "step": 4580
+    },
+    {
+      "epoch": 0.40341008964668656,
+      "grad_norm": 0.3637726604938507,
+      "learning_rate": 2e-05,
+      "loss": 0.5482,
+      "step": 4590
+    },
+    {
+      "epoch": 0.40428897873088415,
+      "grad_norm": 0.4018683433532715,
+      "learning_rate": 2e-05,
+      "loss": 0.5307,
+      "step": 4600
+    },
+    {
+      "epoch": 0.40516786781508174,
+      "grad_norm": 0.38370928168296814,
+      "learning_rate": 2e-05,
+      "loss": 0.5345,
+      "step": 4610
+    },
+    {
+      "epoch": 0.4060467568992793,
+      "grad_norm": 0.40193265676498413,
+      "learning_rate": 2e-05,
+      "loss": 0.5574,
+      "step": 4620
+    },
+    {
+      "epoch": 0.40692564598347686,
+      "grad_norm": 0.35073745250701904,
+      "learning_rate": 2e-05,
+      "loss": 0.544,
+      "step": 4630
+    },
+    {
+      "epoch": 0.40780453506767445,
+      "grad_norm": 0.3612351715564728,
+      "learning_rate": 2e-05,
+      "loss": 0.5297,
+      "step": 4640
+    },
+    {
+      "epoch": 0.40868342415187203,
+      "grad_norm": 0.3514283001422882,
+      "learning_rate": 2e-05,
+      "loss": 0.5577,
+      "step": 4650
+    },
+    {
+      "epoch": 0.4095623132360696,
+      "grad_norm": 0.3579213619232178,
+      "learning_rate": 2e-05,
+      "loss": 0.5274,
+      "step": 4660
+    },
+    {
+      "epoch": 0.4104412023202672,
+      "grad_norm": 0.3235413432121277,
+      "learning_rate": 2e-05,
+      "loss": 0.5421,
+      "step": 4670
+    },
+    {
+      "epoch": 0.41132009140446474,
+      "grad_norm": 0.3696593642234802,
+      "learning_rate": 2e-05,
+      "loss": 0.5365,
+      "step": 4680
+    },
+    {
+      "epoch": 0.4121989804886623,
+      "grad_norm": 0.37844300270080566,
+      "learning_rate": 2e-05,
+      "loss": 0.5359,
+      "step": 4690
+    },
+    {
+      "epoch": 0.4130778695728599,
+      "grad_norm": 0.3614518940448761,
+      "learning_rate": 2e-05,
+      "loss": 0.5325,
+      "step": 4700
+    },
+    {
+      "epoch": 0.4139567586570575,
+      "grad_norm": 0.35735979676246643,
+      "learning_rate": 2e-05,
+      "loss": 0.5411,
+      "step": 4710
+    },
+    {
+      "epoch": 0.41483564774125503,
+      "grad_norm": 0.3506315052509308,
+      "learning_rate": 2e-05,
+      "loss": 0.5416,
+      "step": 4720
+    },
+    {
+      "epoch": 0.4157145368254526,
+      "grad_norm": 0.33660030364990234,
+      "learning_rate": 2e-05,
+      "loss": 0.5641,
+      "step": 4730
+    },
+    {
+      "epoch": 0.4165934259096502,
+      "grad_norm": 0.3599470555782318,
+      "learning_rate": 2e-05,
+      "loss": 0.555,
+      "step": 4740
+    },
+    {
+      "epoch": 0.4174723149938478,
+      "grad_norm": 0.34064146876335144,
+      "learning_rate": 2e-05,
+      "loss": 0.5394,
+      "step": 4750
+    },
+    {
+      "epoch": 0.4183512040780453,
+      "grad_norm": 0.39736467599868774,
+      "learning_rate": 2e-05,
+      "loss": 0.5514,
+      "step": 4760
+    },
+    {
+      "epoch": 0.4192300931622429,
+      "grad_norm": 0.34742310643196106,
+      "learning_rate": 2e-05,
+      "loss": 0.5481,
+      "step": 4770
+    },
+    {
+      "epoch": 0.4201089822464405,
+      "grad_norm": 0.39449450373649597,
+      "learning_rate": 2e-05,
+      "loss": 0.5515,
+      "step": 4780
+    },
+    {
+      "epoch": 0.4209878713306381,
+      "grad_norm": 0.46250203251838684,
+      "learning_rate": 2e-05,
+      "loss": 0.5621,
+      "step": 4790
+    },
+    {
+      "epoch": 0.4218667604148357,
+      "grad_norm": 0.4337044060230255,
+      "learning_rate": 2e-05,
+      "loss": 0.5538,
+      "step": 4800
+    },
+    {
+      "epoch": 0.4227456494990332,
+      "grad_norm": 0.4036795496940613,
+      "learning_rate": 2e-05,
+      "loss": 0.5553,
+      "step": 4810
+    },
+    {
+      "epoch": 0.4236245385832308,
+      "grad_norm": 0.34802335500717163,
+      "learning_rate": 2e-05,
+      "loss": 0.5426,
+      "step": 4820
+    },
+    {
+      "epoch": 0.4245034276674284,
+      "grad_norm": 0.3582998812198639,
+      "learning_rate": 2e-05,
+      "loss": 0.5545,
+      "step": 4830
+    },
+    {
+      "epoch": 0.42538231675162597,
+      "grad_norm": 0.38593223690986633,
+      "learning_rate": 2e-05,
+      "loss": 0.5508,
+      "step": 4840
+    },
+    {
+      "epoch": 0.4262612058358235,
+      "grad_norm": 0.33207061886787415,
+      "learning_rate": 2e-05,
+      "loss": 0.5376,
+      "step": 4850
+    },
+    {
+      "epoch": 0.4271400949200211,
+      "grad_norm": 0.3460117280483246,
+      "learning_rate": 2e-05,
+      "loss": 0.5302,
+      "step": 4860
+    },
+    {
+      "epoch": 0.42801898400421867,
+      "grad_norm": 0.35605132579803467,
+      "learning_rate": 2e-05,
+      "loss": 0.5191,
+      "step": 4870
+    },
+    {
+      "epoch": 0.42889787308841626,
+      "grad_norm": 0.34891632199287415,
+      "learning_rate": 2e-05,
+      "loss": 0.5504,
+      "step": 4880
+    },
+    {
+      "epoch": 0.4297767621726138,
+      "grad_norm": 0.41499564051628113,
+      "learning_rate": 2e-05,
+      "loss": 0.5672,
+      "step": 4890
+    },
+    {
+      "epoch": 0.4306556512568114,
+      "grad_norm": 0.37195125222206116,
+      "learning_rate": 2e-05,
+      "loss": 0.5288,
+      "step": 4900
+    },
+    {
+      "epoch": 0.43153454034100897,
+      "grad_norm": 0.32856228947639465,
+      "learning_rate": 2e-05,
+      "loss": 0.5477,
+      "step": 4910
+    },
+    {
+      "epoch": 0.43241342942520655,
+      "grad_norm": 0.3535173833370209,
+      "learning_rate": 2e-05,
+      "loss": 0.5398,
+      "step": 4920
+    },
+    {
+      "epoch": 0.43329231850940414,
+      "grad_norm": 0.3696601986885071,
+      "learning_rate": 2e-05,
+      "loss": 0.541,
+      "step": 4930
+    },
+    {
+      "epoch": 0.43417120759360167,
+      "grad_norm": 0.379461407661438,
+      "learning_rate": 2e-05,
+      "loss": 0.5546,
+      "step": 4940
+    },
+    {
+      "epoch": 0.43505009667779926,
+      "grad_norm": 0.36268851161003113,
+      "learning_rate": 2e-05,
+      "loss": 0.5456,
+      "step": 4950
+    },
+    {
+      "epoch": 0.43592898576199685,
+      "grad_norm": 0.4394989609718323,
+      "learning_rate": 2e-05,
+      "loss": 0.5368,
+      "step": 4960
+    },
+    {
+      "epoch": 0.43680787484619443,
+      "grad_norm": 0.35105985403060913,
+      "learning_rate": 2e-05,
+      "loss": 0.5086,
+      "step": 4970
+    },
+    {
+      "epoch": 0.43768676393039196,
+      "grad_norm": 0.37677186727523804,
+      "learning_rate": 2e-05,
+      "loss": 0.5373,
+      "step": 4980
+    },
+    {
+      "epoch": 0.43856565301458955,
+      "grad_norm": 0.3612132668495178,
+      "learning_rate": 2e-05,
+      "loss": 0.5389,
+      "step": 4990
+    },
+    {
+      "epoch": 0.43944454209878714,
+      "grad_norm": 0.3109592795372009,
+      "learning_rate": 2e-05,
+      "loss": 0.5237,
+      "step": 5000
+    },
+    {
+      "epoch": 0.4403234311829847,
+      "grad_norm": 0.42327675223350525,
+      "learning_rate": 2e-05,
+      "loss": 0.5209,
+      "step": 5010
+    },
+    {
+      "epoch": 0.44120232026718226,
+      "grad_norm": 0.32862475514411926,
+      "learning_rate": 2e-05,
+      "loss": 0.5364,
+      "step": 5020
+    },
+    {
+      "epoch": 0.44208120935137984,
+      "grad_norm": 0.3787497878074646,
+      "learning_rate": 2e-05,
+      "loss": 0.5379,
+      "step": 5030
+    },
+    {
+      "epoch": 0.44296009843557743,
+      "grad_norm": 0.4235406816005707,
+      "learning_rate": 2e-05,
+      "loss": 0.5118,
+      "step": 5040
+    },
+    {
+      "epoch": 0.443838987519775,
+      "grad_norm": 0.34716659784317017,
+      "learning_rate": 2e-05,
+      "loss": 0.5367,
+      "step": 5050
+    },
+    {
+      "epoch": 0.4447178766039726,
+      "grad_norm": 0.3423392176628113,
+      "learning_rate": 2e-05,
+      "loss": 0.5378,
+      "step": 5060
+    },
+    {
+      "epoch": 0.44559676568817014,
+      "grad_norm": 0.36973053216934204,
+      "learning_rate": 2e-05,
+      "loss": 0.5516,
+      "step": 5070
+    },
+    {
+      "epoch": 0.4464756547723677,
+      "grad_norm": 0.4135674238204956,
+      "learning_rate": 2e-05,
+      "loss": 0.5289,
+      "step": 5080
+    },
+    {
+      "epoch": 0.4473545438565653,
+      "grad_norm": 0.34592947363853455,
+      "learning_rate": 1.9999810479633703e-05,
+      "loss": 0.589,
+      "step": 5090
+    },
+    {
+      "epoch": 0.4482334329407629,
+      "grad_norm": 0.3470785319805145,
+      "learning_rate": 1.997707677020786e-05,
+      "loss": 0.5323,
+      "step": 5100
+    },
+    {
+      "epoch": 0.44911232202496043,
+      "grad_norm": 0.33374693989753723,
+      "learning_rate": 1.9916538196124637e-05,
+      "loss": 0.5446,
+      "step": 5110
+    },
+    {
+      "epoch": 0.449991211109158,
+      "grad_norm": 0.36847248673439026,
+      "learning_rate": 1.9818425303863402e-05,
+      "loss": 0.517,
+      "step": 5120
+    },
+    {
+      "epoch": 0.4508701001933556,
+      "grad_norm": 0.36847418546676636,
+      "learning_rate": 1.9683111732582887e-05,
+      "loss": 0.5261,
+      "step": 5130
+    },
+    {
+      "epoch": 0.4517489892775532,
+      "grad_norm": 0.4955618679523468,
+      "learning_rate": 1.951111279120706e-05,
+      "loss": 0.5404,
+      "step": 5140
+    },
+    {
+      "epoch": 0.4526278783617507,
+      "grad_norm": 0.3508554697036743,
+      "learning_rate": 1.9303083495996028e-05,
+      "loss": 0.5269,
+      "step": 5150
+    },
+    {
+      "epoch": 0.4535067674459483,
+      "grad_norm": 0.3382345139980316,
+      "learning_rate": 1.9059816076075482e-05,
+      "loss": 0.5414,
+      "step": 5160
+    },
+    {
+      "epoch": 0.4543856565301459,
+      "grad_norm": 0.35228630900382996,
+      "learning_rate": 1.8782236956424142e-05,
+      "loss": 0.5404,
+      "step": 5170
+    },
+    {
+      "epoch": 0.4552645456143435,
+      "grad_norm": 0.4006800055503845,
+      "learning_rate": 1.8471403229808897e-05,
+      "loss": 0.5278,
+      "step": 5180
+    },
+    {
+      "epoch": 0.456143434698541,
+      "grad_norm": 0.5065578818321228,
+      "learning_rate": 1.8128498631103333e-05,
+      "loss": 0.5479,
+      "step": 5190
+    },
+    {
+      "epoch": 0.4570223237827386,
+      "grad_norm": 0.33684271574020386,
+      "learning_rate": 1.7754829029320507e-05,
+      "loss": 0.515,
+      "step": 5200
+    },
+    {
+      "epoch": 0.4579012128669362,
+      "grad_norm": 0.33451032638549805,
+      "learning_rate": 1.7351817454527473e-05,
+      "loss": 0.5267,
+      "step": 5210
+    },
+    {
+      "epoch": 0.4587801019511338,
+      "grad_norm": 0.3376351594924927,
+      "learning_rate": 1.6920998678580345e-05,
+      "loss": 0.5165,
+      "step": 5220
+    },
+    {
+      "epoch": 0.45965899103533137,
+      "grad_norm": 0.3563227355480194,
+      "learning_rate": 1.6464013370317854e-05,
+      "loss": 0.5316,
+      "step": 5230
+    },
+    {
+      "epoch": 0.4605378801195289,
+      "grad_norm": 0.3721602261066437,
+      "learning_rate": 1.5982601847471915e-05,
+      "loss": 0.535,
+      "step": 5240
+    },
+    {
+      "epoch": 0.4614167692037265,
+      "grad_norm": 0.3335733115673065,
+      "learning_rate": 1.5478597449089553e-05,
+      "loss": 0.5459,
+      "step": 5250
+    },
+    {
+      "epoch": 0.46229565828792407,
+      "grad_norm": 0.37219899892807007,
+      "learning_rate": 1.4953919553705725e-05,
+      "loss": 0.5432,
+      "step": 5260
+    },
+    {
+      "epoch": 0.46317454737212166,
+      "grad_norm": 0.3701734244823456,
+      "learning_rate": 1.441056626985559e-05,
+      "loss": 0.5714,
+      "step": 5270
+    },
+    {
+      "epoch": 0.4640534364563192,
+      "grad_norm": 0.3756079077720642,
+      "learning_rate": 1.3850606826762668e-05,
+      "loss": 0.5473,
+      "step": 5280
+    },
+    {
+      "epoch": 0.4649323255405168,
+      "grad_norm": 0.3513130843639374,
+      "learning_rate": 1.3276173694181128e-05,
+      "loss": 0.5238,
+      "step": 5290
+    },
+    {
+      "epoch": 0.46581121462471436,
+      "grad_norm": 0.373111367225647,
+      "learning_rate": 1.2689454461401822e-05,
+      "loss": 0.5291,
+      "step": 5300
+    },
+    {
+      "epoch": 0.46669010370891195,
+      "grad_norm": 0.3545767664909363,
+      "learning_rate": 1.2092683506348939e-05,
+      "loss": 0.5181,
+      "step": 5310
+    },
+    {
+      "epoch": 0.4675689927931095,
+      "grad_norm": 0.3761849105358124,
+      "learning_rate": 1.14881334864935e-05,
+      "loss": 0.5234,
+      "step": 5320
+    },
+    {
+      "epoch": 0.46844788187730707,
+      "grad_norm": 0.3567139208316803,
+      "learning_rate": 1.0878106683988481e-05,
+      "loss": 0.5611,
+      "step": 5330
+    },
+    {
+      "epoch": 0.46932677096150466,
+      "grad_norm": 0.41744333505630493,
+      "learning_rate": 1.026492623798551e-05,
+      "loss": 0.5543,
+      "step": 5340
+    },
+    {
+      "epoch": 0.47020566004570225,
+      "grad_norm": 0.36612841486930847,
+      "learning_rate": 9.650927297522743e-06,
+      "loss": 0.5379,
+      "step": 5350
+    },
+    {
+      "epoch": 0.47108454912989983,
+      "grad_norm": 0.349392831325531,
+      "learning_rate": 9.038448128676012e-06,
+      "loss": 0.545,
+      "step": 5360
+    },
+    {
+      "epoch": 0.47196343821409736,
+      "grad_norm": 0.3567434847354889,
+      "learning_rate": 8.42982120983949e-06,
+      "loss": 0.531,
+      "step": 5370
+    },
+    {
+      "epoch": 0.47284232729829495,
+      "grad_norm": 0.3052014410495758,
+      "learning_rate": 7.827364349047303e-06,
+      "loss": 0.5297,
+      "step": 5380
+    },
+    {
+      "epoch": 0.47372121638249254,
+      "grad_norm": 0.3599008619785309,
+      "learning_rate": 7.233371857163706e-06,
+      "loss": 0.5202,
+      "step": 5390
+    },
+    {
+      "epoch": 0.4746001054666901,
+      "grad_norm": 0.3340565264225006,
+      "learning_rate": 6.650105810556492e-06,
+      "loss": 0.5233,
+      "step": 5400
+    },
+    {
+      "epoch": 0.47547899455088766,
+      "grad_norm": 0.35919269919395447,
+      "learning_rate": 6.079787436527681e-06,
+      "loss": 0.5213,
+      "step": 5410
+    },
+    {
+      "epoch": 0.47635788363508524,
+      "grad_norm": 0.4195660650730133,
+      "learning_rate": 5.524588654307961e-06,
+      "loss": 0.5345,
+      "step": 5420
+    },
+    {
+      "epoch": 0.47723677271928283,
+      "grad_norm": 0.39947375655174255,
+      "learning_rate": 4.986623803828985e-06,
+      "loss": 0.5417,
+      "step": 5430
+    },
+    {
+      "epoch": 0.4781156618034804,
+      "grad_norm": 0.34144240617752075,
+      "learning_rate": 4.467941593772423e-06,
+      "loss": 0.5196,
+      "step": 5440
+    },
+    {
+      "epoch": 0.47899455088767795,
+      "grad_norm": 0.31601595878601074,
+      "learning_rate": 3.9705172995597606e-06,
+      "loss": 0.5365,
+      "step": 5450
+    },
+    {
+      "epoch": 0.47987343997187554,
+      "grad_norm": 0.3501845896244049,
+      "learning_rate": 3.4962452409948346e-06,
+      "loss": 0.5399,
+      "step": 5460
+    },
+    {
+      "epoch": 0.4807523290560731,
+      "grad_norm": 0.3590399920940399,
+      "learning_rate": 3.046931568206211e-06,
+      "loss": 0.5548,
+      "step": 5470
+    },
+    {
+      "epoch": 0.4816312181402707,
+      "grad_norm": 0.34990110993385315,
+      "learning_rate": 2.624287383362404e-06,
+      "loss": 0.5186,
+      "step": 5480
+    },
+    {
+      "epoch": 0.4825101072244683,
+      "grad_norm": 0.32826465368270874,
+      "learning_rate": 2.2299222243541845e-06,
+      "loss": 0.5369,
+      "step": 5490
+    },
+    {
+      "epoch": 0.48338899630866583,
+      "grad_norm": 0.3703666627407074,
+      "learning_rate": 1.8653379352598246e-06,
+      "loss": 0.524,
+      "step": 5500
+    },
+    {
+      "epoch": 0.4842678853928634,
+      "grad_norm": 0.38898542523384094,
+      "learning_rate": 1.5319229469361075e-06,
+      "loss": 0.5411,
+      "step": 5510
+    },
+    {
+      "epoch": 0.485146774477061,
+      "grad_norm": 0.3752051889896393,
+      "learning_rate": 1.2309469895160917e-06,
+      "loss": 0.5421,
+      "step": 5520
+    },
+    {
+      "epoch": 0.4860256635612586,
+      "grad_norm": 0.3679511249065399,
+      "learning_rate": 9.635562569497974e-07,
+      "loss": 0.5364,
+      "step": 5530
+    },
+    {
+      "epoch": 0.4869045526454561,
+      "grad_norm": 0.3570229113101959,
+      "learning_rate": 7.307690420024867e-07,
+      "loss": 0.5218,
+      "step": 5540
+    },
+    {
+      "epoch": 0.4877834417296537,
+      "grad_norm": 0.3691729009151459,
+      "learning_rate": 5.334718583335942e-07,
+      "loss": 0.5479,
+      "step": 5550
+    },
+    {
+      "epoch": 0.4886623308138513,
+      "grad_norm": 0.3383973240852356,
+      "learning_rate": 3.7241606442442763e-07,
+      "loss": 0.5463,
+      "step": 5560
+    },
+    {
+      "epoch": 0.4895412198980489,
+      "grad_norm": 0.32522642612457275,
+      "learning_rate": 2.482150022115879e-07,
+      "loss": 0.5216,
+      "step": 5570
+    },
+    {
+      "epoch": 0.4904201089822464,
+      "grad_norm": 0.3617769479751587,
+      "learning_rate": 1.6134166132293174e-07,
+      "loss": 0.528,
+      "step": 5580
+    },
+    {
+      "epoch": 0.491298998066444,
+      "grad_norm": 0.34759119153022766,
+      "learning_rate": 1.1212687781127357e-07,
+      "loss": 0.5224,
+      "step": 5590
+    },
+    {
+      "epoch": 0.4919142204253823,
+      "step": 5597,
+      "total_flos": 2.4985159616803373e+19,
+      "train_loss": 0.048519859191283855,
+      "train_runtime": 5685.2195,
+      "train_samples_per_second": 11.814,
+      "train_steps_per_second": 0.984
+    }
+  ],
+  "logging_steps": 10,
+  "max_steps": 5597,
+  "num_input_tokens_seen": 0,
+  "num_train_epochs": 1,
+  "save_steps": 636,
+  "stateful_callbacks": {
+    "TrainerControl": {
+      "args": {
+        "should_epoch_stop": false,
+        "should_evaluate": false,
+        "should_log": false,
+        "should_save": true,
+        "should_training_stop": true
+      },
+      "attributes": {}
+    }
+  },
+  "total_flos": 2.4985159616803373e+19,
+  "train_batch_size": 1,
+  "trial_name": null,
+  "trial_params": null
+}

g2_1_oldcode_cm_newcode/500M/training_args.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:0b9f9d9a491aad8303d5bdfa6115ae64904e66295db374ffe43bc128ecfef85d
+size 9105