Upload folder using huggingface_hub

.gitattributes

@@ -69,3 +69,4 @@ g2_3_plan_oldcode_newcode/1000M/tokenizer.json filter=lfs diff=lfs merge=lfs -te
 g2_4_plan_cm_oldcode_newcode/250M/tokenizer.json filter=lfs diff=lfs merge=lfs -text
 g2_4_plan_cm_oldcode_newcode/500M/tokenizer.json filter=lfs diff=lfs merge=lfs -text
 g2_4_plan_cm_oldcode_newcode/1000M/tokenizer.json filter=lfs diff=lfs merge=lfs -text
+g2_5_plan_newcode/250M/tokenizer.json filter=lfs diff=lfs merge=lfs -text

g2_5_plan_newcode/250M/README.md

@@ -0,0 +1,60 @@
+---
+library_name: transformers
+license: other
+base_model: Qwen/Qwen3-8B-Base
+tags:
+- llama-factory
+- full
+- generated_from_trainer
+model-index:
+- name: 250M
+  results: []
+---
+
+<!-- This model card has been generated automatically according to the information the Trainer had access to. You
+should probably proofread and complete it, then remove this comment. -->
+
+# 250M
+
+This model is a fine-tuned version of [/lus/flare/projects/MultiModalLLM/model-weights/Qwen/Qwen3-8B-Base](https://huggingface.co//lus/flare/projects/MultiModalLLM/model-weights/Qwen/Qwen3-8B-Base) on the rv5_g2_5_plan_newcode and the instruct_replay datasets.
+
+## Model description
+
+More information needed
+
+## Intended uses & limitations
+
+More information needed
+
+## Training and evaluation data
+
+More information needed
+
+## Training procedure
+
+### Training hyperparameters
+
+The following hyperparameters were used during training:
+- learning_rate: 2e-05
+- train_batch_size: 1
+- eval_batch_size: 8
+- seed: 42
+- distributed_type: multi-GPU
+- num_devices: 12
+- total_train_batch_size: 12
+- total_eval_batch_size: 96
+- optimizer: Use OptimizerNames.ADAMW_TORCH_FUSED with betas=(0.9,0.95) and epsilon=1e-08 and optimizer_args=No additional optimizer arguments
+- lr_scheduler_type: warmup_stable_decay
+- lr_scheduler_warmup_steps: 178
+- training_steps: 2797
+
+### Training results
+
+
+
+### Framework versions
+
+- Transformers 4.57.6
+- Pytorch 2.10.0a0+git449b176
+- Datasets 4.5.0
+- Tokenizers 0.22.2

g2_5_plan_newcode/250M/added_tokens.json

@@ -0,0 +1,28 @@
+{
+  "</think>": 151668,
+  "</tool_call>": 151658,
+  "</tool_response>": 151666,
+  "<think>": 151667,
+  "<tool_call>": 151657,
+  "<tool_response>": 151665,
+  "<|box_end|>": 151649,
+  "<|box_start|>": 151648,
+  "<|endoftext|>": 151643,
+  "<|file_sep|>": 151664,
+  "<|fim_middle|>": 151660,
+  "<|fim_pad|>": 151662,
+  "<|fim_prefix|>": 151659,
+  "<|fim_suffix|>": 151661,
+  "<|im_end|>": 151645,
+  "<|im_start|>": 151644,
+  "<|image_pad|>": 151655,
+  "<|object_ref_end|>": 151647,
+  "<|object_ref_start|>": 151646,
+  "<|quad_end|>": 151651,
+  "<|quad_start|>": 151650,
+  "<|repo_name|>": 151663,
+  "<|video_pad|>": 151656,
+  "<|vision_end|>": 151653,
+  "<|vision_pad|>": 151654,
+  "<|vision_start|>": 151652
+}

g2_5_plan_newcode/250M/all_results.json

@@ -0,0 +1,8 @@
+{
+    "epoch": 0.2459549771368273,
+    "total_flos": 1.2485883762455871e+19,
+    "train_loss": 0.06533449515300774,
+    "train_runtime": 2953.8926,
+    "train_samples_per_second": 11.363,
+    "train_steps_per_second": 0.947
+}

g2_5_plan_newcode/250M/chat_template.jinja

@@ -0,0 +1,85 @@
+{%- if tools %}
+    {{- '<|im_start|>system\n' }}
+    {%- if messages[0].role == 'system' %}
+        {{- messages[0].content + '\n\n' }}
+    {%- endif %}
+    {{- "# Tools\n\nYou may call one or more functions to assist with the user query.\n\nYou are provided with function signatures within <tools></tools> XML tags:\n<tools>" }}
+    {%- for tool in tools %}
+        {{- "\n" }}
+        {{- tool | tojson }}
+    {%- endfor %}
+    {{- "\n</tools>\n\nFor each function call, return a json object with function name and arguments within <tool_call></tool_call> XML tags:\n<tool_call>\n{\"name\": <function-name>, \"arguments\": <args-json-object>}\n</tool_call><|im_end|>\n" }}
+{%- else %}
+    {%- if messages[0].role == 'system' %}
+        {{- '<|im_start|>system\n' + messages[0].content + '<|im_end|>\n' }}
+    {%- endif %}
+{%- endif %}
+{%- set ns = namespace(multi_step_tool=true, last_query_index=messages|length - 1) %}
+{%- for message in messages[::-1] %}
+    {%- set index = (messages|length - 1) - loop.index0 %}
+    {%- if ns.multi_step_tool and message.role == "user" and not(message.content.startswith('<tool_response>') and message.content.endswith('</tool_response>')) %}
+        {%- set ns.multi_step_tool = false %}
+        {%- set ns.last_query_index = index %}
+    {%- endif %}
+{%- endfor %}
+{%- for message in messages %}
+    {%- if (message.role == "user") or (message.role == "system" and not loop.first) %}
+        {{- '<|im_start|>' + message.role + '\n' + message.content + '<|im_end|>' + '\n' }}
+    {%- elif message.role == "assistant" %}
+        {%- set content = message.content %}
+        {%- set reasoning_content = '' %}
+        {%- if message.reasoning_content is defined and message.reasoning_content is not none %}
+            {%- set reasoning_content = message.reasoning_content %}
+        {%- else %}
+            {%- if '</think>' in message.content %}
+                {%- set content = message.content.split('</think>')[-1].lstrip('\n') %}
+                {%- set reasoning_content = message.content.split('</think>')[0].rstrip('\n').split('<think>')[-1].lstrip('\n') %}
+            {%- endif %}
+        {%- endif %}
+        {%- if loop.index0 > ns.last_query_index %}
+            {%- if loop.last or (not loop.last and reasoning_content) %}
+                {{- '<|im_start|>' + message.role + '\n<think>\n' + reasoning_content.strip('\n') + '\n</think>\n\n' + content.lstrip('\n') }}
+            {%- else %}
+                {{- '<|im_start|>' + message.role + '\n' + content }}
+            {%- endif %}
+        {%- else %}
+            {{- '<|im_start|>' + message.role + '\n' + content }}
+        {%- endif %}
+        {%- if message.tool_calls %}
+            {%- for tool_call in message.tool_calls %}
+                {%- if (loop.first and content) or (not loop.first) %}
+                    {{- '\n' }}
+                {%- endif %}
+                {%- if tool_call.function %}
+                    {%- set tool_call = tool_call.function %}
+                {%- endif %}
+                {{- '<tool_call>\n{"name": "' }}
+                {{- tool_call.name }}
+                {{- '", "arguments": ' }}
+                {%- if tool_call.arguments is string %}
+                    {{- tool_call.arguments }}
+                {%- else %}
+                    {{- tool_call.arguments | tojson }}
+                {%- endif %}
+                {{- '}\n</tool_call>' }}
+            {%- endfor %}
+        {%- endif %}
+        {{- '<|im_end|>\n' }}
+    {%- elif message.role == "tool" %}
+        {%- if loop.first or (messages[loop.index0 - 1].role != "tool") %}
+            {{- '<|im_start|>user' }}
+        {%- endif %}
+        {{- '\n<tool_response>\n' }}
+        {{- message.content }}
+        {{- '\n</tool_response>' }}
+        {%- if loop.last or (messages[loop.index0 + 1].role != "tool") %}
+            {{- '<|im_end|>\n' }}
+        {%- endif %}
+    {%- endif %}
+{%- endfor %}
+{%- if add_generation_prompt %}
+    {{- '<|im_start|>assistant\n' }}
+    {%- if enable_thinking is defined and enable_thinking is false %}
+        {{- '<think>\n\n</think>\n\n' }}
+    {%- endif %}
+{%- endif %}

g2_5_plan_newcode/250M/config.json

@@ -0,0 +1,68 @@
+{
+  "architectures": [
+    "Qwen3ForCausalLM"
+  ],
+  "attention_bias": false,
+  "attention_dropout": 0.0,
+  "dtype": "bfloat16",
+  "eos_token_id": 151643,
+  "head_dim": 128,
+  "hidden_act": "silu",
+  "hidden_size": 4096,
+  "initializer_range": 0.02,
+  "intermediate_size": 12288,
+  "layer_types": [
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention"
+  ],
+  "max_position_embeddings": 32768,
+  "max_window_layers": 36,
+  "model_type": "qwen3",
+  "num_attention_heads": 32,
+  "num_hidden_layers": 36,
+  "num_key_value_heads": 8,
+  "pad_token_id": 151643,
+  "rms_norm_eps": 1e-06,
+  "rope_scaling": null,
+  "rope_theta": 1000000,
+  "sliding_window": null,
+  "tie_word_embeddings": false,
+  "transformers_version": "4.57.6",
+  "use_cache": false,
+  "use_sliding_window": false,
+  "vocab_size": 151936
+}

g2_5_plan_newcode/250M/generation_config.json

@@ -0,0 +1,8 @@
+{
+  "eos_token_id": [
+    151643
+  ],
+  "max_new_tokens": 2048,
+  "pad_token_id": 151643,
+  "transformers_version": "4.57.6"
+}

g2_5_plan_newcode/250M/model-00001-of-00004.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:1336479d8aa6e241f0b5b35ff29dcca272debb37cb50afa388719e10a34456cf
+size 4902257696

g2_5_plan_newcode/250M/model-00002-of-00004.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6299ed1d4ac8789090c0f207b260954d7005a6fdff2296abafed39f7925d9ab4
+size 4915960368

g2_5_plan_newcode/250M/model-00003-of-00004.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ac770a20c821b5e51d043b04c8d480ba130e79a0d98368d7df926a0e0099f617
+size 4983068496

g2_5_plan_newcode/250M/model-00004-of-00004.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:1e68be65e4890746a048c77d8b38766f4a67548def41f363d7c9bb2f4f4e39f2
+size 1580230264

g2_5_plan_newcode/250M/model.safetensors.index.json

@@ -0,0 +1,407 @@
+{
+  "metadata": {
+    "total_parameters": 8190735360,
+    "total_size": 16381470720
+  },
+  "weight_map": {
+    "lm_head.weight": "model-00004-of-00004.safetensors",
+    "model.embed_tokens.weight": "model-00001-of-00004.safetensors",
+    "model.layers.0.input_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.0.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.0.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.0.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.0.post_attention_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.0.self_attn.k_norm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.0.self_attn.k_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.0.self_attn.o_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.0.self_attn.q_norm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.0.self_attn.q_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.0.self_attn.v_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.1.input_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.1.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.1.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.1.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.1.post_attention_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.1.self_attn.k_norm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.1.self_attn.k_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.1.self_attn.o_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.1.self_attn.q_norm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.1.self_attn.q_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.1.self_attn.v_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.10.input_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.10.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.10.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.10.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.10.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.10.self_attn.k_norm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.10.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.10.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.10.self_attn.q_norm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.10.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.10.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.11.input_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.11.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.11.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.11.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.11.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.11.self_attn.k_norm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.11.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.11.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.11.self_attn.q_norm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.11.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.11.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.12.input_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.12.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.12.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.12.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.12.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.12.self_attn.k_norm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.12.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.12.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.12.self_attn.q_norm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.12.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.12.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.13.input_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.13.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.13.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.13.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.13.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.13.self_attn.k_norm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.13.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.13.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.13.self_attn.q_norm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.13.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.13.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.14.input_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.14.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.14.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.14.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.14.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.14.self_attn.k_norm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.14.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.14.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.14.self_attn.q_norm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.14.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.14.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.15.input_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.15.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.15.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.15.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.15.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.15.self_attn.k_norm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.15.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.15.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.15.self_attn.q_norm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.15.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.15.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.16.input_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.16.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.16.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.16.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.16.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.16.self_attn.k_norm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.16.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.16.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.16.self_attn.q_norm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.16.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.16.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.17.input_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.17.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.17.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.17.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.17.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.17.self_attn.k_norm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.17.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.17.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.17.self_attn.q_norm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.17.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.17.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.18.input_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.18.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.18.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.18.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.18.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.18.self_attn.k_norm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.18.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.18.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.18.self_attn.q_norm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.18.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.18.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.19.input_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.19.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.19.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.19.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.19.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.19.self_attn.k_norm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.19.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.19.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.19.self_attn.q_norm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.19.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.19.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.2.input_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.2.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.2.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.2.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.2.post_attention_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.2.self_attn.k_norm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.2.self_attn.k_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.2.self_attn.o_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.2.self_attn.q_norm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.2.self_attn.q_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.2.self_attn.v_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.20.input_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.20.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.20.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.20.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.20.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.20.self_attn.k_norm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.20.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.20.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.20.self_attn.q_norm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.20.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.20.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.21.input_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.21.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.21.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.21.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.21.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.21.self_attn.k_norm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.21.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.21.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.21.self_attn.q_norm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.21.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.21.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.22.input_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.22.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.22.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.22.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.22.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.22.self_attn.k_norm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.22.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.22.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.22.self_attn.q_norm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.22.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.22.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.23.input_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.23.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.23.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.23.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.23.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.23.self_attn.k_norm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.23.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.23.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.23.self_attn.q_norm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.23.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.23.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.24.input_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.24.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.24.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.24.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.24.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.24.self_attn.k_norm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.24.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.24.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.24.self_attn.q_norm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.24.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.24.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.25.input_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.25.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.25.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.25.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.25.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.25.self_attn.k_norm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.25.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.25.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.25.self_attn.q_norm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.25.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.25.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.26.input_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.26.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.26.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.26.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.26.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.26.self_attn.k_norm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.26.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.26.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.26.self_attn.q_norm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.26.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.26.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.27.input_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.27.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.27.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.27.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.27.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.27.self_attn.k_norm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.27.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.27.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.27.self_attn.q_norm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.27.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.27.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.28.input_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.28.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.28.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.28.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.28.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.28.self_attn.k_norm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.28.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.28.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.28.self_attn.q_norm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.28.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.28.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.29.input_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.29.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.29.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.29.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.29.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.29.self_attn.k_norm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.29.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.29.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.29.self_attn.q_norm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.29.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.29.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.3.input_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.3.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.3.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.3.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.3.post_attention_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.3.self_attn.k_norm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.3.self_attn.k_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.3.self_attn.o_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.3.self_attn.q_norm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.3.self_attn.q_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.3.self_attn.v_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.30.input_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.30.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.30.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.30.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.30.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.30.self_attn.k_norm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.30.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.30.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.30.self_attn.q_norm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.30.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.30.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.31.input_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.31.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.31.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.31.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.31.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.31.self_attn.k_norm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.31.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.31.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.31.self_attn.q_norm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.31.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.31.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.32.input_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.32.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.32.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.32.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.32.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.32.self_attn.k_norm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.32.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.32.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.32.self_attn.q_norm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.32.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.32.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.33.input_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.33.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.33.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.33.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.33.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.33.self_attn.k_norm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.33.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.33.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.33.self_attn.q_norm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.33.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.33.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.34.input_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.34.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.34.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.34.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.34.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.34.self_attn.k_norm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.34.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.34.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.34.self_attn.q_norm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.34.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.34.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.35.input_layernorm.weight": "model-00004-of-00004.safetensors",
+    "model.layers.35.mlp.down_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.35.mlp.gate_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.35.mlp.up_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.35.post_attention_layernorm.weight": "model-00004-of-00004.safetensors",
+    "model.layers.35.self_attn.k_norm.weight": "model-00004-of-00004.safetensors",
+    "model.layers.35.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.35.self_attn.o_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.35.self_attn.q_norm.weight": "model-00004-of-00004.safetensors",
+    "model.layers.35.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.35.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.4.input_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.4.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.4.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.4.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.4.post_attention_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.4.self_attn.k_norm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.4.self_attn.k_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.4.self_attn.o_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.4.self_attn.q_norm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.4.self_attn.q_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.4.self_attn.v_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.5.input_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.5.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.5.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.5.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.5.post_attention_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.5.self_attn.k_norm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.5.self_attn.k_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.5.self_attn.o_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.5.self_attn.q_norm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.5.self_attn.q_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.5.self_attn.v_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.6.input_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.6.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.6.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.6.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.6.post_attention_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.6.self_attn.k_norm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.6.self_attn.k_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.6.self_attn.o_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.6.self_attn.q_norm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.6.self_attn.q_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.6.self_attn.v_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.7.input_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.7.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.7.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.7.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.7.post_attention_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.7.self_attn.k_norm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.7.self_attn.k_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.7.self_attn.o_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.7.self_attn.q_norm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.7.self_attn.q_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.7.self_attn.v_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.8.input_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.8.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.8.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.8.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.8.post_attention_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.8.self_attn.k_norm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.8.self_attn.k_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.8.self_attn.o_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.8.self_attn.q_norm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.8.self_attn.q_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.8.self_attn.v_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.9.input_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.9.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.9.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.9.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.9.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.9.self_attn.k_norm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.9.self_attn.k_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.9.self_attn.o_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.9.self_attn.q_norm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.9.self_attn.q_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.9.self_attn.v_proj.weight": "model-00001-of-00004.safetensors",
+    "model.norm.weight": "model-00004-of-00004.safetensors"
+  }
+}

g2_5_plan_newcode/250M/special_tokens_map.json

@@ -0,0 +1,31 @@
+{
+  "additional_special_tokens": [
+    "<|im_start|>",
+    "<|im_end|>",
+    "<|object_ref_start|>",
+    "<|object_ref_end|>",
+    "<|box_start|>",
+    "<|box_end|>",
+    "<|quad_start|>",
+    "<|quad_end|>",
+    "<|vision_start|>",
+    "<|vision_end|>",
+    "<|vision_pad|>",
+    "<|image_pad|>",
+    "<|video_pad|>"
+  ],
+  "eos_token": {
+    "content": "<|endoftext|>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "pad_token": {
+    "content": "<|endoftext|>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  }
+}

g2_5_plan_newcode/250M/tokenizer.json

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:aeb13307a71acd8fe81861d94ad54ab689df773318809eed3cbe794b4492dae4
+size 11422654

g2_5_plan_newcode/250M/tokenizer_config.json

@@ -0,0 +1,240 @@
+{
+  "add_bos_token": false,
+  "add_prefix_space": false,
+  "added_tokens_decoder": {
+    "151643": {
+      "content": "<|endoftext|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151644": {
+      "content": "<|im_start|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151645": {
+      "content": "<|im_end|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151646": {
+      "content": "<|object_ref_start|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151647": {
+      "content": "<|object_ref_end|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151648": {
+      "content": "<|box_start|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151649": {
+      "content": "<|box_end|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151650": {
+      "content": "<|quad_start|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151651": {
+      "content": "<|quad_end|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151652": {
+      "content": "<|vision_start|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151653": {
+      "content": "<|vision_end|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151654": {
+      "content": "<|vision_pad|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151655": {
+      "content": "<|image_pad|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151656": {
+      "content": "<|video_pad|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151657": {
+      "content": "<tool_call>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151658": {
+      "content": "</tool_call>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151659": {
+      "content": "<|fim_prefix|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151660": {
+      "content": "<|fim_middle|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151661": {
+      "content": "<|fim_suffix|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151662": {
+      "content": "<|fim_pad|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151663": {
+      "content": "<|repo_name|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151664": {
+      "content": "<|file_sep|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151665": {
+      "content": "<tool_response>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151666": {
+      "content": "</tool_response>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151667": {
+      "content": "<think>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151668": {
+      "content": "</think>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    }
+  },
+  "additional_special_tokens": [
+    "<|im_start|>",
+    "<|im_end|>",
+    "<|object_ref_start|>",
+    "<|object_ref_end|>",
+    "<|box_start|>",
+    "<|box_end|>",
+    "<|quad_start|>",
+    "<|quad_end|>",
+    "<|vision_start|>",
+    "<|vision_end|>",
+    "<|vision_pad|>",
+    "<|image_pad|>",
+    "<|video_pad|>"
+  ],
+  "bos_token": null,
+  "clean_up_tokenization_spaces": false,
+  "eos_token": "<|endoftext|>",
+  "errors": "replace",
+  "extra_special_tokens": {},
+  "model_max_length": 131072,
+  "pad_token": "<|endoftext|>",
+  "padding_side": "right",
+  "split_special_tokens": false,
+  "tokenizer_class": "Qwen2Tokenizer",
+  "unk_token": null
+}

g2_5_plan_newcode/250M/train_results.json

@@ -0,0 +1,8 @@
+{
+    "epoch": 0.2459549771368273,
+    "total_flos": 1.2485883762455871e+19,
+    "train_loss": 0.06533449515300774,
+    "train_runtime": 2953.8926,
+    "train_samples_per_second": 11.363,
+    "train_steps_per_second": 0.947
+}

g2_5_plan_newcode/250M/trainer_log.jsonl

@@ -0,0 +1,26 @@
+{"current_steps": 2550, "total_steps": 2797, "loss": 0.7326, "lr": 1.997261400182901e-05, "epoch": 0.22423496306718255, "percentage": 91.17, "elapsed_time": "0:02:22", "remaining_time": "0:00:13"}
+{"current_steps": 2560, "total_steps": 2797, "loss": 0.7138, "lr": 1.9805800746810423e-05, "epoch": 0.22511431586352446, "percentage": 91.53, "elapsed_time": "0:04:15", "remaining_time": "0:00:23"}
+{"current_steps": 2570, "total_steps": 2797, "loss": 0.722, "lr": 1.9489934220110274e-05, "epoch": 0.22599366865986634, "percentage": 91.88, "elapsed_time": "0:06:08", "remaining_time": "0:00:32"}
+{"current_steps": 2580, "total_steps": 2797, "loss": 0.738, "lr": 1.9029840364820392e-05, "epoch": 0.22687302145620822, "percentage": 92.24, "elapsed_time": "0:08:01", "remaining_time": "0:00:40"}
+{"current_steps": 2590, "total_steps": 2797, "loss": 0.7144, "lr": 1.8432548690479908e-05, "epoch": 0.22775237425255013, "percentage": 92.6, "elapsed_time": "0:09:54", "remaining_time": "0:00:47"}
+{"current_steps": 2600, "total_steps": 2797, "loss": 0.7187, "lr": 1.7707184873239772e-05, "epoch": 0.228631727048892, "percentage": 92.96, "elapsed_time": "0:11:47", "remaining_time": "0:00:53"}
+{"current_steps": 2610, "total_steps": 2797, "loss": 0.7173, "lr": 1.6864831329902565e-05, "epoch": 0.22951107984523392, "percentage": 93.31, "elapsed_time": "0:13:40", "remaining_time": "0:00:58"}
+{"current_steps": 2620, "total_steps": 2797, "loss": 0.7408, "lr": 1.591835789604667e-05, "epoch": 0.2303904326415758, "percentage": 93.67, "elapsed_time": "0:15:33", "remaining_time": "0:01:03"}
+{"current_steps": 2630, "total_steps": 2797, "loss": 0.73, "lr": 1.4882225195206459e-05, "epoch": 0.23126978543791768, "percentage": 94.03, "elapsed_time": "0:17:27", "remaining_time": "0:01:06"}
+{"current_steps": 2640, "total_steps": 2797, "loss": 0.7141, "lr": 1.3772263703317695e-05, "epoch": 0.2321491382342596, "percentage": 94.39, "elapsed_time": "0:19:20", "remaining_time": "0:01:08"}
+{"current_steps": 2650, "total_steps": 2797, "loss": 0.7126, "lr": 1.2605431883975547e-05, "epoch": 0.23302849103060147, "percentage": 94.74, "elapsed_time": "0:21:13", "remaining_time": "0:01:10"}
+{"current_steps": 2660, "total_steps": 2797, "loss": 0.7233, "lr": 1.1399557089817574e-05, "epoch": 0.23390784382694338, "percentage": 95.1, "elapsed_time": "0:23:06", "remaining_time": "0:01:11"}
+{"current_steps": 2670, "total_steps": 2797, "loss": 0.7135, "lr": 1.0173063188650463e-05, "epoch": 0.23478719662328526, "percentage": 95.46, "elapsed_time": "0:24:59", "remaining_time": "0:01:11"}
+{"current_steps": 2680, "total_steps": 2797, "loss": 0.7382, "lr": 8.944689075764396e-06, "epoch": 0.23566654941962714, "percentage": 95.82, "elapsed_time": "0:26:52", "remaining_time": "0:01:10"}
+{"current_steps": 2690, "total_steps": 2797, "loss": 0.7013, "lr": 7.73320237312362e-06, "epoch": 0.23654590221596905, "percentage": 96.17, "elapsed_time": "0:28:45", "remaining_time": "0:01:08"}
+{"current_steps": 2700, "total_steps": 2797, "loss": 0.7246, "lr": 6.557112689658629e-06, "epoch": 0.23742525501231093, "percentage": 96.53, "elapsed_time": "0:30:38", "remaining_time": "0:01:06"}
+{"current_steps": 2710, "total_steps": 2797, "loss": 0.7149, "lr": 5.434388823591213e-06, "epoch": 0.23830460780865284, "percentage": 96.89, "elapsed_time": "0:32:32", "remaining_time": "0:01:02"}
+{"current_steps": 2720, "total_steps": 2797, "loss": 0.7215, "lr": 4.382184227495424e-06, "epoch": 0.23918396060499472, "percentage": 97.25, "elapsed_time": "0:34:25", "remaining_time": "0:00:58"}
+{"current_steps": 2730, "total_steps": 2797, "loss": 0.7156, "lr": 3.4165749305560203e-06, "epoch": 0.2400633134013366, "percentage": 97.6, "elapsed_time": "0:36:18", "remaining_time": "0:00:53"}
+{"current_steps": 2740, "total_steps": 2797, "loss": 0.7004, "lr": 2.5523139221595556e-06, "epoch": 0.2409426661976785, "percentage": 97.96, "elapsed_time": "0:38:11", "remaining_time": "0:00:47"}
+{"current_steps": 2750, "total_steps": 2797, "loss": 0.7005, "lr": 1.8026057494499821e-06, "epoch": 0.2418220189940204, "percentage": 98.32, "elapsed_time": "0:40:04", "remaining_time": "0:00:41"}
+{"current_steps": 2760, "total_steps": 2797, "loss": 0.7192, "lr": 1.178904772643223e-06, "epoch": 0.2427013717903623, "percentage": 98.68, "elapsed_time": "0:41:57", "remaining_time": "0:00:33"}
+{"current_steps": 2770, "total_steps": 2797, "loss": 0.7016, "lr": 6.90740160441754e-07, "epoch": 0.24358072458670418, "percentage": 99.03, "elapsed_time": "0:43:50", "remaining_time": "0:00:25"}
+{"current_steps": 2780, "total_steps": 2797, "loss": 0.7231, "lr": 3.45570299343632e-07, "epoch": 0.24446007738304606, "percentage": 99.39, "elapsed_time": "0:45:43", "remaining_time": "0:00:16"}
+{"current_steps": 2790, "total_steps": 2797, "loss": 0.7384, "lr": 1.486688412425152e-07, "epoch": 0.24533943017938797, "percentage": 99.75, "elapsed_time": "0:47:36", "remaining_time": "0:00:07"}
+{"current_steps": 2797, "total_steps": 2797, "epoch": 0.2459549771368273, "percentage": 100.0, "elapsed_time": "0:49:13", "remaining_time": "0:00:00"}

g2_5_plan_newcode/250M/trainer_state.json

@@ -0,0 +1,1996 @@
+{
+  "best_global_step": null,
+  "best_metric": null,
+  "best_model_checkpoint": null,
+  "epoch": 0.2459549771368273,
+  "eval_steps": 500,
+  "global_step": 2797,
+  "is_hyper_param_search": false,
+  "is_local_process_zero": true,
+  "is_world_process_zero": true,
+  "log_history": [
+    {
+      "epoch": 0.0008793527963418924,
+      "grad_norm": 2.1166908740997314,
+      "learning_rate": 1.1061797752808989e-06,
+      "loss": 1.0021,
+      "step": 10
+    },
+    {
+      "epoch": 0.0017587055926837848,
+      "grad_norm": 1.536681056022644,
+      "learning_rate": 2.2241573033707866e-06,
+      "loss": 0.9972,
+      "step": 20
+    },
+    {
+      "epoch": 0.002638058389025677,
+      "grad_norm": 1.2352495193481445,
+      "learning_rate": 3.342134831460675e-06,
+      "loss": 0.931,
+      "step": 30
+    },
+    {
+      "epoch": 0.0035174111853675696,
+      "grad_norm": 1.0416210889816284,
+      "learning_rate": 4.4601123595505624e-06,
+      "loss": 0.8887,
+      "step": 40
+    },
+    {
+      "epoch": 0.004396763981709462,
+      "grad_norm": 0.9612919688224792,
+      "learning_rate": 5.5780898876404495e-06,
+      "loss": 0.8434,
+      "step": 50
+    },
+    {
+      "epoch": 0.005276116778051354,
+      "grad_norm": 0.9480857253074646,
+      "learning_rate": 6.6960674157303374e-06,
+      "loss": 0.8409,
+      "step": 60
+    },
+    {
+      "epoch": 0.006155469574393247,
+      "grad_norm": 0.8386456966400146,
+      "learning_rate": 7.814044943820226e-06,
+      "loss": 0.8454,
+      "step": 70
+    },
+    {
+      "epoch": 0.007034822370735139,
+      "grad_norm": 0.9042791724205017,
+      "learning_rate": 8.932022471910112e-06,
+      "loss": 0.8192,
+      "step": 80
+    },
+    {
+      "epoch": 0.007914175167077032,
+      "grad_norm": 0.9393767714500427,
+      "learning_rate": 1.005e-05,
+      "loss": 0.7924,
+      "step": 90
+    },
+    {
+      "epoch": 0.008793527963418923,
+      "grad_norm": 0.8629812002182007,
+      "learning_rate": 1.1167977528089888e-05,
+      "loss": 0.8112,
+      "step": 100
+    },
+    {
+      "epoch": 0.009672880759760817,
+      "grad_norm": 0.916533350944519,
+      "learning_rate": 1.2285955056179776e-05,
+      "loss": 0.8029,
+      "step": 110
+    },
+    {
+      "epoch": 0.010552233556102708,
+      "grad_norm": 0.8504175543785095,
+      "learning_rate": 1.3403932584269664e-05,
+      "loss": 0.8107,
+      "step": 120
+    },
+    {
+      "epoch": 0.0114315863524446,
+      "grad_norm": 0.8702086210250854,
+      "learning_rate": 1.4521910112359552e-05,
+      "loss": 0.8049,
+      "step": 130
+    },
+    {
+      "epoch": 0.012310939148786493,
+      "grad_norm": 0.9796447157859802,
+      "learning_rate": 1.563988764044944e-05,
+      "loss": 0.797,
+      "step": 140
+    },
+    {
+      "epoch": 0.013190291945128385,
+      "grad_norm": 0.8172125816345215,
+      "learning_rate": 1.6757865168539328e-05,
+      "loss": 0.7974,
+      "step": 150
+    },
+    {
+      "epoch": 0.014069644741470278,
+      "grad_norm": 0.8361979126930237,
+      "learning_rate": 1.7875842696629214e-05,
+      "loss": 0.7993,
+      "step": 160
+    },
+    {
+      "epoch": 0.01494899753781217,
+      "grad_norm": 0.8309073448181152,
+      "learning_rate": 1.89938202247191e-05,
+      "loss": 0.7973,
+      "step": 170
+    },
+    {
+      "epoch": 0.015828350334154064,
+      "grad_norm": 0.7882160544395447,
+      "learning_rate": 2e-05,
+      "loss": 0.7721,
+      "step": 180
+    },
+    {
+      "epoch": 0.016707703130495955,
+      "grad_norm": 0.750810444355011,
+      "learning_rate": 2e-05,
+      "loss": 0.7777,
+      "step": 190
+    },
+    {
+      "epoch": 0.017587055926837847,
+      "grad_norm": 0.7132768034934998,
+      "learning_rate": 2e-05,
+      "loss": 0.8015,
+      "step": 200
+    },
+    {
+      "epoch": 0.01846640872317974,
+      "grad_norm": 0.6944462656974792,
+      "learning_rate": 2e-05,
+      "loss": 0.7845,
+      "step": 210
+    },
+    {
+      "epoch": 0.019345761519521634,
+      "grad_norm": 0.8518295288085938,
+      "learning_rate": 2e-05,
+      "loss": 0.7943,
+      "step": 220
+    },
+    {
+      "epoch": 0.020225114315863525,
+      "grad_norm": 0.7759379148483276,
+      "learning_rate": 2e-05,
+      "loss": 0.792,
+      "step": 230
+    },
+    {
+      "epoch": 0.021104467112205417,
+      "grad_norm": 0.6462895274162292,
+      "learning_rate": 2e-05,
+      "loss": 0.7865,
+      "step": 240
+    },
+    {
+      "epoch": 0.02198381990854731,
+      "grad_norm": 0.6220803260803223,
+      "learning_rate": 2e-05,
+      "loss": 0.7624,
+      "step": 250
+    },
+    {
+      "epoch": 0.0228631727048892,
+      "grad_norm": 0.7267147302627563,
+      "learning_rate": 2e-05,
+      "loss": 0.7778,
+      "step": 260
+    },
+    {
+      "epoch": 0.023742525501231095,
+      "grad_norm": 0.5965536236763,
+      "learning_rate": 2e-05,
+      "loss": 0.7757,
+      "step": 270
+    },
+    {
+      "epoch": 0.024621878297572987,
+      "grad_norm": 0.6194908022880554,
+      "learning_rate": 2e-05,
+      "loss": 0.7661,
+      "step": 280
+    },
+    {
+      "epoch": 0.02550123109391488,
+      "grad_norm": 0.6582121849060059,
+      "learning_rate": 2e-05,
+      "loss": 0.7804,
+      "step": 290
+    },
+    {
+      "epoch": 0.02638058389025677,
+      "grad_norm": 0.6081708073616028,
+      "learning_rate": 2e-05,
+      "loss": 0.7959,
+      "step": 300
+    },
+    {
+      "epoch": 0.027259936686598662,
+      "grad_norm": 0.5630402565002441,
+      "learning_rate": 2e-05,
+      "loss": 0.7803,
+      "step": 310
+    },
+    {
+      "epoch": 0.028139289482940557,
+      "grad_norm": 0.6008561849594116,
+      "learning_rate": 2e-05,
+      "loss": 0.7884,
+      "step": 320
+    },
+    {
+      "epoch": 0.02901864227928245,
+      "grad_norm": 0.669390857219696,
+      "learning_rate": 2e-05,
+      "loss": 0.7687,
+      "step": 330
+    },
+    {
+      "epoch": 0.02989799507562434,
+      "grad_norm": 0.6076467037200928,
+      "learning_rate": 2e-05,
+      "loss": 0.7697,
+      "step": 340
+    },
+    {
+      "epoch": 0.030777347871966232,
+      "grad_norm": 0.5559185743331909,
+      "learning_rate": 2e-05,
+      "loss": 0.777,
+      "step": 350
+    },
+    {
+      "epoch": 0.03165670066830813,
+      "grad_norm": 0.5372075438499451,
+      "learning_rate": 2e-05,
+      "loss": 0.7702,
+      "step": 360
+    },
+    {
+      "epoch": 0.032536053464650015,
+      "grad_norm": 0.5853421688079834,
+      "learning_rate": 2e-05,
+      "loss": 0.7664,
+      "step": 370
+    },
+    {
+      "epoch": 0.03341540626099191,
+      "grad_norm": 0.525594174861908,
+      "learning_rate": 2e-05,
+      "loss": 0.7541,
+      "step": 380
+    },
+    {
+      "epoch": 0.034294759057333805,
+      "grad_norm": 0.5938466191291809,
+      "learning_rate": 2e-05,
+      "loss": 0.7783,
+      "step": 390
+    },
+    {
+      "epoch": 0.035174111853675694,
+      "grad_norm": 0.5581024289131165,
+      "learning_rate": 2e-05,
+      "loss": 0.7484,
+      "step": 400
+    },
+    {
+      "epoch": 0.03605346465001759,
+      "grad_norm": 0.5327697992324829,
+      "learning_rate": 2e-05,
+      "loss": 0.7836,
+      "step": 410
+    },
+    {
+      "epoch": 0.03693281744635948,
+      "grad_norm": 0.538017749786377,
+      "learning_rate": 2e-05,
+      "loss": 0.7631,
+      "step": 420
+    },
+    {
+      "epoch": 0.03781217024270137,
+      "grad_norm": 0.5272985696792603,
+      "learning_rate": 2e-05,
+      "loss": 0.7618,
+      "step": 430
+    },
+    {
+      "epoch": 0.03869152303904327,
+      "grad_norm": 0.6369595527648926,
+      "learning_rate": 2e-05,
+      "loss": 0.7641,
+      "step": 440
+    },
+    {
+      "epoch": 0.039570875835385155,
+      "grad_norm": 0.6350115537643433,
+      "learning_rate": 2e-05,
+      "loss": 0.7555,
+      "step": 450
+    },
+    {
+      "epoch": 0.04045022863172705,
+      "grad_norm": 0.5966277718544006,
+      "learning_rate": 2e-05,
+      "loss": 0.7663,
+      "step": 460
+    },
+    {
+      "epoch": 0.04132958142806894,
+      "grad_norm": 0.5093820095062256,
+      "learning_rate": 2e-05,
+      "loss": 0.7696,
+      "step": 470
+    },
+    {
+      "epoch": 0.042208934224410834,
+      "grad_norm": 0.518803596496582,
+      "learning_rate": 2e-05,
+      "loss": 0.7562,
+      "step": 480
+    },
+    {
+      "epoch": 0.04308828702075273,
+      "grad_norm": 0.591204822063446,
+      "learning_rate": 2e-05,
+      "loss": 0.7594,
+      "step": 490
+    },
+    {
+      "epoch": 0.04396763981709462,
+      "grad_norm": 0.5149183869361877,
+      "learning_rate": 2e-05,
+      "loss": 0.7725,
+      "step": 500
+    },
+    {
+      "epoch": 0.04484699261343651,
+      "grad_norm": 0.528752326965332,
+      "learning_rate": 2e-05,
+      "loss": 0.7713,
+      "step": 510
+    },
+    {
+      "epoch": 0.0457263454097784,
+      "grad_norm": 0.5549413561820984,
+      "learning_rate": 2e-05,
+      "loss": 0.7823,
+      "step": 520
+    },
+    {
+      "epoch": 0.046605698206120295,
+      "grad_norm": 0.5397849678993225,
+      "learning_rate": 2e-05,
+      "loss": 0.7348,
+      "step": 530
+    },
+    {
+      "epoch": 0.04748505100246219,
+      "grad_norm": 0.5254166722297668,
+      "learning_rate": 2e-05,
+      "loss": 0.7641,
+      "step": 540
+    },
+    {
+      "epoch": 0.04836440379880408,
+      "grad_norm": 0.5081286430358887,
+      "learning_rate": 2e-05,
+      "loss": 0.7655,
+      "step": 550
+    },
+    {
+      "epoch": 0.049243756595145974,
+      "grad_norm": 0.5173667073249817,
+      "learning_rate": 2e-05,
+      "loss": 0.7491,
+      "step": 560
+    },
+    {
+      "epoch": 0.05012310939148786,
+      "grad_norm": 0.5136865973472595,
+      "learning_rate": 2e-05,
+      "loss": 0.7604,
+      "step": 570
+    },
+    {
+      "epoch": 0.05100246218782976,
+      "grad_norm": 0.5271310210227966,
+      "learning_rate": 2e-05,
+      "loss": 0.7628,
+      "step": 580
+    },
+    {
+      "epoch": 0.05188181498417165,
+      "grad_norm": 0.648137629032135,
+      "learning_rate": 2e-05,
+      "loss": 0.7725,
+      "step": 590
+    },
+    {
+      "epoch": 0.05276116778051354,
+      "grad_norm": 0.5719968676567078,
+      "learning_rate": 2e-05,
+      "loss": 0.7788,
+      "step": 600
+    },
+    {
+      "epoch": 0.053640520576855436,
+      "grad_norm": 0.5201082229614258,
+      "learning_rate": 2e-05,
+      "loss": 0.7564,
+      "step": 610
+    },
+    {
+      "epoch": 0.054519873373197324,
+      "grad_norm": 0.5054596066474915,
+      "learning_rate": 2e-05,
+      "loss": 0.7518,
+      "step": 620
+    },
+    {
+      "epoch": 0.05539922616953922,
+      "grad_norm": 0.4945407807826996,
+      "learning_rate": 2e-05,
+      "loss": 0.7322,
+      "step": 630
+    },
+    {
+      "epoch": 0.056278578965881114,
+      "grad_norm": 0.5015317797660828,
+      "learning_rate": 2e-05,
+      "loss": 0.7638,
+      "step": 640
+    },
+    {
+      "epoch": 0.057157931762223,
+      "grad_norm": 0.5158993601799011,
+      "learning_rate": 2e-05,
+      "loss": 0.7508,
+      "step": 650
+    },
+    {
+      "epoch": 0.0580372845585649,
+      "grad_norm": 0.7721976637840271,
+      "learning_rate": 2e-05,
+      "loss": 0.744,
+      "step": 660
+    },
+    {
+      "epoch": 0.058916637354906785,
+      "grad_norm": 0.4921807646751404,
+      "learning_rate": 2e-05,
+      "loss": 0.748,
+      "step": 670
+    },
+    {
+      "epoch": 0.05979599015124868,
+      "grad_norm": 0.48958244919776917,
+      "learning_rate": 2e-05,
+      "loss": 0.7679,
+      "step": 680
+    },
+    {
+      "epoch": 0.060675342947590576,
+      "grad_norm": 0.5455403923988342,
+      "learning_rate": 2e-05,
+      "loss": 0.75,
+      "step": 690
+    },
+    {
+      "epoch": 0.061554695743932464,
+      "grad_norm": 0.5371265411376953,
+      "learning_rate": 2e-05,
+      "loss": 0.7622,
+      "step": 700
+    },
+    {
+      "epoch": 0.06243404854027436,
+      "grad_norm": 0.5122358202934265,
+      "learning_rate": 2e-05,
+      "loss": 0.7665,
+      "step": 710
+    },
+    {
+      "epoch": 0.06331340133661625,
+      "grad_norm": 0.47188127040863037,
+      "learning_rate": 2e-05,
+      "loss": 0.7684,
+      "step": 720
+    },
+    {
+      "epoch": 0.06419275413295815,
+      "grad_norm": 0.5171350836753845,
+      "learning_rate": 2e-05,
+      "loss": 0.7599,
+      "step": 730
+    },
+    {
+      "epoch": 0.06507210692930003,
+      "grad_norm": 0.5169857740402222,
+      "learning_rate": 2e-05,
+      "loss": 0.7634,
+      "step": 740
+    },
+    {
+      "epoch": 0.06595145972564193,
+      "grad_norm": 0.4981776773929596,
+      "learning_rate": 2e-05,
+      "loss": 0.7528,
+      "step": 750
+    },
+    {
+      "epoch": 0.06683081252198382,
+      "grad_norm": 0.4966522753238678,
+      "learning_rate": 2e-05,
+      "loss": 0.7453,
+      "step": 760
+    },
+    {
+      "epoch": 0.06771016531832572,
+      "grad_norm": 0.5899744629859924,
+      "learning_rate": 2e-05,
+      "loss": 0.7471,
+      "step": 770
+    },
+    {
+      "epoch": 0.06858951811466761,
+      "grad_norm": 0.5051476359367371,
+      "learning_rate": 2e-05,
+      "loss": 0.7455,
+      "step": 780
+    },
+    {
+      "epoch": 0.06946887091100949,
+      "grad_norm": 0.4886556565761566,
+      "learning_rate": 2e-05,
+      "loss": 0.767,
+      "step": 790
+    },
+    {
+      "epoch": 0.07034822370735139,
+      "grad_norm": 0.45599037408828735,
+      "learning_rate": 2e-05,
+      "loss": 0.7427,
+      "step": 800
+    },
+    {
+      "epoch": 0.07122757650369328,
+      "grad_norm": 0.6188297271728516,
+      "learning_rate": 2e-05,
+      "loss": 0.7566,
+      "step": 810
+    },
+    {
+      "epoch": 0.07210692930003518,
+      "grad_norm": 0.47206422686576843,
+      "learning_rate": 2e-05,
+      "loss": 0.7573,
+      "step": 820
+    },
+    {
+      "epoch": 0.07298628209637707,
+      "grad_norm": 0.5380171537399292,
+      "learning_rate": 2e-05,
+      "loss": 0.7648,
+      "step": 830
+    },
+    {
+      "epoch": 0.07386563489271895,
+      "grad_norm": 0.4992441236972809,
+      "learning_rate": 2e-05,
+      "loss": 0.7384,
+      "step": 840
+    },
+    {
+      "epoch": 0.07474498768906085,
+      "grad_norm": 0.5099149346351624,
+      "learning_rate": 2e-05,
+      "loss": 0.7636,
+      "step": 850
+    },
+    {
+      "epoch": 0.07562434048540274,
+      "grad_norm": 0.4976833462715149,
+      "learning_rate": 2e-05,
+      "loss": 0.7494,
+      "step": 860
+    },
+    {
+      "epoch": 0.07650369328174464,
+      "grad_norm": 0.4676611125469208,
+      "learning_rate": 2e-05,
+      "loss": 0.7458,
+      "step": 870
+    },
+    {
+      "epoch": 0.07738304607808653,
+      "grad_norm": 0.5090939402580261,
+      "learning_rate": 2e-05,
+      "loss": 0.7474,
+      "step": 880
+    },
+    {
+      "epoch": 0.07826239887442842,
+      "grad_norm": 0.47629624605178833,
+      "learning_rate": 2e-05,
+      "loss": 0.7507,
+      "step": 890
+    },
+    {
+      "epoch": 0.07914175167077031,
+      "grad_norm": 0.512294590473175,
+      "learning_rate": 2e-05,
+      "loss": 0.7304,
+      "step": 900
+    },
+    {
+      "epoch": 0.0800211044671122,
+      "grad_norm": 0.5154348015785217,
+      "learning_rate": 2e-05,
+      "loss": 0.7335,
+      "step": 910
+    },
+    {
+      "epoch": 0.0809004572634541,
+      "grad_norm": 0.7091978788375854,
+      "learning_rate": 2e-05,
+      "loss": 0.7182,
+      "step": 920
+    },
+    {
+      "epoch": 0.081779810059796,
+      "grad_norm": 0.4717979431152344,
+      "learning_rate": 2e-05,
+      "loss": 0.7423,
+      "step": 930
+    },
+    {
+      "epoch": 0.08265916285613788,
+      "grad_norm": 0.5156192779541016,
+      "learning_rate": 2e-05,
+      "loss": 0.7503,
+      "step": 940
+    },
+    {
+      "epoch": 0.08353851565247977,
+      "grad_norm": 0.5224230885505676,
+      "learning_rate": 2e-05,
+      "loss": 0.7488,
+      "step": 950
+    },
+    {
+      "epoch": 0.08441786844882167,
+      "grad_norm": 0.5670732259750366,
+      "learning_rate": 2e-05,
+      "loss": 0.7406,
+      "step": 960
+    },
+    {
+      "epoch": 0.08529722124516356,
+      "grad_norm": 0.4843250513076782,
+      "learning_rate": 2e-05,
+      "loss": 0.7588,
+      "step": 970
+    },
+    {
+      "epoch": 0.08617657404150546,
+      "grad_norm": 0.5184202194213867,
+      "learning_rate": 2e-05,
+      "loss": 0.7406,
+      "step": 980
+    },
+    {
+      "epoch": 0.08705592683784734,
+      "grad_norm": 0.47303125262260437,
+      "learning_rate": 2e-05,
+      "loss": 0.745,
+      "step": 990
+    },
+    {
+      "epoch": 0.08793527963418923,
+      "grad_norm": 0.5098772644996643,
+      "learning_rate": 2e-05,
+      "loss": 0.7598,
+      "step": 1000
+    },
+    {
+      "epoch": 0.08881463243053113,
+      "grad_norm": 0.5410617589950562,
+      "learning_rate": 2e-05,
+      "loss": 0.7452,
+      "step": 1010
+    },
+    {
+      "epoch": 0.08969398522687302,
+      "grad_norm": 0.5066492557525635,
+      "learning_rate": 2e-05,
+      "loss": 0.7507,
+      "step": 1020
+    },
+    {
+      "epoch": 0.09057333802321492,
+      "grad_norm": 0.49481093883514404,
+      "learning_rate": 2e-05,
+      "loss": 0.7416,
+      "step": 1030
+    },
+    {
+      "epoch": 0.0914526908195568,
+      "grad_norm": 0.4778386652469635,
+      "learning_rate": 2e-05,
+      "loss": 0.7365,
+      "step": 1040
+    },
+    {
+      "epoch": 0.0923320436158987,
+      "grad_norm": 0.7310119867324829,
+      "learning_rate": 2e-05,
+      "loss": 0.7504,
+      "step": 1050
+    },
+    {
+      "epoch": 0.09321139641224059,
+      "grad_norm": 0.5311281681060791,
+      "learning_rate": 2e-05,
+      "loss": 0.7462,
+      "step": 1060
+    },
+    {
+      "epoch": 0.09409074920858249,
+      "grad_norm": 0.4910159409046173,
+      "learning_rate": 2e-05,
+      "loss": 0.7453,
+      "step": 1070
+    },
+    {
+      "epoch": 0.09497010200492438,
+      "grad_norm": 0.4807991683483124,
+      "learning_rate": 2e-05,
+      "loss": 0.7396,
+      "step": 1080
+    },
+    {
+      "epoch": 0.09584945480126626,
+      "grad_norm": 0.46889784932136536,
+      "learning_rate": 2e-05,
+      "loss": 0.7247,
+      "step": 1090
+    },
+    {
+      "epoch": 0.09672880759760816,
+      "grad_norm": 0.4893566966056824,
+      "learning_rate": 2e-05,
+      "loss": 0.7441,
+      "step": 1100
+    },
+    {
+      "epoch": 0.09760816039395005,
+      "grad_norm": 0.49207502603530884,
+      "learning_rate": 2e-05,
+      "loss": 0.7547,
+      "step": 1110
+    },
+    {
+      "epoch": 0.09848751319029195,
+      "grad_norm": 0.5297726988792419,
+      "learning_rate": 2e-05,
+      "loss": 0.746,
+      "step": 1120
+    },
+    {
+      "epoch": 0.09936686598663384,
+      "grad_norm": 0.4764488935470581,
+      "learning_rate": 2e-05,
+      "loss": 0.7488,
+      "step": 1130
+    },
+    {
+      "epoch": 0.10024621878297572,
+      "grad_norm": 0.5078687071800232,
+      "learning_rate": 2e-05,
+      "loss": 0.7619,
+      "step": 1140
+    },
+    {
+      "epoch": 0.10112557157931762,
+      "grad_norm": 0.5074195861816406,
+      "learning_rate": 2e-05,
+      "loss": 0.7367,
+      "step": 1150
+    },
+    {
+      "epoch": 0.10200492437565951,
+      "grad_norm": 0.4807511568069458,
+      "learning_rate": 2e-05,
+      "loss": 0.7571,
+      "step": 1160
+    },
+    {
+      "epoch": 0.10288427717200141,
+      "grad_norm": 0.4617295563220978,
+      "learning_rate": 2e-05,
+      "loss": 0.7528,
+      "step": 1170
+    },
+    {
+      "epoch": 0.1037636299683433,
+      "grad_norm": 0.5354571342468262,
+      "learning_rate": 2e-05,
+      "loss": 0.7434,
+      "step": 1180
+    },
+    {
+      "epoch": 0.10464298276468519,
+      "grad_norm": 0.49258777499198914,
+      "learning_rate": 2e-05,
+      "loss": 0.7374,
+      "step": 1190
+    },
+    {
+      "epoch": 0.10552233556102708,
+      "grad_norm": 0.4696492850780487,
+      "learning_rate": 2e-05,
+      "loss": 0.7514,
+      "step": 1200
+    },
+    {
+      "epoch": 0.10640168835736898,
+      "grad_norm": 0.4495760202407837,
+      "learning_rate": 2e-05,
+      "loss": 0.7214,
+      "step": 1210
+    },
+    {
+      "epoch": 0.10728104115371087,
+      "grad_norm": 0.4988597631454468,
+      "learning_rate": 2e-05,
+      "loss": 0.7313,
+      "step": 1220
+    },
+    {
+      "epoch": 0.10816039395005277,
+      "grad_norm": 0.6068702340126038,
+      "learning_rate": 2e-05,
+      "loss": 0.7312,
+      "step": 1230
+    },
+    {
+      "epoch": 0.10903974674639465,
+      "grad_norm": 0.4932054579257965,
+      "learning_rate": 2e-05,
+      "loss": 0.7609,
+      "step": 1240
+    },
+    {
+      "epoch": 0.10991909954273654,
+      "grad_norm": 0.4723249077796936,
+      "learning_rate": 2e-05,
+      "loss": 0.7471,
+      "step": 1250
+    },
+    {
+      "epoch": 0.11079845233907844,
+      "grad_norm": 0.49244555830955505,
+      "learning_rate": 2e-05,
+      "loss": 0.7559,
+      "step": 1260
+    },
+    {
+      "epoch": 0.11167780513542033,
+      "grad_norm": 0.4663756787776947,
+      "learning_rate": 2e-05,
+      "loss": 0.7363,
+      "step": 1270
+    },
+    {
+      "epoch": 0.11255715793176223,
+      "grad_norm": 0.4718684256076813,
+      "learning_rate": 2e-05,
+      "loss": 0.7429,
+      "step": 1280
+    },
+    {
+      "epoch": 0.11343651072810411,
+      "grad_norm": 0.48385754227638245,
+      "learning_rate": 2e-05,
+      "loss": 0.7108,
+      "step": 1290
+    },
+    {
+      "epoch": 0.114315863524446,
+      "grad_norm": 1.039713740348816,
+      "learning_rate": 2e-05,
+      "loss": 0.7379,
+      "step": 1300
+    },
+    {
+      "epoch": 0.1151952163207879,
+      "grad_norm": 0.46157896518707275,
+      "learning_rate": 2e-05,
+      "loss": 0.7458,
+      "step": 1310
+    },
+    {
+      "epoch": 0.1160745691171298,
+      "grad_norm": 0.5224857330322266,
+      "learning_rate": 2e-05,
+      "loss": 0.7342,
+      "step": 1320
+    },
+    {
+      "epoch": 0.11695392191347169,
+      "grad_norm": 0.5942857265472412,
+      "learning_rate": 2e-05,
+      "loss": 0.7525,
+      "step": 1330
+    },
+    {
+      "epoch": 0.11783327470981357,
+      "grad_norm": 0.4735294282436371,
+      "learning_rate": 2e-05,
+      "loss": 0.7295,
+      "step": 1340
+    },
+    {
+      "epoch": 0.11871262750615547,
+      "grad_norm": 0.4908852279186249,
+      "learning_rate": 2e-05,
+      "loss": 0.7348,
+      "step": 1350
+    },
+    {
+      "epoch": 0.11959198030249736,
+      "grad_norm": 0.46380698680877686,
+      "learning_rate": 2e-05,
+      "loss": 0.7628,
+      "step": 1360
+    },
+    {
+      "epoch": 0.12047133309883926,
+      "grad_norm": 0.48500046133995056,
+      "learning_rate": 2e-05,
+      "loss": 0.7522,
+      "step": 1370
+    },
+    {
+      "epoch": 0.12135068589518115,
+      "grad_norm": 0.4701886475086212,
+      "learning_rate": 2e-05,
+      "loss": 0.7365,
+      "step": 1380
+    },
+    {
+      "epoch": 0.12223003869152303,
+      "grad_norm": 0.46431034803390503,
+      "learning_rate": 2e-05,
+      "loss": 0.7404,
+      "step": 1390
+    },
+    {
+      "epoch": 0.12310939148786493,
+      "grad_norm": 0.4731382429599762,
+      "learning_rate": 2e-05,
+      "loss": 0.7174,
+      "step": 1400
+    },
+    {
+      "epoch": 0.12398874428420682,
+      "grad_norm": 0.4926745891571045,
+      "learning_rate": 2e-05,
+      "loss": 0.7396,
+      "step": 1410
+    },
+    {
+      "epoch": 0.12486809708054872,
+      "grad_norm": 0.4711820185184479,
+      "learning_rate": 2e-05,
+      "loss": 0.7431,
+      "step": 1420
+    },
+    {
+      "epoch": 0.1257474498768906,
+      "grad_norm": 0.47734490036964417,
+      "learning_rate": 2e-05,
+      "loss": 0.7574,
+      "step": 1430
+    },
+    {
+      "epoch": 0.1266268026732325,
+      "grad_norm": 0.4614863991737366,
+      "learning_rate": 2e-05,
+      "loss": 0.7462,
+      "step": 1440
+    },
+    {
+      "epoch": 0.1275061554695744,
+      "grad_norm": 0.4701177775859833,
+      "learning_rate": 2e-05,
+      "loss": 0.7237,
+      "step": 1450
+    },
+    {
+      "epoch": 0.1283855082659163,
+      "grad_norm": 0.4762236177921295,
+      "learning_rate": 2e-05,
+      "loss": 0.7262,
+      "step": 1460
+    },
+    {
+      "epoch": 0.12926486106225818,
+      "grad_norm": 0.4886734187602997,
+      "learning_rate": 2e-05,
+      "loss": 0.7587,
+      "step": 1470
+    },
+    {
+      "epoch": 0.13014421385860006,
+      "grad_norm": 0.519328773021698,
+      "learning_rate": 2e-05,
+      "loss": 0.7306,
+      "step": 1480
+    },
+    {
+      "epoch": 0.13102356665494197,
+      "grad_norm": 0.47738534212112427,
+      "learning_rate": 2e-05,
+      "loss": 0.7359,
+      "step": 1490
+    },
+    {
+      "epoch": 0.13190291945128385,
+      "grad_norm": 0.4676651954650879,
+      "learning_rate": 2e-05,
+      "loss": 0.7258,
+      "step": 1500
+    },
+    {
+      "epoch": 0.13278227224762576,
+      "grad_norm": 0.4624931216239929,
+      "learning_rate": 2e-05,
+      "loss": 0.7425,
+      "step": 1510
+    },
+    {
+      "epoch": 0.13366162504396764,
+      "grad_norm": 0.45219478011131287,
+      "learning_rate": 2e-05,
+      "loss": 0.7211,
+      "step": 1520
+    },
+    {
+      "epoch": 0.13454097784030952,
+      "grad_norm": 0.4627293348312378,
+      "learning_rate": 2e-05,
+      "loss": 0.7471,
+      "step": 1530
+    },
+    {
+      "epoch": 0.13542033063665143,
+      "grad_norm": 0.4875096380710602,
+      "learning_rate": 2e-05,
+      "loss": 0.7234,
+      "step": 1540
+    },
+    {
+      "epoch": 0.1362996834329933,
+      "grad_norm": 0.4549151659011841,
+      "learning_rate": 2e-05,
+      "loss": 0.7251,
+      "step": 1550
+    },
+    {
+      "epoch": 0.13717903622933522,
+      "grad_norm": 0.4761291444301605,
+      "learning_rate": 2e-05,
+      "loss": 0.7236,
+      "step": 1560
+    },
+    {
+      "epoch": 0.1380583890256771,
+      "grad_norm": 0.45360177755355835,
+      "learning_rate": 2e-05,
+      "loss": 0.7314,
+      "step": 1570
+    },
+    {
+      "epoch": 0.13893774182201898,
+      "grad_norm": 0.4698902368545532,
+      "learning_rate": 2e-05,
+      "loss": 0.7101,
+      "step": 1580
+    },
+    {
+      "epoch": 0.1398170946183609,
+      "grad_norm": 0.4637377858161926,
+      "learning_rate": 2e-05,
+      "loss": 0.7447,
+      "step": 1590
+    },
+    {
+      "epoch": 0.14069644741470277,
+      "grad_norm": 0.4513353109359741,
+      "learning_rate": 2e-05,
+      "loss": 0.7373,
+      "step": 1600
+    },
+    {
+      "epoch": 0.14157580021104468,
+      "grad_norm": 0.4970129430294037,
+      "learning_rate": 2e-05,
+      "loss": 0.7334,
+      "step": 1610
+    },
+    {
+      "epoch": 0.14245515300738656,
+      "grad_norm": 0.600988507270813,
+      "learning_rate": 2e-05,
+      "loss": 0.7557,
+      "step": 1620
+    },
+    {
+      "epoch": 0.14333450580372845,
+      "grad_norm": 0.4312986135482788,
+      "learning_rate": 2e-05,
+      "loss": 0.7586,
+      "step": 1630
+    },
+    {
+      "epoch": 0.14421385860007035,
+      "grad_norm": 0.4679642617702484,
+      "learning_rate": 2e-05,
+      "loss": 0.742,
+      "step": 1640
+    },
+    {
+      "epoch": 0.14509321139641224,
+      "grad_norm": 0.4587564468383789,
+      "learning_rate": 2e-05,
+      "loss": 0.7207,
+      "step": 1650
+    },
+    {
+      "epoch": 0.14597256419275415,
+      "grad_norm": 0.42665863037109375,
+      "learning_rate": 2e-05,
+      "loss": 0.7403,
+      "step": 1660
+    },
+    {
+      "epoch": 0.14685191698909603,
+      "grad_norm": 0.49361926317214966,
+      "learning_rate": 2e-05,
+      "loss": 0.7523,
+      "step": 1670
+    },
+    {
+      "epoch": 0.1477312697854379,
+      "grad_norm": 0.4327141344547272,
+      "learning_rate": 2e-05,
+      "loss": 0.7471,
+      "step": 1680
+    },
+    {
+      "epoch": 0.14861062258177982,
+      "grad_norm": 0.4570845067501068,
+      "learning_rate": 2e-05,
+      "loss": 0.7409,
+      "step": 1690
+    },
+    {
+      "epoch": 0.1494899753781217,
+      "grad_norm": 0.4543391466140747,
+      "learning_rate": 2e-05,
+      "loss": 0.7394,
+      "step": 1700
+    },
+    {
+      "epoch": 0.1503693281744636,
+      "grad_norm": 0.4563099145889282,
+      "learning_rate": 2e-05,
+      "loss": 0.7379,
+      "step": 1710
+    },
+    {
+      "epoch": 0.1512486809708055,
+      "grad_norm": 0.4498841464519501,
+      "learning_rate": 2e-05,
+      "loss": 0.7075,
+      "step": 1720
+    },
+    {
+      "epoch": 0.15212803376714737,
+      "grad_norm": 0.43725326657295227,
+      "learning_rate": 2e-05,
+      "loss": 0.72,
+      "step": 1730
+    },
+    {
+      "epoch": 0.15300738656348928,
+      "grad_norm": 0.4398544728755951,
+      "learning_rate": 2e-05,
+      "loss": 0.7502,
+      "step": 1740
+    },
+    {
+      "epoch": 0.15388673935983116,
+      "grad_norm": 0.48898088932037354,
+      "learning_rate": 2e-05,
+      "loss": 0.75,
+      "step": 1750
+    },
+    {
+      "epoch": 0.15476609215617307,
+      "grad_norm": 0.4510757327079773,
+      "learning_rate": 2e-05,
+      "loss": 0.7322,
+      "step": 1760
+    },
+    {
+      "epoch": 0.15564544495251495,
+      "grad_norm": 0.44645431637763977,
+      "learning_rate": 2e-05,
+      "loss": 0.7338,
+      "step": 1770
+    },
+    {
+      "epoch": 0.15652479774885683,
+      "grad_norm": 0.4352797269821167,
+      "learning_rate": 2e-05,
+      "loss": 0.744,
+      "step": 1780
+    },
+    {
+      "epoch": 0.15740415054519874,
+      "grad_norm": 0.5218507647514343,
+      "learning_rate": 2e-05,
+      "loss": 0.7396,
+      "step": 1790
+    },
+    {
+      "epoch": 0.15828350334154062,
+      "grad_norm": 0.4577134847640991,
+      "learning_rate": 2e-05,
+      "loss": 0.7362,
+      "step": 1800
+    },
+    {
+      "epoch": 0.15916285613788253,
+      "grad_norm": 0.4389902353286743,
+      "learning_rate": 2e-05,
+      "loss": 0.7305,
+      "step": 1810
+    },
+    {
+      "epoch": 0.1600422089342244,
+      "grad_norm": 0.4352608323097229,
+      "learning_rate": 2e-05,
+      "loss": 0.7284,
+      "step": 1820
+    },
+    {
+      "epoch": 0.1609215617305663,
+      "grad_norm": 0.43718910217285156,
+      "learning_rate": 2e-05,
+      "loss": 0.7258,
+      "step": 1830
+    },
+    {
+      "epoch": 0.1618009145269082,
+      "grad_norm": 0.4347856044769287,
+      "learning_rate": 2e-05,
+      "loss": 0.7275,
+      "step": 1840
+    },
+    {
+      "epoch": 0.16268026732325008,
+      "grad_norm": 0.46117907762527466,
+      "learning_rate": 2e-05,
+      "loss": 0.7463,
+      "step": 1850
+    },
+    {
+      "epoch": 0.163559620119592,
+      "grad_norm": 0.4587339162826538,
+      "learning_rate": 2e-05,
+      "loss": 0.7274,
+      "step": 1860
+    },
+    {
+      "epoch": 0.16443897291593387,
+      "grad_norm": 0.503758430480957,
+      "learning_rate": 2e-05,
+      "loss": 0.7345,
+      "step": 1870
+    },
+    {
+      "epoch": 0.16531832571227575,
+      "grad_norm": 0.4994986653327942,
+      "learning_rate": 2e-05,
+      "loss": 0.7239,
+      "step": 1880
+    },
+    {
+      "epoch": 0.16619767850861766,
+      "grad_norm": 0.4347689747810364,
+      "learning_rate": 2e-05,
+      "loss": 0.7598,
+      "step": 1890
+    },
+    {
+      "epoch": 0.16707703130495954,
+      "grad_norm": 0.41167309880256653,
+      "learning_rate": 2e-05,
+      "loss": 0.7077,
+      "step": 1900
+    },
+    {
+      "epoch": 0.16795638410130145,
+      "grad_norm": 0.43565014004707336,
+      "learning_rate": 2e-05,
+      "loss": 0.7171,
+      "step": 1910
+    },
+    {
+      "epoch": 0.16883573689764333,
+      "grad_norm": 0.43655893206596375,
+      "learning_rate": 2e-05,
+      "loss": 0.7222,
+      "step": 1920
+    },
+    {
+      "epoch": 0.16971508969398522,
+      "grad_norm": 0.42987534403800964,
+      "learning_rate": 2e-05,
+      "loss": 0.7076,
+      "step": 1930
+    },
+    {
+      "epoch": 0.17059444249032713,
+      "grad_norm": 0.4263608753681183,
+      "learning_rate": 2e-05,
+      "loss": 0.7333,
+      "step": 1940
+    },
+    {
+      "epoch": 0.171473795286669,
+      "grad_norm": 0.449835866689682,
+      "learning_rate": 2e-05,
+      "loss": 0.7334,
+      "step": 1950
+    },
+    {
+      "epoch": 0.17235314808301092,
+      "grad_norm": 0.5108100175857544,
+      "learning_rate": 2e-05,
+      "loss": 0.7365,
+      "step": 1960
+    },
+    {
+      "epoch": 0.1732325008793528,
+      "grad_norm": 0.4657161235809326,
+      "learning_rate": 2e-05,
+      "loss": 0.7409,
+      "step": 1970
+    },
+    {
+      "epoch": 0.17411185367569468,
+      "grad_norm": 0.4280586838722229,
+      "learning_rate": 2e-05,
+      "loss": 0.7129,
+      "step": 1980
+    },
+    {
+      "epoch": 0.1749912064720366,
+      "grad_norm": 0.718816876411438,
+      "learning_rate": 2e-05,
+      "loss": 0.7624,
+      "step": 1990
+    },
+    {
+      "epoch": 0.17587055926837847,
+      "grad_norm": 0.44081947207450867,
+      "learning_rate": 2e-05,
+      "loss": 0.7498,
+      "step": 2000
+    },
+    {
+      "epoch": 0.17674991206472038,
+      "grad_norm": 0.44327473640441895,
+      "learning_rate": 2e-05,
+      "loss": 0.7344,
+      "step": 2010
+    },
+    {
+      "epoch": 0.17762926486106226,
+      "grad_norm": 0.45156973600387573,
+      "learning_rate": 2e-05,
+      "loss": 0.737,
+      "step": 2020
+    },
+    {
+      "epoch": 0.17850861765740414,
+      "grad_norm": 0.5566264986991882,
+      "learning_rate": 2e-05,
+      "loss": 0.7206,
+      "step": 2030
+    },
+    {
+      "epoch": 0.17938797045374605,
+      "grad_norm": 0.4235565662384033,
+      "learning_rate": 2e-05,
+      "loss": 0.7236,
+      "step": 2040
+    },
+    {
+      "epoch": 0.18026732325008793,
+      "grad_norm": 1.122708797454834,
+      "learning_rate": 2e-05,
+      "loss": 0.7219,
+      "step": 2050
+    },
+    {
+      "epoch": 0.18114667604642984,
+      "grad_norm": 0.4141223728656769,
+      "learning_rate": 2e-05,
+      "loss": 0.7252,
+      "step": 2060
+    },
+    {
+      "epoch": 0.18202602884277172,
+      "grad_norm": 0.4148985743522644,
+      "learning_rate": 2e-05,
+      "loss": 0.7306,
+      "step": 2070
+    },
+    {
+      "epoch": 0.1829053816391136,
+      "grad_norm": 0.4730316996574402,
+      "learning_rate": 2e-05,
+      "loss": 0.7389,
+      "step": 2080
+    },
+    {
+      "epoch": 0.1837847344354555,
+      "grad_norm": 0.4608680307865143,
+      "learning_rate": 2e-05,
+      "loss": 0.7258,
+      "step": 2090
+    },
+    {
+      "epoch": 0.1846640872317974,
+      "grad_norm": 0.4247531294822693,
+      "learning_rate": 2e-05,
+      "loss": 0.722,
+      "step": 2100
+    },
+    {
+      "epoch": 0.1855434400281393,
+      "grad_norm": 0.4581758379936218,
+      "learning_rate": 2e-05,
+      "loss": 0.7275,
+      "step": 2110
+    },
+    {
+      "epoch": 0.18642279282448118,
+      "grad_norm": 0.5954698920249939,
+      "learning_rate": 2e-05,
+      "loss": 0.7108,
+      "step": 2120
+    },
+    {
+      "epoch": 0.18730214562082306,
+      "grad_norm": 0.42900434136390686,
+      "learning_rate": 2e-05,
+      "loss": 0.723,
+      "step": 2130
+    },
+    {
+      "epoch": 0.18818149841716497,
+      "grad_norm": 0.4176800549030304,
+      "learning_rate": 2e-05,
+      "loss": 0.7529,
+      "step": 2140
+    },
+    {
+      "epoch": 0.18906085121350685,
+      "grad_norm": 0.4418163299560547,
+      "learning_rate": 2e-05,
+      "loss": 0.7308,
+      "step": 2150
+    },
+    {
+      "epoch": 0.18994020400984876,
+      "grad_norm": 0.4328523874282837,
+      "learning_rate": 2e-05,
+      "loss": 0.7546,
+      "step": 2160
+    },
+    {
+      "epoch": 0.19081955680619064,
+      "grad_norm": 0.44161149859428406,
+      "learning_rate": 2e-05,
+      "loss": 0.7034,
+      "step": 2170
+    },
+    {
+      "epoch": 0.19169890960253252,
+      "grad_norm": 0.45360103249549866,
+      "learning_rate": 2e-05,
+      "loss": 0.7212,
+      "step": 2180
+    },
+    {
+      "epoch": 0.19257826239887443,
+      "grad_norm": 0.47255977988243103,
+      "learning_rate": 2e-05,
+      "loss": 0.7219,
+      "step": 2190
+    },
+    {
+      "epoch": 0.19345761519521631,
+      "grad_norm": 0.4865022301673889,
+      "learning_rate": 2e-05,
+      "loss": 0.7307,
+      "step": 2200
+    },
+    {
+      "epoch": 0.19433696799155822,
+      "grad_norm": 0.4334704875946045,
+      "learning_rate": 2e-05,
+      "loss": 0.7496,
+      "step": 2210
+    },
+    {
+      "epoch": 0.1952163207879001,
+      "grad_norm": 0.4357849359512329,
+      "learning_rate": 2e-05,
+      "loss": 0.7277,
+      "step": 2220
+    },
+    {
+      "epoch": 0.196095673584242,
+      "grad_norm": 0.4584740400314331,
+      "learning_rate": 2e-05,
+      "loss": 0.7182,
+      "step": 2230
+    },
+    {
+      "epoch": 0.1969750263805839,
+      "grad_norm": 0.4310411810874939,
+      "learning_rate": 2e-05,
+      "loss": 0.752,
+      "step": 2240
+    },
+    {
+      "epoch": 0.19785437917692578,
+      "grad_norm": 0.47864219546318054,
+      "learning_rate": 2e-05,
+      "loss": 0.7264,
+      "step": 2250
+    },
+    {
+      "epoch": 0.19873373197326769,
+      "grad_norm": 0.4456532299518585,
+      "learning_rate": 2e-05,
+      "loss": 0.7088,
+      "step": 2260
+    },
+    {
+      "epoch": 0.19961308476960957,
+      "grad_norm": 0.42451727390289307,
+      "learning_rate": 2e-05,
+      "loss": 0.729,
+      "step": 2270
+    },
+    {
+      "epoch": 0.20049243756595145,
+      "grad_norm": 0.43368861079216003,
+      "learning_rate": 2e-05,
+      "loss": 0.7179,
+      "step": 2280
+    },
+    {
+      "epoch": 0.20137179036229336,
+      "grad_norm": 0.4401801526546478,
+      "learning_rate": 2e-05,
+      "loss": 0.7239,
+      "step": 2290
+    },
+    {
+      "epoch": 0.20225114315863524,
+      "grad_norm": 0.4528639614582062,
+      "learning_rate": 2e-05,
+      "loss": 0.7372,
+      "step": 2300
+    },
+    {
+      "epoch": 0.20313049595497715,
+      "grad_norm": 0.44536641240119934,
+      "learning_rate": 2e-05,
+      "loss": 0.71,
+      "step": 2310
+    },
+    {
+      "epoch": 0.20400984875131903,
+      "grad_norm": 0.43853285908699036,
+      "learning_rate": 2e-05,
+      "loss": 0.7467,
+      "step": 2320
+    },
+    {
+      "epoch": 0.2048892015476609,
+      "grad_norm": 0.42381903529167175,
+      "learning_rate": 2e-05,
+      "loss": 0.7211,
+      "step": 2330
+    },
+    {
+      "epoch": 0.20576855434400282,
+      "grad_norm": 0.4553108513355255,
+      "learning_rate": 2e-05,
+      "loss": 0.7341,
+      "step": 2340
+    },
+    {
+      "epoch": 0.2066479071403447,
+      "grad_norm": 0.4345291256904602,
+      "learning_rate": 2e-05,
+      "loss": 0.6967,
+      "step": 2350
+    },
+    {
+      "epoch": 0.2075272599366866,
+      "grad_norm": 0.4219886064529419,
+      "learning_rate": 2e-05,
+      "loss": 0.711,
+      "step": 2360
+    },
+    {
+      "epoch": 0.2084066127330285,
+      "grad_norm": 0.4513852894306183,
+      "learning_rate": 2e-05,
+      "loss": 0.7241,
+      "step": 2370
+    },
+    {
+      "epoch": 0.20928596552937037,
+      "grad_norm": 0.44483205676078796,
+      "learning_rate": 2e-05,
+      "loss": 0.7289,
+      "step": 2380
+    },
+    {
+      "epoch": 0.21016531832571228,
+      "grad_norm": 0.45381051301956177,
+      "learning_rate": 2e-05,
+      "loss": 0.7315,
+      "step": 2390
+    },
+    {
+      "epoch": 0.21104467112205416,
+      "grad_norm": 0.47932013869285583,
+      "learning_rate": 2e-05,
+      "loss": 0.7363,
+      "step": 2400
+    },
+    {
+      "epoch": 0.21192402391839607,
+      "grad_norm": 0.49082091450691223,
+      "learning_rate": 2e-05,
+      "loss": 0.7223,
+      "step": 2410
+    },
+    {
+      "epoch": 0.21280337671473795,
+      "grad_norm": 0.4648228585720062,
+      "learning_rate": 2e-05,
+      "loss": 0.7161,
+      "step": 2420
+    },
+    {
+      "epoch": 0.21368272951107983,
+      "grad_norm": 0.4377770721912384,
+      "learning_rate": 2e-05,
+      "loss": 0.7205,
+      "step": 2430
+    },
+    {
+      "epoch": 0.21456208230742174,
+      "grad_norm": 0.47402605414390564,
+      "learning_rate": 2e-05,
+      "loss": 0.7074,
+      "step": 2440
+    },
+    {
+      "epoch": 0.21544143510376362,
+      "grad_norm": 0.5225353837013245,
+      "learning_rate": 2e-05,
+      "loss": 0.7254,
+      "step": 2450
+    },
+    {
+      "epoch": 0.21632078790010553,
+      "grad_norm": 0.4494861364364624,
+      "learning_rate": 2e-05,
+      "loss": 0.734,
+      "step": 2460
+    },
+    {
+      "epoch": 0.2172001406964474,
+      "grad_norm": 0.45226505398750305,
+      "learning_rate": 2e-05,
+      "loss": 0.7216,
+      "step": 2470
+    },
+    {
+      "epoch": 0.2180794934927893,
+      "grad_norm": 0.44078952074050903,
+      "learning_rate": 2e-05,
+      "loss": 0.7338,
+      "step": 2480
+    },
+    {
+      "epoch": 0.2189588462891312,
+      "grad_norm": 0.5343892574310303,
+      "learning_rate": 2e-05,
+      "loss": 0.737,
+      "step": 2490
+    },
+    {
+      "epoch": 0.21983819908547308,
+      "grad_norm": 0.43594157695770264,
+      "learning_rate": 2e-05,
+      "loss": 0.7265,
+      "step": 2500
+    },
+    {
+      "epoch": 0.220717551881815,
+      "grad_norm": 0.44618386030197144,
+      "learning_rate": 2e-05,
+      "loss": 0.7031,
+      "step": 2510
+    },
+    {
+      "epoch": 0.22159690467815688,
+      "grad_norm": 0.47326207160949707,
+      "learning_rate": 2e-05,
+      "loss": 0.7074,
+      "step": 2520
+    },
+    {
+      "epoch": 0.22247625747449876,
+      "grad_norm": 0.48806890845298767,
+      "learning_rate": 2e-05,
+      "loss": 0.7365,
+      "step": 2530
+    },
+    {
+      "epoch": 0.22335561027084067,
+      "grad_norm": 0.42458051443099976,
+      "learning_rate": 2e-05,
+      "loss": 0.7161,
+      "step": 2540
+    },
+    {
+      "epoch": 0.22423496306718255,
+      "grad_norm": 0.473349004983902,
+      "learning_rate": 1.997261400182901e-05,
+      "loss": 0.7326,
+      "step": 2550
+    },
+    {
+      "epoch": 0.22511431586352446,
+      "grad_norm": 0.48617324233055115,
+      "learning_rate": 1.9805800746810423e-05,
+      "loss": 0.7138,
+      "step": 2560
+    },
+    {
+      "epoch": 0.22599366865986634,
+      "grad_norm": 0.4620824456214905,
+      "learning_rate": 1.9489934220110274e-05,
+      "loss": 0.722,
+      "step": 2570
+    },
+    {
+      "epoch": 0.22687302145620822,
+      "grad_norm": 0.43454283475875854,
+      "learning_rate": 1.9029840364820392e-05,
+      "loss": 0.738,
+      "step": 2580
+    },
+    {
+      "epoch": 0.22775237425255013,
+      "grad_norm": 0.44115927815437317,
+      "learning_rate": 1.8432548690479908e-05,
+      "loss": 0.7144,
+      "step": 2590
+    },
+    {
+      "epoch": 0.228631727048892,
+      "grad_norm": 0.42175745964050293,
+      "learning_rate": 1.7707184873239772e-05,
+      "loss": 0.7187,
+      "step": 2600
+    },
+    {
+      "epoch": 0.22951107984523392,
+      "grad_norm": 0.3962736129760742,
+      "learning_rate": 1.6864831329902565e-05,
+      "loss": 0.7173,
+      "step": 2610
+    },
+    {
+      "epoch": 0.2303904326415758,
+      "grad_norm": 0.41026633977890015,
+      "learning_rate": 1.591835789604667e-05,
+      "loss": 0.7408,
+      "step": 2620
+    },
+    {
+      "epoch": 0.23126978543791768,
+      "grad_norm": 0.42476922273635864,
+      "learning_rate": 1.4882225195206459e-05,
+      "loss": 0.73,
+      "step": 2630
+    },
+    {
+      "epoch": 0.2321491382342596,
+      "grad_norm": 0.4424360692501068,
+      "learning_rate": 1.3772263703317695e-05,
+      "loss": 0.7141,
+      "step": 2640
+    },
+    {
+      "epoch": 0.23302849103060147,
+      "grad_norm": 0.40228933095932007,
+      "learning_rate": 1.2605431883975547e-05,
+      "loss": 0.7126,
+      "step": 2650
+    },
+    {
+      "epoch": 0.23390784382694338,
+      "grad_norm": 0.4428738057613373,
+      "learning_rate": 1.1399557089817574e-05,
+      "loss": 0.7233,
+      "step": 2660
+    },
+    {
+      "epoch": 0.23478719662328526,
+      "grad_norm": 0.4310200810432434,
+      "learning_rate": 1.0173063188650463e-05,
+      "loss": 0.7135,
+      "step": 2670
+    },
+    {
+      "epoch": 0.23566654941962714,
+      "grad_norm": 0.47649016976356506,
+      "learning_rate": 8.944689075764396e-06,
+      "loss": 0.7382,
+      "step": 2680
+    },
+    {
+      "epoch": 0.23654590221596905,
+      "grad_norm": 0.4229397177696228,
+      "learning_rate": 7.73320237312362e-06,
+      "loss": 0.7013,
+      "step": 2690
+    },
+    {
+      "epoch": 0.23742525501231093,
+      "grad_norm": 0.41789790987968445,
+      "learning_rate": 6.557112689658629e-06,
+      "loss": 0.7246,
+      "step": 2700
+    },
+    {
+      "epoch": 0.23830460780865284,
+      "grad_norm": 1.0431950092315674,
+      "learning_rate": 5.434388823591213e-06,
+      "loss": 0.7149,
+      "step": 2710
+    },
+    {
+      "epoch": 0.23918396060499472,
+      "grad_norm": 0.422404408454895,
+      "learning_rate": 4.382184227495424e-06,
+      "loss": 0.7215,
+      "step": 2720
+    },
+    {
+      "epoch": 0.2400633134013366,
+      "grad_norm": 0.38634011149406433,
+      "learning_rate": 3.4165749305560203e-06,
+      "loss": 0.7156,
+      "step": 2730
+    },
+    {
+      "epoch": 0.2409426661976785,
+      "grad_norm": 0.4125402569770813,
+      "learning_rate": 2.5523139221595556e-06,
+      "loss": 0.7004,
+      "step": 2740
+    },
+    {
+      "epoch": 0.2418220189940204,
+      "grad_norm": 0.4066152572631836,
+      "learning_rate": 1.8026057494499821e-06,
+      "loss": 0.7005,
+      "step": 2750
+    },
+    {
+      "epoch": 0.2427013717903623,
+      "grad_norm": 0.40452519059181213,
+      "learning_rate": 1.178904772643223e-06,
+      "loss": 0.7192,
+      "step": 2760
+    },
+    {
+      "epoch": 0.24358072458670418,
+      "grad_norm": 0.4185871183872223,
+      "learning_rate": 6.90740160441754e-07,
+      "loss": 0.7016,
+      "step": 2770
+    },
+    {
+      "epoch": 0.24446007738304606,
+      "grad_norm": 0.4322320818901062,
+      "learning_rate": 3.45570299343632e-07,
+      "loss": 0.7231,
+      "step": 2780
+    },
+    {
+      "epoch": 0.24533943017938797,
+      "grad_norm": 0.4022022485733032,
+      "learning_rate": 1.486688412425152e-07,
+      "loss": 0.7384,
+      "step": 2790
+    },
+    {
+      "epoch": 0.2459549771368273,
+      "step": 2797,
+      "total_flos": 1.2485883762455871e+19,
+      "train_loss": 0.06533449515300774,
+      "train_runtime": 2953.8926,
+      "train_samples_per_second": 11.363,
+      "train_steps_per_second": 0.947
+    }
+  ],
+  "logging_steps": 10,
+  "max_steps": 2797,
+  "num_input_tokens_seen": 0,
+  "num_train_epochs": 1,
+  "save_steps": 2543,
+  "stateful_callbacks": {
+    "TrainerControl": {
+      "args": {
+        "should_epoch_stop": false,
+        "should_evaluate": false,
+        "should_log": false,
+        "should_save": true,
+        "should_training_stop": true
+      },
+      "attributes": {}
+    }
+  },
+  "total_flos": 1.2485883762455871e+19,
+  "train_batch_size": 1,
+  "trial_name": null,
+  "trial_params": null
+}

g2_5_plan_newcode/250M/training_args.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:19a9a12b028d057b0b17f54005596f98ad8dd21a0afa47036caf68f3d5c8f513
+size 9105