Add files using upload-large-folder tool

g1_jupyter_notebook__edit/1000M/chat_template.jinja

@@ -0,0 +1 @@
+{% if messages[0]['role'] == 'system' %}{% set loop_messages = messages[1:] %}{% set system_message = messages[0]['content'] %}{% else %}{% set loop_messages = messages %}{% endif %}{% if system_message is defined %}{{ system_message }}{% endif %}{% for message in loop_messages %}{% set content = message['content'] %}{% if message['role'] == 'user' %}{{ content }}{% elif message['role'] == 'assistant' %}{{ content + '<|endoftext|>' }}{% endif %}{% endfor %}

g1_jupyter_notebook__edit/1000M/model.safetensors.index.json

@@ -0,0 +1,363 @@
+{
+  "metadata": {
+    "total_parameters": 7298617344,
+    "total_size": 14597234688
+  },
+  "weight_map": {
+    "lm_head.weight": "model-00003-of-00003.safetensors",
+    "model.embed_tokens.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.post_feedforward_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.self_attn.k_norm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.self_attn.q_norm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.post_feedforward_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.self_attn.k_norm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.self_attn.q_norm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.10.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.10.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.10.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.10.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.10.post_feedforward_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.10.self_attn.k_norm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.10.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.10.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.10.self_attn.q_norm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.10.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.10.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.11.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.11.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.11.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.11.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.11.post_feedforward_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.11.self_attn.k_norm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.11.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.11.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.11.self_attn.q_norm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.11.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.11.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.post_feedforward_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.self_attn.k_norm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.self_attn.q_norm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.post_feedforward_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.self_attn.k_norm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.self_attn.q_norm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.post_feedforward_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.self_attn.k_norm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.self_attn.q_norm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.post_feedforward_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.self_attn.k_norm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.self_attn.q_norm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.post_feedforward_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.self_attn.k_norm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.self_attn.q_norm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.post_feedforward_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.self_attn.k_norm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.self_attn.q_norm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.post_feedforward_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.self_attn.k_norm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.self_attn.q_norm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.post_feedforward_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.self_attn.k_norm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.self_attn.q_norm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.2.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.2.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.2.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.2.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.2.post_feedforward_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.2.self_attn.k_norm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.2.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.2.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.2.self_attn.q_norm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.2.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.2.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.20.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.20.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.20.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.20.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.20.post_feedforward_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.20.self_attn.k_norm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.20.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.20.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.20.self_attn.q_norm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.20.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.20.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.post_feedforward_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.self_attn.k_norm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.self_attn.q_norm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.22.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.22.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.22.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.22.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.22.post_feedforward_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.22.self_attn.k_norm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.22.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.22.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.22.self_attn.q_norm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.22.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.22.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.23.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.23.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.23.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.23.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.23.post_feedforward_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.23.self_attn.k_norm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.23.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.23.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.23.self_attn.q_norm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.23.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.23.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.post_feedforward_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.self_attn.k_norm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.self_attn.q_norm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.post_feedforward_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.self_attn.k_norm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.self_attn.q_norm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.post_feedforward_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.self_attn.k_norm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.self_attn.q_norm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.post_feedforward_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.self_attn.k_norm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.self_attn.q_norm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.28.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.28.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.28.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.28.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.28.post_feedforward_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.28.self_attn.k_norm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.28.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.28.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.28.self_attn.q_norm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.28.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.28.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.29.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.29.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.29.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.29.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.29.post_feedforward_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.29.self_attn.k_norm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.29.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.29.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.29.self_attn.q_norm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.29.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.29.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.3.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.3.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.3.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.3.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.3.post_feedforward_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.3.self_attn.k_norm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.3.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.3.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.3.self_attn.q_norm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.3.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.3.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.30.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.30.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.30.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.30.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.30.post_feedforward_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.30.self_attn.k_norm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.30.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.30.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.30.self_attn.q_norm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.30.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.30.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.31.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.31.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.31.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.31.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.31.post_feedforward_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.31.self_attn.k_norm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.31.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.31.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.31.self_attn.q_norm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.31.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.31.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.4.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.4.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.4.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.4.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.4.post_feedforward_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.4.self_attn.k_norm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.4.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.4.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.4.self_attn.q_norm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.4.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.4.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.post_feedforward_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.self_attn.k_norm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.self_attn.q_norm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.post_feedforward_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.self_attn.k_norm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.self_attn.q_norm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.post_feedforward_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.self_attn.k_norm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.self_attn.q_norm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.post_feedforward_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.self_attn.k_norm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.self_attn.q_norm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.post_feedforward_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.self_attn.k_norm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.self_attn.q_norm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.norm.weight": "model-00003-of-00003.safetensors"
+  }
+}

g1_jupyter_notebook__edit/1000M/train_results.json

@@ -0,0 +1,8 @@
+{
+    "epoch": 0.9333778037188359,
+    "total_flos": 4.547514943786923e+19,
+    "train_loss": 0.0031442601318250325,
+    "train_runtime": 611.8759,
+    "train_samples_per_second": 219.535,
+    "train_steps_per_second": 18.295
+}

g2_2_oldcode_synthcm_newcode__edit/1000M/all_results.json

@@ -0,0 +1,8 @@
+{
+    "epoch": 0.9369716246756508,
+    "total_flos": 4.547514943786923e+19,
+    "train_loss": 0.003452657950399432,
+    "train_runtime": 609.5586,
+    "train_samples_per_second": 220.369,
+    "train_steps_per_second": 18.364
+}

g2_2_oldcode_synthcm_newcode__edit/1000M/chat_template.jinja

@@ -0,0 +1 @@
+{% if messages[0]['role'] == 'system' %}{% set loop_messages = messages[1:] %}{% set system_message = messages[0]['content'] %}{% else %}{% set loop_messages = messages %}{% endif %}{% if system_message is defined %}{{ system_message }}{% endif %}{% for message in loop_messages %}{% set content = message['content'] %}{% if message['role'] == 'user' %}{{ content }}{% elif message['role'] == 'assistant' %}{{ content + '<|endoftext|>' }}{% endif %}{% endfor %}

g2_2_oldcode_synthcm_newcode__edit/1000M/config.json

@@ -0,0 +1,27 @@
+{
+  "architectures": [
+    "Olmo2ForCausalLM"
+  ],
+  "attention_bias": false,
+  "attention_dropout": 0.0,
+  "bos_token_id": 100257,
+  "dtype": "bfloat16",
+  "eos_token_id": 100257,
+  "hidden_act": "silu",
+  "hidden_size": 4096,
+  "initializer_range": 0.02,
+  "intermediate_size": 11008,
+  "max_position_embeddings": 4096,
+  "model_type": "olmo2",
+  "num_attention_heads": 32,
+  "num_hidden_layers": 32,
+  "num_key_value_heads": 32,
+  "pad_token_id": 100277,
+  "rms_norm_eps": 1e-06,
+  "rope_scaling": null,
+  "rope_theta": 500000,
+  "tie_word_embeddings": false,
+  "transformers_version": "4.57.6",
+  "use_cache": false,
+  "vocab_size": 100352
+}

g2_2_oldcode_synthcm_newcode__edit/1000M/generation_config.json

@@ -0,0 +1,9 @@
+{
+  "_from_model_config": true,
+  "bos_token_id": 100257,
+  "eos_token_id": [
+    100257
+  ],
+  "pad_token_id": 100277,
+  "transformers_version": "4.57.6"
+}

g2_2_oldcode_synthcm_newcode__edit/1000M/model.safetensors.index.json

@@ -0,0 +1,363 @@
+{
+  "metadata": {
+    "total_parameters": 7298617344,
+    "total_size": 14597234688
+  },
+  "weight_map": {
+    "lm_head.weight": "model-00003-of-00003.safetensors",
+    "model.embed_tokens.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.post_feedforward_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.self_attn.k_norm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.self_attn.q_norm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.post_feedforward_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.self_attn.k_norm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.self_attn.q_norm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.10.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.10.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.10.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.10.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.10.post_feedforward_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.10.self_attn.k_norm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.10.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.10.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.10.self_attn.q_norm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.10.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.10.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.11.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.11.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.11.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.11.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.11.post_feedforward_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.11.self_attn.k_norm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.11.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.11.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.11.self_attn.q_norm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.11.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.11.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.post_feedforward_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.self_attn.k_norm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.self_attn.q_norm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.post_feedforward_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.self_attn.k_norm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.self_attn.q_norm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.post_feedforward_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.self_attn.k_norm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.self_attn.q_norm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.post_feedforward_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.self_attn.k_norm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.self_attn.q_norm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.post_feedforward_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.self_attn.k_norm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.self_attn.q_norm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.post_feedforward_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.self_attn.k_norm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.self_attn.q_norm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.post_feedforward_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.self_attn.k_norm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.self_attn.q_norm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.post_feedforward_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.self_attn.k_norm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.self_attn.q_norm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.2.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.2.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.2.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.2.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.2.post_feedforward_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.2.self_attn.k_norm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.2.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.2.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.2.self_attn.q_norm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.2.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.2.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.20.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.20.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.20.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.20.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.20.post_feedforward_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.20.self_attn.k_norm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.20.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.20.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.20.self_attn.q_norm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.20.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.20.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.post_feedforward_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.self_attn.k_norm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.self_attn.q_norm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.22.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.22.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.22.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.22.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.22.post_feedforward_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.22.self_attn.k_norm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.22.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.22.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.22.self_attn.q_norm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.22.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.22.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.23.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.23.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.23.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.23.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.23.post_feedforward_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.23.self_attn.k_norm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.23.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.23.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.23.self_attn.q_norm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.23.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.23.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.post_feedforward_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.self_attn.k_norm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.self_attn.q_norm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.post_feedforward_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.self_attn.k_norm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.self_attn.q_norm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.post_feedforward_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.self_attn.k_norm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.self_attn.q_norm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.post_feedforward_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.self_attn.k_norm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.self_attn.q_norm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.28.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.28.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.28.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.28.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.28.post_feedforward_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.28.self_attn.k_norm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.28.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.28.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.28.self_attn.q_norm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.28.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.28.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.29.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.29.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.29.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.29.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.29.post_feedforward_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.29.self_attn.k_norm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.29.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.29.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.29.self_attn.q_norm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.29.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.29.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.3.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.3.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.3.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.3.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.3.post_feedforward_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.3.self_attn.k_norm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.3.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.3.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.3.self_attn.q_norm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.3.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.3.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.30.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.30.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.30.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.30.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.30.post_feedforward_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.30.self_attn.k_norm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.30.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.30.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.30.self_attn.q_norm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.30.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.30.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.31.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.31.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.31.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.31.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.31.post_feedforward_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.31.self_attn.k_norm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.31.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.31.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.31.self_attn.q_norm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.31.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.31.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.4.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.4.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.4.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.4.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.4.post_feedforward_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.4.self_attn.k_norm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.4.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.4.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.4.self_attn.q_norm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.4.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.4.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.post_feedforward_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.self_attn.k_norm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.self_attn.q_norm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.post_feedforward_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.self_attn.k_norm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.self_attn.q_norm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.post_feedforward_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.self_attn.k_norm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.self_attn.q_norm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.post_feedforward_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.self_attn.k_norm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.self_attn.q_norm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.post_feedforward_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.self_attn.k_norm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.self_attn.q_norm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.norm.weight": "model-00003-of-00003.safetensors"
+  }
+}

g2_2_oldcode_synthcm_newcode__edit/1000M/special_tokens_map.json

@@ -0,0 +1,30 @@
+{
+  "bos_token": {
+    "content": "<|endoftext|>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "eos_token": {
+    "content": "<|endoftext|>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "pad_token": {
+    "content": "<|pad|>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "unk_token": {
+    "content": "<|endoftext|>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  }
+}

g2_2_oldcode_synthcm_newcode__edit/1000M/tokenizer_config.json

@@ -0,0 +1,191 @@
+{
+  "add_prefix_space": false,
+  "added_tokens_decoder": {
+    "100256": {
+      "content": "<|extra_id_0|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "100257": {
+      "content": "<|endoftext|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "100258": {
+      "content": "<|fim_prefix|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "100259": {
+      "content": "<|fim_middle|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "100260": {
+      "content": "<|fim_suffix|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "100261": {
+      "content": "|||PHONE_NUMBER|||",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "100262": {
+      "content": "|||EMAIL_ADDRESS|||",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "100263": {
+      "content": "|||IP_ADDRESS|||",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "100264": {
+      "content": "<|im_start|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "100265": {
+      "content": "<|im_end|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "100266": {
+      "content": "<|extra_id_1|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "100267": {
+      "content": "<|extra_id_2|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "100268": {
+      "content": "<|extra_id_3|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "100269": {
+      "content": "<|extra_id_4|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "100270": {
+      "content": "<|extra_id_5|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "100271": {
+      "content": "<|extra_id_6|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "100272": {
+      "content": "<|extra_id_7|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "100273": {
+      "content": "<|extra_id_8|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "100274": {
+      "content": "<|extra_id_9|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "100275": {
+      "content": "<|extra_id_10|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "100276": {
+      "content": "<|endofprompt|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "100277": {
+      "content": "<|pad|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    }
+  },
+  "bos_token": "<|endoftext|>",
+  "clean_up_tokenization_spaces": false,
+  "eos_token": "<|endoftext|>",
+  "extra_special_tokens": {},
+  "model_max_length": 1000000000000000019884624838656,
+  "pad_token": "<|pad|>",
+  "padding_side": "right",
+  "split_special_tokens": false,
+  "tokenizer_class": "GPT2Tokenizer",
+  "unk_token": "<|endoftext|>"
+}

g2_2_oldcode_synthcm_newcode__edit/1000M/train_results.json

@@ -0,0 +1,8 @@
+{
+    "epoch": 0.9369716246756508,
+    "total_flos": 4.547514943786923e+19,
+    "train_loss": 0.003452657950399432,
+    "train_runtime": 609.5586,
+    "train_samples_per_second": 220.369,
+    "train_steps_per_second": 18.364
+}

g2_2_oldcode_synthcm_newcode__edit/1000M/trainer_log.jsonl

@@ -0,0 +1,110 @@
+{"current_steps": 10180, "total_steps": 11194, "loss": 0.5828, "lr": 1.999957358086795e-05, "epoch": 0.8520967606930611, "percentage": 90.94, "elapsed_time": "0:00:37", "remaining_time": "0:00:03"}
+{"current_steps": 10190, "total_steps": 11194, "loss": 0.6095, "lr": 1.9991993813020483e-05, "epoch": 0.8529337909098519, "percentage": 91.03, "elapsed_time": "0:02:05", "remaining_time": "0:00:12"}
+{"current_steps": 10200, "total_steps": 11194, "loss": 0.6153, "lr": 1.9974946373028628e-05, "epoch": 0.8537708211266427, "percentage": 91.12, "elapsed_time": "0:03:34", "remaining_time": "0:00:20"}
+{"current_steps": 10210, "total_steps": 11194, "loss": 0.6221, "lr": 1.994844749501764e-05, "epoch": 0.8546078513434335, "percentage": 91.21, "elapsed_time": "0:05:02", "remaining_time": "0:00:29"}
+{"current_steps": 10220, "total_steps": 11194, "loss": 0.6086, "lr": 1.9912522413632988e-05, "epoch": 0.8554448815602244, "percentage": 91.3, "elapsed_time": "0:06:30", "remaining_time": "0:00:37"}
+{"current_steps": 10230, "total_steps": 11194, "loss": 0.611, "lr": 1.9867205340009656e-05, "epoch": 0.8562819117770152, "percentage": 91.39, "elapsed_time": "0:07:59", "remaining_time": "0:00:45"}
+{"current_steps": 10240, "total_steps": 11194, "loss": 0.5898, "lr": 1.9812539429193164e-05, "epoch": 0.857118941993806, "percentage": 91.48, "elapsed_time": "0:09:27", "remaining_time": "0:00:52"}
+{"current_steps": 10250, "total_steps": 11194, "loss": 0.6305, "lr": 1.9748576739043423e-05, "epoch": 0.8579559722105968, "percentage": 91.57, "elapsed_time": "0:10:56", "remaining_time": "0:01:00"}
+{"current_steps": 10260, "total_steps": 11194, "loss": 0.6252, "lr": 1.967537818066047e-05, "epoch": 0.8587930024273877, "percentage": 91.66, "elapsed_time": "0:12:24", "remaining_time": "0:01:07"}
+{"current_steps": 10270, "total_steps": 11194, "loss": 0.6127, "lr": 1.959301346037941e-05, "epoch": 0.8596300326441785, "percentage": 91.75, "elapsed_time": "0:13:53", "remaining_time": "0:01:14"}
+{"current_steps": 10280, "total_steps": 11194, "loss": 0.6008, "lr": 1.9501561013389646e-05, "epoch": 0.8604670628609693, "percentage": 91.83, "elapsed_time": "0:15:21", "remaining_time": "0:01:21"}
+{"current_steps": 10290, "total_steps": 11194, "loss": 0.6294, "lr": 1.9401107929041795e-05, "epoch": 0.8613040930777601, "percentage": 91.92, "elapsed_time": "0:16:50", "remaining_time": "0:01:28"}
+{"current_steps": 10300, "total_steps": 11194, "loss": 0.6056, "lr": 1.9291749867913225e-05, "epoch": 0.862141123294551, "percentage": 92.01, "elapsed_time": "0:18:18", "remaining_time": "0:01:35"}
+{"current_steps": 10310, "total_steps": 11194, "loss": 0.6066, "lr": 1.91735909707114e-05, "epoch": 0.8629781535113418, "percentage": 92.1, "elapsed_time": "0:19:47", "remaining_time": "0:01:41"}
+{"current_steps": 10320, "total_steps": 11194, "loss": 0.5989, "lr": 1.9046743759101545e-05, "epoch": 0.8638151837281326, "percentage": 92.19, "elapsed_time": "0:21:15", "remaining_time": "0:01:48"}
+{"current_steps": 10330, "total_steps": 11194, "loss": 0.6415, "lr": 1.8911329028553324e-05, "epoch": 0.8646522139449234, "percentage": 92.28, "elapsed_time": "0:22:44", "remaining_time": "0:01:54"}
+{"current_steps": 10340, "total_steps": 11194, "loss": 0.6381, "lr": 1.876747573330835e-05, "epoch": 0.8654892441617142, "percentage": 92.37, "elapsed_time": "0:24:49", "remaining_time": "0:02:03"}
+{"current_steps": 10350, "total_steps": 11194, "loss": 0.6182, "lr": 1.8615320863578244e-05, "epoch": 0.8663262743785051, "percentage": 92.46, "elapsed_time": "0:26:18", "remaining_time": "0:02:08"}
+{"current_steps": 10360, "total_steps": 11194, "loss": 0.6138, "lr": 1.8455009315090036e-05, "epoch": 0.8671633045952959, "percentage": 92.55, "elapsed_time": "0:27:46", "remaining_time": "0:02:14"}
+{"current_steps": 10370, "total_steps": 11194, "loss": 0.6369, "lr": 1.8286693751103294e-05, "epoch": 0.8680003348120867, "percentage": 92.64, "elapsed_time": "0:29:15", "remaining_time": "0:02:19"}
+{"current_steps": 10380, "total_steps": 11194, "loss": 0.5812, "lr": 1.811053445703024e-05, "epoch": 0.8688373650288775, "percentage": 92.73, "elapsed_time": "0:30:43", "remaining_time": "0:02:24"}
+{"current_steps": 10390, "total_steps": 11194, "loss": 0.6012, "lr": 1.7926699187797375e-05, "epoch": 0.8696743952456684, "percentage": 92.82, "elapsed_time": "0:32:12", "remaining_time": "0:02:29"}
+{"current_steps": 10400, "total_steps": 11194, "loss": 0.6154, "lr": 1.7735363008093995e-05, "epoch": 0.8705114254624592, "percentage": 92.91, "elapsed_time": "0:33:40", "remaining_time": "0:02:34"}
+{"current_steps": 10410, "total_steps": 11194, "loss": 0.6595, "lr": 1.7536708125659577e-05, "epoch": 0.87134845567925, "percentage": 93.0, "elapsed_time": "0:35:09", "remaining_time": "0:02:38"}
+{"current_steps": 10420, "total_steps": 11194, "loss": 0.6393, "lr": 1.733092371776905e-05, "epoch": 0.8721854858960408, "percentage": 93.09, "elapsed_time": "0:36:37", "remaining_time": "0:02:43"}
+{"current_steps": 10430, "total_steps": 11194, "loss": 0.609, "lr": 1.7118205751080915e-05, "epoch": 0.8730225161128317, "percentage": 93.17, "elapsed_time": "0:38:05", "remaining_time": "0:02:47"}
+{"current_steps": 10440, "total_steps": 11194, "loss": 0.6142, "lr": 1.6898756795019935e-05, "epoch": 0.8738595463296225, "percentage": 93.26, "elapsed_time": "0:39:34", "remaining_time": "0:02:51"}
+{"current_steps": 10450, "total_steps": 11194, "loss": 0.5902, "lr": 1.667278582887211e-05, "epoch": 0.8746965765464133, "percentage": 93.35, "elapsed_time": "0:41:02", "remaining_time": "0:02:55"}
+{"current_steps": 10460, "total_steps": 11194, "loss": 0.634, "lr": 1.6440508042775545e-05, "epoch": 0.8755336067632041, "percentage": 93.44, "elapsed_time": "0:42:31", "remaining_time": "0:02:59"}
+{"current_steps": 10470, "total_steps": 11194, "loss": 0.6097, "lr": 1.6202144632796796e-05, "epoch": 0.876370636979995, "percentage": 93.53, "elapsed_time": "0:43:59", "remaining_time": "0:03:02"}
+{"current_steps": 10480, "total_steps": 11194, "loss": 0.608, "lr": 1.595792259028783e-05, "epoch": 0.8772076671967858, "percentage": 93.62, "elapsed_time": "0:45:28", "remaining_time": "0:03:05"}
+{"current_steps": 10490, "total_steps": 11194, "loss": 0.6294, "lr": 1.570807448572419e-05, "epoch": 0.8780446974135766, "percentage": 93.71, "elapsed_time": "0:46:56", "remaining_time": "0:03:09"}
+{"current_steps": 10500, "total_steps": 11194, "loss": 0.607, "lr": 1.5452838247230197e-05, "epoch": 0.8788817276303674, "percentage": 93.8, "elapsed_time": "0:49:02", "remaining_time": "0:03:14"}
+{"current_steps": 10510, "total_steps": 11194, "loss": 0.5915, "lr": 1.5192456934002106e-05, "epoch": 0.8797187578471583, "percentage": 93.89, "elapsed_time": "0:50:30", "remaining_time": "0:03:17"}
+{"current_steps": 10520, "total_steps": 11194, "loss": 0.6131, "lr": 1.492717850484502e-05, "epoch": 0.8805557880639491, "percentage": 93.98, "elapsed_time": "0:51:58", "remaining_time": "0:03:19"}
+{"current_steps": 10500, "total_steps": 11194, "loss": 0.6224, "lr": 1.5452838247230197e-05, "epoch": 0.8788817276303674, "percentage": 93.8, "elapsed_time": "0:01:04", "remaining_time": "0:00:04"}
+{"current_steps": 10510, "total_steps": 11194, "loss": 0.5915, "lr": 1.5192456934002106e-05, "epoch": 0.8797187578471583, "percentage": 93.89, "elapsed_time": "0:02:35", "remaining_time": "0:00:10"}
+{"current_steps": 10520, "total_steps": 11194, "loss": 0.6131, "lr": 1.492717850484502e-05, "epoch": 0.8805557880639491, "percentage": 93.98, "elapsed_time": "0:04:05", "remaining_time": "0:00:15"}
+{"current_steps": 10530, "total_steps": 11194, "loss": 0.6242, "lr": 1.4657255582043904e-05, "epoch": 0.8813928182807399, "percentage": 94.07, "elapsed_time": "0:05:35", "remaining_time": "0:00:21"}
+{"current_steps": 10540, "total_steps": 11194, "loss": 0.6279, "lr": 1.4382945210793663e-05, "epoch": 0.8822298484975307, "percentage": 94.16, "elapsed_time": "0:07:05", "remaining_time": "0:00:26"}
+{"current_steps": 10550, "total_steps": 11194, "loss": 0.6086, "lr": 1.4104508614417267e-05, "epoch": 0.8830668787143215, "percentage": 94.25, "elapsed_time": "0:08:35", "remaining_time": "0:00:31"}
+{"current_steps": 10560, "total_steps": 11194, "loss": 0.6205, "lr": 1.382221094560512e-05, "epoch": 0.8839039089311124, "percentage": 94.34, "elapsed_time": "0:10:05", "remaining_time": "0:00:36"}
+{"current_steps": 10570, "total_steps": 11194, "loss": 0.6442, "lr": 1.3536321033912506e-05, "epoch": 0.8847409391479032, "percentage": 94.43, "elapsed_time": "0:11:35", "remaining_time": "0:00:41"}
+{"current_steps": 10580, "total_steps": 11194, "loss": 0.5914, "lr": 1.3247111129755609e-05, "epoch": 0.885577969364694, "percentage": 94.51, "elapsed_time": "0:13:05", "remaining_time": "0:00:45"}
+{"current_steps": 10590, "total_steps": 11194, "loss": 0.6201, "lr": 1.2954856645149868e-05, "epoch": 0.8864149995814848, "percentage": 94.6, "elapsed_time": "0:14:35", "remaining_time": "0:00:49"}
+{"current_steps": 10600, "total_steps": 11194, "loss": 0.6264, "lr": 1.2659835891437553e-05, "epoch": 0.8872520297982757, "percentage": 94.69, "elapsed_time": "0:16:05", "remaining_time": "0:00:54"}
+{"current_steps": 10610, "total_steps": 11194, "loss": 0.5637, "lr": 1.236232981425436e-05, "epoch": 0.8880890600150665, "percentage": 94.78, "elapsed_time": "0:17:35", "remaining_time": "0:00:58"}
+{"current_steps": 10620, "total_steps": 11194, "loss": 0.592, "lr": 1.2062621725987402e-05, "epoch": 0.8889260902318574, "percentage": 94.87, "elapsed_time": "0:19:05", "remaining_time": "0:01:01"}
+{"current_steps": 10630, "total_steps": 11194, "loss": 0.6297, "lr": 1.1760997035979318e-05, "epoch": 0.8897631204486482, "percentage": 94.96, "elapsed_time": "0:20:35", "remaining_time": "0:01:05"}
+{"current_steps": 10640, "total_steps": 11194, "loss": 0.6192, "lr": 1.1457742978735514e-05, "epoch": 0.8906001506654391, "percentage": 95.05, "elapsed_time": "0:22:05", "remaining_time": "0:01:09"}
+{"current_steps": 10650, "total_steps": 11194, "loss": 0.5706, "lr": 1.1153148340393298e-05, "epoch": 0.8914371808822299, "percentage": 95.14, "elapsed_time": "0:23:35", "remaining_time": "0:01:12"}
+{"current_steps": 10660, "total_steps": 11194, "loss": 0.6118, "lr": 1.0847503183713424e-05, "epoch": 0.8922742110990207, "percentage": 95.23, "elapsed_time": "0:25:43", "remaining_time": "0:01:17"}
+{"current_steps": 10670, "total_steps": 11194, "loss": 0.5964, "lr": 1.0541098571855896e-05, "epoch": 0.8931112413158115, "percentage": 95.32, "elapsed_time": "0:27:13", "remaining_time": "0:01:20"}
+{"current_steps": 10680, "total_steps": 11194, "loss": 0.5827, "lr": 1.0234226291203123e-05, "epoch": 0.8939482715326024, "percentage": 95.41, "elapsed_time": "0:28:43", "remaining_time": "0:01:22"}
+{"current_steps": 10690, "total_steps": 11194, "loss": 0.6307, "lr": 9.927178573494333e-06, "epoch": 0.8947853017493932, "percentage": 95.5, "elapsed_time": "0:30:13", "remaining_time": "0:01:25"}
+{"current_steps": 10700, "total_steps": 11194, "loss": 0.608, "lr": 9.620247817535902e-06, "epoch": 0.895622331966184, "percentage": 95.59, "elapsed_time": "0:31:43", "remaining_time": "0:01:27"}
+{"current_steps": 10710, "total_steps": 11194, "loss": 0.6169, "lr": 9.313726310752581e-06, "epoch": 0.8964593621829748, "percentage": 95.68, "elapsed_time": "0:33:14", "remaining_time": "0:01:30"}
+{"current_steps": 10720, "total_steps": 11194, "loss": 0.603, "lr": 9.00790595084477e-06, "epoch": 0.8972963923997657, "percentage": 95.77, "elapsed_time": "0:34:44", "remaining_time": "0:01:32"}
+{"current_steps": 10730, "total_steps": 11194, "loss": 0.5913, "lr": 8.703077967816944e-06, "epoch": 0.8981334226165565, "percentage": 95.85, "elapsed_time": "0:36:14", "remaining_time": "0:01:34"}
+{"current_steps": 10740, "total_steps": 11194, "loss": 0.5932, "lr": 8.39953264664192e-06, "epoch": 0.8989704528333473, "percentage": 95.94, "elapsed_time": "0:37:44", "remaining_time": "0:01:35"}
+{"current_steps": 10750, "total_steps": 11194, "loss": 0.5913, "lr": 8.097559050825058e-06, "epoch": 0.8998074830501381, "percentage": 96.03, "elapsed_time": "0:39:14", "remaining_time": "0:01:37"}
+{"current_steps": 10760, "total_steps": 11194, "loss": 0.5874, "lr": 7.79744474713166e-06, "epoch": 0.900644513266929, "percentage": 96.12, "elapsed_time": "0:40:44", "remaining_time": "0:01:38"}
+{"current_steps": 10770, "total_steps": 11194, "loss": 0.6148, "lr": 7.499475531739696e-06, "epoch": 0.9014815434837198, "percentage": 96.21, "elapsed_time": "0:42:14", "remaining_time": "0:01:39"}
+{"current_steps": 10780, "total_steps": 11194, "loss": 0.6082, "lr": 7.203935158078662e-06, "epoch": 0.9023185737005106, "percentage": 96.3, "elapsed_time": "0:43:44", "remaining_time": "0:01:40"}
+{"current_steps": 10790, "total_steps": 11194, "loss": 0.6039, "lr": 6.911105066613709e-06, "epoch": 0.9031556039173014, "percentage": 96.39, "elapsed_time": "0:45:14", "remaining_time": "0:01:41"}
+{"current_steps": 10800, "total_steps": 11194, "loss": 0.6, "lr": 6.621264116832394e-06, "epoch": 0.9039926341340923, "percentage": 96.48, "elapsed_time": "0:46:44", "remaining_time": "0:01:42"}
+{"current_steps": 10810, "total_steps": 11194, "loss": 0.6136, "lr": 6.334688321689274e-06, "epoch": 0.9048296643508831, "percentage": 96.57, "elapsed_time": "0:48:14", "remaining_time": "0:01:42"}
+{"current_steps": 10820, "total_steps": 11194, "loss": 0.6158, "lr": 6.051650584761237e-06, "epoch": 0.9056666945676739, "percentage": 96.66, "elapsed_time": "0:50:21", "remaining_time": "0:01:44"}
+{"current_steps": 10830, "total_steps": 11194, "loss": 0.6179, "lr": 5.772420440363841e-06, "epoch": 0.9065037247844647, "percentage": 96.75, "elapsed_time": "0:51:51", "remaining_time": "0:01:44"}
+{"current_steps": 10820, "total_steps": 11194, "loss": 0.5971, "lr": 6.051650584761237e-06, "epoch": 0.9056666945676739, "percentage": 96.66, "elapsed_time": "0:01:14", "remaining_time": "0:00:02"}
+{"current_steps": 10830, "total_steps": 11194, "loss": 0.6179, "lr": 5.772420440363841e-06, "epoch": 0.9065037247844647, "percentage": 96.75, "elapsed_time": "0:02:43", "remaining_time": "0:00:05"}
+{"current_steps": 10840, "total_steps": 11194, "loss": 0.5645, "lr": 5.497263796876207e-06, "epoch": 0.9073407550012556, "percentage": 96.84, "elapsed_time": "0:04:13", "remaining_time": "0:00:08"}
+{"current_steps": 10850, "total_steps": 11194, "loss": 0.6092, "lr": 5.226442683518813e-06, "epoch": 0.9081777852180464, "percentage": 96.93, "elapsed_time": "0:05:42", "remaining_time": "0:00:10"}
+{"current_steps": 10860, "total_steps": 11194, "loss": 0.6008, "lr": 4.960215000825432e-06, "epoch": 0.9090148154348372, "percentage": 97.02, "elapsed_time": "0:07:11", "remaining_time": "0:00:13"}
+{"current_steps": 10870, "total_steps": 11194, "loss": 0.6254, "lr": 4.698834275046711e-06, "epoch": 0.909851845651628, "percentage": 97.11, "elapsed_time": "0:08:41", "remaining_time": "0:00:15"}
+{"current_steps": 10880, "total_steps": 11194, "loss": 0.6048, "lr": 4.442549416719423e-06, "epoch": 0.9106888758684188, "percentage": 97.19, "elapsed_time": "0:10:10", "remaining_time": "0:00:17"}
+{"current_steps": 10890, "total_steps": 11194, "loss": 0.6195, "lr": 4.1916044836311565e-06, "epoch": 0.9115259060852097, "percentage": 97.28, "elapsed_time": "0:11:39", "remaining_time": "0:00:19"}
+{"current_steps": 10900, "total_steps": 11194, "loss": 0.563, "lr": 3.94623844840626e-06, "epoch": 0.9123629363020005, "percentage": 97.37, "elapsed_time": "0:13:09", "remaining_time": "0:00:21"}
+{"current_steps": 10910, "total_steps": 11194, "loss": 0.5984, "lr": 3.7066849709343467e-06, "epoch": 0.9131999665187913, "percentage": 97.46, "elapsed_time": "0:14:38", "remaining_time": "0:00:22"}
+{"current_steps": 10920, "total_steps": 11194, "loss": 0.5985, "lr": 3.473172175858059e-06, "epoch": 0.9140369967355821, "percentage": 97.55, "elapsed_time": "0:16:07", "remaining_time": "0:00:24"}
+{"current_steps": 10930, "total_steps": 11194, "loss": 0.6132, "lr": 3.2459224353320046e-06, "epoch": 0.914874026952373, "percentage": 97.64, "elapsed_time": "0:17:37", "remaining_time": "0:00:25"}
+{"current_steps": 10940, "total_steps": 11194, "loss": 0.6138, "lr": 3.025152157259735e-06, "epoch": 0.9157110571691638, "percentage": 97.73, "elapsed_time": "0:19:06", "remaining_time": "0:00:26"}
+{"current_steps": 10950, "total_steps": 11194, "loss": 0.5878, "lr": 2.811071579210425e-06, "epoch": 0.9165480873859546, "percentage": 97.82, "elapsed_time": "0:20:36", "remaining_time": "0:00:27"}
+{"current_steps": 10960, "total_steps": 11194, "loss": 0.5899, "lr": 2.603884568211513e-06, "epoch": 0.9173851176027454, "percentage": 97.91, "elapsed_time": "0:22:05", "remaining_time": "0:00:28"}
+{"current_steps": 10970, "total_steps": 11194, "loss": 0.6166, "lr": 2.4037884266079434e-06, "epoch": 0.9182221478195363, "percentage": 98.0, "elapsed_time": "0:23:34", "remaining_time": "0:00:28"}
+{"current_steps": 10980, "total_steps": 11194, "loss": 0.5803, "lr": 2.2109737041728955e-06, "epoch": 0.9190591780363271, "percentage": 98.09, "elapsed_time": "0:25:42", "remaining_time": "0:00:30"}
+{"current_steps": 10990, "total_steps": 11194, "loss": 0.5816, "lr": 2.0256240166489454e-06, "epoch": 0.9198962082531179, "percentage": 98.18, "elapsed_time": "0:27:11", "remaining_time": "0:00:30"}
+{"current_steps": 11000, "total_steps": 11194, "loss": 0.6243, "lr": 1.847915870892413e-06, "epoch": 0.9207332384699087, "percentage": 98.27, "elapsed_time": "0:28:41", "remaining_time": "0:00:30"}
+{"current_steps": 11010, "total_steps": 11194, "loss": 0.6135, "lr": 1.6780184967874715e-06, "epoch": 0.9215702686866996, "percentage": 98.36, "elapsed_time": "0:30:10", "remaining_time": "0:00:30"}
+{"current_steps": 11020, "total_steps": 11194, "loss": 0.6305, "lr": 1.5160936860900294e-06, "epoch": 0.9224072989034904, "percentage": 98.45, "elapsed_time": "0:31:39", "remaining_time": "0:00:29"}
+{"current_steps": 11030, "total_steps": 11194, "loss": 0.6266, "lr": 1.3622956383548762e-06, "epoch": 0.9232443291202812, "percentage": 98.53, "elapsed_time": "0:33:09", "remaining_time": "0:00:29"}
+{"current_steps": 11040, "total_steps": 11194, "loss": 0.6163, "lr": 1.216770814092834e-06, "epoch": 0.924081359337072, "percentage": 98.62, "elapsed_time": "0:34:38", "remaining_time": "0:00:28"}
+{"current_steps": 11050, "total_steps": 11194, "loss": 0.5669, "lr": 1.0796577952977013e-06, "epoch": 0.9249183895538629, "percentage": 98.71, "elapsed_time": "0:36:08", "remaining_time": "0:00:28"}
+{"current_steps": 11060, "total_steps": 11194, "loss": 0.6015, "lr": 9.510871534758705e-07, "epoch": 0.9257554197706537, "percentage": 98.8, "elapsed_time": "0:37:37", "remaining_time": "0:00:27"}
+{"current_steps": 11070, "total_steps": 11194, "loss": 0.5854, "lr": 8.311813253042334e-07, "epoch": 0.9265924499874445, "percentage": 98.89, "elapsed_time": "0:39:06", "remaining_time": "0:00:26"}
+{"current_steps": 11080, "total_steps": 11194, "loss": 0.5813, "lr": 7.200544960348308e-07, "epoch": 0.9274294802042353, "percentage": 98.98, "elapsed_time": "0:40:36", "remaining_time": "0:00:25"}
+{"current_steps": 11090, "total_steps": 11194, "loss": 0.6052, "lr": 6.178124907572545e-07, "epoch": 0.9282665104210261, "percentage": 99.07, "elapsed_time": "0:42:05", "remaining_time": "0:00:23"}
+{"current_steps": 11100, "total_steps": 11194, "loss": 0.5952, "lr": 5.245526736223528e-07, "epoch": 0.929103540637817, "percentage": 99.16, "elapsed_time": "0:43:34", "remaining_time": "0:00:22"}
+{"current_steps": 11110, "total_steps": 11194, "loss": 0.5812, "lr": 4.403638551232205e-07, "epoch": 0.9299405708546079, "percentage": 99.25, "elapsed_time": "0:45:04", "remaining_time": "0:00:20"}
+{"current_steps": 11120, "total_steps": 11194, "loss": 0.5956, "lr": 3.6532620752175616e-07, "epoch": 0.9307776010713987, "percentage": 99.34, "elapsed_time": "0:46:33", "remaining_time": "0:00:18"}
+{"current_steps": 11130, "total_steps": 11194, "loss": 0.6201, "lr": 2.9951118850133247e-07, "epoch": 0.9316146312881896, "percentage": 99.43, "elapsed_time": "0:48:02", "remaining_time": "0:00:16"}
+{"current_steps": 11140, "total_steps": 11194, "loss": 0.5721, "lr": 2.429814731182774e-07, "epoch": 0.9324516615049804, "percentage": 99.52, "elapsed_time": "0:50:10", "remaining_time": "0:00:14"}
+{"current_steps": 11150, "total_steps": 11194, "loss": 0.6218, "lr": 1.957908941169813e-07, "epoch": 0.9332886917217712, "percentage": 99.61, "elapsed_time": "0:51:40", "remaining_time": "0:00:12"}
+{"current_steps": 11140, "total_steps": 11194, "loss": 0.5721, "lr": 2.429814731182774e-07, "epoch": 0.9324516615049804, "percentage": 99.52, "elapsed_time": "0:01:31", "remaining_time": "0:00:00"}
+{"current_steps": 11150, "total_steps": 11194, "loss": 0.6219, "lr": 1.957908941169813e-07, "epoch": 0.9332886917217712, "percentage": 99.61, "elapsed_time": "0:03:00", "remaining_time": "0:00:00"}
+{"current_steps": 11160, "total_steps": 11194, "loss": 0.5932, "lr": 1.5798439066545254e-07, "epoch": 0.934125721938562, "percentage": 99.7, "elapsed_time": "0:04:29", "remaining_time": "0:00:00"}
+{"current_steps": 11170, "total_steps": 11194, "loss": 0.589, "lr": 1.2959796556014471e-07, "epoch": 0.9349627521553528, "percentage": 99.79, "elapsed_time": "0:05:58", "remaining_time": "0:00:00"}
+{"current_steps": 11180, "total_steps": 11194, "loss": 0.6063, "lr": 1.1065865094081884e-07, "epoch": 0.9357997823721437, "percentage": 99.87, "elapsed_time": "0:07:27", "remaining_time": "0:00:00"}
+{"current_steps": 11190, "total_steps": 11194, "loss": 0.5984, "lr": 1.011844825480734e-07, "epoch": 0.9366368125889345, "percentage": 99.96, "elapsed_time": "0:08:56", "remaining_time": "0:00:00"}
+{"current_steps": 11194, "total_steps": 11194, "epoch": 0.9369716246756508, "percentage": 100.0, "elapsed_time": "0:10:09", "remaining_time": "0:00:00"}

g2_2_oldcode_synthcm_newcode__edit/250M/all_results.json

@@ -0,0 +1,8 @@
+{
+    "epoch": 0.23420105465807314,
+    "total_flos": 1.1366756130249245e+19,
+    "train_loss": 0.06283065401885746,
+    "train_runtime": 2304.4445,
+    "train_samples_per_second": 14.57,
+    "train_steps_per_second": 1.214
+}

g2_2_oldcode_synthcm_newcode__edit/250M/chat_template.jinja

@@ -0,0 +1 @@
+{% if messages[0]['role'] == 'system' %}{% set loop_messages = messages[1:] %}{% set system_message = messages[0]['content'] %}{% else %}{% set loop_messages = messages %}{% endif %}{% if system_message is defined %}{{ system_message }}{% endif %}{% for message in loop_messages %}{% set content = message['content'] %}{% if message['role'] == 'user' %}{{ content }}{% elif message['role'] == 'assistant' %}{{ content + '<|endoftext|>' }}{% endif %}{% endfor %}

g2_2_oldcode_synthcm_newcode__edit/250M/config.json

@@ -0,0 +1,27 @@
+{
+  "architectures": [
+    "Olmo2ForCausalLM"
+  ],
+  "attention_bias": false,
+  "attention_dropout": 0.0,
+  "bos_token_id": 100257,
+  "dtype": "bfloat16",
+  "eos_token_id": 100257,
+  "hidden_act": "silu",
+  "hidden_size": 4096,
+  "initializer_range": 0.02,
+  "intermediate_size": 11008,
+  "max_position_embeddings": 4096,
+  "model_type": "olmo2",
+  "num_attention_heads": 32,
+  "num_hidden_layers": 32,
+  "num_key_value_heads": 32,
+  "pad_token_id": 100277,
+  "rms_norm_eps": 1e-06,
+  "rope_scaling": null,
+  "rope_theta": 500000,
+  "tie_word_embeddings": false,
+  "transformers_version": "4.57.6",
+  "use_cache": false,
+  "vocab_size": 100352
+}

g2_2_oldcode_synthcm_newcode__edit/250M/generation_config.json

@@ -0,0 +1,9 @@
+{
+  "_from_model_config": true,
+  "bos_token_id": 100257,
+  "eos_token_id": [
+    100257
+  ],
+  "pad_token_id": 100277,
+  "transformers_version": "4.57.6"
+}

g2_2_oldcode_synthcm_newcode__edit/250M/model.safetensors.index.json

@@ -0,0 +1,363 @@
+{
+  "metadata": {
+    "total_parameters": 7298617344,
+    "total_size": 14597234688
+  },
+  "weight_map": {
+    "lm_head.weight": "model-00003-of-00003.safetensors",
+    "model.embed_tokens.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.post_feedforward_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.self_attn.k_norm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.self_attn.q_norm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.post_feedforward_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.self_attn.k_norm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.self_attn.q_norm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.10.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.10.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.10.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.10.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.10.post_feedforward_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.10.self_attn.k_norm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.10.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.10.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.10.self_attn.q_norm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.10.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.10.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.11.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.11.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.11.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.11.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.11.post_feedforward_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.11.self_attn.k_norm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.11.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.11.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.11.self_attn.q_norm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.11.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.11.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.post_feedforward_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.self_attn.k_norm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.self_attn.q_norm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.post_feedforward_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.self_attn.k_norm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.self_attn.q_norm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.post_feedforward_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.self_attn.k_norm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.self_attn.q_norm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.post_feedforward_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.self_attn.k_norm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.self_attn.q_norm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.post_feedforward_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.self_attn.k_norm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.self_attn.q_norm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.post_feedforward_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.self_attn.k_norm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.self_attn.q_norm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.post_feedforward_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.self_attn.k_norm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.self_attn.q_norm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.post_feedforward_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.self_attn.k_norm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.self_attn.q_norm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.2.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.2.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.2.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.2.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.2.post_feedforward_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.2.self_attn.k_norm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.2.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.2.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.2.self_attn.q_norm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.2.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.2.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.20.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.20.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.20.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.20.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.20.post_feedforward_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.20.self_attn.k_norm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.20.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.20.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.20.self_attn.q_norm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.20.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.20.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.post_feedforward_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.self_attn.k_norm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.self_attn.q_norm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.22.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.22.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.22.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.22.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.22.post_feedforward_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.22.self_attn.k_norm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.22.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.22.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.22.self_attn.q_norm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.22.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.22.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.23.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.23.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.23.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.23.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.23.post_feedforward_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.23.self_attn.k_norm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.23.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.23.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.23.self_attn.q_norm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.23.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.23.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.post_feedforward_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.self_attn.k_norm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.self_attn.q_norm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.post_feedforward_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.self_attn.k_norm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.self_attn.q_norm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.post_feedforward_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.self_attn.k_norm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.self_attn.q_norm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.post_feedforward_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.self_attn.k_norm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.self_attn.q_norm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.28.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.28.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.28.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.28.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.28.post_feedforward_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.28.self_attn.k_norm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.28.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.28.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.28.self_attn.q_norm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.28.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.28.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.29.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.29.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.29.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.29.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.29.post_feedforward_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.29.self_attn.k_norm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.29.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.29.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.29.self_attn.q_norm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.29.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.29.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.3.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.3.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.3.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.3.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.3.post_feedforward_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.3.self_attn.k_norm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.3.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.3.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.3.self_attn.q_norm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.3.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.3.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.30.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.30.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.30.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.30.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.30.post_feedforward_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.30.self_attn.k_norm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.30.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.30.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.30.self_attn.q_norm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.30.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.30.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.31.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.31.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.31.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.31.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.31.post_feedforward_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.31.self_attn.k_norm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.31.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.31.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.31.self_attn.q_norm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.31.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.31.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.4.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.4.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.4.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.4.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.4.post_feedforward_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.4.self_attn.k_norm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.4.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.4.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.4.self_attn.q_norm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.4.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.4.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.post_feedforward_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.self_attn.k_norm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.self_attn.q_norm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.post_feedforward_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.self_attn.k_norm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.self_attn.q_norm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.post_feedforward_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.self_attn.k_norm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.self_attn.q_norm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.post_feedforward_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.self_attn.k_norm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.self_attn.q_norm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.post_feedforward_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.self_attn.k_norm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.self_attn.q_norm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.norm.weight": "model-00003-of-00003.safetensors"
+  }
+}

g2_2_oldcode_synthcm_newcode__edit/250M/special_tokens_map.json

@@ -0,0 +1,30 @@
+{
+  "bos_token": {
+    "content": "<|endoftext|>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "eos_token": {
+    "content": "<|endoftext|>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "pad_token": {
+    "content": "<|pad|>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "unk_token": {
+    "content": "<|endoftext|>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  }
+}

g2_2_oldcode_synthcm_newcode__edit/250M/tokenizer_config.json

@@ -0,0 +1,191 @@
+{
+  "add_prefix_space": false,
+  "added_tokens_decoder": {
+    "100256": {
+      "content": "<|extra_id_0|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "100257": {
+      "content": "<|endoftext|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "100258": {
+      "content": "<|fim_prefix|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "100259": {
+      "content": "<|fim_middle|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "100260": {
+      "content": "<|fim_suffix|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "100261": {
+      "content": "|||PHONE_NUMBER|||",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "100262": {
+      "content": "|||EMAIL_ADDRESS|||",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "100263": {
+      "content": "|||IP_ADDRESS|||",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "100264": {
+      "content": "<|im_start|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "100265": {
+      "content": "<|im_end|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "100266": {
+      "content": "<|extra_id_1|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "100267": {
+      "content": "<|extra_id_2|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "100268": {
+      "content": "<|extra_id_3|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "100269": {
+      "content": "<|extra_id_4|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "100270": {
+      "content": "<|extra_id_5|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "100271": {
+      "content": "<|extra_id_6|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "100272": {
+      "content": "<|extra_id_7|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "100273": {
+      "content": "<|extra_id_8|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "100274": {
+      "content": "<|extra_id_9|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "100275": {
+      "content": "<|extra_id_10|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "100276": {
+      "content": "<|endofprompt|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "100277": {
+      "content": "<|pad|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    }
+  },
+  "bos_token": "<|endoftext|>",
+  "clean_up_tokenization_spaces": false,
+  "eos_token": "<|endoftext|>",
+  "extra_special_tokens": {},
+  "model_max_length": 1000000000000000019884624838656,
+  "pad_token": "<|pad|>",
+  "padding_side": "right",
+  "split_special_tokens": false,
+  "tokenizer_class": "GPT2Tokenizer",
+  "unk_token": "<|endoftext|>"
+}

g2_2_oldcode_synthcm_newcode__edit/250M/train_results.json

@@ -0,0 +1,8 @@
+{
+    "epoch": 0.23420105465807314,
+    "total_flos": 1.1366756130249245e+19,
+    "train_loss": 0.06283065401885746,
+    "train_runtime": 2304.4445,
+    "train_samples_per_second": 14.57,
+    "train_steps_per_second": 1.214
+}

g2_2_oldcode_synthcm_newcode__edit/250M/trainer_log.jsonl

@@ -0,0 +1,26 @@
+{"current_steps": 2550, "total_steps": 2798, "loss": 0.7023, "lr": 1.9980979278401095e-05, "epoch": 0.21344270528166068, "percentage": 91.14, "elapsed_time": "0:00:56", "remaining_time": "0:00:05"}
+{"current_steps": 2560, "total_steps": 2798, "loss": 0.704, "lr": 1.982924955463327e-05, "epoch": 0.2142797354984515, "percentage": 91.49, "elapsed_time": "0:02:25", "remaining_time": "0:00:13"}
+{"current_steps": 2570, "total_steps": 2798, "loss": 0.675, "lr": 1.9528108298328235e-05, "epoch": 0.21511676571524233, "percentage": 91.85, "elapsed_time": "0:03:54", "remaining_time": "0:00:20"}
+{"current_steps": 2580, "total_steps": 2798, "loss": 0.6927, "lr": 1.9082156473635994e-05, "epoch": 0.21595379593203315, "percentage": 92.21, "elapsed_time": "0:05:23", "remaining_time": "0:00:27"}
+{"current_steps": 2590, "total_steps": 2798, "loss": 0.722, "lr": 1.8498207522139434e-05, "epoch": 0.21679082614882397, "percentage": 92.57, "elapsed_time": "0:06:52", "remaining_time": "0:00:33"}
+{"current_steps": 2600, "total_steps": 2798, "loss": 0.6936, "lr": 1.7785183264196977e-05, "epoch": 0.2176278563656148, "percentage": 92.92, "elapsed_time": "0:08:22", "remaining_time": "0:00:38"}
+{"current_steps": 2610, "total_steps": 2798, "loss": 0.6909, "lr": 1.6953977587577846e-05, "epoch": 0.21846488658240562, "percentage": 93.28, "elapsed_time": "0:09:51", "remaining_time": "0:00:42"}
+{"current_steps": 2620, "total_steps": 2798, "loss": 0.682, "lr": 1.601729000601297e-05, "epoch": 0.21930191679919644, "percentage": 93.64, "elapsed_time": "0:11:20", "remaining_time": "0:00:46"}
+{"current_steps": 2630, "total_steps": 2798, "loss": 0.6635, "lr": 1.4989431630624763e-05, "epoch": 0.22013894701598727, "percentage": 94.0, "elapsed_time": "0:12:49", "remaining_time": "0:00:49"}
+{"current_steps": 2640, "total_steps": 2798, "loss": 0.7174, "lr": 1.3886106518686746e-05, "epoch": 0.2209759772327781, "percentage": 94.35, "elapsed_time": "0:14:18", "remaining_time": "0:00:51"}
+{"current_steps": 2650, "total_steps": 2798, "loss": 0.6907, "lr": 1.2724171740359612e-05, "epoch": 0.22181300744956892, "percentage": 94.71, "elapsed_time": "0:15:48", "remaining_time": "0:00:52"}
+{"current_steps": 2660, "total_steps": 2798, "loss": 0.7166, "lr": 1.1521379829206218e-05, "epoch": 0.22265003766635977, "percentage": 95.07, "elapsed_time": "0:17:17", "remaining_time": "0:00:53"}
+{"current_steps": 2670, "total_steps": 2798, "loss": 0.6995, "lr": 1.029610755143595e-05, "epoch": 0.2234870678831506, "percentage": 95.43, "elapsed_time": "0:18:46", "remaining_time": "0:00:54"}
+{"current_steps": 2680, "total_steps": 2798, "loss": 0.7024, "lr": 9.067075137857348e-06, "epoch": 0.22432409809994142, "percentage": 95.78, "elapsed_time": "0:20:15", "remaining_time": "0:00:53"}
+{"current_steps": 2690, "total_steps": 2798, "loss": 0.6813, "lr": 7.853060268232525e-06, "epoch": 0.22516112831673224, "percentage": 96.14, "elapsed_time": "0:21:45", "remaining_time": "0:00:52"}
+{"current_steps": 2700, "total_steps": 2798, "loss": 0.6568, "lr": 6.672611177902054e-06, "epoch": 0.22599815853352306, "percentage": 96.5, "elapsed_time": "0:23:14", "remaining_time": "0:00:50"}
+{"current_steps": 2710, "total_steps": 2798, "loss": 0.6786, "lr": 5.543763269959312e-06, "epoch": 0.2268351887503139, "percentage": 96.85, "elapsed_time": "0:24:59", "remaining_time": "0:00:48"}
+{"current_steps": 2720, "total_steps": 2798, "loss": 0.6893, "lr": 4.483763562693867e-06, "epoch": 0.2276722189671047, "percentage": 97.21, "elapsed_time": "0:26:29", "remaining_time": "0:00:45"}
+{"current_steps": 2730, "total_steps": 2798, "loss": 0.6892, "lr": 3.508807182313187e-06, "epoch": 0.22850924918389554, "percentage": 97.57, "elapsed_time": "0:27:58", "remaining_time": "0:00:41"}
+{"current_steps": 2740, "total_steps": 2798, "loss": 0.7054, "lr": 2.6337899269187294e-06, "epoch": 0.22934627940068636, "percentage": 97.93, "elapsed_time": "0:29:27", "remaining_time": "0:00:37"}
+{"current_steps": 2750, "total_steps": 2798, "loss": 0.7102, "lr": 1.8720806821693126e-06, "epoch": 0.23018330961747718, "percentage": 98.28, "elapsed_time": "0:30:56", "remaining_time": "0:00:32"}
+{"current_steps": 2760, "total_steps": 2798, "loss": 0.7018, "lr": 1.2353171657619887e-06, "epoch": 0.231020339834268, "percentage": 98.64, "elapsed_time": "0:32:25", "remaining_time": "0:00:26"}
+{"current_steps": 2770, "total_steps": 2798, "loss": 0.7009, "lr": 7.332281214330356e-07, "epoch": 0.23185737005105883, "percentage": 99.0, "elapsed_time": "0:33:55", "remaining_time": "0:00:20"}
+{"current_steps": 2780, "total_steps": 2798, "loss": 0.7039, "lr": 3.73484679074808e-07, "epoch": 0.23269440026784968, "percentage": 99.36, "elapsed_time": "0:35:24", "remaining_time": "0:00:13"}
+{"current_steps": 2790, "total_steps": 2798, "loss": 0.6481, "lr": 1.61583151951702e-07, "epoch": 0.2335314304846405, "percentage": 99.71, "elapsed_time": "0:36:53", "remaining_time": "0:00:06"}
+{"current_steps": 2798, "total_steps": 2798, "epoch": 0.23420105465807314, "percentage": 100.0, "elapsed_time": "0:38:24", "remaining_time": "0:00:00"}

g2_2_oldcode_synthcm_newcode__edit/250M/trainer_state.json

@@ -0,0 +1,1996 @@
+{
+  "best_global_step": null,
+  "best_metric": null,
+  "best_model_checkpoint": null,
+  "epoch": 0.23420105465807314,
+  "eval_steps": 500,
+  "global_step": 2798,
+  "is_hyper_param_search": false,
+  "is_local_process_zero": true,
+  "is_world_process_zero": true,
+  "log_history": [
+    {
+      "epoch": 0.0008370302167908262,
+      "grad_norm": 5.156661510467529,
+      "learning_rate": 1.1061797752808989e-06,
+      "loss": 1.5667,
+      "step": 10
+    },
+    {
+      "epoch": 0.0016740604335816524,
+      "grad_norm": 2.557840347290039,
+      "learning_rate": 2.2241573033707866e-06,
+      "loss": 1.3469,
+      "step": 20
+    },
+    {
+      "epoch": 0.0025110906503724786,
+      "grad_norm": 1.6066818237304688,
+      "learning_rate": 3.342134831460675e-06,
+      "loss": 1.2057,
+      "step": 30
+    },
+    {
+      "epoch": 0.0033481208671633047,
+      "grad_norm": 1.1234897375106812,
+      "learning_rate": 4.4601123595505624e-06,
+      "loss": 1.0592,
+      "step": 40
+    },
+    {
+      "epoch": 0.004185151083954131,
+      "grad_norm": 1.0128376483917236,
+      "learning_rate": 5.5780898876404495e-06,
+      "loss": 0.9068,
+      "step": 50
+    },
+    {
+      "epoch": 0.005022181300744957,
+      "grad_norm": 0.7160766124725342,
+      "learning_rate": 6.6960674157303374e-06,
+      "loss": 0.8015,
+      "step": 60
+    },
+    {
+      "epoch": 0.005859211517535783,
+      "grad_norm": 0.47716090083122253,
+      "learning_rate": 7.814044943820226e-06,
+      "loss": 0.8152,
+      "step": 70
+    },
+    {
+      "epoch": 0.006696241734326609,
+      "grad_norm": 0.5273013114929199,
+      "learning_rate": 8.932022471910112e-06,
+      "loss": 0.7991,
+      "step": 80
+    },
+    {
+      "epoch": 0.007533271951117435,
+      "grad_norm": 0.4626390039920807,
+      "learning_rate": 1.005e-05,
+      "loss": 0.7387,
+      "step": 90
+    },
+    {
+      "epoch": 0.008370302167908262,
+      "grad_norm": 0.5121504664421082,
+      "learning_rate": 1.1167977528089888e-05,
+      "loss": 0.7605,
+      "step": 100
+    },
+    {
+      "epoch": 0.009207332384699087,
+      "grad_norm": 0.4082605540752411,
+      "learning_rate": 1.2285955056179776e-05,
+      "loss": 0.7623,
+      "step": 110
+    },
+    {
+      "epoch": 0.010044362601489915,
+      "grad_norm": 0.5059930682182312,
+      "learning_rate": 1.3403932584269664e-05,
+      "loss": 0.764,
+      "step": 120
+    },
+    {
+      "epoch": 0.01088139281828074,
+      "grad_norm": 0.751139223575592,
+      "learning_rate": 1.4521910112359552e-05,
+      "loss": 0.7645,
+      "step": 130
+    },
+    {
+      "epoch": 0.011718423035071566,
+      "grad_norm": 0.492674320936203,
+      "learning_rate": 1.563988764044944e-05,
+      "loss": 0.781,
+      "step": 140
+    },
+    {
+      "epoch": 0.012555453251862391,
+      "grad_norm": 0.566781222820282,
+      "learning_rate": 1.6757865168539328e-05,
+      "loss": 0.7548,
+      "step": 150
+    },
+    {
+      "epoch": 0.013392483468653219,
+      "grad_norm": 0.5136772990226746,
+      "learning_rate": 1.7875842696629214e-05,
+      "loss": 0.7799,
+      "step": 160
+    },
+    {
+      "epoch": 0.014229513685444044,
+      "grad_norm": 0.40454524755477905,
+      "learning_rate": 1.89938202247191e-05,
+      "loss": 0.7748,
+      "step": 170
+    },
+    {
+      "epoch": 0.01506654390223487,
+      "grad_norm": 0.4504382014274597,
+      "learning_rate": 2e-05,
+      "loss": 0.7757,
+      "step": 180
+    },
+    {
+      "epoch": 0.015903574119025696,
+      "grad_norm": 0.5842396020889282,
+      "learning_rate": 2e-05,
+      "loss": 0.7491,
+      "step": 190
+    },
+    {
+      "epoch": 0.016740604335816523,
+      "grad_norm": 0.42499402165412903,
+      "learning_rate": 2e-05,
+      "loss": 0.7533,
+      "step": 200
+    },
+    {
+      "epoch": 0.01757763455260735,
+      "grad_norm": 0.4922196567058563,
+      "learning_rate": 2e-05,
+      "loss": 0.7639,
+      "step": 210
+    },
+    {
+      "epoch": 0.018414664769398174,
+      "grad_norm": 0.4512643814086914,
+      "learning_rate": 2e-05,
+      "loss": 0.7663,
+      "step": 220
+    },
+    {
+      "epoch": 0.019251694986189002,
+      "grad_norm": 0.4370189607143402,
+      "learning_rate": 2e-05,
+      "loss": 0.765,
+      "step": 230
+    },
+    {
+      "epoch": 0.02008872520297983,
+      "grad_norm": 0.41970548033714294,
+      "learning_rate": 2e-05,
+      "loss": 0.745,
+      "step": 240
+    },
+    {
+      "epoch": 0.020925755419770653,
+      "grad_norm": 0.45108288526535034,
+      "learning_rate": 2e-05,
+      "loss": 0.7364,
+      "step": 250
+    },
+    {
+      "epoch": 0.02176278563656148,
+      "grad_norm": 0.42641422152519226,
+      "learning_rate": 2e-05,
+      "loss": 0.732,
+      "step": 260
+    },
+    {
+      "epoch": 0.022599815853352308,
+      "grad_norm": 0.3930014371871948,
+      "learning_rate": 2e-05,
+      "loss": 0.7634,
+      "step": 270
+    },
+    {
+      "epoch": 0.02343684607014313,
+      "grad_norm": 0.43919554352760315,
+      "learning_rate": 2e-05,
+      "loss": 0.7621,
+      "step": 280
+    },
+    {
+      "epoch": 0.02427387628693396,
+      "grad_norm": 0.4371900260448456,
+      "learning_rate": 2e-05,
+      "loss": 0.7379,
+      "step": 290
+    },
+    {
+      "epoch": 0.025110906503724783,
+      "grad_norm": 0.4431065618991852,
+      "learning_rate": 2e-05,
+      "loss": 0.7306,
+      "step": 300
+    },
+    {
+      "epoch": 0.02594793672051561,
+      "grad_norm": 0.4054403603076935,
+      "learning_rate": 2e-05,
+      "loss": 0.7532,
+      "step": 310
+    },
+    {
+      "epoch": 0.026784966937306438,
+      "grad_norm": 0.43971535563468933,
+      "learning_rate": 2e-05,
+      "loss": 0.7527,
+      "step": 320
+    },
+    {
+      "epoch": 0.02762199715409726,
+      "grad_norm": 1.0866317749023438,
+      "learning_rate": 2e-05,
+      "loss": 0.7705,
+      "step": 330
+    },
+    {
+      "epoch": 0.02845902737088809,
+      "grad_norm": 0.4061005115509033,
+      "learning_rate": 2e-05,
+      "loss": 0.7739,
+      "step": 340
+    },
+    {
+      "epoch": 0.029296057587678916,
+      "grad_norm": 0.5896549820899963,
+      "learning_rate": 2e-05,
+      "loss": 0.7655,
+      "step": 350
+    },
+    {
+      "epoch": 0.03013308780446974,
+      "grad_norm": 0.44174519181251526,
+      "learning_rate": 2e-05,
+      "loss": 0.7502,
+      "step": 360
+    },
+    {
+      "epoch": 0.030970118021260568,
+      "grad_norm": 0.4336935877799988,
+      "learning_rate": 2e-05,
+      "loss": 0.7119,
+      "step": 370
+    },
+    {
+      "epoch": 0.03180714823805139,
+      "grad_norm": 0.43263721466064453,
+      "learning_rate": 2e-05,
+      "loss": 0.7308,
+      "step": 380
+    },
+    {
+      "epoch": 0.03264417845484222,
+      "grad_norm": 0.4120350480079651,
+      "learning_rate": 2e-05,
+      "loss": 0.7464,
+      "step": 390
+    },
+    {
+      "epoch": 0.033481208671633046,
+      "grad_norm": 0.4036485254764557,
+      "learning_rate": 2e-05,
+      "loss": 0.7601,
+      "step": 400
+    },
+    {
+      "epoch": 0.03431823888842387,
+      "grad_norm": 0.41021424531936646,
+      "learning_rate": 2e-05,
+      "loss": 0.7281,
+      "step": 410
+    },
+    {
+      "epoch": 0.0351552691052147,
+      "grad_norm": 0.4482649564743042,
+      "learning_rate": 2e-05,
+      "loss": 0.7316,
+      "step": 420
+    },
+    {
+      "epoch": 0.035992299322005525,
+      "grad_norm": 0.4150967299938202,
+      "learning_rate": 2e-05,
+      "loss": 0.7365,
+      "step": 430
+    },
+    {
+      "epoch": 0.03682932953879635,
+      "grad_norm": 0.3876197934150696,
+      "learning_rate": 2e-05,
+      "loss": 0.7511,
+      "step": 440
+    },
+    {
+      "epoch": 0.03766635975558718,
+      "grad_norm": 0.40040096640586853,
+      "learning_rate": 2e-05,
+      "loss": 0.793,
+      "step": 450
+    },
+    {
+      "epoch": 0.038503389972378003,
+      "grad_norm": 0.4496796727180481,
+      "learning_rate": 2e-05,
+      "loss": 0.7423,
+      "step": 460
+    },
+    {
+      "epoch": 0.03934042018916883,
+      "grad_norm": 0.4478641152381897,
+      "learning_rate": 2e-05,
+      "loss": 0.7485,
+      "step": 470
+    },
+    {
+      "epoch": 0.04017745040595966,
+      "grad_norm": 0.4376203715801239,
+      "learning_rate": 2e-05,
+      "loss": 0.7909,
+      "step": 480
+    },
+    {
+      "epoch": 0.04101448062275048,
+      "grad_norm": 0.42852625250816345,
+      "learning_rate": 2e-05,
+      "loss": 0.7159,
+      "step": 490
+    },
+    {
+      "epoch": 0.041851510839541306,
+      "grad_norm": 0.42360687255859375,
+      "learning_rate": 2e-05,
+      "loss": 0.7401,
+      "step": 500
+    },
+    {
+      "epoch": 0.04268854105633214,
+      "grad_norm": 0.38901394605636597,
+      "learning_rate": 2e-05,
+      "loss": 0.7284,
+      "step": 510
+    },
+    {
+      "epoch": 0.04352557127312296,
+      "grad_norm": 0.4375343322753906,
+      "learning_rate": 2e-05,
+      "loss": 0.738,
+      "step": 520
+    },
+    {
+      "epoch": 0.044362601489913785,
+      "grad_norm": 0.44532549381256104,
+      "learning_rate": 2e-05,
+      "loss": 0.7469,
+      "step": 530
+    },
+    {
+      "epoch": 0.045199631706704615,
+      "grad_norm": 0.46848392486572266,
+      "learning_rate": 2e-05,
+      "loss": 0.7366,
+      "step": 540
+    },
+    {
+      "epoch": 0.04603666192349544,
+      "grad_norm": 0.3851851224899292,
+      "learning_rate": 2e-05,
+      "loss": 0.7335,
+      "step": 550
+    },
+    {
+      "epoch": 0.04687369214028626,
+      "grad_norm": 0.40839746594429016,
+      "learning_rate": 2e-05,
+      "loss": 0.7053,
+      "step": 560
+    },
+    {
+      "epoch": 0.04771072235707709,
+      "grad_norm": 0.4012123644351959,
+      "learning_rate": 2e-05,
+      "loss": 0.7137,
+      "step": 570
+    },
+    {
+      "epoch": 0.04854775257386792,
+      "grad_norm": 0.3855404555797577,
+      "learning_rate": 2e-05,
+      "loss": 0.7133,
+      "step": 580
+    },
+    {
+      "epoch": 0.04938478279065874,
+      "grad_norm": 0.3988204002380371,
+      "learning_rate": 2e-05,
+      "loss": 0.7541,
+      "step": 590
+    },
+    {
+      "epoch": 0.050221813007449566,
+      "grad_norm": 0.4450335204601288,
+      "learning_rate": 2e-05,
+      "loss": 0.7238,
+      "step": 600
+    },
+    {
+      "epoch": 0.0510588432242404,
+      "grad_norm": 0.4375969469547272,
+      "learning_rate": 2e-05,
+      "loss": 0.7044,
+      "step": 610
+    },
+    {
+      "epoch": 0.05189587344103122,
+      "grad_norm": 0.4379328787326813,
+      "learning_rate": 2e-05,
+      "loss": 0.726,
+      "step": 620
+    },
+    {
+      "epoch": 0.052732903657822044,
+      "grad_norm": 0.39580798149108887,
+      "learning_rate": 2e-05,
+      "loss": 0.7463,
+      "step": 630
+    },
+    {
+      "epoch": 0.053569933874612875,
+      "grad_norm": 0.41373711824417114,
+      "learning_rate": 2e-05,
+      "loss": 0.7499,
+      "step": 640
+    },
+    {
+      "epoch": 0.0544069640914037,
+      "grad_norm": 0.4301176369190216,
+      "learning_rate": 2e-05,
+      "loss": 0.7341,
+      "step": 650
+    },
+    {
+      "epoch": 0.05524399430819452,
+      "grad_norm": 0.45098692178726196,
+      "learning_rate": 2e-05,
+      "loss": 0.7318,
+      "step": 660
+    },
+    {
+      "epoch": 0.056081024524985354,
+      "grad_norm": 0.4196617305278778,
+      "learning_rate": 2e-05,
+      "loss": 0.734,
+      "step": 670
+    },
+    {
+      "epoch": 0.05691805474177618,
+      "grad_norm": 0.4272335171699524,
+      "learning_rate": 2e-05,
+      "loss": 0.7083,
+      "step": 680
+    },
+    {
+      "epoch": 0.057755084958567,
+      "grad_norm": 0.4190222918987274,
+      "learning_rate": 2e-05,
+      "loss": 0.7272,
+      "step": 690
+    },
+    {
+      "epoch": 0.05859211517535783,
+      "grad_norm": 0.42260268330574036,
+      "learning_rate": 2e-05,
+      "loss": 0.7266,
+      "step": 700
+    },
+    {
+      "epoch": 0.059429145392148656,
+      "grad_norm": 0.3979990780353546,
+      "learning_rate": 2e-05,
+      "loss": 0.7265,
+      "step": 710
+    },
+    {
+      "epoch": 0.06026617560893948,
+      "grad_norm": 0.41610896587371826,
+      "learning_rate": 2e-05,
+      "loss": 0.7308,
+      "step": 720
+    },
+    {
+      "epoch": 0.06110320582573031,
+      "grad_norm": 0.44141194224357605,
+      "learning_rate": 2e-05,
+      "loss": 0.752,
+      "step": 730
+    },
+    {
+      "epoch": 0.061940236042521135,
+      "grad_norm": 0.40890923142433167,
+      "learning_rate": 2e-05,
+      "loss": 0.7552,
+      "step": 740
+    },
+    {
+      "epoch": 0.06277726625931196,
+      "grad_norm": 0.37664929032325745,
+      "learning_rate": 2e-05,
+      "loss": 0.7363,
+      "step": 750
+    },
+    {
+      "epoch": 0.06361429647610278,
+      "grad_norm": 0.3718191683292389,
+      "learning_rate": 2e-05,
+      "loss": 0.7458,
+      "step": 760
+    },
+    {
+      "epoch": 0.0644513266928936,
+      "grad_norm": 0.4162319004535675,
+      "learning_rate": 2e-05,
+      "loss": 0.7587,
+      "step": 770
+    },
+    {
+      "epoch": 0.06528835690968444,
+      "grad_norm": 0.5250889658927917,
+      "learning_rate": 2e-05,
+      "loss": 0.733,
+      "step": 780
+    },
+    {
+      "epoch": 0.06612538712647527,
+      "grad_norm": 0.3899112343788147,
+      "learning_rate": 2e-05,
+      "loss": 0.731,
+      "step": 790
+    },
+    {
+      "epoch": 0.06696241734326609,
+      "grad_norm": 0.39714515209198,
+      "learning_rate": 2e-05,
+      "loss": 0.7397,
+      "step": 800
+    },
+    {
+      "epoch": 0.06779944756005692,
+      "grad_norm": 0.4384140372276306,
+      "learning_rate": 2e-05,
+      "loss": 0.7334,
+      "step": 810
+    },
+    {
+      "epoch": 0.06863647777684774,
+      "grad_norm": 0.37715354561805725,
+      "learning_rate": 2e-05,
+      "loss": 0.7442,
+      "step": 820
+    },
+    {
+      "epoch": 0.06947350799363856,
+      "grad_norm": 0.4676320552825928,
+      "learning_rate": 2e-05,
+      "loss": 0.729,
+      "step": 830
+    },
+    {
+      "epoch": 0.0703105382104294,
+      "grad_norm": 0.4101060926914215,
+      "learning_rate": 2e-05,
+      "loss": 0.7412,
+      "step": 840
+    },
+    {
+      "epoch": 0.07114756842722023,
+      "grad_norm": 0.40823832154273987,
+      "learning_rate": 2e-05,
+      "loss": 0.7404,
+      "step": 850
+    },
+    {
+      "epoch": 0.07198459864401105,
+      "grad_norm": 0.35889768600463867,
+      "learning_rate": 2e-05,
+      "loss": 0.7301,
+      "step": 860
+    },
+    {
+      "epoch": 0.07282162886080187,
+      "grad_norm": 0.35591280460357666,
+      "learning_rate": 2e-05,
+      "loss": 0.7434,
+      "step": 870
+    },
+    {
+      "epoch": 0.0736586590775927,
+      "grad_norm": 0.3851705491542816,
+      "learning_rate": 2e-05,
+      "loss": 0.7002,
+      "step": 880
+    },
+    {
+      "epoch": 0.07449568929438352,
+      "grad_norm": 0.3983948230743408,
+      "learning_rate": 2e-05,
+      "loss": 0.7274,
+      "step": 890
+    },
+    {
+      "epoch": 0.07533271951117436,
+      "grad_norm": 0.3635563254356384,
+      "learning_rate": 2e-05,
+      "loss": 0.6975,
+      "step": 900
+    },
+    {
+      "epoch": 0.07616974972796518,
+      "grad_norm": 0.3900563418865204,
+      "learning_rate": 2e-05,
+      "loss": 0.7467,
+      "step": 910
+    },
+    {
+      "epoch": 0.07700677994475601,
+      "grad_norm": 0.4121996760368347,
+      "learning_rate": 2e-05,
+      "loss": 0.7181,
+      "step": 920
+    },
+    {
+      "epoch": 0.07784381016154683,
+      "grad_norm": 0.35744690895080566,
+      "learning_rate": 2e-05,
+      "loss": 0.72,
+      "step": 930
+    },
+    {
+      "epoch": 0.07868084037833765,
+      "grad_norm": 0.44155263900756836,
+      "learning_rate": 2e-05,
+      "loss": 0.714,
+      "step": 940
+    },
+    {
+      "epoch": 0.07951787059512848,
+      "grad_norm": 0.415809690952301,
+      "learning_rate": 2e-05,
+      "loss": 0.7518,
+      "step": 950
+    },
+    {
+      "epoch": 0.08035490081191932,
+      "grad_norm": 0.3967406153678894,
+      "learning_rate": 2e-05,
+      "loss": 0.7445,
+      "step": 960
+    },
+    {
+      "epoch": 0.08119193102871014,
+      "grad_norm": 0.4007624089717865,
+      "learning_rate": 2e-05,
+      "loss": 0.747,
+      "step": 970
+    },
+    {
+      "epoch": 0.08202896124550096,
+      "grad_norm": 0.38344693183898926,
+      "learning_rate": 2e-05,
+      "loss": 0.7295,
+      "step": 980
+    },
+    {
+      "epoch": 0.08286599146229179,
+      "grad_norm": 0.45661893486976624,
+      "learning_rate": 2e-05,
+      "loss": 0.7111,
+      "step": 990
+    },
+    {
+      "epoch": 0.08370302167908261,
+      "grad_norm": 0.4255678951740265,
+      "learning_rate": 2e-05,
+      "loss": 0.775,
+      "step": 1000
+    },
+    {
+      "epoch": 0.08454005189587344,
+      "grad_norm": 0.49089524149894714,
+      "learning_rate": 2e-05,
+      "loss": 0.7387,
+      "step": 1010
+    },
+    {
+      "epoch": 0.08537708211266427,
+      "grad_norm": 0.3930380642414093,
+      "learning_rate": 2e-05,
+      "loss": 0.7601,
+      "step": 1020
+    },
+    {
+      "epoch": 0.0862141123294551,
+      "grad_norm": 0.4231725037097931,
+      "learning_rate": 2e-05,
+      "loss": 0.6822,
+      "step": 1030
+    },
+    {
+      "epoch": 0.08705114254624592,
+      "grad_norm": 0.39020201563835144,
+      "learning_rate": 2e-05,
+      "loss": 0.7325,
+      "step": 1040
+    },
+    {
+      "epoch": 0.08788817276303675,
+      "grad_norm": 0.429443359375,
+      "learning_rate": 2e-05,
+      "loss": 0.7194,
+      "step": 1050
+    },
+    {
+      "epoch": 0.08872520297982757,
+      "grad_norm": 0.38595613837242126,
+      "learning_rate": 2e-05,
+      "loss": 0.7336,
+      "step": 1060
+    },
+    {
+      "epoch": 0.0895622331966184,
+      "grad_norm": 0.371380090713501,
+      "learning_rate": 2e-05,
+      "loss": 0.7325,
+      "step": 1070
+    },
+    {
+      "epoch": 0.09039926341340923,
+      "grad_norm": 0.396846204996109,
+      "learning_rate": 2e-05,
+      "loss": 0.7281,
+      "step": 1080
+    },
+    {
+      "epoch": 0.09123629363020005,
+      "grad_norm": 0.42279285192489624,
+      "learning_rate": 2e-05,
+      "loss": 0.7383,
+      "step": 1090
+    },
+    {
+      "epoch": 0.09207332384699088,
+      "grad_norm": 0.41259169578552246,
+      "learning_rate": 2e-05,
+      "loss": 0.7398,
+      "step": 1100
+    },
+    {
+      "epoch": 0.0929103540637817,
+      "grad_norm": 0.43002089858055115,
+      "learning_rate": 2e-05,
+      "loss": 0.7323,
+      "step": 1110
+    },
+    {
+      "epoch": 0.09374738428057253,
+      "grad_norm": 0.4018864929676056,
+      "learning_rate": 2e-05,
+      "loss": 0.7542,
+      "step": 1120
+    },
+    {
+      "epoch": 0.09458441449736335,
+      "grad_norm": 0.3879416882991791,
+      "learning_rate": 2e-05,
+      "loss": 0.724,
+      "step": 1130
+    },
+    {
+      "epoch": 0.09542144471415417,
+      "grad_norm": 0.40612295269966125,
+      "learning_rate": 2e-05,
+      "loss": 0.7181,
+      "step": 1140
+    },
+    {
+      "epoch": 0.09625847493094501,
+      "grad_norm": 0.3889906406402588,
+      "learning_rate": 2e-05,
+      "loss": 0.7206,
+      "step": 1150
+    },
+    {
+      "epoch": 0.09709550514773584,
+      "grad_norm": 0.3631397783756256,
+      "learning_rate": 2e-05,
+      "loss": 0.7608,
+      "step": 1160
+    },
+    {
+      "epoch": 0.09793253536452666,
+      "grad_norm": 0.3583589494228363,
+      "learning_rate": 2e-05,
+      "loss": 0.7257,
+      "step": 1170
+    },
+    {
+      "epoch": 0.09876956558131748,
+      "grad_norm": 0.38138115406036377,
+      "learning_rate": 2e-05,
+      "loss": 0.7296,
+      "step": 1180
+    },
+    {
+      "epoch": 0.09960659579810831,
+      "grad_norm": 0.381708025932312,
+      "learning_rate": 2e-05,
+      "loss": 0.7275,
+      "step": 1190
+    },
+    {
+      "epoch": 0.10044362601489913,
+      "grad_norm": 0.3952488601207733,
+      "learning_rate": 2e-05,
+      "loss": 0.7211,
+      "step": 1200
+    },
+    {
+      "epoch": 0.10128065623168997,
+      "grad_norm": 0.369489848613739,
+      "learning_rate": 2e-05,
+      "loss": 0.6827,
+      "step": 1210
+    },
+    {
+      "epoch": 0.1021176864484808,
+      "grad_norm": 0.3854788541793823,
+      "learning_rate": 2e-05,
+      "loss": 0.7141,
+      "step": 1220
+    },
+    {
+      "epoch": 0.10295471666527162,
+      "grad_norm": 0.44547921419143677,
+      "learning_rate": 2e-05,
+      "loss": 0.7465,
+      "step": 1230
+    },
+    {
+      "epoch": 0.10379174688206244,
+      "grad_norm": 0.3807377219200134,
+      "learning_rate": 2e-05,
+      "loss": 0.6965,
+      "step": 1240
+    },
+    {
+      "epoch": 0.10462877709885327,
+      "grad_norm": 1.0734481811523438,
+      "learning_rate": 2e-05,
+      "loss": 0.7228,
+      "step": 1250
+    },
+    {
+      "epoch": 0.10546580731564409,
+      "grad_norm": 0.3832617700099945,
+      "learning_rate": 2e-05,
+      "loss": 0.76,
+      "step": 1260
+    },
+    {
+      "epoch": 0.10630283753243493,
+      "grad_norm": 0.4228065013885498,
+      "learning_rate": 2e-05,
+      "loss": 0.7379,
+      "step": 1270
+    },
+    {
+      "epoch": 0.10713986774922575,
+      "grad_norm": 0.39900723099708557,
+      "learning_rate": 2e-05,
+      "loss": 0.6866,
+      "step": 1280
+    },
+    {
+      "epoch": 0.10797689796601657,
+      "grad_norm": 0.5197460055351257,
+      "learning_rate": 2e-05,
+      "loss": 0.6915,
+      "step": 1290
+    },
+    {
+      "epoch": 0.1088139281828074,
+      "grad_norm": 0.6023410558700562,
+      "learning_rate": 2e-05,
+      "loss": 0.7019,
+      "step": 1300
+    },
+    {
+      "epoch": 0.10965095839959822,
+      "grad_norm": 0.4066433310508728,
+      "learning_rate": 2e-05,
+      "loss": 0.7001,
+      "step": 1310
+    },
+    {
+      "epoch": 0.11048798861638905,
+      "grad_norm": 0.382357656955719,
+      "learning_rate": 2e-05,
+      "loss": 0.7287,
+      "step": 1320
+    },
+    {
+      "epoch": 0.11132501883317988,
+      "grad_norm": 0.3749663829803467,
+      "learning_rate": 2e-05,
+      "loss": 0.7429,
+      "step": 1330
+    },
+    {
+      "epoch": 0.11216204904997071,
+      "grad_norm": 0.3689495325088501,
+      "learning_rate": 2e-05,
+      "loss": 0.6842,
+      "step": 1340
+    },
+    {
+      "epoch": 0.11299907926676153,
+      "grad_norm": 0.41168835759162903,
+      "learning_rate": 2e-05,
+      "loss": 0.7533,
+      "step": 1350
+    },
+    {
+      "epoch": 0.11383610948355236,
+      "grad_norm": 0.39960694313049316,
+      "learning_rate": 2e-05,
+      "loss": 0.7312,
+      "step": 1360
+    },
+    {
+      "epoch": 0.11467313970034318,
+      "grad_norm": 0.3908539414405823,
+      "learning_rate": 2e-05,
+      "loss": 0.7085,
+      "step": 1370
+    },
+    {
+      "epoch": 0.115510169917134,
+      "grad_norm": 0.38182100653648376,
+      "learning_rate": 2e-05,
+      "loss": 0.7271,
+      "step": 1380
+    },
+    {
+      "epoch": 0.11634720013392484,
+      "grad_norm": 0.405620276927948,
+      "learning_rate": 2e-05,
+      "loss": 0.7235,
+      "step": 1390
+    },
+    {
+      "epoch": 0.11718423035071567,
+      "grad_norm": 0.3966982364654541,
+      "learning_rate": 2e-05,
+      "loss": 0.7371,
+      "step": 1400
+    },
+    {
+      "epoch": 0.11802126056750649,
+      "grad_norm": 0.3768328130245209,
+      "learning_rate": 2e-05,
+      "loss": 0.7172,
+      "step": 1410
+    },
+    {
+      "epoch": 0.11885829078429731,
+      "grad_norm": 0.4282349944114685,
+      "learning_rate": 2e-05,
+      "loss": 0.7318,
+      "step": 1420
+    },
+    {
+      "epoch": 0.11969532100108814,
+      "grad_norm": 0.39045482873916626,
+      "learning_rate": 2e-05,
+      "loss": 0.704,
+      "step": 1430
+    },
+    {
+      "epoch": 0.12053235121787896,
+      "grad_norm": 0.3846886456012726,
+      "learning_rate": 2e-05,
+      "loss": 0.7069,
+      "step": 1440
+    },
+    {
+      "epoch": 0.12136938143466978,
+      "grad_norm": 0.3845898509025574,
+      "learning_rate": 2e-05,
+      "loss": 0.703,
+      "step": 1450
+    },
+    {
+      "epoch": 0.12220641165146062,
+      "grad_norm": 0.4043080508708954,
+      "learning_rate": 2e-05,
+      "loss": 0.7329,
+      "step": 1460
+    },
+    {
+      "epoch": 0.12304344186825145,
+      "grad_norm": 0.353148877620697,
+      "learning_rate": 2e-05,
+      "loss": 0.7039,
+      "step": 1470
+    },
+    {
+      "epoch": 0.12388047208504227,
+      "grad_norm": 0.3626961410045624,
+      "learning_rate": 2e-05,
+      "loss": 0.7125,
+      "step": 1480
+    },
+    {
+      "epoch": 0.1247175023018331,
+      "grad_norm": 0.40537577867507935,
+      "learning_rate": 2e-05,
+      "loss": 0.6874,
+      "step": 1490
+    },
+    {
+      "epoch": 0.12555453251862392,
+      "grad_norm": 0.38374269008636475,
+      "learning_rate": 2e-05,
+      "loss": 0.7166,
+      "step": 1500
+    },
+    {
+      "epoch": 0.12639156273541474,
+      "grad_norm": 0.3687508702278137,
+      "learning_rate": 2e-05,
+      "loss": 0.7042,
+      "step": 1510
+    },
+    {
+      "epoch": 0.12722859295220557,
+      "grad_norm": 0.38297727704048157,
+      "learning_rate": 2e-05,
+      "loss": 0.7276,
+      "step": 1520
+    },
+    {
+      "epoch": 0.1280656231689964,
+      "grad_norm": 0.40739986300468445,
+      "learning_rate": 2e-05,
+      "loss": 0.7352,
+      "step": 1530
+    },
+    {
+      "epoch": 0.1289026533857872,
+      "grad_norm": 0.5073732733726501,
+      "learning_rate": 2e-05,
+      "loss": 0.7407,
+      "step": 1540
+    },
+    {
+      "epoch": 0.12973968360257807,
+      "grad_norm": 0.3655388653278351,
+      "learning_rate": 2e-05,
+      "loss": 0.707,
+      "step": 1550
+    },
+    {
+      "epoch": 0.1305767138193689,
+      "grad_norm": 0.38478103280067444,
+      "learning_rate": 2e-05,
+      "loss": 0.7155,
+      "step": 1560
+    },
+    {
+      "epoch": 0.1314137440361597,
+      "grad_norm": 0.3790566027164459,
+      "learning_rate": 2e-05,
+      "loss": 0.7048,
+      "step": 1570
+    },
+    {
+      "epoch": 0.13225077425295054,
+      "grad_norm": 0.3985552489757538,
+      "learning_rate": 2e-05,
+      "loss": 0.7469,
+      "step": 1580
+    },
+    {
+      "epoch": 0.13308780446974136,
+      "grad_norm": 0.3573896586894989,
+      "learning_rate": 2e-05,
+      "loss": 0.7128,
+      "step": 1590
+    },
+    {
+      "epoch": 0.13392483468653218,
+      "grad_norm": 0.3679613471031189,
+      "learning_rate": 2e-05,
+      "loss": 0.7063,
+      "step": 1600
+    },
+    {
+      "epoch": 0.134761864903323,
+      "grad_norm": 0.4195422828197479,
+      "learning_rate": 2e-05,
+      "loss": 0.7214,
+      "step": 1610
+    },
+    {
+      "epoch": 0.13559889512011383,
+      "grad_norm": 0.4262971878051758,
+      "learning_rate": 2e-05,
+      "loss": 0.7214,
+      "step": 1620
+    },
+    {
+      "epoch": 0.13643592533690466,
+      "grad_norm": 0.3953056335449219,
+      "learning_rate": 2e-05,
+      "loss": 0.7457,
+      "step": 1630
+    },
+    {
+      "epoch": 0.13727295555369548,
+      "grad_norm": 0.3896598517894745,
+      "learning_rate": 2e-05,
+      "loss": 0.68,
+      "step": 1640
+    },
+    {
+      "epoch": 0.1381099857704863,
+      "grad_norm": 0.39134523272514343,
+      "learning_rate": 2e-05,
+      "loss": 0.7233,
+      "step": 1650
+    },
+    {
+      "epoch": 0.13894701598727713,
+      "grad_norm": 0.38543522357940674,
+      "learning_rate": 2e-05,
+      "loss": 0.7078,
+      "step": 1660
+    },
+    {
+      "epoch": 0.13978404620406798,
+      "grad_norm": 0.37859565019607544,
+      "learning_rate": 2e-05,
+      "loss": 0.7141,
+      "step": 1670
+    },
+    {
+      "epoch": 0.1406210764208588,
+      "grad_norm": 0.3654524087905884,
+      "learning_rate": 2e-05,
+      "loss": 0.6817,
+      "step": 1680
+    },
+    {
+      "epoch": 0.14145810663764963,
+      "grad_norm": 0.3827599585056305,
+      "learning_rate": 2e-05,
+      "loss": 0.6995,
+      "step": 1690
+    },
+    {
+      "epoch": 0.14229513685444045,
+      "grad_norm": 0.3548571467399597,
+      "learning_rate": 2e-05,
+      "loss": 0.7106,
+      "step": 1700
+    },
+    {
+      "epoch": 0.14313216707123128,
+      "grad_norm": 0.4650750756263733,
+      "learning_rate": 2e-05,
+      "loss": 0.7498,
+      "step": 1710
+    },
+    {
+      "epoch": 0.1439691972880221,
+      "grad_norm": 0.38478001952171326,
+      "learning_rate": 2e-05,
+      "loss": 0.7699,
+      "step": 1720
+    },
+    {
+      "epoch": 0.14480622750481292,
+      "grad_norm": 0.37154585123062134,
+      "learning_rate": 2e-05,
+      "loss": 0.6941,
+      "step": 1730
+    },
+    {
+      "epoch": 0.14564325772160375,
+      "grad_norm": 0.38709065318107605,
+      "learning_rate": 2e-05,
+      "loss": 0.7337,
+      "step": 1740
+    },
+    {
+      "epoch": 0.14648028793839457,
+      "grad_norm": 1.7314486503601074,
+      "learning_rate": 2e-05,
+      "loss": 0.7005,
+      "step": 1750
+    },
+    {
+      "epoch": 0.1473173181551854,
+      "grad_norm": 0.3551817834377289,
+      "learning_rate": 2e-05,
+      "loss": 0.7132,
+      "step": 1760
+    },
+    {
+      "epoch": 0.14815434837197622,
+      "grad_norm": 0.3846317529678345,
+      "learning_rate": 2e-05,
+      "loss": 0.6762,
+      "step": 1770
+    },
+    {
+      "epoch": 0.14899137858876704,
+      "grad_norm": 0.37913134694099426,
+      "learning_rate": 2e-05,
+      "loss": 0.7101,
+      "step": 1780
+    },
+    {
+      "epoch": 0.1498284088055579,
+      "grad_norm": 0.3794521391391754,
+      "learning_rate": 2e-05,
+      "loss": 0.7327,
+      "step": 1790
+    },
+    {
+      "epoch": 0.15066543902234872,
+      "grad_norm": 0.3727302551269531,
+      "learning_rate": 2e-05,
+      "loss": 0.7198,
+      "step": 1800
+    },
+    {
+      "epoch": 0.15150246923913954,
+      "grad_norm": 0.39975666999816895,
+      "learning_rate": 2e-05,
+      "loss": 0.7283,
+      "step": 1810
+    },
+    {
+      "epoch": 0.15233949945593037,
+      "grad_norm": 0.37394288182258606,
+      "learning_rate": 2e-05,
+      "loss": 0.6815,
+      "step": 1820
+    },
+    {
+      "epoch": 0.1531765296727212,
+      "grad_norm": 0.38191404938697815,
+      "learning_rate": 2e-05,
+      "loss": 0.723,
+      "step": 1830
+    },
+    {
+      "epoch": 0.15401355988951201,
+      "grad_norm": 0.3914385437965393,
+      "learning_rate": 2e-05,
+      "loss": 0.6959,
+      "step": 1840
+    },
+    {
+      "epoch": 0.15485059010630284,
+      "grad_norm": 0.36960726976394653,
+      "learning_rate": 2e-05,
+      "loss": 0.6981,
+      "step": 1850
+    },
+    {
+      "epoch": 0.15568762032309366,
+      "grad_norm": 0.4533153474330902,
+      "learning_rate": 2e-05,
+      "loss": 0.7042,
+      "step": 1860
+    },
+    {
+      "epoch": 0.15652465053988449,
+      "grad_norm": 0.390516459941864,
+      "learning_rate": 2e-05,
+      "loss": 0.7285,
+      "step": 1870
+    },
+    {
+      "epoch": 0.1573616807566753,
+      "grad_norm": 0.36687996983528137,
+      "learning_rate": 2e-05,
+      "loss": 0.7039,
+      "step": 1880
+    },
+    {
+      "epoch": 0.15819871097346613,
+      "grad_norm": 0.3998259902000427,
+      "learning_rate": 2e-05,
+      "loss": 0.7045,
+      "step": 1890
+    },
+    {
+      "epoch": 0.15903574119025696,
+      "grad_norm": 0.3630969226360321,
+      "learning_rate": 2e-05,
+      "loss": 0.7156,
+      "step": 1900
+    },
+    {
+      "epoch": 0.15987277140704778,
+      "grad_norm": 0.49144166707992554,
+      "learning_rate": 2e-05,
+      "loss": 0.7492,
+      "step": 1910
+    },
+    {
+      "epoch": 0.16070980162383863,
+      "grad_norm": 0.3666890263557434,
+      "learning_rate": 2e-05,
+      "loss": 0.6957,
+      "step": 1920
+    },
+    {
+      "epoch": 0.16154683184062946,
+      "grad_norm": 0.36240652203559875,
+      "learning_rate": 2e-05,
+      "loss": 0.7208,
+      "step": 1930
+    },
+    {
+      "epoch": 0.16238386205742028,
+      "grad_norm": 0.37068548798561096,
+      "learning_rate": 2e-05,
+      "loss": 0.7593,
+      "step": 1940
+    },
+    {
+      "epoch": 0.1632208922742111,
+      "grad_norm": 0.38422319293022156,
+      "learning_rate": 2e-05,
+      "loss": 0.718,
+      "step": 1950
+    },
+    {
+      "epoch": 0.16405792249100193,
+      "grad_norm": 0.4526020884513855,
+      "learning_rate": 2e-05,
+      "loss": 0.7152,
+      "step": 1960
+    },
+    {
+      "epoch": 0.16489495270779275,
+      "grad_norm": 0.42693546414375305,
+      "learning_rate": 2e-05,
+      "loss": 0.7031,
+      "step": 1970
+    },
+    {
+      "epoch": 0.16573198292458358,
+      "grad_norm": 0.352283239364624,
+      "learning_rate": 2e-05,
+      "loss": 0.6788,
+      "step": 1980
+    },
+    {
+      "epoch": 0.1665690131413744,
+      "grad_norm": 0.4254308342933655,
+      "learning_rate": 2e-05,
+      "loss": 0.7004,
+      "step": 1990
+    },
+    {
+      "epoch": 0.16740604335816522,
+      "grad_norm": 0.37109827995300293,
+      "learning_rate": 2e-05,
+      "loss": 0.7183,
+      "step": 2000
+    },
+    {
+      "epoch": 0.16824307357495605,
+      "grad_norm": 0.3963578939437866,
+      "learning_rate": 2e-05,
+      "loss": 0.7152,
+      "step": 2010
+    },
+    {
+      "epoch": 0.16908010379174687,
+      "grad_norm": 0.4323606491088867,
+      "learning_rate": 2e-05,
+      "loss": 0.7275,
+      "step": 2020
+    },
+    {
+      "epoch": 0.1699171340085377,
+      "grad_norm": 0.38154488801956177,
+      "learning_rate": 2e-05,
+      "loss": 0.7043,
+      "step": 2030
+    },
+    {
+      "epoch": 0.17075416422532855,
+      "grad_norm": 0.37403401732444763,
+      "learning_rate": 2e-05,
+      "loss": 0.6961,
+      "step": 2040
+    },
+    {
+      "epoch": 0.17159119444211937,
+      "grad_norm": 0.4297429919242859,
+      "learning_rate": 2e-05,
+      "loss": 0.7279,
+      "step": 2050
+    },
+    {
+      "epoch": 0.1724282246589102,
+      "grad_norm": 0.3764204978942871,
+      "learning_rate": 2e-05,
+      "loss": 0.7169,
+      "step": 2060
+    },
+    {
+      "epoch": 0.17326525487570102,
+      "grad_norm": 0.4536464512348175,
+      "learning_rate": 2e-05,
+      "loss": 0.7076,
+      "step": 2070
+    },
+    {
+      "epoch": 0.17410228509249184,
+      "grad_norm": 0.3891889750957489,
+      "learning_rate": 2e-05,
+      "loss": 0.6937,
+      "step": 2080
+    },
+    {
+      "epoch": 0.17493931530928267,
+      "grad_norm": 0.3979426920413971,
+      "learning_rate": 2e-05,
+      "loss": 0.7117,
+      "step": 2090
+    },
+    {
+      "epoch": 0.1757763455260735,
+      "grad_norm": 0.33533552289009094,
+      "learning_rate": 2e-05,
+      "loss": 0.6911,
+      "step": 2100
+    },
+    {
+      "epoch": 0.17661337574286431,
+      "grad_norm": 0.4086405038833618,
+      "learning_rate": 2e-05,
+      "loss": 0.6961,
+      "step": 2110
+    },
+    {
+      "epoch": 0.17745040595965514,
+      "grad_norm": 0.32550907135009766,
+      "learning_rate": 2e-05,
+      "loss": 0.7189,
+      "step": 2120
+    },
+    {
+      "epoch": 0.17828743617644596,
+      "grad_norm": 0.41129428148269653,
+      "learning_rate": 2e-05,
+      "loss": 0.7132,
+      "step": 2130
+    },
+    {
+      "epoch": 0.1791244663932368,
+      "grad_norm": 0.38379132747650146,
+      "learning_rate": 2e-05,
+      "loss": 0.7083,
+      "step": 2140
+    },
+    {
+      "epoch": 0.1799614966100276,
+      "grad_norm": 0.33447548747062683,
+      "learning_rate": 2e-05,
+      "loss": 0.6957,
+      "step": 2150
+    },
+    {
+      "epoch": 0.18079852682681846,
+      "grad_norm": 0.36413493752479553,
+      "learning_rate": 2e-05,
+      "loss": 0.6845,
+      "step": 2160
+    },
+    {
+      "epoch": 0.18163555704360929,
+      "grad_norm": 0.3704940676689148,
+      "learning_rate": 2e-05,
+      "loss": 0.7045,
+      "step": 2170
+    },
+    {
+      "epoch": 0.1824725872604001,
+      "grad_norm": 0.37158524990081787,
+      "learning_rate": 2e-05,
+      "loss": 0.6945,
+      "step": 2180
+    },
+    {
+      "epoch": 0.18330961747719093,
+      "grad_norm": 0.37925466895103455,
+      "learning_rate": 2e-05,
+      "loss": 0.7088,
+      "step": 2190
+    },
+    {
+      "epoch": 0.18414664769398176,
+      "grad_norm": 0.3649779260158539,
+      "learning_rate": 2e-05,
+      "loss": 0.7342,
+      "step": 2200
+    },
+    {
+      "epoch": 0.18498367791077258,
+      "grad_norm": 0.38125574588775635,
+      "learning_rate": 2e-05,
+      "loss": 0.7045,
+      "step": 2210
+    },
+    {
+      "epoch": 0.1858207081275634,
+      "grad_norm": 0.4088246524333954,
+      "learning_rate": 2e-05,
+      "loss": 0.7006,
+      "step": 2220
+    },
+    {
+      "epoch": 0.18665773834435423,
+      "grad_norm": 0.4514538645744324,
+      "learning_rate": 2e-05,
+      "loss": 0.7126,
+      "step": 2230
+    },
+    {
+      "epoch": 0.18749476856114505,
+      "grad_norm": 0.3832446336746216,
+      "learning_rate": 2e-05,
+      "loss": 0.6912,
+      "step": 2240
+    },
+    {
+      "epoch": 0.18833179877793588,
+      "grad_norm": 0.37790173292160034,
+      "learning_rate": 2e-05,
+      "loss": 0.6738,
+      "step": 2250
+    },
+    {
+      "epoch": 0.1891688289947267,
+      "grad_norm": 0.3716677129268646,
+      "learning_rate": 2e-05,
+      "loss": 0.7122,
+      "step": 2260
+    },
+    {
+      "epoch": 0.19000585921151752,
+      "grad_norm": 0.3565468490123749,
+      "learning_rate": 2e-05,
+      "loss": 0.679,
+      "step": 2270
+    },
+    {
+      "epoch": 0.19084288942830835,
+      "grad_norm": 0.37897953391075134,
+      "learning_rate": 2e-05,
+      "loss": 0.6941,
+      "step": 2280
+    },
+    {
+      "epoch": 0.1916799196450992,
+      "grad_norm": 0.3777736723423004,
+      "learning_rate": 2e-05,
+      "loss": 0.7166,
+      "step": 2290
+    },
+    {
+      "epoch": 0.19251694986189002,
+      "grad_norm": 0.3585146367549896,
+      "learning_rate": 2e-05,
+      "loss": 0.6928,
+      "step": 2300
+    },
+    {
+      "epoch": 0.19335398007868085,
+      "grad_norm": 0.3698466122150421,
+      "learning_rate": 2e-05,
+      "loss": 0.6937,
+      "step": 2310
+    },
+    {
+      "epoch": 0.19419101029547167,
+      "grad_norm": 0.3647162914276123,
+      "learning_rate": 2e-05,
+      "loss": 0.7128,
+      "step": 2320
+    },
+    {
+      "epoch": 0.1950280405122625,
+      "grad_norm": 0.4572893977165222,
+      "learning_rate": 2e-05,
+      "loss": 0.6978,
+      "step": 2330
+    },
+    {
+      "epoch": 0.19586507072905332,
+      "grad_norm": 0.42052674293518066,
+      "learning_rate": 2e-05,
+      "loss": 0.6901,
+      "step": 2340
+    },
+    {
+      "epoch": 0.19670210094584414,
+      "grad_norm": 0.4102165102958679,
+      "learning_rate": 2e-05,
+      "loss": 0.7097,
+      "step": 2350
+    },
+    {
+      "epoch": 0.19753913116263497,
+      "grad_norm": 0.3792290687561035,
+      "learning_rate": 2e-05,
+      "loss": 0.6917,
+      "step": 2360
+    },
+    {
+      "epoch": 0.1983761613794258,
+      "grad_norm": 0.41515079140663147,
+      "learning_rate": 2e-05,
+      "loss": 0.7033,
+      "step": 2370
+    },
+    {
+      "epoch": 0.19921319159621662,
+      "grad_norm": 0.44105610251426697,
+      "learning_rate": 2e-05,
+      "loss": 0.7093,
+      "step": 2380
+    },
+    {
+      "epoch": 0.20005022181300744,
+      "grad_norm": 0.3615840971469879,
+      "learning_rate": 2e-05,
+      "loss": 0.7512,
+      "step": 2390
+    },
+    {
+      "epoch": 0.20088725202979826,
+      "grad_norm": 0.3563784062862396,
+      "learning_rate": 2e-05,
+      "loss": 0.7046,
+      "step": 2400
+    },
+    {
+      "epoch": 0.20172428224658911,
+      "grad_norm": 0.3593883216381073,
+      "learning_rate": 2e-05,
+      "loss": 0.7143,
+      "step": 2410
+    },
+    {
+      "epoch": 0.20256131246337994,
+      "grad_norm": 0.375445157289505,
+      "learning_rate": 2e-05,
+      "loss": 0.678,
+      "step": 2420
+    },
+    {
+      "epoch": 0.20339834268017076,
+      "grad_norm": 0.3783409595489502,
+      "learning_rate": 2e-05,
+      "loss": 0.6926,
+      "step": 2430
+    },
+    {
+      "epoch": 0.2042353728969616,
+      "grad_norm": 0.3709818720817566,
+      "learning_rate": 2e-05,
+      "loss": 0.6974,
+      "step": 2440
+    },
+    {
+      "epoch": 0.2050724031137524,
+      "grad_norm": 0.3591563403606415,
+      "learning_rate": 2e-05,
+      "loss": 0.7022,
+      "step": 2450
+    },
+    {
+      "epoch": 0.20590943333054323,
+      "grad_norm": 0.42944005131721497,
+      "learning_rate": 2e-05,
+      "loss": 0.7151,
+      "step": 2460
+    },
+    {
+      "epoch": 0.20674646354733406,
+      "grad_norm": 0.36380648612976074,
+      "learning_rate": 2e-05,
+      "loss": 0.6938,
+      "step": 2470
+    },
+    {
+      "epoch": 0.20758349376412488,
+      "grad_norm": 0.3823877274990082,
+      "learning_rate": 2e-05,
+      "loss": 0.6853,
+      "step": 2480
+    },
+    {
+      "epoch": 0.2084205239809157,
+      "grad_norm": 0.40340110659599304,
+      "learning_rate": 2e-05,
+      "loss": 0.6983,
+      "step": 2490
+    },
+    {
+      "epoch": 0.20925755419770653,
+      "grad_norm": 0.3656036853790283,
+      "learning_rate": 2e-05,
+      "loss": 0.6927,
+      "step": 2500
+    },
+    {
+      "epoch": 0.21009458441449735,
+      "grad_norm": 0.3953785002231598,
+      "learning_rate": 2e-05,
+      "loss": 0.6779,
+      "step": 2510
+    },
+    {
+      "epoch": 0.21093161463128818,
+      "grad_norm": 0.40045875310897827,
+      "learning_rate": 2e-05,
+      "loss": 0.6812,
+      "step": 2520
+    },
+    {
+      "epoch": 0.211768644848079,
+      "grad_norm": 0.3604848086833954,
+      "learning_rate": 2e-05,
+      "loss": 0.6908,
+      "step": 2530
+    },
+    {
+      "epoch": 0.21260567506486985,
+      "grad_norm": 0.40517356991767883,
+      "learning_rate": 2e-05,
+      "loss": 0.7074,
+      "step": 2540
+    },
+    {
+      "epoch": 0.21344270528166068,
+      "grad_norm": 0.40945446491241455,
+      "learning_rate": 1.9980979278401095e-05,
+      "loss": 0.7023,
+      "step": 2550
+    },
+    {
+      "epoch": 0.2142797354984515,
+      "grad_norm": 0.35332345962524414,
+      "learning_rate": 1.982924955463327e-05,
+      "loss": 0.704,
+      "step": 2560
+    },
+    {
+      "epoch": 0.21511676571524233,
+      "grad_norm": 0.32888680696487427,
+      "learning_rate": 1.9528108298328235e-05,
+      "loss": 0.675,
+      "step": 2570
+    },
+    {
+      "epoch": 0.21595379593203315,
+      "grad_norm": 0.3869115114212036,
+      "learning_rate": 1.9082156473635994e-05,
+      "loss": 0.6927,
+      "step": 2580
+    },
+    {
+      "epoch": 0.21679082614882397,
+      "grad_norm": 0.38035112619400024,
+      "learning_rate": 1.8498207522139434e-05,
+      "loss": 0.722,
+      "step": 2590
+    },
+    {
+      "epoch": 0.2176278563656148,
+      "grad_norm": 0.3800182044506073,
+      "learning_rate": 1.7785183264196977e-05,
+      "loss": 0.6936,
+      "step": 2600
+    },
+    {
+      "epoch": 0.21846488658240562,
+      "grad_norm": 0.3644275367259979,
+      "learning_rate": 1.6953977587577846e-05,
+      "loss": 0.6909,
+      "step": 2610
+    },
+    {
+      "epoch": 0.21930191679919644,
+      "grad_norm": 0.38952329754829407,
+      "learning_rate": 1.601729000601297e-05,
+      "loss": 0.682,
+      "step": 2620
+    },
+    {
+      "epoch": 0.22013894701598727,
+      "grad_norm": 0.37770339846611023,
+      "learning_rate": 1.4989431630624763e-05,
+      "loss": 0.6635,
+      "step": 2630
+    },
+    {
+      "epoch": 0.2209759772327781,
+      "grad_norm": 0.34742140769958496,
+      "learning_rate": 1.3886106518686746e-05,
+      "loss": 0.7174,
+      "step": 2640
+    },
+    {
+      "epoch": 0.22181300744956892,
+      "grad_norm": 0.3670879304409027,
+      "learning_rate": 1.2724171740359612e-05,
+      "loss": 0.6907,
+      "step": 2650
+    },
+    {
+      "epoch": 0.22265003766635977,
+      "grad_norm": 0.3617439270019531,
+      "learning_rate": 1.1521379829206218e-05,
+      "loss": 0.7166,
+      "step": 2660
+    },
+    {
+      "epoch": 0.2234870678831506,
+      "grad_norm": 0.36148807406425476,
+      "learning_rate": 1.029610755143595e-05,
+      "loss": 0.6995,
+      "step": 2670
+    },
+    {
+      "epoch": 0.22432409809994142,
+      "grad_norm": 0.34924519062042236,
+      "learning_rate": 9.067075137857348e-06,
+      "loss": 0.7024,
+      "step": 2680
+    },
+    {
+      "epoch": 0.22516112831673224,
+      "grad_norm": 0.3570590317249298,
+      "learning_rate": 7.853060268232525e-06,
+      "loss": 0.6813,
+      "step": 2690
+    },
+    {
+      "epoch": 0.22599815853352306,
+      "grad_norm": 0.37608057260513306,
+      "learning_rate": 6.672611177902054e-06,
+      "loss": 0.6568,
+      "step": 2700
+    },
+    {
+      "epoch": 0.2268351887503139,
+      "grad_norm": 0.350795716047287,
+      "learning_rate": 5.543763269959312e-06,
+      "loss": 0.6786,
+      "step": 2710
+    },
+    {
+      "epoch": 0.2276722189671047,
+      "grad_norm": 0.3413582742214203,
+      "learning_rate": 4.483763562693867e-06,
+      "loss": 0.6893,
+      "step": 2720
+    },
+    {
+      "epoch": 0.22850924918389554,
+      "grad_norm": 0.38526901602745056,
+      "learning_rate": 3.508807182313187e-06,
+      "loss": 0.6892,
+      "step": 2730
+    },
+    {
+      "epoch": 0.22934627940068636,
+      "grad_norm": 0.36428093910217285,
+      "learning_rate": 2.6337899269187294e-06,
+      "loss": 0.7054,
+      "step": 2740
+    },
+    {
+      "epoch": 0.23018330961747718,
+      "grad_norm": 0.3428427278995514,
+      "learning_rate": 1.8720806821693126e-06,
+      "loss": 0.7102,
+      "step": 2750
+    },
+    {
+      "epoch": 0.231020339834268,
+      "grad_norm": 0.4259762465953827,
+      "learning_rate": 1.2353171657619887e-06,
+      "loss": 0.7018,
+      "step": 2760
+    },
+    {
+      "epoch": 0.23185737005105883,
+      "grad_norm": 0.35835063457489014,
+      "learning_rate": 7.332281214330356e-07,
+      "loss": 0.7009,
+      "step": 2770
+    },
+    {
+      "epoch": 0.23269440026784968,
+      "grad_norm": 0.3557145595550537,
+      "learning_rate": 3.73484679074808e-07,
+      "loss": 0.7039,
+      "step": 2780
+    },
+    {
+      "epoch": 0.2335314304846405,
+      "grad_norm": 0.35716041922569275,
+      "learning_rate": 1.61583151951702e-07,
+      "loss": 0.6481,
+      "step": 2790
+    },
+    {
+      "epoch": 0.23420105465807314,
+      "step": 2798,
+      "total_flos": 1.1366756130249245e+19,
+      "train_loss": 0.06283065401885746,
+      "train_runtime": 2304.4445,
+      "train_samples_per_second": 14.57,
+      "train_steps_per_second": 1.214
+    }
+  ],
+  "logging_steps": 10,
+  "max_steps": 2798,
+  "num_input_tokens_seen": 0,
+  "num_train_epochs": 1,
+  "save_steps": 159,
+  "stateful_callbacks": {
+    "TrainerControl": {
+      "args": {
+        "should_epoch_stop": false,
+        "should_evaluate": false,
+        "should_log": false,
+        "should_save": true,
+        "should_training_stop": true
+      },
+      "attributes": {}
+    }
+  },
+  "total_flos": 1.1366756130249245e+19,
+  "train_batch_size": 1,
+  "trial_name": null,
+  "trial_params": null
+}

g2_2_oldcode_synthcm_newcode__edit/500M/all_results.json

@@ -0,0 +1,8 @@
+{
+    "epoch": 0.4684858123378254,
+    "total_flos": 2.2737574718397743e+19,
+    "train_loss": 0.02192150756293415,
+    "train_runtime": 1760.7014,
+    "train_samples_per_second": 38.146,
+    "train_steps_per_second": 3.179
+}

g2_2_oldcode_synthcm_newcode__edit/500M/chat_template.jinja

@@ -0,0 +1 @@
+{% if messages[0]['role'] == 'system' %}{% set loop_messages = messages[1:] %}{% set system_message = messages[0]['content'] %}{% else %}{% set loop_messages = messages %}{% endif %}{% if system_message is defined %}{{ system_message }}{% endif %}{% for message in loop_messages %}{% set content = message['content'] %}{% if message['role'] == 'user' %}{{ content }}{% elif message['role'] == 'assistant' %}{{ content + '<|endoftext|>' }}{% endif %}{% endfor %}

g2_2_oldcode_synthcm_newcode__edit/500M/config.json

@@ -0,0 +1,27 @@
+{
+  "architectures": [
+    "Olmo2ForCausalLM"
+  ],
+  "attention_bias": false,
+  "attention_dropout": 0.0,
+  "bos_token_id": 100257,
+  "dtype": "bfloat16",
+  "eos_token_id": 100257,
+  "hidden_act": "silu",
+  "hidden_size": 4096,
+  "initializer_range": 0.02,
+  "intermediate_size": 11008,
+  "max_position_embeddings": 4096,
+  "model_type": "olmo2",
+  "num_attention_heads": 32,
+  "num_hidden_layers": 32,
+  "num_key_value_heads": 32,
+  "pad_token_id": 100277,
+  "rms_norm_eps": 1e-06,
+  "rope_scaling": null,
+  "rope_theta": 500000,
+  "tie_word_embeddings": false,
+  "transformers_version": "4.57.6",
+  "use_cache": false,
+  "vocab_size": 100352
+}

g2_2_oldcode_synthcm_newcode__edit/500M/generation_config.json

@@ -0,0 +1,9 @@
+{
+  "_from_model_config": true,
+  "bos_token_id": 100257,
+  "eos_token_id": [
+    100257
+  ],
+  "pad_token_id": 100277,
+  "transformers_version": "4.57.6"
+}

g2_2_oldcode_synthcm_newcode__edit/500M/model.safetensors.index.json

@@ -0,0 +1,363 @@
+{
+  "metadata": {
+    "total_parameters": 7298617344,
+    "total_size": 14597234688
+  },
+  "weight_map": {
+    "lm_head.weight": "model-00003-of-00003.safetensors",
+    "model.embed_tokens.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.post_feedforward_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.self_attn.k_norm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.self_attn.q_norm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.post_feedforward_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.self_attn.k_norm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.self_attn.q_norm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.10.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.10.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.10.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.10.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.10.post_feedforward_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.10.self_attn.k_norm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.10.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.10.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.10.self_attn.q_norm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.10.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.10.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.11.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.11.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.11.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.11.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.11.post_feedforward_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.11.self_attn.k_norm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.11.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.11.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.11.self_attn.q_norm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.11.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.11.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.post_feedforward_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.self_attn.k_norm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.self_attn.q_norm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.post_feedforward_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.self_attn.k_norm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.self_attn.q_norm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.post_feedforward_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.self_attn.k_norm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.self_attn.q_norm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.post_feedforward_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.self_attn.k_norm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.self_attn.q_norm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.post_feedforward_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.self_attn.k_norm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.self_attn.q_norm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.post_feedforward_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.self_attn.k_norm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.self_attn.q_norm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.post_feedforward_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.self_attn.k_norm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.self_attn.q_norm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.post_feedforward_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.self_attn.k_norm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.self_attn.q_norm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.2.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.2.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.2.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.2.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.2.post_feedforward_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.2.self_attn.k_norm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.2.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.2.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.2.self_attn.q_norm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.2.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.2.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.20.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.20.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.20.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.20.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.20.post_feedforward_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.20.self_attn.k_norm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.20.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.20.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.20.self_attn.q_norm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.20.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.20.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.post_feedforward_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.self_attn.k_norm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.self_attn.q_norm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.22.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.22.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.22.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.22.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.22.post_feedforward_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.22.self_attn.k_norm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.22.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.22.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.22.self_attn.q_norm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.22.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.22.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.23.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.23.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.23.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.23.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.23.post_feedforward_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.23.self_attn.k_norm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.23.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.23.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.23.self_attn.q_norm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.23.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.23.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.post_feedforward_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.self_attn.k_norm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.self_attn.q_norm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.post_feedforward_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.self_attn.k_norm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.self_attn.q_norm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.post_feedforward_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.self_attn.k_norm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.self_attn.q_norm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.post_feedforward_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.self_attn.k_norm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.self_attn.q_norm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.28.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.28.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.28.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.28.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.28.post_feedforward_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.28.self_attn.k_norm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.28.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.28.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.28.self_attn.q_norm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.28.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.28.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.29.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.29.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.29.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.29.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.29.post_feedforward_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.29.self_attn.k_norm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.29.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.29.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.29.self_attn.q_norm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.29.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.29.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.3.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.3.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.3.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.3.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.3.post_feedforward_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.3.self_attn.k_norm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.3.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.3.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.3.self_attn.q_norm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.3.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.3.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.30.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.30.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.30.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.30.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.30.post_feedforward_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.30.self_attn.k_norm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.30.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.30.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.30.self_attn.q_norm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.30.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.30.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.31.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.31.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.31.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.31.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.31.post_feedforward_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.31.self_attn.k_norm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.31.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.31.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.31.self_attn.q_norm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.31.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.31.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.4.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.4.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.4.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.4.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.4.post_feedforward_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.4.self_attn.k_norm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.4.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.4.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.4.self_attn.q_norm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.4.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.4.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.post_feedforward_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.self_attn.k_norm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.self_attn.q_norm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.post_feedforward_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.self_attn.k_norm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.self_attn.q_norm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.post_feedforward_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.self_attn.k_norm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.self_attn.q_norm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.post_feedforward_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.self_attn.k_norm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.self_attn.q_norm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.post_feedforward_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.self_attn.k_norm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.self_attn.q_norm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.norm.weight": "model-00003-of-00003.safetensors"
+  }
+}

g2_2_oldcode_synthcm_newcode__edit/500M/special_tokens_map.json

@@ -0,0 +1,30 @@
+{
+  "bos_token": {
+    "content": "<|endoftext|>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "eos_token": {
+    "content": "<|endoftext|>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "pad_token": {
+    "content": "<|pad|>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "unk_token": {
+    "content": "<|endoftext|>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  }
+}

g2_2_oldcode_synthcm_newcode__edit/500M/tokenizer_config.json

@@ -0,0 +1,191 @@
+{
+  "add_prefix_space": false,
+  "added_tokens_decoder": {
+    "100256": {
+      "content": "<|extra_id_0|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "100257": {
+      "content": "<|endoftext|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "100258": {
+      "content": "<|fim_prefix|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "100259": {
+      "content": "<|fim_middle|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "100260": {
+      "content": "<|fim_suffix|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "100261": {
+      "content": "|||PHONE_NUMBER|||",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "100262": {
+      "content": "|||EMAIL_ADDRESS|||",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "100263": {
+      "content": "|||IP_ADDRESS|||",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "100264": {
+      "content": "<|im_start|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "100265": {
+      "content": "<|im_end|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "100266": {
+      "content": "<|extra_id_1|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "100267": {
+      "content": "<|extra_id_2|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "100268": {
+      "content": "<|extra_id_3|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "100269": {
+      "content": "<|extra_id_4|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "100270": {
+      "content": "<|extra_id_5|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "100271": {
+      "content": "<|extra_id_6|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "100272": {
+      "content": "<|extra_id_7|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "100273": {
+      "content": "<|extra_id_8|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "100274": {
+      "content": "<|extra_id_9|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "100275": {
+      "content": "<|extra_id_10|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "100276": {
+      "content": "<|endofprompt|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "100277": {
+      "content": "<|pad|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    }
+  },
+  "bos_token": "<|endoftext|>",
+  "clean_up_tokenization_spaces": false,
+  "eos_token": "<|endoftext|>",
+  "extra_special_tokens": {},
+  "model_max_length": 1000000000000000019884624838656,
+  "pad_token": "<|pad|>",
+  "padding_side": "right",
+  "split_special_tokens": false,
+  "tokenizer_class": "GPT2Tokenizer",
+  "unk_token": "<|endoftext|>"
+}

g2_2_oldcode_synthcm_newcode__edit/500M/train_results.json

@@ -0,0 +1,8 @@
+{
+    "epoch": 0.4684858123378254,
+    "total_flos": 2.2737574718397743e+19,
+    "train_loss": 0.02192150756293415,
+    "train_runtime": 1760.7014,
+    "train_samples_per_second": 38.146,
+    "train_steps_per_second": 3.179
+}

g2_2_oldcode_synthcm_newcode__edit/500M/trainer_log.jsonl

@@ -0,0 +1,61 @@
+{"current_steps": 5090, "total_steps": 5597, "loss": 0.5847, "lr": 1.9999810479633703e-05, "epoch": 0.42604838034653053, "percentage": 90.94, "elapsed_time": "0:00:19", "remaining_time": "0:00:01"}
+{"current_steps": 5100, "total_steps": 5597, "loss": 0.6512, "lr": 1.997707677020786e-05, "epoch": 0.42688541056332135, "percentage": 91.12, "elapsed_time": "0:01:48", "remaining_time": "0:00:10"}
+{"current_steps": 5110, "total_steps": 5597, "loss": 0.6726, "lr": 1.9916538196124637e-05, "epoch": 0.4277224407801122, "percentage": 91.3, "elapsed_time": "0:03:18", "remaining_time": "0:00:18"}
+{"current_steps": 5120, "total_steps": 5597, "loss": 0.6721, "lr": 1.9818425303863402e-05, "epoch": 0.428559470996903, "percentage": 91.48, "elapsed_time": "0:04:47", "remaining_time": "0:00:26"}
+{"current_steps": 5130, "total_steps": 5597, "loss": 0.6925, "lr": 1.9683111732582887e-05, "epoch": 0.4293965012136938, "percentage": 91.66, "elapsed_time": "0:06:16", "remaining_time": "0:00:34"}
+{"current_steps": 5140, "total_steps": 5597, "loss": 0.6565, "lr": 1.951111279120706e-05, "epoch": 0.43023353143048465, "percentage": 91.83, "elapsed_time": "0:07:45", "remaining_time": "0:00:41"}
+{"current_steps": 5090, "total_steps": 5597, "loss": 0.5847, "lr": 1.9999810479633703e-05, "epoch": 0.42604838034653053, "percentage": 90.94, "elapsed_time": "0:00:21", "remaining_time": "0:00:02"}
+{"current_steps": 5100, "total_steps": 5597, "loss": 0.6512, "lr": 1.997707677020786e-05, "epoch": 0.42688541056332135, "percentage": 91.12, "elapsed_time": "0:01:50", "remaining_time": "0:00:10"}
+{"current_steps": 5110, "total_steps": 5597, "loss": 0.6725, "lr": 1.9916538196124637e-05, "epoch": 0.4277224407801122, "percentage": 91.3, "elapsed_time": "0:03:18", "remaining_time": "0:00:18"}
+{"current_steps": 5120, "total_steps": 5597, "loss": 0.6721, "lr": 1.9818425303863402e-05, "epoch": 0.428559470996903, "percentage": 91.48, "elapsed_time": "0:04:47", "remaining_time": "0:00:26"}
+{"current_steps": 5130, "total_steps": 5597, "loss": 0.6925, "lr": 1.9683111732582887e-05, "epoch": 0.4293965012136938, "percentage": 91.66, "elapsed_time": "0:06:16", "remaining_time": "0:00:34"}
+{"current_steps": 5140, "total_steps": 5597, "loss": 0.6566, "lr": 1.951111279120706e-05, "epoch": 0.43023353143048465, "percentage": 91.83, "elapsed_time": "0:07:45", "remaining_time": "0:00:41"}
+{"current_steps": 5150, "total_steps": 5597, "loss": 0.653, "lr": 1.9303083495996028e-05, "epoch": 0.4310705616472755, "percentage": 92.01, "elapsed_time": "0:09:14", "remaining_time": "0:00:48"}
+{"current_steps": 5160, "total_steps": 5597, "loss": 0.6435, "lr": 1.9059816076075482e-05, "epoch": 0.4319075918640663, "percentage": 92.19, "elapsed_time": "0:10:43", "remaining_time": "0:00:54"}
+{"current_steps": 5170, "total_steps": 5597, "loss": 0.6457, "lr": 1.8782236956424142e-05, "epoch": 0.4327446220808571, "percentage": 92.37, "elapsed_time": "0:12:12", "remaining_time": "0:01:00"}
+{"current_steps": 5180, "total_steps": 5597, "loss": 0.6873, "lr": 1.8471403229808897e-05, "epoch": 0.43358165229764795, "percentage": 92.55, "elapsed_time": "0:13:41", "remaining_time": "0:01:06"}
+{"current_steps": 5190, "total_steps": 5597, "loss": 0.6788, "lr": 1.8128498631103333e-05, "epoch": 0.43441868251443877, "percentage": 92.73, "elapsed_time": "0:15:09", "remaining_time": "0:01:11"}
+{"current_steps": 5200, "total_steps": 5597, "loss": 0.6728, "lr": 1.7754829029320507e-05, "epoch": 0.4352557127312296, "percentage": 92.91, "elapsed_time": "0:16:38", "remaining_time": "0:01:16"}
+{"current_steps": 5210, "total_steps": 5597, "loss": 0.6409, "lr": 1.7351817454527473e-05, "epoch": 0.4360927429480204, "percentage": 93.09, "elapsed_time": "0:18:07", "remaining_time": "0:01:20"}
+{"current_steps": 5220, "total_steps": 5597, "loss": 0.6645, "lr": 1.6920998678580345e-05, "epoch": 0.43692977316481124, "percentage": 93.26, "elapsed_time": "0:19:36", "remaining_time": "0:01:24"}
+{"current_steps": 5230, "total_steps": 5597, "loss": 0.6616, "lr": 1.6464013370317854e-05, "epoch": 0.43776680338160207, "percentage": 93.44, "elapsed_time": "0:21:05", "remaining_time": "0:01:28"}
+{"current_steps": 5240, "total_steps": 5597, "loss": 0.6364, "lr": 1.5982601847471915e-05, "epoch": 0.4386038335983929, "percentage": 93.62, "elapsed_time": "0:22:34", "remaining_time": "0:01:32"}
+{"current_steps": 5250, "total_steps": 5597, "loss": 0.6473, "lr": 1.5478597449089553e-05, "epoch": 0.4394408638151837, "percentage": 93.8, "elapsed_time": "0:24:42", "remaining_time": "0:01:37"}
+{"current_steps": 5260, "total_steps": 5597, "loss": 0.6731, "lr": 1.4953919553705725e-05, "epoch": 0.44027789403197454, "percentage": 93.98, "elapsed_time": "0:26:10", "remaining_time": "0:01:40"}
+{"current_steps": 5270, "total_steps": 5597, "loss": 0.6668, "lr": 1.441056626985559e-05, "epoch": 0.44111492424876536, "percentage": 94.16, "elapsed_time": "0:27:39", "remaining_time": "0:01:42"}
+{"current_steps": 5280, "total_steps": 5597, "loss": 0.6675, "lr": 1.3850606826762668e-05, "epoch": 0.4419519544655562, "percentage": 94.34, "elapsed_time": "0:29:08", "remaining_time": "0:01:44"}
+{"current_steps": 5290, "total_steps": 5597, "loss": 0.6675, "lr": 1.3276173694181128e-05, "epoch": 0.442788984682347, "percentage": 94.51, "elapsed_time": "0:30:37", "remaining_time": "0:01:46"}
+{"current_steps": 5300, "total_steps": 5597, "loss": 0.648, "lr": 1.2689454461401822e-05, "epoch": 0.44362601489913783, "percentage": 94.69, "elapsed_time": "0:32:06", "remaining_time": "0:01:47"}
+{"current_steps": 5310, "total_steps": 5597, "loss": 0.6463, "lr": 1.2092683506348939e-05, "epoch": 0.4444630451159287, "percentage": 94.87, "elapsed_time": "0:33:35", "remaining_time": "0:01:48"}
+{"current_steps": 5320, "total_steps": 5597, "loss": 0.6575, "lr": 1.14881334864935e-05, "epoch": 0.44530007533271954, "percentage": 95.05, "elapsed_time": "0:35:04", "remaining_time": "0:01:49"}
+{"current_steps": 5330, "total_steps": 5597, "loss": 0.6597, "lr": 1.0878106683988481e-05, "epoch": 0.44613710554951036, "percentage": 95.23, "elapsed_time": "0:36:33", "remaining_time": "0:01:49"}
+{"current_steps": 5340, "total_steps": 5597, "loss": 0.6277, "lr": 1.026492623798551e-05, "epoch": 0.4469741357663012, "percentage": 95.41, "elapsed_time": "0:38:02", "remaining_time": "0:01:49"}
+{"current_steps": 5350, "total_steps": 5597, "loss": 0.6637, "lr": 9.650927297522743e-06, "epoch": 0.447811165983092, "percentage": 95.59, "elapsed_time": "0:39:31", "remaining_time": "0:01:49"}
+{"current_steps": 5360, "total_steps": 5597, "loss": 0.6529, "lr": 9.038448128676012e-06, "epoch": 0.44864819619988283, "percentage": 95.77, "elapsed_time": "0:41:00", "remaining_time": "0:01:48"}
+{"current_steps": 5370, "total_steps": 5597, "loss": 0.6497, "lr": 8.42982120983949e-06, "epoch": 0.44948522641667366, "percentage": 95.94, "elapsed_time": "0:42:29", "remaining_time": "0:01:47"}
+{"current_steps": 5380, "total_steps": 5597, "loss": 0.6752, "lr": 7.827364349047303e-06, "epoch": 0.4503222566334645, "percentage": 96.12, "elapsed_time": "0:43:58", "remaining_time": "0:01:46"}
+{"current_steps": 5390, "total_steps": 5597, "loss": 0.6394, "lr": 7.233371857163706e-06, "epoch": 0.4511592868502553, "percentage": 96.3, "elapsed_time": "0:45:27", "remaining_time": "0:01:44"}
+{"current_steps": 5400, "total_steps": 5597, "loss": 0.6787, "lr": 6.650105810556492e-06, "epoch": 0.4519963170670461, "percentage": 96.48, "elapsed_time": "0:46:55", "remaining_time": "0:01:42"}
+{"current_steps": 5410, "total_steps": 5597, "loss": 0.6692, "lr": 6.079787436527681e-06, "epoch": 0.45283334728383695, "percentage": 96.66, "elapsed_time": "0:49:03", "remaining_time": "0:01:41"}
+{"current_steps": 5420, "total_steps": 5597, "loss": 0.6682, "lr": 5.524588654307961e-06, "epoch": 0.4536703775006278, "percentage": 96.84, "elapsed_time": "0:50:32", "remaining_time": "0:01:39"}
+{"current_steps": 5430, "total_steps": 5597, "loss": 0.6573, "lr": 4.986623803828985e-06, "epoch": 0.4545074077174186, "percentage": 97.02, "elapsed_time": "0:52:01", "remaining_time": "0:01:35"}
+{"current_steps": 5410, "total_steps": 5597, "loss": 0.6971, "lr": 6.079787436527681e-06, "epoch": 0.45283334728383695, "percentage": 96.66, "elapsed_time": "0:00:38", "remaining_time": "0:00:01"}
+{"current_steps": 5420, "total_steps": 5597, "loss": 0.6682, "lr": 5.524588654307961e-06, "epoch": 0.4536703775006278, "percentage": 96.84, "elapsed_time": "0:02:06", "remaining_time": "0:00:04"}
+{"current_steps": 5430, "total_steps": 5597, "loss": 0.6573, "lr": 4.986623803828985e-06, "epoch": 0.4545074077174186, "percentage": 97.02, "elapsed_time": "0:03:34", "remaining_time": "0:00:06"}
+{"current_steps": 5440, "total_steps": 5597, "loss": 0.6321, "lr": 4.467941593772423e-06, "epoch": 0.4553444379342094, "percentage": 97.19, "elapsed_time": "0:05:02", "remaining_time": "0:00:08"}
+{"current_steps": 5450, "total_steps": 5597, "loss": 0.6756, "lr": 3.9705172995597606e-06, "epoch": 0.45618146815100025, "percentage": 97.37, "elapsed_time": "0:06:30", "remaining_time": "0:00:10"}
+{"current_steps": 5460, "total_steps": 5597, "loss": 0.6385, "lr": 3.4962452409948346e-06, "epoch": 0.45701849836779107, "percentage": 97.55, "elapsed_time": "0:07:58", "remaining_time": "0:00:12"}
+{"current_steps": 5470, "total_steps": 5597, "loss": 0.6826, "lr": 3.046931568206211e-06, "epoch": 0.4578555285845819, "percentage": 97.73, "elapsed_time": "0:09:26", "remaining_time": "0:00:13"}
+{"current_steps": 5480, "total_steps": 5597, "loss": 0.6485, "lr": 2.624287383362404e-06, "epoch": 0.4586925588013727, "percentage": 97.91, "elapsed_time": "0:10:54", "remaining_time": "0:00:13"}
+{"current_steps": 5490, "total_steps": 5597, "loss": 0.6697, "lr": 2.2299222243541845e-06, "epoch": 0.45952958901816354, "percentage": 98.09, "elapsed_time": "0:12:22", "remaining_time": "0:00:14"}
+{"current_steps": 5500, "total_steps": 5597, "loss": 0.6132, "lr": 1.8653379352598246e-06, "epoch": 0.46036661923495437, "percentage": 98.27, "elapsed_time": "0:13:50", "remaining_time": "0:00:14"}
+{"current_steps": 5510, "total_steps": 5597, "loss": 0.6436, "lr": 1.5319229469361075e-06, "epoch": 0.4612036494517452, "percentage": 98.45, "elapsed_time": "0:15:18", "remaining_time": "0:00:14"}
+{"current_steps": 5520, "total_steps": 5597, "loss": 0.6035, "lr": 1.2309469895160917e-06, "epoch": 0.462040679668536, "percentage": 98.62, "elapsed_time": "0:16:46", "remaining_time": "0:00:14"}
+{"current_steps": 5530, "total_steps": 5597, "loss": 0.616, "lr": 9.635562569497974e-07, "epoch": 0.46287770988532684, "percentage": 98.8, "elapsed_time": "0:18:14", "remaining_time": "0:00:13"}
+{"current_steps": 5540, "total_steps": 5597, "loss": 0.622, "lr": 7.307690420024867e-07, "epoch": 0.46371474010211766, "percentage": 98.98, "elapsed_time": "0:19:42", "remaining_time": "0:00:12"}
+{"current_steps": 5550, "total_steps": 5597, "loss": 0.6435, "lr": 5.334718583335942e-07, "epoch": 0.4645517703189085, "percentage": 99.16, "elapsed_time": "0:21:10", "remaining_time": "0:00:10"}
+{"current_steps": 5560, "total_steps": 5597, "loss": 0.6681, "lr": 3.7241606442442763e-07, "epoch": 0.46538880053569937, "percentage": 99.34, "elapsed_time": "0:22:38", "remaining_time": "0:00:09"}
+{"current_steps": 5570, "total_steps": 5597, "loss": 0.6293, "lr": 2.482150022115879e-07, "epoch": 0.4662258307524902, "percentage": 99.52, "elapsed_time": "0:24:45", "remaining_time": "0:00:07"}
+{"current_steps": 5580, "total_steps": 5597, "loss": 0.6354, "lr": 1.6134166132293174e-07, "epoch": 0.467062860969281, "percentage": 99.7, "elapsed_time": "0:26:13", "remaining_time": "0:00:04"}
+{"current_steps": 5590, "total_steps": 5597, "loss": 0.6365, "lr": 1.1212687781127357e-07, "epoch": 0.46789989118607184, "percentage": 99.87, "elapsed_time": "0:27:41", "remaining_time": "0:00:02"}
+{"current_steps": 5597, "total_steps": 5597, "epoch": 0.4684858123378254, "percentage": 100.0, "elapsed_time": "0:29:20", "remaining_time": "0:00:00"}

g2_2_oldcode_synthcm_newcode__edit/500M/trainer_state.json

@@ -0,0 +1,3956 @@
+{
+  "best_global_step": null,
+  "best_metric": null,
+  "best_model_checkpoint": null,
+  "epoch": 0.4684858123378254,
+  "eval_steps": 500,
+  "global_step": 5597,
+  "is_hyper_param_search": false,
+  "is_local_process_zero": true,
+  "is_world_process_zero": true,
+  "log_history": [
+    {
+      "epoch": 0.0008370302167908262,
+      "grad_norm": 5.156661510467529,
+      "learning_rate": 1.1061797752808989e-06,
+      "loss": 1.5667,
+      "step": 10
+    },
+    {
+      "epoch": 0.0016740604335816524,
+      "grad_norm": 2.557840347290039,
+      "learning_rate": 2.2241573033707866e-06,
+      "loss": 1.3469,
+      "step": 20
+    },
+    {
+      "epoch": 0.0025110906503724786,
+      "grad_norm": 1.6066818237304688,
+      "learning_rate": 3.342134831460675e-06,
+      "loss": 1.2057,
+      "step": 30
+    },
+    {
+      "epoch": 0.0033481208671633047,
+      "grad_norm": 1.1234897375106812,
+      "learning_rate": 4.4601123595505624e-06,
+      "loss": 1.0592,
+      "step": 40
+    },
+    {
+      "epoch": 0.004185151083954131,
+      "grad_norm": 1.0128376483917236,
+      "learning_rate": 5.5780898876404495e-06,
+      "loss": 0.9068,
+      "step": 50
+    },
+    {
+      "epoch": 0.005022181300744957,
+      "grad_norm": 0.7160766124725342,
+      "learning_rate": 6.6960674157303374e-06,
+      "loss": 0.8015,
+      "step": 60
+    },
+    {
+      "epoch": 0.005859211517535783,
+      "grad_norm": 0.47716090083122253,
+      "learning_rate": 7.814044943820226e-06,
+      "loss": 0.8152,
+      "step": 70
+    },
+    {
+      "epoch": 0.006696241734326609,
+      "grad_norm": 0.5273013114929199,
+      "learning_rate": 8.932022471910112e-06,
+      "loss": 0.7991,
+      "step": 80
+    },
+    {
+      "epoch": 0.007533271951117435,
+      "grad_norm": 0.4626390039920807,
+      "learning_rate": 1.005e-05,
+      "loss": 0.7387,
+      "step": 90
+    },
+    {
+      "epoch": 0.008370302167908262,
+      "grad_norm": 0.5121504664421082,
+      "learning_rate": 1.1167977528089888e-05,
+      "loss": 0.7605,
+      "step": 100
+    },
+    {
+      "epoch": 0.009207332384699087,
+      "grad_norm": 0.4082605540752411,
+      "learning_rate": 1.2285955056179776e-05,
+      "loss": 0.7623,
+      "step": 110
+    },
+    {
+      "epoch": 0.010044362601489915,
+      "grad_norm": 0.5059930682182312,
+      "learning_rate": 1.3403932584269664e-05,
+      "loss": 0.764,
+      "step": 120
+    },
+    {
+      "epoch": 0.01088139281828074,
+      "grad_norm": 0.751139223575592,
+      "learning_rate": 1.4521910112359552e-05,
+      "loss": 0.7645,
+      "step": 130
+    },
+    {
+      "epoch": 0.011718423035071566,
+      "grad_norm": 0.492674320936203,
+      "learning_rate": 1.563988764044944e-05,
+      "loss": 0.781,
+      "step": 140
+    },
+    {
+      "epoch": 0.012555453251862391,
+      "grad_norm": 0.566781222820282,
+      "learning_rate": 1.6757865168539328e-05,
+      "loss": 0.7548,
+      "step": 150
+    },
+    {
+      "epoch": 0.013392483468653219,
+      "grad_norm": 0.5136772990226746,
+      "learning_rate": 1.7875842696629214e-05,
+      "loss": 0.7799,
+      "step": 160
+    },
+    {
+      "epoch": 0.014229513685444044,
+      "grad_norm": 0.40454524755477905,
+      "learning_rate": 1.89938202247191e-05,
+      "loss": 0.7748,
+      "step": 170
+    },
+    {
+      "epoch": 0.01506654390223487,
+      "grad_norm": 0.4504382014274597,
+      "learning_rate": 2e-05,
+      "loss": 0.7757,
+      "step": 180
+    },
+    {
+      "epoch": 0.015903574119025696,
+      "grad_norm": 0.5842396020889282,
+      "learning_rate": 2e-05,
+      "loss": 0.7491,
+      "step": 190
+    },
+    {
+      "epoch": 0.016740604335816523,
+      "grad_norm": 0.42499402165412903,
+      "learning_rate": 2e-05,
+      "loss": 0.7533,
+      "step": 200
+    },
+    {
+      "epoch": 0.01757763455260735,
+      "grad_norm": 0.4922196567058563,
+      "learning_rate": 2e-05,
+      "loss": 0.7639,
+      "step": 210
+    },
+    {
+      "epoch": 0.018414664769398174,
+      "grad_norm": 0.4512643814086914,
+      "learning_rate": 2e-05,
+      "loss": 0.7663,
+      "step": 220
+    },
+    {
+      "epoch": 0.019251694986189002,
+      "grad_norm": 0.4370189607143402,
+      "learning_rate": 2e-05,
+      "loss": 0.765,
+      "step": 230
+    },
+    {
+      "epoch": 0.02008872520297983,
+      "grad_norm": 0.41970548033714294,
+      "learning_rate": 2e-05,
+      "loss": 0.745,
+      "step": 240
+    },
+    {
+      "epoch": 0.020925755419770653,
+      "grad_norm": 0.45108288526535034,
+      "learning_rate": 2e-05,
+      "loss": 0.7364,
+      "step": 250
+    },
+    {
+      "epoch": 0.02176278563656148,
+      "grad_norm": 0.42641422152519226,
+      "learning_rate": 2e-05,
+      "loss": 0.732,
+      "step": 260
+    },
+    {
+      "epoch": 0.022599815853352308,
+      "grad_norm": 0.3930014371871948,
+      "learning_rate": 2e-05,
+      "loss": 0.7634,
+      "step": 270
+    },
+    {
+      "epoch": 0.02343684607014313,
+      "grad_norm": 0.43919554352760315,
+      "learning_rate": 2e-05,
+      "loss": 0.7621,
+      "step": 280
+    },
+    {
+      "epoch": 0.02427387628693396,
+      "grad_norm": 0.4371900260448456,
+      "learning_rate": 2e-05,
+      "loss": 0.7379,
+      "step": 290
+    },
+    {
+      "epoch": 0.025110906503724783,
+      "grad_norm": 0.4431065618991852,
+      "learning_rate": 2e-05,
+      "loss": 0.7306,
+      "step": 300
+    },
+    {
+      "epoch": 0.02594793672051561,
+      "grad_norm": 0.4054403603076935,
+      "learning_rate": 2e-05,
+      "loss": 0.7532,
+      "step": 310
+    },
+    {
+      "epoch": 0.026784966937306438,
+      "grad_norm": 0.43971535563468933,
+      "learning_rate": 2e-05,
+      "loss": 0.7527,
+      "step": 320
+    },
+    {
+      "epoch": 0.02762199715409726,
+      "grad_norm": 1.0866317749023438,
+      "learning_rate": 2e-05,
+      "loss": 0.7705,
+      "step": 330
+    },
+    {
+      "epoch": 0.02845902737088809,
+      "grad_norm": 0.4061005115509033,
+      "learning_rate": 2e-05,
+      "loss": 0.7739,
+      "step": 340
+    },
+    {
+      "epoch": 0.029296057587678916,
+      "grad_norm": 0.5896549820899963,
+      "learning_rate": 2e-05,
+      "loss": 0.7655,
+      "step": 350
+    },
+    {
+      "epoch": 0.03013308780446974,
+      "grad_norm": 0.44174519181251526,
+      "learning_rate": 2e-05,
+      "loss": 0.7502,
+      "step": 360
+    },
+    {
+      "epoch": 0.030970118021260568,
+      "grad_norm": 0.4336935877799988,
+      "learning_rate": 2e-05,
+      "loss": 0.7119,
+      "step": 370
+    },
+    {
+      "epoch": 0.03180714823805139,
+      "grad_norm": 0.43263721466064453,
+      "learning_rate": 2e-05,
+      "loss": 0.7308,
+      "step": 380
+    },
+    {
+      "epoch": 0.03264417845484222,
+      "grad_norm": 0.4120350480079651,
+      "learning_rate": 2e-05,
+      "loss": 0.7464,
+      "step": 390
+    },
+    {
+      "epoch": 0.033481208671633046,
+      "grad_norm": 0.4036485254764557,
+      "learning_rate": 2e-05,
+      "loss": 0.7601,
+      "step": 400
+    },
+    {
+      "epoch": 0.03431823888842387,
+      "grad_norm": 0.41021424531936646,
+      "learning_rate": 2e-05,
+      "loss": 0.7281,
+      "step": 410
+    },
+    {
+      "epoch": 0.0351552691052147,
+      "grad_norm": 0.4482649564743042,
+      "learning_rate": 2e-05,
+      "loss": 0.7316,
+      "step": 420
+    },
+    {
+      "epoch": 0.035992299322005525,
+      "grad_norm": 0.4150967299938202,
+      "learning_rate": 2e-05,
+      "loss": 0.7365,
+      "step": 430
+    },
+    {
+      "epoch": 0.03682932953879635,
+      "grad_norm": 0.3876197934150696,
+      "learning_rate": 2e-05,
+      "loss": 0.7511,
+      "step": 440
+    },
+    {
+      "epoch": 0.03766635975558718,
+      "grad_norm": 0.40040096640586853,
+      "learning_rate": 2e-05,
+      "loss": 0.793,
+      "step": 450
+    },
+    {
+      "epoch": 0.038503389972378003,
+      "grad_norm": 0.4496796727180481,
+      "learning_rate": 2e-05,
+      "loss": 0.7423,
+      "step": 460
+    },
+    {
+      "epoch": 0.03934042018916883,
+      "grad_norm": 0.4478641152381897,
+      "learning_rate": 2e-05,
+      "loss": 0.7485,
+      "step": 470
+    },
+    {
+      "epoch": 0.04017745040595966,
+      "grad_norm": 0.4376203715801239,
+      "learning_rate": 2e-05,
+      "loss": 0.7909,
+      "step": 480
+    },
+    {
+      "epoch": 0.04101448062275048,
+      "grad_norm": 0.42852625250816345,
+      "learning_rate": 2e-05,
+      "loss": 0.7159,
+      "step": 490
+    },
+    {
+      "epoch": 0.041851510839541306,
+      "grad_norm": 0.42360687255859375,
+      "learning_rate": 2e-05,
+      "loss": 0.7401,
+      "step": 500
+    },
+    {
+      "epoch": 0.04268854105633214,
+      "grad_norm": 0.38901394605636597,
+      "learning_rate": 2e-05,
+      "loss": 0.7284,
+      "step": 510
+    },
+    {
+      "epoch": 0.04352557127312296,
+      "grad_norm": 0.4375343322753906,
+      "learning_rate": 2e-05,
+      "loss": 0.738,
+      "step": 520
+    },
+    {
+      "epoch": 0.044362601489913785,
+      "grad_norm": 0.44532549381256104,
+      "learning_rate": 2e-05,
+      "loss": 0.7469,
+      "step": 530
+    },
+    {
+      "epoch": 0.045199631706704615,
+      "grad_norm": 0.46848392486572266,
+      "learning_rate": 2e-05,
+      "loss": 0.7366,
+      "step": 540
+    },
+    {
+      "epoch": 0.04603666192349544,
+      "grad_norm": 0.3851851224899292,
+      "learning_rate": 2e-05,
+      "loss": 0.7335,
+      "step": 550
+    },
+    {
+      "epoch": 0.04687369214028626,
+      "grad_norm": 0.40839746594429016,
+      "learning_rate": 2e-05,
+      "loss": 0.7053,
+      "step": 560
+    },
+    {
+      "epoch": 0.04771072235707709,
+      "grad_norm": 0.4012123644351959,
+      "learning_rate": 2e-05,
+      "loss": 0.7137,
+      "step": 570
+    },
+    {
+      "epoch": 0.04854775257386792,
+      "grad_norm": 0.3855404555797577,
+      "learning_rate": 2e-05,
+      "loss": 0.7133,
+      "step": 580
+    },
+    {
+      "epoch": 0.04938478279065874,
+      "grad_norm": 0.3988204002380371,
+      "learning_rate": 2e-05,
+      "loss": 0.7541,
+      "step": 590
+    },
+    {
+      "epoch": 0.050221813007449566,
+      "grad_norm": 0.4450335204601288,
+      "learning_rate": 2e-05,
+      "loss": 0.7238,
+      "step": 600
+    },
+    {
+      "epoch": 0.0510588432242404,
+      "grad_norm": 0.4375969469547272,
+      "learning_rate": 2e-05,
+      "loss": 0.7044,
+      "step": 610
+    },
+    {
+      "epoch": 0.05189587344103122,
+      "grad_norm": 0.4379328787326813,
+      "learning_rate": 2e-05,
+      "loss": 0.726,
+      "step": 620
+    },
+    {
+      "epoch": 0.052732903657822044,
+      "grad_norm": 0.39580798149108887,
+      "learning_rate": 2e-05,
+      "loss": 0.7463,
+      "step": 630
+    },
+    {
+      "epoch": 0.053569933874612875,
+      "grad_norm": 0.41373711824417114,
+      "learning_rate": 2e-05,
+      "loss": 0.7499,
+      "step": 640
+    },
+    {
+      "epoch": 0.0544069640914037,
+      "grad_norm": 0.4301176369190216,
+      "learning_rate": 2e-05,
+      "loss": 0.7341,
+      "step": 650
+    },
+    {
+      "epoch": 0.05524399430819452,
+      "grad_norm": 0.45098692178726196,
+      "learning_rate": 2e-05,
+      "loss": 0.7318,
+      "step": 660
+    },
+    {
+      "epoch": 0.056081024524985354,
+      "grad_norm": 0.4196617305278778,
+      "learning_rate": 2e-05,
+      "loss": 0.734,
+      "step": 670
+    },
+    {
+      "epoch": 0.05691805474177618,
+      "grad_norm": 0.4272335171699524,
+      "learning_rate": 2e-05,
+      "loss": 0.7083,
+      "step": 680
+    },
+    {
+      "epoch": 0.057755084958567,
+      "grad_norm": 0.4190222918987274,
+      "learning_rate": 2e-05,
+      "loss": 0.7272,
+      "step": 690
+    },
+    {
+      "epoch": 0.05859211517535783,
+      "grad_norm": 0.42260268330574036,
+      "learning_rate": 2e-05,
+      "loss": 0.7266,
+      "step": 700
+    },
+    {
+      "epoch": 0.059429145392148656,
+      "grad_norm": 0.3979990780353546,
+      "learning_rate": 2e-05,
+      "loss": 0.7265,
+      "step": 710
+    },
+    {
+      "epoch": 0.06026617560893948,
+      "grad_norm": 0.41610896587371826,
+      "learning_rate": 2e-05,
+      "loss": 0.7308,
+      "step": 720
+    },
+    {
+      "epoch": 0.06110320582573031,
+      "grad_norm": 0.44141194224357605,
+      "learning_rate": 2e-05,
+      "loss": 0.752,
+      "step": 730
+    },
+    {
+      "epoch": 0.061940236042521135,
+      "grad_norm": 0.40890923142433167,
+      "learning_rate": 2e-05,
+      "loss": 0.7552,
+      "step": 740
+    },
+    {
+      "epoch": 0.06277726625931196,
+      "grad_norm": 0.37664929032325745,
+      "learning_rate": 2e-05,
+      "loss": 0.7363,
+      "step": 750
+    },
+    {
+      "epoch": 0.06361429647610278,
+      "grad_norm": 0.3718191683292389,
+      "learning_rate": 2e-05,
+      "loss": 0.7458,
+      "step": 760
+    },
+    {
+      "epoch": 0.0644513266928936,
+      "grad_norm": 0.4162319004535675,
+      "learning_rate": 2e-05,
+      "loss": 0.7587,
+      "step": 770
+    },
+    {
+      "epoch": 0.06528835690968444,
+      "grad_norm": 0.5250889658927917,
+      "learning_rate": 2e-05,
+      "loss": 0.733,
+      "step": 780
+    },
+    {
+      "epoch": 0.06612538712647527,
+      "grad_norm": 0.3899112343788147,
+      "learning_rate": 2e-05,
+      "loss": 0.731,
+      "step": 790
+    },
+    {
+      "epoch": 0.06696241734326609,
+      "grad_norm": 0.39714515209198,
+      "learning_rate": 2e-05,
+      "loss": 0.7397,
+      "step": 800
+    },
+    {
+      "epoch": 0.06779944756005692,
+      "grad_norm": 0.4384140372276306,
+      "learning_rate": 2e-05,
+      "loss": 0.7334,
+      "step": 810
+    },
+    {
+      "epoch": 0.06863647777684774,
+      "grad_norm": 0.37715354561805725,
+      "learning_rate": 2e-05,
+      "loss": 0.7442,
+      "step": 820
+    },
+    {
+      "epoch": 0.06947350799363856,
+      "grad_norm": 0.4676320552825928,
+      "learning_rate": 2e-05,
+      "loss": 0.729,
+      "step": 830
+    },
+    {
+      "epoch": 0.0703105382104294,
+      "grad_norm": 0.4101060926914215,
+      "learning_rate": 2e-05,
+      "loss": 0.7412,
+      "step": 840
+    },
+    {
+      "epoch": 0.07114756842722023,
+      "grad_norm": 0.40823832154273987,
+      "learning_rate": 2e-05,
+      "loss": 0.7404,
+      "step": 850
+    },
+    {
+      "epoch": 0.07198459864401105,
+      "grad_norm": 0.35889768600463867,
+      "learning_rate": 2e-05,
+      "loss": 0.7301,
+      "step": 860
+    },
+    {
+      "epoch": 0.07282162886080187,
+      "grad_norm": 0.35591280460357666,
+      "learning_rate": 2e-05,
+      "loss": 0.7434,
+      "step": 870
+    },
+    {
+      "epoch": 0.0736586590775927,
+      "grad_norm": 0.3851705491542816,
+      "learning_rate": 2e-05,
+      "loss": 0.7002,
+      "step": 880
+    },
+    {
+      "epoch": 0.07449568929438352,
+      "grad_norm": 0.3983948230743408,
+      "learning_rate": 2e-05,
+      "loss": 0.7274,
+      "step": 890
+    },
+    {
+      "epoch": 0.07533271951117436,
+      "grad_norm": 0.3635563254356384,
+      "learning_rate": 2e-05,
+      "loss": 0.6975,
+      "step": 900
+    },
+    {
+      "epoch": 0.07616974972796518,
+      "grad_norm": 0.3900563418865204,
+      "learning_rate": 2e-05,
+      "loss": 0.7467,
+      "step": 910
+    },
+    {
+      "epoch": 0.07700677994475601,
+      "grad_norm": 0.4121996760368347,
+      "learning_rate": 2e-05,
+      "loss": 0.7181,
+      "step": 920
+    },
+    {
+      "epoch": 0.07784381016154683,
+      "grad_norm": 0.35744690895080566,
+      "learning_rate": 2e-05,
+      "loss": 0.72,
+      "step": 930
+    },
+    {
+      "epoch": 0.07868084037833765,
+      "grad_norm": 0.44155263900756836,
+      "learning_rate": 2e-05,
+      "loss": 0.714,
+      "step": 940
+    },
+    {
+      "epoch": 0.07951787059512848,
+      "grad_norm": 0.415809690952301,
+      "learning_rate": 2e-05,
+      "loss": 0.7518,
+      "step": 950
+    },
+    {
+      "epoch": 0.08035490081191932,
+      "grad_norm": 0.3967406153678894,
+      "learning_rate": 2e-05,
+      "loss": 0.7445,
+      "step": 960
+    },
+    {
+      "epoch": 0.08119193102871014,
+      "grad_norm": 0.4007624089717865,
+      "learning_rate": 2e-05,
+      "loss": 0.747,
+      "step": 970
+    },
+    {
+      "epoch": 0.08202896124550096,
+      "grad_norm": 0.38344693183898926,
+      "learning_rate": 2e-05,
+      "loss": 0.7295,
+      "step": 980
+    },
+    {
+      "epoch": 0.08286599146229179,
+      "grad_norm": 0.45661893486976624,
+      "learning_rate": 2e-05,
+      "loss": 0.7111,
+      "step": 990
+    },
+    {
+      "epoch": 0.08370302167908261,
+      "grad_norm": 0.4255678951740265,
+      "learning_rate": 2e-05,
+      "loss": 0.775,
+      "step": 1000
+    },
+    {
+      "epoch": 0.08454005189587344,
+      "grad_norm": 0.49089524149894714,
+      "learning_rate": 2e-05,
+      "loss": 0.7387,
+      "step": 1010
+    },
+    {
+      "epoch": 0.08537708211266427,
+      "grad_norm": 0.3930380642414093,
+      "learning_rate": 2e-05,
+      "loss": 0.7601,
+      "step": 1020
+    },
+    {
+      "epoch": 0.0862141123294551,
+      "grad_norm": 0.4231725037097931,
+      "learning_rate": 2e-05,
+      "loss": 0.6822,
+      "step": 1030
+    },
+    {
+      "epoch": 0.08705114254624592,
+      "grad_norm": 0.39020201563835144,
+      "learning_rate": 2e-05,
+      "loss": 0.7325,
+      "step": 1040
+    },
+    {
+      "epoch": 0.08788817276303675,
+      "grad_norm": 0.429443359375,
+      "learning_rate": 2e-05,
+      "loss": 0.7194,
+      "step": 1050
+    },
+    {
+      "epoch": 0.08872520297982757,
+      "grad_norm": 0.38595613837242126,
+      "learning_rate": 2e-05,
+      "loss": 0.7336,
+      "step": 1060
+    },
+    {
+      "epoch": 0.0895622331966184,
+      "grad_norm": 0.371380090713501,
+      "learning_rate": 2e-05,
+      "loss": 0.7325,
+      "step": 1070
+    },
+    {
+      "epoch": 0.09039926341340923,
+      "grad_norm": 0.396846204996109,
+      "learning_rate": 2e-05,
+      "loss": 0.7281,
+      "step": 1080
+    },
+    {
+      "epoch": 0.09123629363020005,
+      "grad_norm": 0.42279285192489624,
+      "learning_rate": 2e-05,
+      "loss": 0.7383,
+      "step": 1090
+    },
+    {
+      "epoch": 0.09207332384699088,
+      "grad_norm": 0.41259169578552246,
+      "learning_rate": 2e-05,
+      "loss": 0.7398,
+      "step": 1100
+    },
+    {
+      "epoch": 0.0929103540637817,
+      "grad_norm": 0.43002089858055115,
+      "learning_rate": 2e-05,
+      "loss": 0.7323,
+      "step": 1110
+    },
+    {
+      "epoch": 0.09374738428057253,
+      "grad_norm": 0.4018864929676056,
+      "learning_rate": 2e-05,
+      "loss": 0.7542,
+      "step": 1120
+    },
+    {
+      "epoch": 0.09458441449736335,
+      "grad_norm": 0.3879416882991791,
+      "learning_rate": 2e-05,
+      "loss": 0.724,
+      "step": 1130
+    },
+    {
+      "epoch": 0.09542144471415417,
+      "grad_norm": 0.40612295269966125,
+      "learning_rate": 2e-05,
+      "loss": 0.7181,
+      "step": 1140
+    },
+    {
+      "epoch": 0.09625847493094501,
+      "grad_norm": 0.3889906406402588,
+      "learning_rate": 2e-05,
+      "loss": 0.7206,
+      "step": 1150
+    },
+    {
+      "epoch": 0.09709550514773584,
+      "grad_norm": 0.3631397783756256,
+      "learning_rate": 2e-05,
+      "loss": 0.7608,
+      "step": 1160
+    },
+    {
+      "epoch": 0.09793253536452666,
+      "grad_norm": 0.3583589494228363,
+      "learning_rate": 2e-05,
+      "loss": 0.7257,
+      "step": 1170
+    },
+    {
+      "epoch": 0.09876956558131748,
+      "grad_norm": 0.38138115406036377,
+      "learning_rate": 2e-05,
+      "loss": 0.7296,
+      "step": 1180
+    },
+    {
+      "epoch": 0.09960659579810831,
+      "grad_norm": 0.381708025932312,
+      "learning_rate": 2e-05,
+      "loss": 0.7275,
+      "step": 1190
+    },
+    {
+      "epoch": 0.10044362601489913,
+      "grad_norm": 0.3952488601207733,
+      "learning_rate": 2e-05,
+      "loss": 0.7211,
+      "step": 1200
+    },
+    {
+      "epoch": 0.10128065623168997,
+      "grad_norm": 0.369489848613739,
+      "learning_rate": 2e-05,
+      "loss": 0.6827,
+      "step": 1210
+    },
+    {
+      "epoch": 0.1021176864484808,
+      "grad_norm": 0.3854788541793823,
+      "learning_rate": 2e-05,
+      "loss": 0.7141,
+      "step": 1220
+    },
+    {
+      "epoch": 0.10295471666527162,
+      "grad_norm": 0.44547921419143677,
+      "learning_rate": 2e-05,
+      "loss": 0.7465,
+      "step": 1230
+    },
+    {
+      "epoch": 0.10379174688206244,
+      "grad_norm": 0.3807377219200134,
+      "learning_rate": 2e-05,
+      "loss": 0.6965,
+      "step": 1240
+    },
+    {
+      "epoch": 0.10462877709885327,
+      "grad_norm": 1.0734481811523438,
+      "learning_rate": 2e-05,
+      "loss": 0.7228,
+      "step": 1250
+    },
+    {
+      "epoch": 0.10546580731564409,
+      "grad_norm": 0.3832617700099945,
+      "learning_rate": 2e-05,
+      "loss": 0.76,
+      "step": 1260
+    },
+    {
+      "epoch": 0.10630283753243493,
+      "grad_norm": 0.4228065013885498,
+      "learning_rate": 2e-05,
+      "loss": 0.7379,
+      "step": 1270
+    },
+    {
+      "epoch": 0.10713986774922575,
+      "grad_norm": 0.39900723099708557,
+      "learning_rate": 2e-05,
+      "loss": 0.6866,
+      "step": 1280
+    },
+    {
+      "epoch": 0.10797689796601657,
+      "grad_norm": 0.5197460055351257,
+      "learning_rate": 2e-05,
+      "loss": 0.6915,
+      "step": 1290
+    },
+    {
+      "epoch": 0.1088139281828074,
+      "grad_norm": 0.6023410558700562,
+      "learning_rate": 2e-05,
+      "loss": 0.7019,
+      "step": 1300
+    },
+    {
+      "epoch": 0.10965095839959822,
+      "grad_norm": 0.4066433310508728,
+      "learning_rate": 2e-05,
+      "loss": 0.7001,
+      "step": 1310
+    },
+    {
+      "epoch": 0.11048798861638905,
+      "grad_norm": 0.382357656955719,
+      "learning_rate": 2e-05,
+      "loss": 0.7287,
+      "step": 1320
+    },
+    {
+      "epoch": 0.11132501883317988,
+      "grad_norm": 0.3749663829803467,
+      "learning_rate": 2e-05,
+      "loss": 0.7429,
+      "step": 1330
+    },
+    {
+      "epoch": 0.11216204904997071,
+      "grad_norm": 0.3689495325088501,
+      "learning_rate": 2e-05,
+      "loss": 0.6842,
+      "step": 1340
+    },
+    {
+      "epoch": 0.11299907926676153,
+      "grad_norm": 0.41168835759162903,
+      "learning_rate": 2e-05,
+      "loss": 0.7533,
+      "step": 1350
+    },
+    {
+      "epoch": 0.11383610948355236,
+      "grad_norm": 0.39960694313049316,
+      "learning_rate": 2e-05,
+      "loss": 0.7312,
+      "step": 1360
+    },
+    {
+      "epoch": 0.11467313970034318,
+      "grad_norm": 0.3908539414405823,
+      "learning_rate": 2e-05,
+      "loss": 0.7085,
+      "step": 1370
+    },
+    {
+      "epoch": 0.115510169917134,
+      "grad_norm": 0.38182100653648376,
+      "learning_rate": 2e-05,
+      "loss": 0.7271,
+      "step": 1380
+    },
+    {
+      "epoch": 0.11634720013392484,
+      "grad_norm": 0.405620276927948,
+      "learning_rate": 2e-05,
+      "loss": 0.7235,
+      "step": 1390
+    },
+    {
+      "epoch": 0.11718423035071567,
+      "grad_norm": 0.3966982364654541,
+      "learning_rate": 2e-05,
+      "loss": 0.7371,
+      "step": 1400
+    },
+    {
+      "epoch": 0.11802126056750649,
+      "grad_norm": 0.3768328130245209,
+      "learning_rate": 2e-05,
+      "loss": 0.7172,
+      "step": 1410
+    },
+    {
+      "epoch": 0.11885829078429731,
+      "grad_norm": 0.4282349944114685,
+      "learning_rate": 2e-05,
+      "loss": 0.7318,
+      "step": 1420
+    },
+    {
+      "epoch": 0.11969532100108814,
+      "grad_norm": 0.39045482873916626,
+      "learning_rate": 2e-05,
+      "loss": 0.704,
+      "step": 1430
+    },
+    {
+      "epoch": 0.12053235121787896,
+      "grad_norm": 0.3846886456012726,
+      "learning_rate": 2e-05,
+      "loss": 0.7069,
+      "step": 1440
+    },
+    {
+      "epoch": 0.12136938143466978,
+      "grad_norm": 0.3845898509025574,
+      "learning_rate": 2e-05,
+      "loss": 0.703,
+      "step": 1450
+    },
+    {
+      "epoch": 0.12220641165146062,
+      "grad_norm": 0.4043080508708954,
+      "learning_rate": 2e-05,
+      "loss": 0.7329,
+      "step": 1460
+    },
+    {
+      "epoch": 0.12304344186825145,
+      "grad_norm": 0.353148877620697,
+      "learning_rate": 2e-05,
+      "loss": 0.7039,
+      "step": 1470
+    },
+    {
+      "epoch": 0.12388047208504227,
+      "grad_norm": 0.3626961410045624,
+      "learning_rate": 2e-05,
+      "loss": 0.7125,
+      "step": 1480
+    },
+    {
+      "epoch": 0.1247175023018331,
+      "grad_norm": 0.40537577867507935,
+      "learning_rate": 2e-05,
+      "loss": 0.6874,
+      "step": 1490
+    },
+    {
+      "epoch": 0.12555453251862392,
+      "grad_norm": 0.38374269008636475,
+      "learning_rate": 2e-05,
+      "loss": 0.7166,
+      "step": 1500
+    },
+    {
+      "epoch": 0.12639156273541474,
+      "grad_norm": 0.3687508702278137,
+      "learning_rate": 2e-05,
+      "loss": 0.7042,
+      "step": 1510
+    },
+    {
+      "epoch": 0.12722859295220557,
+      "grad_norm": 0.38297727704048157,
+      "learning_rate": 2e-05,
+      "loss": 0.7276,
+      "step": 1520
+    },
+    {
+      "epoch": 0.1280656231689964,
+      "grad_norm": 0.40739986300468445,
+      "learning_rate": 2e-05,
+      "loss": 0.7352,
+      "step": 1530
+    },
+    {
+      "epoch": 0.1289026533857872,
+      "grad_norm": 0.5073732733726501,
+      "learning_rate": 2e-05,
+      "loss": 0.7407,
+      "step": 1540
+    },
+    {
+      "epoch": 0.12973968360257807,
+      "grad_norm": 0.3655388653278351,
+      "learning_rate": 2e-05,
+      "loss": 0.707,
+      "step": 1550
+    },
+    {
+      "epoch": 0.1305767138193689,
+      "grad_norm": 0.38478103280067444,
+      "learning_rate": 2e-05,
+      "loss": 0.7155,
+      "step": 1560
+    },
+    {
+      "epoch": 0.1314137440361597,
+      "grad_norm": 0.3790566027164459,
+      "learning_rate": 2e-05,
+      "loss": 0.7048,
+      "step": 1570
+    },
+    {
+      "epoch": 0.13225077425295054,
+      "grad_norm": 0.3985552489757538,
+      "learning_rate": 2e-05,
+      "loss": 0.7469,
+      "step": 1580
+    },
+    {
+      "epoch": 0.13308780446974136,
+      "grad_norm": 0.3573896586894989,
+      "learning_rate": 2e-05,
+      "loss": 0.7128,
+      "step": 1590
+    },
+    {
+      "epoch": 0.13392483468653218,
+      "grad_norm": 0.3679613471031189,
+      "learning_rate": 2e-05,
+      "loss": 0.7063,
+      "step": 1600
+    },
+    {
+      "epoch": 0.134761864903323,
+      "grad_norm": 0.4195422828197479,
+      "learning_rate": 2e-05,
+      "loss": 0.7214,
+      "step": 1610
+    },
+    {
+      "epoch": 0.13559889512011383,
+      "grad_norm": 0.4262971878051758,
+      "learning_rate": 2e-05,
+      "loss": 0.7214,
+      "step": 1620
+    },
+    {
+      "epoch": 0.13643592533690466,
+      "grad_norm": 0.3953056335449219,
+      "learning_rate": 2e-05,
+      "loss": 0.7457,
+      "step": 1630
+    },
+    {
+      "epoch": 0.13727295555369548,
+      "grad_norm": 0.3896598517894745,
+      "learning_rate": 2e-05,
+      "loss": 0.68,
+      "step": 1640
+    },
+    {
+      "epoch": 0.1381099857704863,
+      "grad_norm": 0.39134523272514343,
+      "learning_rate": 2e-05,
+      "loss": 0.7233,
+      "step": 1650
+    },
+    {
+      "epoch": 0.13894701598727713,
+      "grad_norm": 0.38543522357940674,
+      "learning_rate": 2e-05,
+      "loss": 0.7078,
+      "step": 1660
+    },
+    {
+      "epoch": 0.13978404620406798,
+      "grad_norm": 0.37859565019607544,
+      "learning_rate": 2e-05,
+      "loss": 0.7141,
+      "step": 1670
+    },
+    {
+      "epoch": 0.1406210764208588,
+      "grad_norm": 0.3654524087905884,
+      "learning_rate": 2e-05,
+      "loss": 0.6817,
+      "step": 1680
+    },
+    {
+      "epoch": 0.14145810663764963,
+      "grad_norm": 0.3827599585056305,
+      "learning_rate": 2e-05,
+      "loss": 0.6995,
+      "step": 1690
+    },
+    {
+      "epoch": 0.14229513685444045,
+      "grad_norm": 0.3548571467399597,
+      "learning_rate": 2e-05,
+      "loss": 0.7106,
+      "step": 1700
+    },
+    {
+      "epoch": 0.14313216707123128,
+      "grad_norm": 0.4650750756263733,
+      "learning_rate": 2e-05,
+      "loss": 0.7498,
+      "step": 1710
+    },
+    {
+      "epoch": 0.1439691972880221,
+      "grad_norm": 0.38478001952171326,
+      "learning_rate": 2e-05,
+      "loss": 0.7699,
+      "step": 1720
+    },
+    {
+      "epoch": 0.14480622750481292,
+      "grad_norm": 0.37154585123062134,
+      "learning_rate": 2e-05,
+      "loss": 0.6941,
+      "step": 1730
+    },
+    {
+      "epoch": 0.14564325772160375,
+      "grad_norm": 0.38709065318107605,
+      "learning_rate": 2e-05,
+      "loss": 0.7337,
+      "step": 1740
+    },
+    {
+      "epoch": 0.14648028793839457,
+      "grad_norm": 1.7314486503601074,
+      "learning_rate": 2e-05,
+      "loss": 0.7005,
+      "step": 1750
+    },
+    {
+      "epoch": 0.1473173181551854,
+      "grad_norm": 0.3551817834377289,
+      "learning_rate": 2e-05,
+      "loss": 0.7132,
+      "step": 1760
+    },
+    {
+      "epoch": 0.14815434837197622,
+      "grad_norm": 0.3846317529678345,
+      "learning_rate": 2e-05,
+      "loss": 0.6762,
+      "step": 1770
+    },
+    {
+      "epoch": 0.14899137858876704,
+      "grad_norm": 0.37913134694099426,
+      "learning_rate": 2e-05,
+      "loss": 0.7101,
+      "step": 1780
+    },
+    {
+      "epoch": 0.1498284088055579,
+      "grad_norm": 0.3794521391391754,
+      "learning_rate": 2e-05,
+      "loss": 0.7327,
+      "step": 1790
+    },
+    {
+      "epoch": 0.15066543902234872,
+      "grad_norm": 0.3727302551269531,
+      "learning_rate": 2e-05,
+      "loss": 0.7198,
+      "step": 1800
+    },
+    {
+      "epoch": 0.15150246923913954,
+      "grad_norm": 0.39975666999816895,
+      "learning_rate": 2e-05,
+      "loss": 0.7283,
+      "step": 1810
+    },
+    {
+      "epoch": 0.15233949945593037,
+      "grad_norm": 0.37394288182258606,
+      "learning_rate": 2e-05,
+      "loss": 0.6815,
+      "step": 1820
+    },
+    {
+      "epoch": 0.1531765296727212,
+      "grad_norm": 0.38191404938697815,
+      "learning_rate": 2e-05,
+      "loss": 0.723,
+      "step": 1830
+    },
+    {
+      "epoch": 0.15401355988951201,
+      "grad_norm": 0.3914385437965393,
+      "learning_rate": 2e-05,
+      "loss": 0.6959,
+      "step": 1840
+    },
+    {
+      "epoch": 0.15485059010630284,
+      "grad_norm": 0.36960726976394653,
+      "learning_rate": 2e-05,
+      "loss": 0.6981,
+      "step": 1850
+    },
+    {
+      "epoch": 0.15568762032309366,
+      "grad_norm": 0.4533153474330902,
+      "learning_rate": 2e-05,
+      "loss": 0.7042,
+      "step": 1860
+    },
+    {
+      "epoch": 0.15652465053988449,
+      "grad_norm": 0.390516459941864,
+      "learning_rate": 2e-05,
+      "loss": 0.7285,
+      "step": 1870
+    },
+    {
+      "epoch": 0.1573616807566753,
+      "grad_norm": 0.36687996983528137,
+      "learning_rate": 2e-05,
+      "loss": 0.7039,
+      "step": 1880
+    },
+    {
+      "epoch": 0.15819871097346613,
+      "grad_norm": 0.3998259902000427,
+      "learning_rate": 2e-05,
+      "loss": 0.7045,
+      "step": 1890
+    },
+    {
+      "epoch": 0.15903574119025696,
+      "grad_norm": 0.3630969226360321,
+      "learning_rate": 2e-05,
+      "loss": 0.7156,
+      "step": 1900
+    },
+    {
+      "epoch": 0.15987277140704778,
+      "grad_norm": 0.49144166707992554,
+      "learning_rate": 2e-05,
+      "loss": 0.7492,
+      "step": 1910
+    },
+    {
+      "epoch": 0.16070980162383863,
+      "grad_norm": 0.3666890263557434,
+      "learning_rate": 2e-05,
+      "loss": 0.6957,
+      "step": 1920
+    },
+    {
+      "epoch": 0.16154683184062946,
+      "grad_norm": 0.36240652203559875,
+      "learning_rate": 2e-05,
+      "loss": 0.7208,
+      "step": 1930
+    },
+    {
+      "epoch": 0.16238386205742028,
+      "grad_norm": 0.37068548798561096,
+      "learning_rate": 2e-05,
+      "loss": 0.7593,
+      "step": 1940
+    },
+    {
+      "epoch": 0.1632208922742111,
+      "grad_norm": 0.38422319293022156,
+      "learning_rate": 2e-05,
+      "loss": 0.718,
+      "step": 1950
+    },
+    {
+      "epoch": 0.16405792249100193,
+      "grad_norm": 0.4526020884513855,
+      "learning_rate": 2e-05,
+      "loss": 0.7152,
+      "step": 1960
+    },
+    {
+      "epoch": 0.16489495270779275,
+      "grad_norm": 0.42693546414375305,
+      "learning_rate": 2e-05,
+      "loss": 0.7031,
+      "step": 1970
+    },
+    {
+      "epoch": 0.16573198292458358,
+      "grad_norm": 0.352283239364624,
+      "learning_rate": 2e-05,
+      "loss": 0.6788,
+      "step": 1980
+    },
+    {
+      "epoch": 0.1665690131413744,
+      "grad_norm": 0.4254308342933655,
+      "learning_rate": 2e-05,
+      "loss": 0.7004,
+      "step": 1990
+    },
+    {
+      "epoch": 0.16740604335816522,
+      "grad_norm": 0.37109827995300293,
+      "learning_rate": 2e-05,
+      "loss": 0.7183,
+      "step": 2000
+    },
+    {
+      "epoch": 0.16824307357495605,
+      "grad_norm": 0.3963578939437866,
+      "learning_rate": 2e-05,
+      "loss": 0.7152,
+      "step": 2010
+    },
+    {
+      "epoch": 0.16908010379174687,
+      "grad_norm": 0.4323606491088867,
+      "learning_rate": 2e-05,
+      "loss": 0.7275,
+      "step": 2020
+    },
+    {
+      "epoch": 0.1699171340085377,
+      "grad_norm": 0.38154488801956177,
+      "learning_rate": 2e-05,
+      "loss": 0.7043,
+      "step": 2030
+    },
+    {
+      "epoch": 0.17075416422532855,
+      "grad_norm": 0.37403401732444763,
+      "learning_rate": 2e-05,
+      "loss": 0.6961,
+      "step": 2040
+    },
+    {
+      "epoch": 0.17159119444211937,
+      "grad_norm": 0.4297429919242859,
+      "learning_rate": 2e-05,
+      "loss": 0.7279,
+      "step": 2050
+    },
+    {
+      "epoch": 0.1724282246589102,
+      "grad_norm": 0.3764204978942871,
+      "learning_rate": 2e-05,
+      "loss": 0.7169,
+      "step": 2060
+    },
+    {
+      "epoch": 0.17326525487570102,
+      "grad_norm": 0.4536464512348175,
+      "learning_rate": 2e-05,
+      "loss": 0.7076,
+      "step": 2070
+    },
+    {
+      "epoch": 0.17410228509249184,
+      "grad_norm": 0.3891889750957489,
+      "learning_rate": 2e-05,
+      "loss": 0.6937,
+      "step": 2080
+    },
+    {
+      "epoch": 0.17493931530928267,
+      "grad_norm": 0.3979426920413971,
+      "learning_rate": 2e-05,
+      "loss": 0.7117,
+      "step": 2090
+    },
+    {
+      "epoch": 0.1757763455260735,
+      "grad_norm": 0.33533552289009094,
+      "learning_rate": 2e-05,
+      "loss": 0.6911,
+      "step": 2100
+    },
+    {
+      "epoch": 0.17661337574286431,
+      "grad_norm": 0.4086405038833618,
+      "learning_rate": 2e-05,
+      "loss": 0.6961,
+      "step": 2110
+    },
+    {
+      "epoch": 0.17745040595965514,
+      "grad_norm": 0.32550907135009766,
+      "learning_rate": 2e-05,
+      "loss": 0.7189,
+      "step": 2120
+    },
+    {
+      "epoch": 0.17828743617644596,
+      "grad_norm": 0.41129428148269653,
+      "learning_rate": 2e-05,
+      "loss": 0.7132,
+      "step": 2130
+    },
+    {
+      "epoch": 0.1791244663932368,
+      "grad_norm": 0.38379132747650146,
+      "learning_rate": 2e-05,
+      "loss": 0.7083,
+      "step": 2140
+    },
+    {
+      "epoch": 0.1799614966100276,
+      "grad_norm": 0.33447548747062683,
+      "learning_rate": 2e-05,
+      "loss": 0.6957,
+      "step": 2150
+    },
+    {
+      "epoch": 0.18079852682681846,
+      "grad_norm": 0.36413493752479553,
+      "learning_rate": 2e-05,
+      "loss": 0.6845,
+      "step": 2160
+    },
+    {
+      "epoch": 0.18163555704360929,
+      "grad_norm": 0.3704940676689148,
+      "learning_rate": 2e-05,
+      "loss": 0.7045,
+      "step": 2170
+    },
+    {
+      "epoch": 0.1824725872604001,
+      "grad_norm": 0.37158524990081787,
+      "learning_rate": 2e-05,
+      "loss": 0.6945,
+      "step": 2180
+    },
+    {
+      "epoch": 0.18330961747719093,
+      "grad_norm": 0.37925466895103455,
+      "learning_rate": 2e-05,
+      "loss": 0.7088,
+      "step": 2190
+    },
+    {
+      "epoch": 0.18414664769398176,
+      "grad_norm": 0.3649779260158539,
+      "learning_rate": 2e-05,
+      "loss": 0.7342,
+      "step": 2200
+    },
+    {
+      "epoch": 0.18498367791077258,
+      "grad_norm": 0.38125574588775635,
+      "learning_rate": 2e-05,
+      "loss": 0.7045,
+      "step": 2210
+    },
+    {
+      "epoch": 0.1858207081275634,
+      "grad_norm": 0.4088246524333954,
+      "learning_rate": 2e-05,
+      "loss": 0.7006,
+      "step": 2220
+    },
+    {
+      "epoch": 0.18665773834435423,
+      "grad_norm": 0.4514538645744324,
+      "learning_rate": 2e-05,
+      "loss": 0.7126,
+      "step": 2230
+    },
+    {
+      "epoch": 0.18749476856114505,
+      "grad_norm": 0.3832446336746216,
+      "learning_rate": 2e-05,
+      "loss": 0.6912,
+      "step": 2240
+    },
+    {
+      "epoch": 0.18833179877793588,
+      "grad_norm": 0.37790173292160034,
+      "learning_rate": 2e-05,
+      "loss": 0.6738,
+      "step": 2250
+    },
+    {
+      "epoch": 0.1891688289947267,
+      "grad_norm": 0.3716677129268646,
+      "learning_rate": 2e-05,
+      "loss": 0.7122,
+      "step": 2260
+    },
+    {
+      "epoch": 0.19000585921151752,
+      "grad_norm": 0.3565468490123749,
+      "learning_rate": 2e-05,
+      "loss": 0.679,
+      "step": 2270
+    },
+    {
+      "epoch": 0.19084288942830835,
+      "grad_norm": 0.37897953391075134,
+      "learning_rate": 2e-05,
+      "loss": 0.6941,
+      "step": 2280
+    },
+    {
+      "epoch": 0.1916799196450992,
+      "grad_norm": 0.3777736723423004,
+      "learning_rate": 2e-05,
+      "loss": 0.7166,
+      "step": 2290
+    },
+    {
+      "epoch": 0.19251694986189002,
+      "grad_norm": 0.3585146367549896,
+      "learning_rate": 2e-05,
+      "loss": 0.6928,
+      "step": 2300
+    },
+    {
+      "epoch": 0.19335398007868085,
+      "grad_norm": 0.3698466122150421,
+      "learning_rate": 2e-05,
+      "loss": 0.6937,
+      "step": 2310
+    },
+    {
+      "epoch": 0.19419101029547167,
+      "grad_norm": 0.3647162914276123,
+      "learning_rate": 2e-05,
+      "loss": 0.7128,
+      "step": 2320
+    },
+    {
+      "epoch": 0.1950280405122625,
+      "grad_norm": 0.4572893977165222,
+      "learning_rate": 2e-05,
+      "loss": 0.6978,
+      "step": 2330
+    },
+    {
+      "epoch": 0.19586507072905332,
+      "grad_norm": 0.42052674293518066,
+      "learning_rate": 2e-05,
+      "loss": 0.6901,
+      "step": 2340
+    },
+    {
+      "epoch": 0.19670210094584414,
+      "grad_norm": 0.4102165102958679,
+      "learning_rate": 2e-05,
+      "loss": 0.7097,
+      "step": 2350
+    },
+    {
+      "epoch": 0.19753913116263497,
+      "grad_norm": 0.3792290687561035,
+      "learning_rate": 2e-05,
+      "loss": 0.6917,
+      "step": 2360
+    },
+    {
+      "epoch": 0.1983761613794258,
+      "grad_norm": 0.41515079140663147,
+      "learning_rate": 2e-05,
+      "loss": 0.7033,
+      "step": 2370
+    },
+    {
+      "epoch": 0.19921319159621662,
+      "grad_norm": 0.44105610251426697,
+      "learning_rate": 2e-05,
+      "loss": 0.7093,
+      "step": 2380
+    },
+    {
+      "epoch": 0.20005022181300744,
+      "grad_norm": 0.3615840971469879,
+      "learning_rate": 2e-05,
+      "loss": 0.7512,
+      "step": 2390
+    },
+    {
+      "epoch": 0.20088725202979826,
+      "grad_norm": 0.3563784062862396,
+      "learning_rate": 2e-05,
+      "loss": 0.7046,
+      "step": 2400
+    },
+    {
+      "epoch": 0.20172428224658911,
+      "grad_norm": 0.3593883216381073,
+      "learning_rate": 2e-05,
+      "loss": 0.7143,
+      "step": 2410
+    },
+    {
+      "epoch": 0.20256131246337994,
+      "grad_norm": 0.375445157289505,
+      "learning_rate": 2e-05,
+      "loss": 0.678,
+      "step": 2420
+    },
+    {
+      "epoch": 0.20339834268017076,
+      "grad_norm": 0.3783409595489502,
+      "learning_rate": 2e-05,
+      "loss": 0.6926,
+      "step": 2430
+    },
+    {
+      "epoch": 0.2042353728969616,
+      "grad_norm": 0.3709818720817566,
+      "learning_rate": 2e-05,
+      "loss": 0.6974,
+      "step": 2440
+    },
+    {
+      "epoch": 0.2050724031137524,
+      "grad_norm": 0.3591563403606415,
+      "learning_rate": 2e-05,
+      "loss": 0.7022,
+      "step": 2450
+    },
+    {
+      "epoch": 0.20590943333054323,
+      "grad_norm": 0.42944005131721497,
+      "learning_rate": 2e-05,
+      "loss": 0.7151,
+      "step": 2460
+    },
+    {
+      "epoch": 0.20674646354733406,
+      "grad_norm": 0.36380648612976074,
+      "learning_rate": 2e-05,
+      "loss": 0.6938,
+      "step": 2470
+    },
+    {
+      "epoch": 0.20758349376412488,
+      "grad_norm": 0.3823877274990082,
+      "learning_rate": 2e-05,
+      "loss": 0.6853,
+      "step": 2480
+    },
+    {
+      "epoch": 0.2084205239809157,
+      "grad_norm": 0.40340110659599304,
+      "learning_rate": 2e-05,
+      "loss": 0.6983,
+      "step": 2490
+    },
+    {
+      "epoch": 0.20925755419770653,
+      "grad_norm": 0.3656036853790283,
+      "learning_rate": 2e-05,
+      "loss": 0.6927,
+      "step": 2500
+    },
+    {
+      "epoch": 0.21009458441449735,
+      "grad_norm": 0.3953785002231598,
+      "learning_rate": 2e-05,
+      "loss": 0.6779,
+      "step": 2510
+    },
+    {
+      "epoch": 0.21093161463128818,
+      "grad_norm": 0.40045875310897827,
+      "learning_rate": 2e-05,
+      "loss": 0.6812,
+      "step": 2520
+    },
+    {
+      "epoch": 0.211768644848079,
+      "grad_norm": 0.3604848086833954,
+      "learning_rate": 2e-05,
+      "loss": 0.6908,
+      "step": 2530
+    },
+    {
+      "epoch": 0.21260567506486985,
+      "grad_norm": 0.40517356991767883,
+      "learning_rate": 2e-05,
+      "loss": 0.7074,
+      "step": 2540
+    },
+    {
+      "epoch": 0.21344270528166068,
+      "grad_norm": 0.3920334279537201,
+      "learning_rate": 2e-05,
+      "loss": 0.7021,
+      "step": 2550
+    },
+    {
+      "epoch": 0.2142797354984515,
+      "grad_norm": 0.35391831398010254,
+      "learning_rate": 2e-05,
+      "loss": 0.704,
+      "step": 2560
+    },
+    {
+      "epoch": 0.21511676571524233,
+      "grad_norm": 0.33125635981559753,
+      "learning_rate": 2e-05,
+      "loss": 0.6751,
+      "step": 2570
+    },
+    {
+      "epoch": 0.21595379593203315,
+      "grad_norm": 0.3804548382759094,
+      "learning_rate": 2e-05,
+      "loss": 0.6929,
+      "step": 2580
+    },
+    {
+      "epoch": 0.21679082614882397,
+      "grad_norm": 0.41652804613113403,
+      "learning_rate": 2e-05,
+      "loss": 0.7224,
+      "step": 2590
+    },
+    {
+      "epoch": 0.2176278563656148,
+      "grad_norm": 0.37819674611091614,
+      "learning_rate": 2e-05,
+      "loss": 0.6942,
+      "step": 2600
+    },
+    {
+      "epoch": 0.21846488658240562,
+      "grad_norm": 0.3700343668460846,
+      "learning_rate": 2e-05,
+      "loss": 0.6922,
+      "step": 2610
+    },
+    {
+      "epoch": 0.21930191679919644,
+      "grad_norm": 0.39303967356681824,
+      "learning_rate": 2e-05,
+      "loss": 0.6836,
+      "step": 2620
+    },
+    {
+      "epoch": 0.22013894701598727,
+      "grad_norm": 0.3704970180988312,
+      "learning_rate": 2e-05,
+      "loss": 0.6656,
+      "step": 2630
+    },
+    {
+      "epoch": 0.2209759772327781,
+      "grad_norm": 0.35388532280921936,
+      "learning_rate": 2e-05,
+      "loss": 0.7203,
+      "step": 2640
+    },
+    {
+      "epoch": 0.22181300744956892,
+      "grad_norm": 0.3718905746936798,
+      "learning_rate": 2e-05,
+      "loss": 0.6942,
+      "step": 2650
+    },
+    {
+      "epoch": 0.22265003766635977,
+      "grad_norm": 0.36480191349983215,
+      "learning_rate": 2e-05,
+      "loss": 0.7213,
+      "step": 2660
+    },
+    {
+      "epoch": 0.2234870678831506,
+      "grad_norm": 0.5288359522819519,
+      "learning_rate": 2e-05,
+      "loss": 0.7045,
+      "step": 2670
+    },
+    {
+      "epoch": 0.22432409809994142,
+      "grad_norm": 0.3652160167694092,
+      "learning_rate": 2e-05,
+      "loss": 0.708,
+      "step": 2680
+    },
+    {
+      "epoch": 0.22516112831673224,
+      "grad_norm": 0.36390116810798645,
+      "learning_rate": 2e-05,
+      "loss": 0.6881,
+      "step": 2690
+    },
+    {
+      "epoch": 0.22599815853352306,
+      "grad_norm": 0.3840753734111786,
+      "learning_rate": 2e-05,
+      "loss": 0.6633,
+      "step": 2700
+    },
+    {
+      "epoch": 0.2268351887503139,
+      "grad_norm": 0.3530333638191223,
+      "learning_rate": 2e-05,
+      "loss": 0.7124,
+      "step": 2710
+    },
+    {
+      "epoch": 0.2276722189671047,
+      "grad_norm": 0.342545747756958,
+      "learning_rate": 2e-05,
+      "loss": 0.697,
+      "step": 2720
+    },
+    {
+      "epoch": 0.22850924918389554,
+      "grad_norm": 0.5206401348114014,
+      "learning_rate": 2e-05,
+      "loss": 0.6976,
+      "step": 2730
+    },
+    {
+      "epoch": 0.22934627940068636,
+      "grad_norm": 0.3752775192260742,
+      "learning_rate": 2e-05,
+      "loss": 0.7146,
+      "step": 2740
+    },
+    {
+      "epoch": 0.23018330961747718,
+      "grad_norm": 0.3544803857803345,
+      "learning_rate": 2e-05,
+      "loss": 0.7199,
+      "step": 2750
+    },
+    {
+      "epoch": 0.231020339834268,
+      "grad_norm": 0.47715863585472107,
+      "learning_rate": 2e-05,
+      "loss": 0.7116,
+      "step": 2760
+    },
+    {
+      "epoch": 0.23185737005105883,
+      "grad_norm": 0.3936082720756531,
+      "learning_rate": 2e-05,
+      "loss": 0.7104,
+      "step": 2770
+    },
+    {
+      "epoch": 0.23269440026784968,
+      "grad_norm": 0.3845888376235962,
+      "learning_rate": 2e-05,
+      "loss": 0.7153,
+      "step": 2780
+    },
+    {
+      "epoch": 0.2335314304846405,
+      "grad_norm": 0.3471842110157013,
+      "learning_rate": 2e-05,
+      "loss": 0.6582,
+      "step": 2790
+    },
+    {
+      "epoch": 0.23436846070143133,
+      "grad_norm": 0.3849339485168457,
+      "learning_rate": 2e-05,
+      "loss": 0.7134,
+      "step": 2800
+    },
+    {
+      "epoch": 0.23520549091822215,
+      "grad_norm": 0.4041845500469208,
+      "learning_rate": 2e-05,
+      "loss": 0.6917,
+      "step": 2810
+    },
+    {
+      "epoch": 0.23604252113501298,
+      "grad_norm": 0.347912460565567,
+      "learning_rate": 2e-05,
+      "loss": 0.7065,
+      "step": 2820
+    },
+    {
+      "epoch": 0.2368795513518038,
+      "grad_norm": 0.383291631937027,
+      "learning_rate": 2e-05,
+      "loss": 0.7237,
+      "step": 2830
+    },
+    {
+      "epoch": 0.23771658156859463,
+      "grad_norm": 0.3993476927280426,
+      "learning_rate": 2e-05,
+      "loss": 0.7263,
+      "step": 2840
+    },
+    {
+      "epoch": 0.23855361178538545,
+      "grad_norm": 0.364463210105896,
+      "learning_rate": 2e-05,
+      "loss": 0.6825,
+      "step": 2850
+    },
+    {
+      "epoch": 0.23939064200217627,
+      "grad_norm": 0.3854297697544098,
+      "learning_rate": 2e-05,
+      "loss": 0.6828,
+      "step": 2860
+    },
+    {
+      "epoch": 0.2402276722189671,
+      "grad_norm": 0.4553110599517822,
+      "learning_rate": 2e-05,
+      "loss": 0.6841,
+      "step": 2870
+    },
+    {
+      "epoch": 0.24106470243575792,
+      "grad_norm": 0.3404019773006439,
+      "learning_rate": 2e-05,
+      "loss": 0.6824,
+      "step": 2880
+    },
+    {
+      "epoch": 0.24190173265254875,
+      "grad_norm": 0.3552730679512024,
+      "learning_rate": 2e-05,
+      "loss": 0.705,
+      "step": 2890
+    },
+    {
+      "epoch": 0.24273876286933957,
+      "grad_norm": 0.6009185910224915,
+      "learning_rate": 2e-05,
+      "loss": 0.6872,
+      "step": 2900
+    },
+    {
+      "epoch": 0.24357579308613042,
+      "grad_norm": 0.3935545086860657,
+      "learning_rate": 2e-05,
+      "loss": 0.6606,
+      "step": 2910
+    },
+    {
+      "epoch": 0.24441282330292124,
+      "grad_norm": 0.3839946389198303,
+      "learning_rate": 2e-05,
+      "loss": 0.6758,
+      "step": 2920
+    },
+    {
+      "epoch": 0.24524985351971207,
+      "grad_norm": 0.49954575300216675,
+      "learning_rate": 2e-05,
+      "loss": 0.6747,
+      "step": 2930
+    },
+    {
+      "epoch": 0.2460868837365029,
+      "grad_norm": 0.3401426672935486,
+      "learning_rate": 2e-05,
+      "loss": 0.6798,
+      "step": 2940
+    },
+    {
+      "epoch": 0.24692391395329372,
+      "grad_norm": 0.38917040824890137,
+      "learning_rate": 2e-05,
+      "loss": 0.6955,
+      "step": 2950
+    },
+    {
+      "epoch": 0.24776094417008454,
+      "grad_norm": 0.3529467284679413,
+      "learning_rate": 2e-05,
+      "loss": 0.6733,
+      "step": 2960
+    },
+    {
+      "epoch": 0.24859797438687536,
+      "grad_norm": 0.3835350275039673,
+      "learning_rate": 2e-05,
+      "loss": 0.7163,
+      "step": 2970
+    },
+    {
+      "epoch": 0.2494350046036662,
+      "grad_norm": 0.34143391251564026,
+      "learning_rate": 2e-05,
+      "loss": 0.7019,
+      "step": 2980
+    },
+    {
+      "epoch": 0.250272034820457,
+      "grad_norm": 0.39724600315093994,
+      "learning_rate": 2e-05,
+      "loss": 0.6812,
+      "step": 2990
+    },
+    {
+      "epoch": 0.25110906503724784,
+      "grad_norm": 0.3517789840698242,
+      "learning_rate": 2e-05,
+      "loss": 0.6866,
+      "step": 3000
+    },
+    {
+      "epoch": 0.25194609525403866,
+      "grad_norm": 0.457095742225647,
+      "learning_rate": 2e-05,
+      "loss": 0.6892,
+      "step": 3010
+    },
+    {
+      "epoch": 0.2527831254708295,
+      "grad_norm": 0.3822527825832367,
+      "learning_rate": 2e-05,
+      "loss": 0.7063,
+      "step": 3020
+    },
+    {
+      "epoch": 0.2536201556876203,
+      "grad_norm": 0.39527907967567444,
+      "learning_rate": 2e-05,
+      "loss": 0.6757,
+      "step": 3030
+    },
+    {
+      "epoch": 0.25445718590441113,
+      "grad_norm": 0.36703014373779297,
+      "learning_rate": 2e-05,
+      "loss": 0.6991,
+      "step": 3040
+    },
+    {
+      "epoch": 0.25529421612120196,
+      "grad_norm": 0.5913490056991577,
+      "learning_rate": 2e-05,
+      "loss": 0.6808,
+      "step": 3050
+    },
+    {
+      "epoch": 0.2561312463379928,
+      "grad_norm": 0.6630072593688965,
+      "learning_rate": 2e-05,
+      "loss": 0.7075,
+      "step": 3060
+    },
+    {
+      "epoch": 0.2569682765547836,
+      "grad_norm": 0.3538056015968323,
+      "learning_rate": 2e-05,
+      "loss": 0.7061,
+      "step": 3070
+    },
+    {
+      "epoch": 0.2578053067715744,
+      "grad_norm": 0.4272959530353546,
+      "learning_rate": 2e-05,
+      "loss": 0.6843,
+      "step": 3080
+    },
+    {
+      "epoch": 0.2586423369883653,
+      "grad_norm": 0.396391898393631,
+      "learning_rate": 2e-05,
+      "loss": 0.6751,
+      "step": 3090
+    },
+    {
+      "epoch": 0.25947936720515613,
+      "grad_norm": 0.3552750051021576,
+      "learning_rate": 2e-05,
+      "loss": 0.7147,
+      "step": 3100
+    },
+    {
+      "epoch": 0.26031639742194695,
+      "grad_norm": 0.3866838216781616,
+      "learning_rate": 2e-05,
+      "loss": 0.736,
+      "step": 3110
+    },
+    {
+      "epoch": 0.2611534276387378,
+      "grad_norm": 0.37459060549736023,
+      "learning_rate": 2e-05,
+      "loss": 0.6848,
+      "step": 3120
+    },
+    {
+      "epoch": 0.2619904578555286,
+      "grad_norm": 0.3690750300884247,
+      "learning_rate": 2e-05,
+      "loss": 0.6834,
+      "step": 3130
+    },
+    {
+      "epoch": 0.2628274880723194,
+      "grad_norm": 0.4790518879890442,
+      "learning_rate": 2e-05,
+      "loss": 0.6947,
+      "step": 3140
+    },
+    {
+      "epoch": 0.26366451828911025,
+      "grad_norm": 0.3825061023235321,
+      "learning_rate": 2e-05,
+      "loss": 0.7086,
+      "step": 3150
+    },
+    {
+      "epoch": 0.2645015485059011,
+      "grad_norm": 0.43931442499160767,
+      "learning_rate": 2e-05,
+      "loss": 0.7131,
+      "step": 3160
+    },
+    {
+      "epoch": 0.2653385787226919,
+      "grad_norm": 0.36091071367263794,
+      "learning_rate": 2e-05,
+      "loss": 0.6678,
+      "step": 3170
+    },
+    {
+      "epoch": 0.2661756089394827,
+      "grad_norm": 0.35754019021987915,
+      "learning_rate": 2e-05,
+      "loss": 0.6757,
+      "step": 3180
+    },
+    {
+      "epoch": 0.26701263915627355,
+      "grad_norm": 0.34211859107017517,
+      "learning_rate": 2e-05,
+      "loss": 0.6844,
+      "step": 3190
+    },
+    {
+      "epoch": 0.26784966937306437,
+      "grad_norm": 0.3811708390712738,
+      "learning_rate": 2e-05,
+      "loss": 0.6752,
+      "step": 3200
+    },
+    {
+      "epoch": 0.2686866995898552,
+      "grad_norm": 0.37679523229599,
+      "learning_rate": 2e-05,
+      "loss": 0.6768,
+      "step": 3210
+    },
+    {
+      "epoch": 0.269523729806646,
+      "grad_norm": 0.3931836187839508,
+      "learning_rate": 2e-05,
+      "loss": 0.7104,
+      "step": 3220
+    },
+    {
+      "epoch": 0.27036076002343684,
+      "grad_norm": 0.4057767689228058,
+      "learning_rate": 2e-05,
+      "loss": 0.6791,
+      "step": 3230
+    },
+    {
+      "epoch": 0.27119779024022767,
+      "grad_norm": 0.33529049158096313,
+      "learning_rate": 2e-05,
+      "loss": 0.6784,
+      "step": 3240
+    },
+    {
+      "epoch": 0.2720348204570185,
+      "grad_norm": 0.34509772062301636,
+      "learning_rate": 2e-05,
+      "loss": 0.7474,
+      "step": 3250
+    },
+    {
+      "epoch": 0.2728718506738093,
+      "grad_norm": 0.3847754895687103,
+      "learning_rate": 2e-05,
+      "loss": 0.6765,
+      "step": 3260
+    },
+    {
+      "epoch": 0.27370888089060014,
+      "grad_norm": 0.38386350870132446,
+      "learning_rate": 2e-05,
+      "loss": 0.6782,
+      "step": 3270
+    },
+    {
+      "epoch": 0.27454591110739096,
+      "grad_norm": 0.4805894196033478,
+      "learning_rate": 2e-05,
+      "loss": 0.6864,
+      "step": 3280
+    },
+    {
+      "epoch": 0.2753829413241818,
+      "grad_norm": 0.46428972482681274,
+      "learning_rate": 2e-05,
+      "loss": 0.6901,
+      "step": 3290
+    },
+    {
+      "epoch": 0.2762199715409726,
+      "grad_norm": 0.3913302719593048,
+      "learning_rate": 2e-05,
+      "loss": 0.7239,
+      "step": 3300
+    },
+    {
+      "epoch": 0.27705700175776343,
+      "grad_norm": 0.3995335102081299,
+      "learning_rate": 2e-05,
+      "loss": 0.6828,
+      "step": 3310
+    },
+    {
+      "epoch": 0.27789403197455426,
+      "grad_norm": 0.36607876420021057,
+      "learning_rate": 2e-05,
+      "loss": 0.6621,
+      "step": 3320
+    },
+    {
+      "epoch": 0.2787310621913451,
+      "grad_norm": 0.38472291827201843,
+      "learning_rate": 2e-05,
+      "loss": 0.6712,
+      "step": 3330
+    },
+    {
+      "epoch": 0.27956809240813596,
+      "grad_norm": 0.34564754366874695,
+      "learning_rate": 2e-05,
+      "loss": 0.6863,
+      "step": 3340
+    },
+    {
+      "epoch": 0.2804051226249268,
+      "grad_norm": 0.3804911673069,
+      "learning_rate": 2e-05,
+      "loss": 0.7107,
+      "step": 3350
+    },
+    {
+      "epoch": 0.2812421528417176,
+      "grad_norm": 0.3177351653575897,
+      "learning_rate": 2e-05,
+      "loss": 0.6595,
+      "step": 3360
+    },
+    {
+      "epoch": 0.28207918305850843,
+      "grad_norm": 0.3490181863307953,
+      "learning_rate": 2e-05,
+      "loss": 0.6671,
+      "step": 3370
+    },
+    {
+      "epoch": 0.28291621327529926,
+      "grad_norm": 0.5247802734375,
+      "learning_rate": 2e-05,
+      "loss": 0.6614,
+      "step": 3380
+    },
+    {
+      "epoch": 0.2837532434920901,
+      "grad_norm": 0.36323365569114685,
+      "learning_rate": 2e-05,
+      "loss": 0.684,
+      "step": 3390
+    },
+    {
+      "epoch": 0.2845902737088809,
+      "grad_norm": 0.359318345785141,
+      "learning_rate": 2e-05,
+      "loss": 0.661,
+      "step": 3400
+    },
+    {
+      "epoch": 0.2854273039256717,
+      "grad_norm": 0.3711358308792114,
+      "learning_rate": 2e-05,
+      "loss": 0.6748,
+      "step": 3410
+    },
+    {
+      "epoch": 0.28626433414246255,
+      "grad_norm": 0.36873385310173035,
+      "learning_rate": 2e-05,
+      "loss": 0.6655,
+      "step": 3420
+    },
+    {
+      "epoch": 0.2871013643592534,
+      "grad_norm": 0.398560494184494,
+      "learning_rate": 2e-05,
+      "loss": 0.6624,
+      "step": 3430
+    },
+    {
+      "epoch": 0.2879383945760442,
+      "grad_norm": 0.48586469888687134,
+      "learning_rate": 2e-05,
+      "loss": 0.6993,
+      "step": 3440
+    },
+    {
+      "epoch": 0.288775424792835,
+      "grad_norm": 0.3617458641529083,
+      "learning_rate": 2e-05,
+      "loss": 0.6928,
+      "step": 3450
+    },
+    {
+      "epoch": 0.28961245500962585,
+      "grad_norm": 0.3452693521976471,
+      "learning_rate": 2e-05,
+      "loss": 0.6965,
+      "step": 3460
+    },
+    {
+      "epoch": 0.29044948522641667,
+      "grad_norm": 0.39020127058029175,
+      "learning_rate": 2e-05,
+      "loss": 0.691,
+      "step": 3470
+    },
+    {
+      "epoch": 0.2912865154432075,
+      "grad_norm": 0.38490548729896545,
+      "learning_rate": 2e-05,
+      "loss": 0.6824,
+      "step": 3480
+    },
+    {
+      "epoch": 0.2921235456599983,
+      "grad_norm": 0.3624000549316406,
+      "learning_rate": 2e-05,
+      "loss": 0.6849,
+      "step": 3490
+    },
+    {
+      "epoch": 0.29296057587678914,
+      "grad_norm": 0.40137097239494324,
+      "learning_rate": 2e-05,
+      "loss": 0.6779,
+      "step": 3500
+    },
+    {
+      "epoch": 0.29379760609357997,
+      "grad_norm": 0.3723010718822479,
+      "learning_rate": 2e-05,
+      "loss": 0.6639,
+      "step": 3510
+    },
+    {
+      "epoch": 0.2946346363103708,
+      "grad_norm": 0.33410966396331787,
+      "learning_rate": 2e-05,
+      "loss": 0.6849,
+      "step": 3520
+    },
+    {
+      "epoch": 0.2954716665271616,
+      "grad_norm": 0.4087796211242676,
+      "learning_rate": 2e-05,
+      "loss": 0.7198,
+      "step": 3530
+    },
+    {
+      "epoch": 0.29630869674395244,
+      "grad_norm": 0.3803013265132904,
+      "learning_rate": 2e-05,
+      "loss": 0.6705,
+      "step": 3540
+    },
+    {
+      "epoch": 0.29714572696074326,
+      "grad_norm": 0.38683268427848816,
+      "learning_rate": 2e-05,
+      "loss": 0.6848,
+      "step": 3550
+    },
+    {
+      "epoch": 0.2979827571775341,
+      "grad_norm": 0.3987625241279602,
+      "learning_rate": 2e-05,
+      "loss": 0.693,
+      "step": 3560
+    },
+    {
+      "epoch": 0.2988197873943249,
+      "grad_norm": 0.3848333954811096,
+      "learning_rate": 2e-05,
+      "loss": 0.691,
+      "step": 3570
+    },
+    {
+      "epoch": 0.2996568176111158,
+      "grad_norm": 0.3905976116657257,
+      "learning_rate": 2e-05,
+      "loss": 0.6745,
+      "step": 3580
+    },
+    {
+      "epoch": 0.3004938478279066,
+      "grad_norm": 0.36275070905685425,
+      "learning_rate": 2e-05,
+      "loss": 0.6693,
+      "step": 3590
+    },
+    {
+      "epoch": 0.30133087804469744,
+      "grad_norm": 0.3495936691761017,
+      "learning_rate": 2e-05,
+      "loss": 0.6845,
+      "step": 3600
+    },
+    {
+      "epoch": 0.30216790826148826,
+      "grad_norm": 0.36721131205558777,
+      "learning_rate": 2e-05,
+      "loss": 0.6807,
+      "step": 3610
+    },
+    {
+      "epoch": 0.3030049384782791,
+      "grad_norm": 0.42015427350997925,
+      "learning_rate": 2e-05,
+      "loss": 0.6934,
+      "step": 3620
+    },
+    {
+      "epoch": 0.3038419686950699,
+      "grad_norm": 0.41565388441085815,
+      "learning_rate": 2e-05,
+      "loss": 0.6817,
+      "step": 3630
+    },
+    {
+      "epoch": 0.30467899891186073,
+      "grad_norm": 0.3864046633243561,
+      "learning_rate": 2e-05,
+      "loss": 0.6972,
+      "step": 3640
+    },
+    {
+      "epoch": 0.30551602912865156,
+      "grad_norm": 0.3387700021266937,
+      "learning_rate": 2e-05,
+      "loss": 0.6753,
+      "step": 3650
+    },
+    {
+      "epoch": 0.3063530593454424,
+      "grad_norm": 0.37594422698020935,
+      "learning_rate": 2e-05,
+      "loss": 0.6808,
+      "step": 3660
+    },
+    {
+      "epoch": 0.3071900895622332,
+      "grad_norm": 0.3680058419704437,
+      "learning_rate": 2e-05,
+      "loss": 0.6924,
+      "step": 3670
+    },
+    {
+      "epoch": 0.30802711977902403,
+      "grad_norm": 0.37362930178642273,
+      "learning_rate": 2e-05,
+      "loss": 0.6598,
+      "step": 3680
+    },
+    {
+      "epoch": 0.30886414999581485,
+      "grad_norm": 0.3793889880180359,
+      "learning_rate": 2e-05,
+      "loss": 0.6754,
+      "step": 3690
+    },
+    {
+      "epoch": 0.3097011802126057,
+      "grad_norm": 0.35503870248794556,
+      "learning_rate": 2e-05,
+      "loss": 0.6834,
+      "step": 3700
+    },
+    {
+      "epoch": 0.3105382104293965,
+      "grad_norm": 0.3719038665294647,
+      "learning_rate": 2e-05,
+      "loss": 0.6721,
+      "step": 3710
+    },
+    {
+      "epoch": 0.3113752406461873,
+      "grad_norm": 0.38428905606269836,
+      "learning_rate": 2e-05,
+      "loss": 0.6499,
+      "step": 3720
+    },
+    {
+      "epoch": 0.31221227086297815,
+      "grad_norm": 0.374455988407135,
+      "learning_rate": 2e-05,
+      "loss": 0.657,
+      "step": 3730
+    },
+    {
+      "epoch": 0.31304930107976897,
+      "grad_norm": 0.37055298686027527,
+      "learning_rate": 2e-05,
+      "loss": 0.691,
+      "step": 3740
+    },
+    {
+      "epoch": 0.3138863312965598,
+      "grad_norm": 0.37087371945381165,
+      "learning_rate": 2e-05,
+      "loss": 0.6859,
+      "step": 3750
+    },
+    {
+      "epoch": 0.3147233615133506,
+      "grad_norm": 0.44824519753456116,
+      "learning_rate": 2e-05,
+      "loss": 0.689,
+      "step": 3760
+    },
+    {
+      "epoch": 0.31556039173014144,
+      "grad_norm": 0.40546661615371704,
+      "learning_rate": 2e-05,
+      "loss": 0.6715,
+      "step": 3770
+    },
+    {
+      "epoch": 0.31639742194693227,
+      "grad_norm": 0.39258283376693726,
+      "learning_rate": 2e-05,
+      "loss": 0.6877,
+      "step": 3780
+    },
+    {
+      "epoch": 0.3172344521637231,
+      "grad_norm": 0.36289212107658386,
+      "learning_rate": 2e-05,
+      "loss": 0.6668,
+      "step": 3790
+    },
+    {
+      "epoch": 0.3180714823805139,
+      "grad_norm": 0.40301036834716797,
+      "learning_rate": 2e-05,
+      "loss": 0.6785,
+      "step": 3800
+    },
+    {
+      "epoch": 0.31890851259730474,
+      "grad_norm": 0.3613869249820709,
+      "learning_rate": 2e-05,
+      "loss": 0.6792,
+      "step": 3810
+    },
+    {
+      "epoch": 0.31974554281409556,
+      "grad_norm": 0.3650999069213867,
+      "learning_rate": 2e-05,
+      "loss": 0.6548,
+      "step": 3820
+    },
+    {
+      "epoch": 0.32058257303088644,
+      "grad_norm": 0.38625913858413696,
+      "learning_rate": 2e-05,
+      "loss": 0.63,
+      "step": 3830
+    },
+    {
+      "epoch": 0.32141960324767727,
+      "grad_norm": 0.35693198442459106,
+      "learning_rate": 2e-05,
+      "loss": 0.6974,
+      "step": 3840
+    },
+    {
+      "epoch": 0.3222566334644681,
+      "grad_norm": 0.3826311230659485,
+      "learning_rate": 2e-05,
+      "loss": 0.6842,
+      "step": 3850
+    },
+    {
+      "epoch": 0.3230936636812589,
+      "grad_norm": 0.3499303162097931,
+      "learning_rate": 2e-05,
+      "loss": 0.667,
+      "step": 3860
+    },
+    {
+      "epoch": 0.32393069389804974,
+      "grad_norm": 0.33624884486198425,
+      "learning_rate": 2e-05,
+      "loss": 0.6643,
+      "step": 3870
+    },
+    {
+      "epoch": 0.32476772411484056,
+      "grad_norm": 0.4605383574962616,
+      "learning_rate": 2e-05,
+      "loss": 0.6631,
+      "step": 3880
+    },
+    {
+      "epoch": 0.3256047543316314,
+      "grad_norm": 0.3703138828277588,
+      "learning_rate": 2e-05,
+      "loss": 0.7092,
+      "step": 3890
+    },
+    {
+      "epoch": 0.3264417845484222,
+      "grad_norm": 0.3547705411911011,
+      "learning_rate": 2e-05,
+      "loss": 0.674,
+      "step": 3900
+    },
+    {
+      "epoch": 0.32727881476521303,
+      "grad_norm": 0.38592010736465454,
+      "learning_rate": 2e-05,
+      "loss": 0.729,
+      "step": 3910
+    },
+    {
+      "epoch": 0.32811584498200386,
+      "grad_norm": 0.38469675183296204,
+      "learning_rate": 2e-05,
+      "loss": 0.6379,
+      "step": 3920
+    },
+    {
+      "epoch": 0.3289528751987947,
+      "grad_norm": 0.38590508699417114,
+      "learning_rate": 2e-05,
+      "loss": 0.6671,
+      "step": 3930
+    },
+    {
+      "epoch": 0.3297899054155855,
+      "grad_norm": 0.4092082381248474,
+      "learning_rate": 2e-05,
+      "loss": 0.7011,
+      "step": 3940
+    },
+    {
+      "epoch": 0.33062693563237633,
+      "grad_norm": 0.3793179988861084,
+      "learning_rate": 2e-05,
+      "loss": 0.6973,
+      "step": 3950
+    },
+    {
+      "epoch": 0.33146396584916715,
+      "grad_norm": 0.38254857063293457,
+      "learning_rate": 2e-05,
+      "loss": 0.6909,
+      "step": 3960
+    },
+    {
+      "epoch": 0.332300996065958,
+      "grad_norm": 0.3839905261993408,
+      "learning_rate": 2e-05,
+      "loss": 0.6891,
+      "step": 3970
+    },
+    {
+      "epoch": 0.3331380262827488,
+      "grad_norm": 0.38743963837623596,
+      "learning_rate": 2e-05,
+      "loss": 0.7079,
+      "step": 3980
+    },
+    {
+      "epoch": 0.3339750564995396,
+      "grad_norm": 0.381570965051651,
+      "learning_rate": 2e-05,
+      "loss": 0.7051,
+      "step": 3990
+    },
+    {
+      "epoch": 0.33481208671633045,
+      "grad_norm": 0.3812105059623718,
+      "learning_rate": 2e-05,
+      "loss": 0.6781,
+      "step": 4000
+    },
+    {
+      "epoch": 0.33564911693312127,
+      "grad_norm": 0.3621882200241089,
+      "learning_rate": 2e-05,
+      "loss": 0.6923,
+      "step": 4010
+    },
+    {
+      "epoch": 0.3364861471499121,
+      "grad_norm": 0.35323214530944824,
+      "learning_rate": 2e-05,
+      "loss": 0.6851,
+      "step": 4020
+    },
+    {
+      "epoch": 0.3373231773667029,
+      "grad_norm": 0.35751551389694214,
+      "learning_rate": 2e-05,
+      "loss": 0.6655,
+      "step": 4030
+    },
+    {
+      "epoch": 0.33816020758349374,
+      "grad_norm": 0.3949991464614868,
+      "learning_rate": 2e-05,
+      "loss": 0.6621,
+      "step": 4040
+    },
+    {
+      "epoch": 0.33899723780028457,
+      "grad_norm": 0.3553832173347473,
+      "learning_rate": 2e-05,
+      "loss": 0.7026,
+      "step": 4050
+    },
+    {
+      "epoch": 0.3398342680170754,
+      "grad_norm": 0.39129525423049927,
+      "learning_rate": 2e-05,
+      "loss": 0.6727,
+      "step": 4060
+    },
+    {
+      "epoch": 0.3406712982338662,
+      "grad_norm": 0.39109840989112854,
+      "learning_rate": 2e-05,
+      "loss": 0.692,
+      "step": 4070
+    },
+    {
+      "epoch": 0.3415083284506571,
+      "grad_norm": 0.384739488363266,
+      "learning_rate": 2e-05,
+      "loss": 0.6578,
+      "step": 4080
+    },
+    {
+      "epoch": 0.3423453586674479,
+      "grad_norm": 0.34750863909721375,
+      "learning_rate": 2e-05,
+      "loss": 0.6972,
+      "step": 4090
+    },
+    {
+      "epoch": 0.34318238888423874,
+      "grad_norm": 0.3817720115184784,
+      "learning_rate": 2e-05,
+      "loss": 0.6563,
+      "step": 4100
+    },
+    {
+      "epoch": 0.34401941910102957,
+      "grad_norm": 0.38964515924453735,
+      "learning_rate": 2e-05,
+      "loss": 0.6591,
+      "step": 4110
+    },
+    {
+      "epoch": 0.3448564493178204,
+      "grad_norm": 0.38531047105789185,
+      "learning_rate": 2e-05,
+      "loss": 0.6475,
+      "step": 4120
+    },
+    {
+      "epoch": 0.3456934795346112,
+      "grad_norm": 0.4181869626045227,
+      "learning_rate": 2e-05,
+      "loss": 0.6746,
+      "step": 4130
+    },
+    {
+      "epoch": 0.34653050975140204,
+      "grad_norm": 0.36208441853523254,
+      "learning_rate": 2e-05,
+      "loss": 0.6825,
+      "step": 4140
+    },
+    {
+      "epoch": 0.34736753996819286,
+      "grad_norm": 0.3654119074344635,
+      "learning_rate": 2e-05,
+      "loss": 0.6887,
+      "step": 4150
+    },
+    {
+      "epoch": 0.3482045701849837,
+      "grad_norm": 0.39477288722991943,
+      "learning_rate": 2e-05,
+      "loss": 0.6977,
+      "step": 4160
+    },
+    {
+      "epoch": 0.3490416004017745,
+      "grad_norm": 0.36870384216308594,
+      "learning_rate": 2e-05,
+      "loss": 0.6725,
+      "step": 4170
+    },
+    {
+      "epoch": 0.34987863061856533,
+      "grad_norm": 0.35877320170402527,
+      "learning_rate": 2e-05,
+      "loss": 0.679,
+      "step": 4180
+    },
+    {
+      "epoch": 0.35071566083535616,
+      "grad_norm": 0.5562852621078491,
+      "learning_rate": 2e-05,
+      "loss": 0.6439,
+      "step": 4190
+    },
+    {
+      "epoch": 0.351552691052147,
+      "grad_norm": 0.40305373072624207,
+      "learning_rate": 2e-05,
+      "loss": 0.703,
+      "step": 4200
+    },
+    {
+      "epoch": 0.3523897212689378,
+      "grad_norm": 0.41178417205810547,
+      "learning_rate": 2e-05,
+      "loss": 0.6618,
+      "step": 4210
+    },
+    {
+      "epoch": 0.35322675148572863,
+      "grad_norm": 0.3854200541973114,
+      "learning_rate": 2e-05,
+      "loss": 0.6662,
+      "step": 4220
+    },
+    {
+      "epoch": 0.35406378170251945,
+      "grad_norm": 0.3953651487827301,
+      "learning_rate": 2e-05,
+      "loss": 0.7133,
+      "step": 4230
+    },
+    {
+      "epoch": 0.3549008119193103,
+      "grad_norm": 0.3878295123577118,
+      "learning_rate": 2e-05,
+      "loss": 0.701,
+      "step": 4240
+    },
+    {
+      "epoch": 0.3557378421361011,
+      "grad_norm": 0.36245018243789673,
+      "learning_rate": 2e-05,
+      "loss": 0.6104,
+      "step": 4250
+    },
+    {
+      "epoch": 0.3565748723528919,
+      "grad_norm": 0.3872743844985962,
+      "learning_rate": 2e-05,
+      "loss": 0.6878,
+      "step": 4260
+    },
+    {
+      "epoch": 0.35741190256968275,
+      "grad_norm": 0.4376787543296814,
+      "learning_rate": 2e-05,
+      "loss": 0.6741,
+      "step": 4270
+    },
+    {
+      "epoch": 0.3582489327864736,
+      "grad_norm": 0.5118808150291443,
+      "learning_rate": 2e-05,
+      "loss": 0.7126,
+      "step": 4280
+    },
+    {
+      "epoch": 0.3590859630032644,
+      "grad_norm": 0.35995811223983765,
+      "learning_rate": 2e-05,
+      "loss": 0.6814,
+      "step": 4290
+    },
+    {
+      "epoch": 0.3599229932200552,
+      "grad_norm": 0.32932382822036743,
+      "learning_rate": 2e-05,
+      "loss": 0.6686,
+      "step": 4300
+    },
+    {
+      "epoch": 0.36076002343684604,
+      "grad_norm": 0.3927007019519806,
+      "learning_rate": 2e-05,
+      "loss": 0.6526,
+      "step": 4310
+    },
+    {
+      "epoch": 0.3615970536536369,
+      "grad_norm": 0.3740251958370209,
+      "learning_rate": 2e-05,
+      "loss": 0.6658,
+      "step": 4320
+    },
+    {
+      "epoch": 0.36243408387042775,
+      "grad_norm": 0.3808770477771759,
+      "learning_rate": 2e-05,
+      "loss": 0.6923,
+      "step": 4330
+    },
+    {
+      "epoch": 0.36327111408721857,
+      "grad_norm": 0.3733738660812378,
+      "learning_rate": 2e-05,
+      "loss": 0.6758,
+      "step": 4340
+    },
+    {
+      "epoch": 0.3641081443040094,
+      "grad_norm": 0.36265361309051514,
+      "learning_rate": 2e-05,
+      "loss": 0.6714,
+      "step": 4350
+    },
+    {
+      "epoch": 0.3649451745208002,
+      "grad_norm": 0.37637802958488464,
+      "learning_rate": 2e-05,
+      "loss": 0.7047,
+      "step": 4360
+    },
+    {
+      "epoch": 0.36578220473759104,
+      "grad_norm": 0.3615052402019501,
+      "learning_rate": 2e-05,
+      "loss": 0.6887,
+      "step": 4370
+    },
+    {
+      "epoch": 0.36661923495438187,
+      "grad_norm": 0.3507406413555145,
+      "learning_rate": 2e-05,
+      "loss": 0.6817,
+      "step": 4380
+    },
+    {
+      "epoch": 0.3674562651711727,
+      "grad_norm": 0.3859863579273224,
+      "learning_rate": 2e-05,
+      "loss": 0.6908,
+      "step": 4390
+    },
+    {
+      "epoch": 0.3682932953879635,
+      "grad_norm": 0.5150337815284729,
+      "learning_rate": 2e-05,
+      "loss": 0.6829,
+      "step": 4400
+    },
+    {
+      "epoch": 0.36913032560475434,
+      "grad_norm": 0.3702976107597351,
+      "learning_rate": 2e-05,
+      "loss": 0.6628,
+      "step": 4410
+    },
+    {
+      "epoch": 0.36996735582154516,
+      "grad_norm": 0.36254793405532837,
+      "learning_rate": 2e-05,
+      "loss": 0.6735,
+      "step": 4420
+    },
+    {
+      "epoch": 0.370804386038336,
+      "grad_norm": 0.3495381474494934,
+      "learning_rate": 2e-05,
+      "loss": 0.6815,
+      "step": 4430
+    },
+    {
+      "epoch": 0.3716414162551268,
+      "grad_norm": 0.3943999707698822,
+      "learning_rate": 2e-05,
+      "loss": 0.6797,
+      "step": 4440
+    },
+    {
+      "epoch": 0.37247844647191763,
+      "grad_norm": 0.3715941905975342,
+      "learning_rate": 2e-05,
+      "loss": 0.6615,
+      "step": 4450
+    },
+    {
+      "epoch": 0.37331547668870846,
+      "grad_norm": 0.47644540667533875,
+      "learning_rate": 2e-05,
+      "loss": 0.6901,
+      "step": 4460
+    },
+    {
+      "epoch": 0.3741525069054993,
+      "grad_norm": 0.40302789211273193,
+      "learning_rate": 2e-05,
+      "loss": 0.6556,
+      "step": 4470
+    },
+    {
+      "epoch": 0.3749895371222901,
+      "grad_norm": 0.366240531206131,
+      "learning_rate": 2e-05,
+      "loss": 0.6787,
+      "step": 4480
+    },
+    {
+      "epoch": 0.37582656733908093,
+      "grad_norm": 0.3775447607040405,
+      "learning_rate": 2e-05,
+      "loss": 0.6518,
+      "step": 4490
+    },
+    {
+      "epoch": 0.37666359755587175,
+      "grad_norm": 0.3555198609828949,
+      "learning_rate": 2e-05,
+      "loss": 0.6844,
+      "step": 4500
+    },
+    {
+      "epoch": 0.3775006277726626,
+      "grad_norm": 0.40677034854888916,
+      "learning_rate": 2e-05,
+      "loss": 0.6899,
+      "step": 4510
+    },
+    {
+      "epoch": 0.3783376579894534,
+      "grad_norm": 0.33568084239959717,
+      "learning_rate": 2e-05,
+      "loss": 0.6329,
+      "step": 4520
+    },
+    {
+      "epoch": 0.3791746882062442,
+      "grad_norm": 0.4298158586025238,
+      "learning_rate": 2e-05,
+      "loss": 0.6856,
+      "step": 4530
+    },
+    {
+      "epoch": 0.38001171842303505,
+      "grad_norm": 0.3619917333126068,
+      "learning_rate": 2e-05,
+      "loss": 0.655,
+      "step": 4540
+    },
+    {
+      "epoch": 0.3808487486398259,
+      "grad_norm": 0.34173473715782166,
+      "learning_rate": 2e-05,
+      "loss": 0.6386,
+      "step": 4550
+    },
+    {
+      "epoch": 0.3816857788566167,
+      "grad_norm": 0.41530799865722656,
+      "learning_rate": 2e-05,
+      "loss": 0.6436,
+      "step": 4560
+    },
+    {
+      "epoch": 0.3825228090734076,
+      "grad_norm": 0.3594314455986023,
+      "learning_rate": 2e-05,
+      "loss": 0.6765,
+      "step": 4570
+    },
+    {
+      "epoch": 0.3833598392901984,
+      "grad_norm": 0.3838607370853424,
+      "learning_rate": 2e-05,
+      "loss": 0.656,
+      "step": 4580
+    },
+    {
+      "epoch": 0.3841968695069892,
+      "grad_norm": 0.3307185769081116,
+      "learning_rate": 2e-05,
+      "loss": 0.6976,
+      "step": 4590
+    },
+    {
+      "epoch": 0.38503389972378005,
+      "grad_norm": 0.4034730792045593,
+      "learning_rate": 2e-05,
+      "loss": 0.6824,
+      "step": 4600
+    },
+    {
+      "epoch": 0.3858709299405709,
+      "grad_norm": 0.38577455282211304,
+      "learning_rate": 2e-05,
+      "loss": 0.6516,
+      "step": 4610
+    },
+    {
+      "epoch": 0.3867079601573617,
+      "grad_norm": 0.3357555568218231,
+      "learning_rate": 2e-05,
+      "loss": 0.645,
+      "step": 4620
+    },
+    {
+      "epoch": 0.3875449903741525,
+      "grad_norm": 0.37598440051078796,
+      "learning_rate": 2e-05,
+      "loss": 0.7059,
+      "step": 4630
+    },
+    {
+      "epoch": 0.38838202059094334,
+      "grad_norm": 0.3609393239021301,
+      "learning_rate": 2e-05,
+      "loss": 0.669,
+      "step": 4640
+    },
+    {
+      "epoch": 0.38921905080773417,
+      "grad_norm": 0.3757115304470062,
+      "learning_rate": 2e-05,
+      "loss": 0.6395,
+      "step": 4650
+    },
+    {
+      "epoch": 0.390056081024525,
+      "grad_norm": 0.36719217896461487,
+      "learning_rate": 2e-05,
+      "loss": 0.6744,
+      "step": 4660
+    },
+    {
+      "epoch": 0.3908931112413158,
+      "grad_norm": 0.36577510833740234,
+      "learning_rate": 2e-05,
+      "loss": 0.6851,
+      "step": 4670
+    },
+    {
+      "epoch": 0.39173014145810664,
+      "grad_norm": 0.36640894412994385,
+      "learning_rate": 2e-05,
+      "loss": 0.6627,
+      "step": 4680
+    },
+    {
+      "epoch": 0.39256717167489746,
+      "grad_norm": 0.3786374628543854,
+      "learning_rate": 2e-05,
+      "loss": 0.6888,
+      "step": 4690
+    },
+    {
+      "epoch": 0.3934042018916883,
+      "grad_norm": 0.3693200349807739,
+      "learning_rate": 2e-05,
+      "loss": 0.6608,
+      "step": 4700
+    },
+    {
+      "epoch": 0.3942412321084791,
+      "grad_norm": 0.35623887181282043,
+      "learning_rate": 2e-05,
+      "loss": 0.6913,
+      "step": 4710
+    },
+    {
+      "epoch": 0.39507826232526994,
+      "grad_norm": 0.37266993522644043,
+      "learning_rate": 2e-05,
+      "loss": 0.6216,
+      "step": 4720
+    },
+    {
+      "epoch": 0.39591529254206076,
+      "grad_norm": 0.48957139253616333,
+      "learning_rate": 2e-05,
+      "loss": 0.6577,
+      "step": 4730
+    },
+    {
+      "epoch": 0.3967523227588516,
+      "grad_norm": 0.361542284488678,
+      "learning_rate": 2e-05,
+      "loss": 0.6707,
+      "step": 4740
+    },
+    {
+      "epoch": 0.3975893529756424,
+      "grad_norm": 0.3781006932258606,
+      "learning_rate": 2e-05,
+      "loss": 0.6729,
+      "step": 4750
+    },
+    {
+      "epoch": 0.39842638319243323,
+      "grad_norm": 0.3931069076061249,
+      "learning_rate": 2e-05,
+      "loss": 0.6626,
+      "step": 4760
+    },
+    {
+      "epoch": 0.39926341340922405,
+      "grad_norm": 0.3622207045555115,
+      "learning_rate": 2e-05,
+      "loss": 0.6719,
+      "step": 4770
+    },
+    {
+      "epoch": 0.4001004436260149,
+      "grad_norm": 0.43951886892318726,
+      "learning_rate": 2e-05,
+      "loss": 0.6866,
+      "step": 4780
+    },
+    {
+      "epoch": 0.4009374738428057,
+      "grad_norm": 0.36725106835365295,
+      "learning_rate": 2e-05,
+      "loss": 0.6412,
+      "step": 4790
+    },
+    {
+      "epoch": 0.4017745040595965,
+      "grad_norm": 0.3468758761882782,
+      "learning_rate": 2e-05,
+      "loss": 0.656,
+      "step": 4800
+    },
+    {
+      "epoch": 0.40261153427638735,
+      "grad_norm": 0.3753811717033386,
+      "learning_rate": 2e-05,
+      "loss": 0.674,
+      "step": 4810
+    },
+    {
+      "epoch": 0.40344856449317823,
+      "grad_norm": 0.35826578736305237,
+      "learning_rate": 2e-05,
+      "loss": 0.6671,
+      "step": 4820
+    },
+    {
+      "epoch": 0.40428559470996905,
+      "grad_norm": 0.5122092962265015,
+      "learning_rate": 2e-05,
+      "loss": 0.68,
+      "step": 4830
+    },
+    {
+      "epoch": 0.4051226249267599,
+      "grad_norm": 0.34622722864151,
+      "learning_rate": 2e-05,
+      "loss": 0.6371,
+      "step": 4840
+    },
+    {
+      "epoch": 0.4059596551435507,
+      "grad_norm": 0.354408323764801,
+      "learning_rate": 2e-05,
+      "loss": 0.6516,
+      "step": 4850
+    },
+    {
+      "epoch": 0.4067966853603415,
+      "grad_norm": 0.3598283529281616,
+      "learning_rate": 2e-05,
+      "loss": 0.6357,
+      "step": 4860
+    },
+    {
+      "epoch": 0.40763371557713235,
+      "grad_norm": 0.34254366159439087,
+      "learning_rate": 2e-05,
+      "loss": 0.6617,
+      "step": 4870
+    },
+    {
+      "epoch": 0.4084707457939232,
+      "grad_norm": 0.4542747437953949,
+      "learning_rate": 2e-05,
+      "loss": 0.6532,
+      "step": 4880
+    },
+    {
+      "epoch": 0.409307776010714,
+      "grad_norm": 0.35308054089546204,
+      "learning_rate": 2e-05,
+      "loss": 0.6558,
+      "step": 4890
+    },
+    {
+      "epoch": 0.4101448062275048,
+      "grad_norm": 0.37054240703582764,
+      "learning_rate": 2e-05,
+      "loss": 0.6681,
+      "step": 4900
+    },
+    {
+      "epoch": 0.41098183644429565,
+      "grad_norm": 0.3926886022090912,
+      "learning_rate": 2e-05,
+      "loss": 0.6739,
+      "step": 4910
+    },
+    {
+      "epoch": 0.41181886666108647,
+      "grad_norm": 0.4271712601184845,
+      "learning_rate": 2e-05,
+      "loss": 0.655,
+      "step": 4920
+    },
+    {
+      "epoch": 0.4126558968778773,
+      "grad_norm": 0.4078362286090851,
+      "learning_rate": 2e-05,
+      "loss": 0.5931,
+      "step": 4930
+    },
+    {
+      "epoch": 0.4134929270946681,
+      "grad_norm": 0.3612181544303894,
+      "learning_rate": 2e-05,
+      "loss": 0.6747,
+      "step": 4940
+    },
+    {
+      "epoch": 0.41432995731145894,
+      "grad_norm": 0.36205559968948364,
+      "learning_rate": 2e-05,
+      "loss": 0.6643,
+      "step": 4950
+    },
+    {
+      "epoch": 0.41516698752824976,
+      "grad_norm": 0.33852431178092957,
+      "learning_rate": 2e-05,
+      "loss": 0.6516,
+      "step": 4960
+    },
+    {
+      "epoch": 0.4160040177450406,
+      "grad_norm": 0.360599160194397,
+      "learning_rate": 2e-05,
+      "loss": 0.6424,
+      "step": 4970
+    },
+    {
+      "epoch": 0.4168410479618314,
+      "grad_norm": 0.3367842733860016,
+      "learning_rate": 2e-05,
+      "loss": 0.6583,
+      "step": 4980
+    },
+    {
+      "epoch": 0.41767807817862224,
+      "grad_norm": 0.37129393219947815,
+      "learning_rate": 2e-05,
+      "loss": 0.6406,
+      "step": 4990
+    },
+    {
+      "epoch": 0.41851510839541306,
+      "grad_norm": 0.40336930751800537,
+      "learning_rate": 2e-05,
+      "loss": 0.6615,
+      "step": 5000
+    },
+    {
+      "epoch": 0.4193521386122039,
+      "grad_norm": 0.3625902831554413,
+      "learning_rate": 2e-05,
+      "loss": 0.6726,
+      "step": 5010
+    },
+    {
+      "epoch": 0.4201891688289947,
+      "grad_norm": 0.3696928322315216,
+      "learning_rate": 2e-05,
+      "loss": 0.6929,
+      "step": 5020
+    },
+    {
+      "epoch": 0.42102619904578553,
+      "grad_norm": 0.35005995631217957,
+      "learning_rate": 2e-05,
+      "loss": 0.6734,
+      "step": 5030
+    },
+    {
+      "epoch": 0.42186322926257636,
+      "grad_norm": 0.3266897201538086,
+      "learning_rate": 2e-05,
+      "loss": 0.6882,
+      "step": 5040
+    },
+    {
+      "epoch": 0.4227002594793672,
+      "grad_norm": 0.4611261188983917,
+      "learning_rate": 2e-05,
+      "loss": 0.7007,
+      "step": 5050
+    },
+    {
+      "epoch": 0.423537289696158,
+      "grad_norm": 0.39426982402801514,
+      "learning_rate": 2e-05,
+      "loss": 0.6638,
+      "step": 5060
+    },
+    {
+      "epoch": 0.4243743199129489,
+      "grad_norm": 0.3529796302318573,
+      "learning_rate": 2e-05,
+      "loss": 0.6559,
+      "step": 5070
+    },
+    {
+      "epoch": 0.4252113501297397,
+      "grad_norm": 0.39787614345550537,
+      "learning_rate": 2e-05,
+      "loss": 0.6387,
+      "step": 5080
+    },
+    {
+      "epoch": 0.42604838034653053,
+      "grad_norm": 0.3493591547012329,
+      "learning_rate": 1.9999810479633703e-05,
+      "loss": 0.5847,
+      "step": 5090
+    },
+    {
+      "epoch": 0.42688541056332135,
+      "grad_norm": 0.3723395764827728,
+      "learning_rate": 1.997707677020786e-05,
+      "loss": 0.6512,
+      "step": 5100
+    },
+    {
+      "epoch": 0.4277224407801122,
+      "grad_norm": 0.40137779712677,
+      "learning_rate": 1.9916538196124637e-05,
+      "loss": 0.6725,
+      "step": 5110
+    },
+    {
+      "epoch": 0.428559470996903,
+      "grad_norm": 0.3983379900455475,
+      "learning_rate": 1.9818425303863402e-05,
+      "loss": 0.6721,
+      "step": 5120
+    },
+    {
+      "epoch": 0.4293965012136938,
+      "grad_norm": 0.4244665801525116,
+      "learning_rate": 1.9683111732582887e-05,
+      "loss": 0.6925,
+      "step": 5130
+    },
+    {
+      "epoch": 0.43023353143048465,
+      "grad_norm": 0.39193373918533325,
+      "learning_rate": 1.951111279120706e-05,
+      "loss": 0.6566,
+      "step": 5140
+    },
+    {
+      "epoch": 0.4310705616472755,
+      "grad_norm": 0.3795602321624756,
+      "learning_rate": 1.9303083495996028e-05,
+      "loss": 0.653,
+      "step": 5150
+    },
+    {
+      "epoch": 0.4319075918640663,
+      "grad_norm": 0.35238948464393616,
+      "learning_rate": 1.9059816076075482e-05,
+      "loss": 0.6435,
+      "step": 5160
+    },
+    {
+      "epoch": 0.4327446220808571,
+      "grad_norm": 0.3624531030654907,
+      "learning_rate": 1.8782236956424142e-05,
+      "loss": 0.6457,
+      "step": 5170
+    },
+    {
+      "epoch": 0.43358165229764795,
+      "grad_norm": 0.34817200899124146,
+      "learning_rate": 1.8471403229808897e-05,
+      "loss": 0.6873,
+      "step": 5180
+    },
+    {
+      "epoch": 0.43441868251443877,
+      "grad_norm": 0.4171640872955322,
+      "learning_rate": 1.8128498631103333e-05,
+      "loss": 0.6788,
+      "step": 5190
+    },
+    {
+      "epoch": 0.4352557127312296,
+      "grad_norm": 0.37277984619140625,
+      "learning_rate": 1.7754829029320507e-05,
+      "loss": 0.6728,
+      "step": 5200
+    },
+    {
+      "epoch": 0.4360927429480204,
+      "grad_norm": 0.38008707761764526,
+      "learning_rate": 1.7351817454527473e-05,
+      "loss": 0.6409,
+      "step": 5210
+    },
+    {
+      "epoch": 0.43692977316481124,
+      "grad_norm": 0.34851887822151184,
+      "learning_rate": 1.6920998678580345e-05,
+      "loss": 0.6645,
+      "step": 5220
+    },
+    {
+      "epoch": 0.43776680338160207,
+      "grad_norm": 0.3664775788784027,
+      "learning_rate": 1.6464013370317854e-05,
+      "loss": 0.6616,
+      "step": 5230
+    },
+    {
+      "epoch": 0.4386038335983929,
+      "grad_norm": 0.3701815605163574,
+      "learning_rate": 1.5982601847471915e-05,
+      "loss": 0.6364,
+      "step": 5240
+    },
+    {
+      "epoch": 0.4394408638151837,
+      "grad_norm": 0.362636923789978,
+      "learning_rate": 1.5478597449089553e-05,
+      "loss": 0.6473,
+      "step": 5250
+    },
+    {
+      "epoch": 0.44027789403197454,
+      "grad_norm": 0.342526376247406,
+      "learning_rate": 1.4953919553705725e-05,
+      "loss": 0.6731,
+      "step": 5260
+    },
+    {
+      "epoch": 0.44111492424876536,
+      "grad_norm": 0.32191959023475647,
+      "learning_rate": 1.441056626985559e-05,
+      "loss": 0.6668,
+      "step": 5270
+    },
+    {
+      "epoch": 0.4419519544655562,
+      "grad_norm": 0.3483056128025055,
+      "learning_rate": 1.3850606826762668e-05,
+      "loss": 0.6675,
+      "step": 5280
+    },
+    {
+      "epoch": 0.442788984682347,
+      "grad_norm": 0.3659847378730774,
+      "learning_rate": 1.3276173694181128e-05,
+      "loss": 0.6675,
+      "step": 5290
+    },
+    {
+      "epoch": 0.44362601489913783,
+      "grad_norm": 0.36162468791007996,
+      "learning_rate": 1.2689454461401822e-05,
+      "loss": 0.648,
+      "step": 5300
+    },
+    {
+      "epoch": 0.4444630451159287,
+      "grad_norm": 0.36163952946662903,
+      "learning_rate": 1.2092683506348939e-05,
+      "loss": 0.6463,
+      "step": 5310
+    },
+    {
+      "epoch": 0.44530007533271954,
+      "grad_norm": 0.5795994997024536,
+      "learning_rate": 1.14881334864935e-05,
+      "loss": 0.6575,
+      "step": 5320
+    },
+    {
+      "epoch": 0.44613710554951036,
+      "grad_norm": 0.35306987166404724,
+      "learning_rate": 1.0878106683988481e-05,
+      "loss": 0.6597,
+      "step": 5330
+    },
+    {
+      "epoch": 0.4469741357663012,
+      "grad_norm": 0.33831799030303955,
+      "learning_rate": 1.026492623798551e-05,
+      "loss": 0.6277,
+      "step": 5340
+    },
+    {
+      "epoch": 0.447811165983092,
+      "grad_norm": 0.3335249423980713,
+      "learning_rate": 9.650927297522743e-06,
+      "loss": 0.6637,
+      "step": 5350
+    },
+    {
+      "epoch": 0.44864819619988283,
+      "grad_norm": 0.3490908443927765,
+      "learning_rate": 9.038448128676012e-06,
+      "loss": 0.6529,
+      "step": 5360
+    },
+    {
+      "epoch": 0.44948522641667366,
+      "grad_norm": 0.36650386452674866,
+      "learning_rate": 8.42982120983949e-06,
+      "loss": 0.6497,
+      "step": 5370
+    },
+    {
+      "epoch": 0.4503222566334645,
+      "grad_norm": 0.346248596906662,
+      "learning_rate": 7.827364349047303e-06,
+      "loss": 0.6752,
+      "step": 5380
+    },
+    {
+      "epoch": 0.4511592868502553,
+      "grad_norm": 0.34524908661842346,
+      "learning_rate": 7.233371857163706e-06,
+      "loss": 0.6394,
+      "step": 5390
+    },
+    {
+      "epoch": 0.4519963170670461,
+      "grad_norm": 0.3573249578475952,
+      "learning_rate": 6.650105810556492e-06,
+      "loss": 0.6787,
+      "step": 5400
+    },
+    {
+      "epoch": 0.45283334728383695,
+      "grad_norm": 0.35597580671310425,
+      "learning_rate": 6.079787436527681e-06,
+      "loss": 0.6971,
+      "step": 5410
+    },
+    {
+      "epoch": 0.4536703775006278,
+      "grad_norm": 0.36199232935905457,
+      "learning_rate": 5.524588654307961e-06,
+      "loss": 0.6682,
+      "step": 5420
+    },
+    {
+      "epoch": 0.4545074077174186,
+      "grad_norm": 0.393062025308609,
+      "learning_rate": 4.986623803828985e-06,
+      "loss": 0.6573,
+      "step": 5430
+    },
+    {
+      "epoch": 0.4553444379342094,
+      "grad_norm": 0.3516090512275696,
+      "learning_rate": 4.467941593772423e-06,
+      "loss": 0.6321,
+      "step": 5440
+    },
+    {
+      "epoch": 0.45618146815100025,
+      "grad_norm": 0.387460857629776,
+      "learning_rate": 3.9705172995597606e-06,
+      "loss": 0.6756,
+      "step": 5450
+    },
+    {
+      "epoch": 0.45701849836779107,
+      "grad_norm": 0.3646886348724365,
+      "learning_rate": 3.4962452409948346e-06,
+      "loss": 0.6385,
+      "step": 5460
+    },
+    {
+      "epoch": 0.4578555285845819,
+      "grad_norm": 0.33883002400398254,
+      "learning_rate": 3.046931568206211e-06,
+      "loss": 0.6826,
+      "step": 5470
+    },
+    {
+      "epoch": 0.4586925588013727,
+      "grad_norm": 0.3687443137168884,
+      "learning_rate": 2.624287383362404e-06,
+      "loss": 0.6485,
+      "step": 5480
+    },
+    {
+      "epoch": 0.45952958901816354,
+      "grad_norm": 0.36745738983154297,
+      "learning_rate": 2.2299222243541845e-06,
+      "loss": 0.6697,
+      "step": 5490
+    },
+    {
+      "epoch": 0.46036661923495437,
+      "grad_norm": 0.3611207604408264,
+      "learning_rate": 1.8653379352598246e-06,
+      "loss": 0.6132,
+      "step": 5500
+    },
+    {
+      "epoch": 0.4612036494517452,
+      "grad_norm": 0.34954649209976196,
+      "learning_rate": 1.5319229469361075e-06,
+      "loss": 0.6436,
+      "step": 5510
+    },
+    {
+      "epoch": 0.462040679668536,
+      "grad_norm": 0.3694668710231781,
+      "learning_rate": 1.2309469895160917e-06,
+      "loss": 0.6035,
+      "step": 5520
+    },
+    {
+      "epoch": 0.46287770988532684,
+      "grad_norm": 0.33643293380737305,
+      "learning_rate": 9.635562569497974e-07,
+      "loss": 0.616,
+      "step": 5530
+    },
+    {
+      "epoch": 0.46371474010211766,
+      "grad_norm": 0.35816338658332825,
+      "learning_rate": 7.307690420024867e-07,
+      "loss": 0.622,
+      "step": 5540
+    },
+    {
+      "epoch": 0.4645517703189085,
+      "grad_norm": 0.3566610515117645,
+      "learning_rate": 5.334718583335942e-07,
+      "loss": 0.6435,
+      "step": 5550
+    },
+    {
+      "epoch": 0.46538880053569937,
+      "grad_norm": 0.32019272446632385,
+      "learning_rate": 3.7241606442442763e-07,
+      "loss": 0.6681,
+      "step": 5560
+    },
+    {
+      "epoch": 0.4662258307524902,
+      "grad_norm": 0.33671486377716064,
+      "learning_rate": 2.482150022115879e-07,
+      "loss": 0.6293,
+      "step": 5570
+    },
+    {
+      "epoch": 0.467062860969281,
+      "grad_norm": 0.3729787766933441,
+      "learning_rate": 1.6134166132293174e-07,
+      "loss": 0.6354,
+      "step": 5580
+    },
+    {
+      "epoch": 0.46789989118607184,
+      "grad_norm": 0.3181573152542114,
+      "learning_rate": 1.1212687781127357e-07,
+      "loss": 0.6365,
+      "step": 5590
+    },
+    {
+      "epoch": 0.4684858123378254,
+      "step": 5597,
+      "total_flos": 2.2737574718397743e+19,
+      "train_loss": 0.02192150756293415,
+      "train_runtime": 1760.7014,
+      "train_samples_per_second": 38.146,
+      "train_steps_per_second": 3.179
+    }
+  ],
+  "logging_steps": 10,
+  "max_steps": 5597,
+  "num_input_tokens_seen": 0,
+  "num_train_epochs": 1,
+  "save_steps": 159,
+  "stateful_callbacks": {
+    "TrainerControl": {
+      "args": {
+        "should_epoch_stop": false,
+        "should_evaluate": false,
+        "should_log": false,
+        "should_save": true,
+        "should_training_stop": true
+      },
+      "attributes": {}
+    }
+  },
+  "total_flos": 2.2737574718397743e+19,
+  "train_batch_size": 1,
+  "trial_name": null,
+  "trial_params": null
+}

g2_3_plan_oldcode_newcode/1000M/all_results.json

@@ -0,0 +1,8 @@
+{
+    "epoch": 0.9616838487972509,
+    "total_flos": 4.547514943786923e+19,
+    "train_loss": 0.003869807505066616,
+    "train_runtime": 609.7819,
+    "train_samples_per_second": 220.289,
+    "train_steps_per_second": 18.357
+}

g2_3_plan_oldcode_newcode/1000M/chat_template.jinja

@@ -0,0 +1 @@
+{% if messages[0]['role'] == 'system' %}{% set loop_messages = messages[1:] %}{% set system_message = messages[0]['content'] %}{% else %}{% set loop_messages = messages %}{% endif %}{% if system_message is defined %}{{ system_message }}{% endif %}{% for message in loop_messages %}{% set content = message['content'] %}{% if message['role'] == 'user' %}{{ content }}{% elif message['role'] == 'assistant' %}{{ content + '<|endoftext|>' }}{% endif %}{% endfor %}

g2_3_plan_oldcode_newcode/1000M/config.json

@@ -0,0 +1,27 @@
+{
+  "architectures": [
+    "Olmo2ForCausalLM"
+  ],
+  "attention_bias": false,
+  "attention_dropout": 0.0,
+  "bos_token_id": 100257,
+  "dtype": "bfloat16",
+  "eos_token_id": 100257,
+  "hidden_act": "silu",
+  "hidden_size": 4096,
+  "initializer_range": 0.02,
+  "intermediate_size": 11008,
+  "max_position_embeddings": 4096,
+  "model_type": "olmo2",
+  "num_attention_heads": 32,
+  "num_hidden_layers": 32,
+  "num_key_value_heads": 32,
+  "pad_token_id": 100277,
+  "rms_norm_eps": 1e-06,
+  "rope_scaling": null,
+  "rope_theta": 500000,
+  "tie_word_embeddings": false,
+  "transformers_version": "4.57.6",
+  "use_cache": false,
+  "vocab_size": 100352
+}