Add files using upload-large-folder tool

g1_0_baseline_original_code__edit/1000M/all_results.json

@@ -0,0 +1,8 @@
+{
+    "epoch": 0.9399613737509447,
+    "total_flos": 4.547514943786923e+19,
+    "train_loss": 0.004097137745947035,
+    "train_runtime": 612.2405,
+    "train_samples_per_second": 219.404,
+    "train_steps_per_second": 18.284
+}

g1_0_baseline_original_code__edit/1000M/chat_template.jinja

@@ -0,0 +1 @@
+{% if messages[0]['role'] == 'system' %}{% set loop_messages = messages[1:] %}{% set system_message = messages[0]['content'] %}{% else %}{% set loop_messages = messages %}{% endif %}{% if system_message is defined %}{{ system_message }}{% endif %}{% for message in loop_messages %}{% set content = message['content'] %}{% if message['role'] == 'user' %}{{ content }}{% elif message['role'] == 'assistant' %}{{ content + '<|endoftext|>' }}{% endif %}{% endfor %}

g1_0_baseline_original_code__edit/1000M/config.json

@@ -0,0 +1,27 @@
+{
+  "architectures": [
+    "Olmo2ForCausalLM"
+  ],
+  "attention_bias": false,
+  "attention_dropout": 0.0,
+  "bos_token_id": 100257,
+  "dtype": "bfloat16",
+  "eos_token_id": 100257,
+  "hidden_act": "silu",
+  "hidden_size": 4096,
+  "initializer_range": 0.02,
+  "intermediate_size": 11008,
+  "max_position_embeddings": 4096,
+  "model_type": "olmo2",
+  "num_attention_heads": 32,
+  "num_hidden_layers": 32,
+  "num_key_value_heads": 32,
+  "pad_token_id": 100277,
+  "rms_norm_eps": 1e-06,
+  "rope_scaling": null,
+  "rope_theta": 500000,
+  "tie_word_embeddings": false,
+  "transformers_version": "4.57.6",
+  "use_cache": false,
+  "vocab_size": 100352
+}

g1_0_baseline_original_code__edit/1000M/generation_config.json

@@ -0,0 +1,9 @@
+{
+  "_from_model_config": true,
+  "bos_token_id": 100257,
+  "eos_token_id": [
+    100257
+  ],
+  "pad_token_id": 100277,
+  "transformers_version": "4.57.6"
+}

g1_0_baseline_original_code__edit/1000M/model.safetensors.index.json

@@ -0,0 +1,363 @@
+{
+  "metadata": {
+    "total_parameters": 7298617344,
+    "total_size": 14597234688
+  },
+  "weight_map": {
+    "lm_head.weight": "model-00003-of-00003.safetensors",
+    "model.embed_tokens.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.post_feedforward_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.self_attn.k_norm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.self_attn.q_norm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.post_feedforward_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.self_attn.k_norm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.self_attn.q_norm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.10.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.10.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.10.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.10.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.10.post_feedforward_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.10.self_attn.k_norm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.10.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.10.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.10.self_attn.q_norm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.10.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.10.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.11.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.11.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.11.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.11.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.11.post_feedforward_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.11.self_attn.k_norm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.11.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.11.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.11.self_attn.q_norm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.11.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.11.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.post_feedforward_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.self_attn.k_norm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.self_attn.q_norm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.post_feedforward_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.self_attn.k_norm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.self_attn.q_norm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.post_feedforward_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.self_attn.k_norm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.self_attn.q_norm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.post_feedforward_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.self_attn.k_norm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.self_attn.q_norm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.post_feedforward_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.self_attn.k_norm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.self_attn.q_norm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.post_feedforward_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.self_attn.k_norm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.self_attn.q_norm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.post_feedforward_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.self_attn.k_norm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.self_attn.q_norm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.post_feedforward_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.self_attn.k_norm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.self_attn.q_norm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.2.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.2.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.2.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.2.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.2.post_feedforward_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.2.self_attn.k_norm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.2.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.2.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.2.self_attn.q_norm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.2.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.2.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.20.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.20.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.20.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.20.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.20.post_feedforward_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.20.self_attn.k_norm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.20.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.20.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.20.self_attn.q_norm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.20.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.20.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.post_feedforward_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.self_attn.k_norm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.self_attn.q_norm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.22.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.22.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.22.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.22.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.22.post_feedforward_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.22.self_attn.k_norm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.22.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.22.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.22.self_attn.q_norm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.22.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.22.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.23.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.23.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.23.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.23.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.23.post_feedforward_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.23.self_attn.k_norm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.23.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.23.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.23.self_attn.q_norm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.23.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.23.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.post_feedforward_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.self_attn.k_norm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.self_attn.q_norm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.post_feedforward_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.self_attn.k_norm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.self_attn.q_norm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.post_feedforward_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.self_attn.k_norm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.self_attn.q_norm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.post_feedforward_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.self_attn.k_norm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.self_attn.q_norm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.28.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.28.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.28.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.28.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.28.post_feedforward_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.28.self_attn.k_norm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.28.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.28.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.28.self_attn.q_norm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.28.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.28.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.29.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.29.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.29.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.29.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.29.post_feedforward_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.29.self_attn.k_norm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.29.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.29.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.29.self_attn.q_norm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.29.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.29.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.3.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.3.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.3.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.3.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.3.post_feedforward_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.3.self_attn.k_norm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.3.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.3.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.3.self_attn.q_norm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.3.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.3.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.30.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.30.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.30.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.30.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.30.post_feedforward_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.30.self_attn.k_norm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.30.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.30.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.30.self_attn.q_norm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.30.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.30.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.31.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.31.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.31.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.31.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.31.post_feedforward_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.31.self_attn.k_norm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.31.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.31.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.31.self_attn.q_norm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.31.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.31.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.4.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.4.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.4.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.4.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.4.post_feedforward_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.4.self_attn.k_norm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.4.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.4.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.4.self_attn.q_norm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.4.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.4.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.post_feedforward_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.self_attn.k_norm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.self_attn.q_norm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.post_feedforward_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.self_attn.k_norm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.self_attn.q_norm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.post_feedforward_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.self_attn.k_norm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.self_attn.q_norm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.post_feedforward_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.self_attn.k_norm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.self_attn.q_norm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.post_feedforward_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.self_attn.k_norm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.self_attn.q_norm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.norm.weight": "model-00003-of-00003.safetensors"
+  }
+}

g1_0_baseline_original_code__edit/1000M/special_tokens_map.json

@@ -0,0 +1,30 @@
+{
+  "bos_token": {
+    "content": "<|endoftext|>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "eos_token": {
+    "content": "<|endoftext|>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "pad_token": {
+    "content": "<|pad|>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "unk_token": {
+    "content": "<|endoftext|>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  }
+}

g1_0_baseline_original_code__edit/1000M/tokenizer_config.json

@@ -0,0 +1,191 @@
+{
+  "add_prefix_space": false,
+  "added_tokens_decoder": {
+    "100256": {
+      "content": "<|extra_id_0|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "100257": {
+      "content": "<|endoftext|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "100258": {
+      "content": "<|fim_prefix|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "100259": {
+      "content": "<|fim_middle|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "100260": {
+      "content": "<|fim_suffix|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "100261": {
+      "content": "|||PHONE_NUMBER|||",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "100262": {
+      "content": "|||EMAIL_ADDRESS|||",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "100263": {
+      "content": "|||IP_ADDRESS|||",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "100264": {
+      "content": "<|im_start|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "100265": {
+      "content": "<|im_end|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "100266": {
+      "content": "<|extra_id_1|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "100267": {
+      "content": "<|extra_id_2|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "100268": {
+      "content": "<|extra_id_3|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "100269": {
+      "content": "<|extra_id_4|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "100270": {
+      "content": "<|extra_id_5|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "100271": {
+      "content": "<|extra_id_6|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "100272": {
+      "content": "<|extra_id_7|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "100273": {
+      "content": "<|extra_id_8|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "100274": {
+      "content": "<|extra_id_9|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "100275": {
+      "content": "<|extra_id_10|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "100276": {
+      "content": "<|endofprompt|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "100277": {
+      "content": "<|pad|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    }
+  },
+  "bos_token": "<|endoftext|>",
+  "clean_up_tokenization_spaces": false,
+  "eos_token": "<|endoftext|>",
+  "extra_special_tokens": {},
+  "model_max_length": 1000000000000000019884624838656,
+  "pad_token": "<|pad|>",
+  "padding_side": "right",
+  "split_special_tokens": false,
+  "tokenizer_class": "GPT2Tokenizer",
+  "unk_token": "<|endoftext|>"
+}

g1_0_baseline_original_code__edit/1000M/train_results.json

@@ -0,0 +1,8 @@
+{
+    "epoch": 0.9399613737509447,
+    "total_flos": 4.547514943786923e+19,
+    "train_loss": 0.004097137745947035,
+    "train_runtime": 612.2405,
+    "train_samples_per_second": 219.404,
+    "train_steps_per_second": 18.284
+}

g1_0_baseline_original_code__edit/1000M/trainer_log.jsonl

@@ -0,0 +1,110 @@
+{"current_steps": 10180, "total_steps": 11194, "loss": 0.7284, "lr": 1.999957358086795e-05, "epoch": 0.8548156856159207, "percentage": 90.94, "elapsed_time": "0:00:39", "remaining_time": "0:00:03"}
+{"current_steps": 10190, "total_steps": 11194, "loss": 0.7559, "lr": 1.9991993813020483e-05, "epoch": 0.8556553866823411, "percentage": 91.03, "elapsed_time": "0:02:08", "remaining_time": "0:00:12"}
+{"current_steps": 10200, "total_steps": 11194, "loss": 0.7478, "lr": 1.9974946373028628e-05, "epoch": 0.8564950877487615, "percentage": 91.12, "elapsed_time": "0:03:38", "remaining_time": "0:00:21"}
+{"current_steps": 10210, "total_steps": 11194, "loss": 0.7396, "lr": 1.994844749501764e-05, "epoch": 0.8573347888151818, "percentage": 91.21, "elapsed_time": "0:05:07", "remaining_time": "0:00:29"}
+{"current_steps": 10220, "total_steps": 11194, "loss": 0.7583, "lr": 1.9912522413632988e-05, "epoch": 0.8581744898816022, "percentage": 91.3, "elapsed_time": "0:06:36", "remaining_time": "0:00:37"}
+{"current_steps": 10230, "total_steps": 11194, "loss": 0.7252, "lr": 1.9867205340009656e-05, "epoch": 0.8590141909480224, "percentage": 91.39, "elapsed_time": "0:08:06", "remaining_time": "0:00:45"}
+{"current_steps": 10240, "total_steps": 11194, "loss": 0.7013, "lr": 1.9812539429193164e-05, "epoch": 0.8598538920144428, "percentage": 91.48, "elapsed_time": "0:09:35", "remaining_time": "0:00:53"}
+{"current_steps": 10250, "total_steps": 11194, "loss": 0.7613, "lr": 1.9748576739043423e-05, "epoch": 0.8606935930808632, "percentage": 91.57, "elapsed_time": "0:11:04", "remaining_time": "0:01:01"}
+{"current_steps": 10260, "total_steps": 11194, "loss": 0.7591, "lr": 1.967537818066047e-05, "epoch": 0.8615332941472835, "percentage": 91.66, "elapsed_time": "0:12:34", "remaining_time": "0:01:08"}
+{"current_steps": 10270, "total_steps": 11194, "loss": 0.7604, "lr": 1.959301346037941e-05, "epoch": 0.8623729952137039, "percentage": 91.75, "elapsed_time": "0:14:03", "remaining_time": "0:01:15"}
+{"current_steps": 10280, "total_steps": 11194, "loss": 0.6827, "lr": 1.9501561013389646e-05, "epoch": 0.8632126962801243, "percentage": 91.83, "elapsed_time": "0:15:33", "remaining_time": "0:01:22"}
+{"current_steps": 10290, "total_steps": 11194, "loss": 0.6854, "lr": 1.9401107929041795e-05, "epoch": 0.8640523973465446, "percentage": 91.92, "elapsed_time": "0:17:02", "remaining_time": "0:01:29"}
+{"current_steps": 10300, "total_steps": 11194, "loss": 0.7629, "lr": 1.9291749867913225e-05, "epoch": 0.864892098412965, "percentage": 92.01, "elapsed_time": "0:18:31", "remaining_time": "0:01:36"}
+{"current_steps": 10310, "total_steps": 11194, "loss": 0.7661, "lr": 1.91735909707114e-05, "epoch": 0.8657317994793854, "percentage": 92.1, "elapsed_time": "0:20:01", "remaining_time": "0:01:42"}
+{"current_steps": 10320, "total_steps": 11194, "loss": 0.7883, "lr": 1.9046743759101545e-05, "epoch": 0.8665715005458057, "percentage": 92.19, "elapsed_time": "0:21:30", "remaining_time": "0:01:49"}
+{"current_steps": 10330, "total_steps": 11194, "loss": 0.702, "lr": 1.8911329028553324e-05, "epoch": 0.8674112016122261, "percentage": 92.28, "elapsed_time": "0:22:59", "remaining_time": "0:01:55"}
+{"current_steps": 10340, "total_steps": 11194, "loss": 0.7278, "lr": 1.876747573330835e-05, "epoch": 0.8682509026786464, "percentage": 92.37, "elapsed_time": "0:25:07", "remaining_time": "0:02:04"}
+{"current_steps": 10350, "total_steps": 11194, "loss": 0.6744, "lr": 1.8615320863578244e-05, "epoch": 0.8690906037450667, "percentage": 92.46, "elapsed_time": "0:26:37", "remaining_time": "0:02:10"}
+{"current_steps": 10360, "total_steps": 11194, "loss": 0.6707, "lr": 1.8455009315090036e-05, "epoch": 0.8699303048114871, "percentage": 92.55, "elapsed_time": "0:28:06", "remaining_time": "0:02:15"}
+{"current_steps": 10370, "total_steps": 11194, "loss": 0.75, "lr": 1.8286693751103294e-05, "epoch": 0.8707700058779074, "percentage": 92.64, "elapsed_time": "0:29:35", "remaining_time": "0:02:21"}
+{"current_steps": 10380, "total_steps": 11194, "loss": 0.7231, "lr": 1.811053445703024e-05, "epoch": 0.8716097069443278, "percentage": 92.73, "elapsed_time": "0:31:05", "remaining_time": "0:02:26"}
+{"current_steps": 10390, "total_steps": 11194, "loss": 0.7458, "lr": 1.7926699187797375e-05, "epoch": 0.8724494080107482, "percentage": 92.82, "elapsed_time": "0:32:34", "remaining_time": "0:02:31"}
+{"current_steps": 10400, "total_steps": 11194, "loss": 0.7509, "lr": 1.7735363008093995e-05, "epoch": 0.8732891090771685, "percentage": 92.91, "elapsed_time": "0:34:03", "remaining_time": "0:02:36"}
+{"current_steps": 10410, "total_steps": 11194, "loss": 0.7645, "lr": 1.7536708125659577e-05, "epoch": 0.8741288101435889, "percentage": 93.0, "elapsed_time": "0:35:33", "remaining_time": "0:02:40"}
+{"current_steps": 10420, "total_steps": 11194, "loss": 0.7743, "lr": 1.733092371776905e-05, "epoch": 0.8749685112100093, "percentage": 93.09, "elapsed_time": "0:37:02", "remaining_time": "0:02:45"}
+{"current_steps": 10430, "total_steps": 11194, "loss": 0.7355, "lr": 1.7118205751080915e-05, "epoch": 0.8758082122764296, "percentage": 93.17, "elapsed_time": "0:38:32", "remaining_time": "0:02:49"}
+{"current_steps": 10440, "total_steps": 11194, "loss": 0.7349, "lr": 1.6898756795019935e-05, "epoch": 0.87664791334285, "percentage": 93.26, "elapsed_time": "0:40:01", "remaining_time": "0:02:53"}
+{"current_steps": 10450, "total_steps": 11194, "loss": 0.7531, "lr": 1.667278582887211e-05, "epoch": 0.8774876144092703, "percentage": 93.35, "elapsed_time": "0:41:30", "remaining_time": "0:02:57"}
+{"current_steps": 10460, "total_steps": 11194, "loss": 0.7168, "lr": 1.6440508042775545e-05, "epoch": 0.8783273154756907, "percentage": 93.44, "elapsed_time": "0:43:00", "remaining_time": "0:03:01"}
+{"current_steps": 10470, "total_steps": 11194, "loss": 0.6811, "lr": 1.6202144632796796e-05, "epoch": 0.879167016542111, "percentage": 93.53, "elapsed_time": "0:44:29", "remaining_time": "0:03:04"}
+{"current_steps": 10480, "total_steps": 11194, "loss": 0.7511, "lr": 1.595792259028783e-05, "epoch": 0.8800067176085313, "percentage": 93.62, "elapsed_time": "0:45:58", "remaining_time": "0:03:07"}
+{"current_steps": 10490, "total_steps": 11194, "loss": 0.6897, "lr": 1.570807448572419e-05, "epoch": 0.8808464186749517, "percentage": 93.71, "elapsed_time": "0:47:28", "remaining_time": "0:03:11"}
+{"current_steps": 10500, "total_steps": 11194, "loss": 0.6794, "lr": 1.5452838247230197e-05, "epoch": 0.8816861197413721, "percentage": 93.8, "elapsed_time": "0:49:35", "remaining_time": "0:03:16"}
+{"current_steps": 10510, "total_steps": 11194, "loss": 0.773, "lr": 1.5192456934002106e-05, "epoch": 0.8825258208077924, "percentage": 93.89, "elapsed_time": "0:51:04", "remaining_time": "0:03:19"}
+{"current_steps": 10520, "total_steps": 11194, "loss": 0.7436, "lr": 1.492717850484502e-05, "epoch": 0.8833655218742128, "percentage": 93.98, "elapsed_time": "0:52:33", "remaining_time": "0:03:22"}
+{"current_steps": 10500, "total_steps": 11194, "loss": 0.6718, "lr": 1.5452838247230197e-05, "epoch": 0.8816861197413721, "percentage": 93.8, "elapsed_time": "0:00:55", "remaining_time": "0:00:03"}
+{"current_steps": 10510, "total_steps": 11194, "loss": 0.773, "lr": 1.5192456934002106e-05, "epoch": 0.8825258208077924, "percentage": 93.89, "elapsed_time": "0:02:24", "remaining_time": "0:00:09"}
+{"current_steps": 10520, "total_steps": 11194, "loss": 0.7436, "lr": 1.492717850484502e-05, "epoch": 0.8833655218742128, "percentage": 93.98, "elapsed_time": "0:03:53", "remaining_time": "0:00:14"}
+{"current_steps": 10530, "total_steps": 11194, "loss": 0.7151, "lr": 1.4657255582043904e-05, "epoch": 0.8842052229406331, "percentage": 94.07, "elapsed_time": "0:05:21", "remaining_time": "0:00:20"}
+{"current_steps": 10540, "total_steps": 11194, "loss": 0.7645, "lr": 1.4382945210793663e-05, "epoch": 0.8850449240070535, "percentage": 94.16, "elapsed_time": "0:06:50", "remaining_time": "0:00:25"}
+{"current_steps": 10550, "total_steps": 11194, "loss": 0.7145, "lr": 1.4104508614417267e-05, "epoch": 0.8858846250734739, "percentage": 94.25, "elapsed_time": "0:08:19", "remaining_time": "0:00:30"}
+{"current_steps": 10560, "total_steps": 11194, "loss": 0.7155, "lr": 1.382221094560512e-05, "epoch": 0.8867243261398942, "percentage": 94.34, "elapsed_time": "0:09:47", "remaining_time": "0:00:35"}
+{"current_steps": 10570, "total_steps": 11194, "loss": 0.6733, "lr": 1.3536321033912506e-05, "epoch": 0.8875640272063146, "percentage": 94.43, "elapsed_time": "0:11:16", "remaining_time": "0:00:39"}
+{"current_steps": 10580, "total_steps": 11194, "loss": 0.7246, "lr": 1.3247111129755609e-05, "epoch": 0.888403728272735, "percentage": 94.51, "elapsed_time": "0:12:45", "remaining_time": "0:00:44"}
+{"current_steps": 10590, "total_steps": 11194, "loss": 0.696, "lr": 1.2954856645149868e-05, "epoch": 0.8892434293391552, "percentage": 94.6, "elapsed_time": "0:14:14", "remaining_time": "0:00:48"}
+{"current_steps": 10600, "total_steps": 11194, "loss": 0.7253, "lr": 1.2659835891437553e-05, "epoch": 0.8900831304055756, "percentage": 94.69, "elapsed_time": "0:15:42", "remaining_time": "0:00:52"}
+{"current_steps": 10610, "total_steps": 11194, "loss": 0.7176, "lr": 1.236232981425436e-05, "epoch": 0.890922831471996, "percentage": 94.78, "elapsed_time": "0:17:11", "remaining_time": "0:00:56"}
+{"current_steps": 10620, "total_steps": 11194, "loss": 0.7417, "lr": 1.2062621725987402e-05, "epoch": 0.8917625325384163, "percentage": 94.87, "elapsed_time": "0:18:40", "remaining_time": "0:01:00"}
+{"current_steps": 10630, "total_steps": 11194, "loss": 0.7464, "lr": 1.1760997035979318e-05, "epoch": 0.8926022336048367, "percentage": 94.96, "elapsed_time": "0:20:09", "remaining_time": "0:01:04"}
+{"current_steps": 10640, "total_steps": 11194, "loss": 0.6794, "lr": 1.1457742978735514e-05, "epoch": 0.893441934671257, "percentage": 95.05, "elapsed_time": "0:21:37", "remaining_time": "0:01:07"}
+{"current_steps": 10650, "total_steps": 11194, "loss": 0.7094, "lr": 1.1153148340393298e-05, "epoch": 0.8942816357376774, "percentage": 95.14, "elapsed_time": "0:23:06", "remaining_time": "0:01:10"}
+{"current_steps": 10660, "total_steps": 11194, "loss": 0.7117, "lr": 1.0847503183713424e-05, "epoch": 0.8951213368040978, "percentage": 95.23, "elapsed_time": "0:25:12", "remaining_time": "0:01:15"}
+{"current_steps": 10670, "total_steps": 11194, "loss": 0.7438, "lr": 1.0541098571855896e-05, "epoch": 0.8959610378705181, "percentage": 95.32, "elapsed_time": "0:26:41", "remaining_time": "0:01:18"}
+{"current_steps": 10680, "total_steps": 11194, "loss": 0.718, "lr": 1.0234226291203123e-05, "epoch": 0.8968007389369385, "percentage": 95.41, "elapsed_time": "0:28:09", "remaining_time": "0:01:21"}
+{"current_steps": 10690, "total_steps": 11194, "loss": 0.7187, "lr": 9.927178573494333e-06, "epoch": 0.8976404400033589, "percentage": 95.5, "elapsed_time": "0:29:38", "remaining_time": "0:01:23"}
+{"current_steps": 10700, "total_steps": 11194, "loss": 0.7045, "lr": 9.620247817535902e-06, "epoch": 0.8984801410697791, "percentage": 95.59, "elapsed_time": "0:31:07", "remaining_time": "0:01:26"}
+{"current_steps": 10710, "total_steps": 11194, "loss": 0.7105, "lr": 9.313726310752581e-06, "epoch": 0.8993198421361995, "percentage": 95.68, "elapsed_time": "0:32:36", "remaining_time": "0:01:28"}
+{"current_steps": 10720, "total_steps": 11194, "loss": 0.6803, "lr": 9.00790595084477e-06, "epoch": 0.9001595432026198, "percentage": 95.77, "elapsed_time": "0:34:04", "remaining_time": "0:01:30"}
+{"current_steps": 10730, "total_steps": 11194, "loss": 0.7057, "lr": 8.703077967816944e-06, "epoch": 0.9009992442690402, "percentage": 95.85, "elapsed_time": "0:35:33", "remaining_time": "0:01:32"}
+{"current_steps": 10740, "total_steps": 11194, "loss": 0.7233, "lr": 8.39953264664192e-06, "epoch": 0.9018389453354606, "percentage": 95.94, "elapsed_time": "0:37:02", "remaining_time": "0:01:33"}
+{"current_steps": 10750, "total_steps": 11194, "loss": 0.7084, "lr": 8.097559050825058e-06, "epoch": 0.9026786464018809, "percentage": 96.03, "elapsed_time": "0:38:30", "remaining_time": "0:01:35"}
+{"current_steps": 10760, "total_steps": 11194, "loss": 0.6907, "lr": 7.79744474713166e-06, "epoch": 0.9035183474683013, "percentage": 96.12, "elapsed_time": "0:39:59", "remaining_time": "0:01:36"}
+{"current_steps": 10770, "total_steps": 11194, "loss": 0.7159, "lr": 7.499475531739696e-06, "epoch": 0.9043580485347217, "percentage": 96.21, "elapsed_time": "0:41:28", "remaining_time": "0:01:37"}
+{"current_steps": 10780, "total_steps": 11194, "loss": 0.7161, "lr": 7.203935158078662e-06, "epoch": 0.905197749601142, "percentage": 96.3, "elapsed_time": "0:42:56", "remaining_time": "0:01:38"}
+{"current_steps": 10790, "total_steps": 11194, "loss": 0.6993, "lr": 6.911105066613709e-06, "epoch": 0.9060374506675624, "percentage": 96.39, "elapsed_time": "0:44:25", "remaining_time": "0:01:39"}
+{"current_steps": 10800, "total_steps": 11194, "loss": 0.7098, "lr": 6.621264116832394e-06, "epoch": 0.9068771517339828, "percentage": 96.48, "elapsed_time": "0:45:54", "remaining_time": "0:01:40"}
+{"current_steps": 10810, "total_steps": 11194, "loss": 0.6805, "lr": 6.334688321689274e-06, "epoch": 0.907716852800403, "percentage": 96.57, "elapsed_time": "0:47:22", "remaining_time": "0:01:40"}
+{"current_steps": 10820, "total_steps": 11194, "loss": 0.7564, "lr": 6.051650584761237e-06, "epoch": 0.9085565538668234, "percentage": 96.66, "elapsed_time": "0:49:28", "remaining_time": "0:01:42"}
+{"current_steps": 10830, "total_steps": 11194, "loss": 0.6956, "lr": 5.772420440363841e-06, "epoch": 0.9093962549332437, "percentage": 96.75, "elapsed_time": "0:50:57", "remaining_time": "0:01:42"}
+{"current_steps": 10820, "total_steps": 11194, "loss": 0.6876, "lr": 6.051650584761237e-06, "epoch": 0.9085565538668234, "percentage": 96.66, "elapsed_time": "0:01:13", "remaining_time": "0:00:02"}
+{"current_steps": 10830, "total_steps": 11194, "loss": 0.6956, "lr": 5.772420440363841e-06, "epoch": 0.9093962549332437, "percentage": 96.75, "elapsed_time": "0:02:42", "remaining_time": "0:00:05"}
+{"current_steps": 10840, "total_steps": 11194, "loss": 0.6904, "lr": 5.497263796876207e-06, "epoch": 0.9102359559996641, "percentage": 96.84, "elapsed_time": "0:04:11", "remaining_time": "0:00:08"}
+{"current_steps": 10850, "total_steps": 11194, "loss": 0.6077, "lr": 5.226442683518813e-06, "epoch": 0.9110756570660845, "percentage": 96.93, "elapsed_time": "0:05:40", "remaining_time": "0:00:10"}
+{"current_steps": 10860, "total_steps": 11194, "loss": 0.78, "lr": 4.960215000825432e-06, "epoch": 0.9119153581325048, "percentage": 97.02, "elapsed_time": "0:07:09", "remaining_time": "0:00:13"}
+{"current_steps": 10870, "total_steps": 11194, "loss": 0.7262, "lr": 4.698834275046711e-06, "epoch": 0.9127550591989252, "percentage": 97.11, "elapsed_time": "0:08:38", "remaining_time": "0:00:15"}
+{"current_steps": 10880, "total_steps": 11194, "loss": 0.7429, "lr": 4.442549416719423e-06, "epoch": 0.9135947602653456, "percentage": 97.19, "elapsed_time": "0:10:07", "remaining_time": "0:00:17"}
+{"current_steps": 10890, "total_steps": 11194, "loss": 0.6897, "lr": 4.1916044836311565e-06, "epoch": 0.9144344613317659, "percentage": 97.28, "elapsed_time": "0:11:36", "remaining_time": "0:00:19"}
+{"current_steps": 10900, "total_steps": 11194, "loss": 0.6589, "lr": 3.94623844840626e-06, "epoch": 0.9152741623981863, "percentage": 97.37, "elapsed_time": "0:13:05", "remaining_time": "0:00:21"}
+{"current_steps": 10910, "total_steps": 11194, "loss": 0.7797, "lr": 3.7066849709343467e-06, "epoch": 0.9161138634646065, "percentage": 97.46, "elapsed_time": "0:14:34", "remaining_time": "0:00:22"}
+{"current_steps": 10920, "total_steps": 11194, "loss": 0.6941, "lr": 3.473172175858059e-06, "epoch": 0.9169535645310269, "percentage": 97.55, "elapsed_time": "0:16:04", "remaining_time": "0:00:24"}
+{"current_steps": 10930, "total_steps": 11194, "loss": 0.7145, "lr": 3.2459224353320046e-06, "epoch": 0.9177932655974473, "percentage": 97.64, "elapsed_time": "0:17:33", "remaining_time": "0:00:25"}
+{"current_steps": 10940, "total_steps": 11194, "loss": 0.7135, "lr": 3.025152157259735e-06, "epoch": 0.9186329666638676, "percentage": 97.73, "elapsed_time": "0:19:02", "remaining_time": "0:00:26"}
+{"current_steps": 10950, "total_steps": 11194, "loss": 0.7507, "lr": 2.811071579210425e-06, "epoch": 0.919472667730288, "percentage": 97.82, "elapsed_time": "0:20:31", "remaining_time": "0:00:27"}
+{"current_steps": 10960, "total_steps": 11194, "loss": 0.7501, "lr": 2.603884568211513e-06, "epoch": 0.9203123687967084, "percentage": 97.91, "elapsed_time": "0:22:00", "remaining_time": "0:00:28"}
+{"current_steps": 10970, "total_steps": 11194, "loss": 0.6804, "lr": 2.4037884266079434e-06, "epoch": 0.9211520698631287, "percentage": 98.0, "elapsed_time": "0:23:29", "remaining_time": "0:00:28"}
+{"current_steps": 10980, "total_steps": 11194, "loss": 0.7234, "lr": 2.2109737041728955e-06, "epoch": 0.9219917709295491, "percentage": 98.09, "elapsed_time": "0:25:36", "remaining_time": "0:00:29"}
+{"current_steps": 10990, "total_steps": 11194, "loss": 0.634, "lr": 2.0256240166489454e-06, "epoch": 0.9228314719959695, "percentage": 98.18, "elapsed_time": "0:27:05", "remaining_time": "0:00:30"}
+{"current_steps": 11000, "total_steps": 11194, "loss": 0.7315, "lr": 1.847915870892413e-06, "epoch": 0.9236711730623898, "percentage": 98.27, "elapsed_time": "0:28:34", "remaining_time": "0:00:30"}
+{"current_steps": 11010, "total_steps": 11194, "loss": 0.665, "lr": 1.6780184967874715e-06, "epoch": 0.9245108741288102, "percentage": 98.36, "elapsed_time": "0:30:03", "remaining_time": "0:00:30"}
+{"current_steps": 11020, "total_steps": 11194, "loss": 0.7135, "lr": 1.5160936860900294e-06, "epoch": 0.9253505751952305, "percentage": 98.45, "elapsed_time": "0:31:32", "remaining_time": "0:00:29"}
+{"current_steps": 11030, "total_steps": 11194, "loss": 0.6915, "lr": 1.3622956383548762e-06, "epoch": 0.9261902762616508, "percentage": 98.53, "elapsed_time": "0:33:01", "remaining_time": "0:00:29"}
+{"current_steps": 11040, "total_steps": 11194, "loss": 0.7423, "lr": 1.216770814092834e-06, "epoch": 0.9270299773280712, "percentage": 98.62, "elapsed_time": "0:34:30", "remaining_time": "0:00:28"}
+{"current_steps": 11050, "total_steps": 11194, "loss": 0.6938, "lr": 1.0796577952977013e-06, "epoch": 0.9278696783944915, "percentage": 98.71, "elapsed_time": "0:35:59", "remaining_time": "0:00:28"}
+{"current_steps": 11060, "total_steps": 11194, "loss": 0.7208, "lr": 9.510871534758705e-07, "epoch": 0.9287093794609119, "percentage": 98.8, "elapsed_time": "0:37:28", "remaining_time": "0:00:27"}
+{"current_steps": 11070, "total_steps": 11194, "loss": 0.7655, "lr": 8.311813253042334e-07, "epoch": 0.9295490805273323, "percentage": 98.89, "elapsed_time": "0:38:57", "remaining_time": "0:00:26"}
+{"current_steps": 11080, "total_steps": 11194, "loss": 0.6588, "lr": 7.200544960348308e-07, "epoch": 0.9303887815937526, "percentage": 98.98, "elapsed_time": "0:40:26", "remaining_time": "0:00:24"}
+{"current_steps": 11090, "total_steps": 11194, "loss": 0.6495, "lr": 6.178124907572545e-07, "epoch": 0.931228482660173, "percentage": 99.07, "elapsed_time": "0:41:55", "remaining_time": "0:00:23"}
+{"current_steps": 11100, "total_steps": 11194, "loss": 0.6966, "lr": 5.245526736223528e-07, "epoch": 0.9320681837265933, "percentage": 99.16, "elapsed_time": "0:43:24", "remaining_time": "0:00:22"}
+{"current_steps": 11110, "total_steps": 11194, "loss": 0.7007, "lr": 4.403638551232205e-07, "epoch": 0.9329078847930137, "percentage": 99.25, "elapsed_time": "0:44:53", "remaining_time": "0:00:20"}
+{"current_steps": 11120, "total_steps": 11194, "loss": 0.6614, "lr": 3.6532620752175616e-07, "epoch": 0.9337475858594341, "percentage": 99.34, "elapsed_time": "0:46:22", "remaining_time": "0:00:18"}
+{"current_steps": 11130, "total_steps": 11194, "loss": 0.7339, "lr": 2.9951118850133247e-07, "epoch": 0.9345872869258544, "percentage": 99.43, "elapsed_time": "0:47:52", "remaining_time": "0:00:16"}
+{"current_steps": 11140, "total_steps": 11194, "loss": 0.6504, "lr": 2.429814731182774e-07, "epoch": 0.9354269879922747, "percentage": 99.52, "elapsed_time": "0:49:58", "remaining_time": "0:00:14"}
+{"current_steps": 11150, "total_steps": 11194, "loss": 0.6877, "lr": 1.957908941169813e-07, "epoch": 0.9362666890586951, "percentage": 99.61, "elapsed_time": "0:51:27", "remaining_time": "0:00:12"}
+{"current_steps": 11140, "total_steps": 11194, "loss": 0.6504, "lr": 2.429814731182774e-07, "epoch": 0.9354269879922747, "percentage": 99.52, "elapsed_time": "0:01:32", "remaining_time": "0:00:00"}
+{"current_steps": 11150, "total_steps": 11194, "loss": 0.6877, "lr": 1.957908941169813e-07, "epoch": 0.9362666890586951, "percentage": 99.61, "elapsed_time": "0:03:01", "remaining_time": "0:00:00"}
+{"current_steps": 11160, "total_steps": 11194, "loss": 0.6981, "lr": 1.5798439066545254e-07, "epoch": 0.9371063901251154, "percentage": 99.7, "elapsed_time": "0:04:31", "remaining_time": "0:00:00"}
+{"current_steps": 11170, "total_steps": 11194, "loss": 0.7283, "lr": 1.2959796556014471e-07, "epoch": 0.9379460911915358, "percentage": 99.79, "elapsed_time": "0:06:00", "remaining_time": "0:00:00"}
+{"current_steps": 11180, "total_steps": 11194, "loss": 0.7261, "lr": 1.1065865094081884e-07, "epoch": 0.9387857922579562, "percentage": 99.87, "elapsed_time": "0:07:29", "remaining_time": "0:00:00"}
+{"current_steps": 11190, "total_steps": 11194, "loss": 0.6849, "lr": 1.011844825480734e-07, "epoch": 0.9396254933243765, "percentage": 99.96, "elapsed_time": "0:08:58", "remaining_time": "0:00:00"}
+{"current_steps": 11194, "total_steps": 11194, "epoch": 0.9399613737509447, "percentage": 100.0, "elapsed_time": "0:10:12", "remaining_time": "0:00:00"}

g1_0_baseline_original_code__edit/250M/all_results.json

@@ -0,0 +1,8 @@
+{
+    "epoch": 0.23494835838441513,
+    "total_flos": 1.1366756130249245e+19,
+    "train_loss": 0.08313362047960282,
+    "train_runtime": 2289.8517,
+    "train_samples_per_second": 14.663,
+    "train_steps_per_second": 1.222
+}

g1_0_baseline_original_code__edit/250M/chat_template.jinja

@@ -0,0 +1 @@
+{% if messages[0]['role'] == 'system' %}{% set loop_messages = messages[1:] %}{% set system_message = messages[0]['content'] %}{% else %}{% set loop_messages = messages %}{% endif %}{% if system_message is defined %}{{ system_message }}{% endif %}{% for message in loop_messages %}{% set content = message['content'] %}{% if message['role'] == 'user' %}{{ content }}{% elif message['role'] == 'assistant' %}{{ content + '<|endoftext|>' }}{% endif %}{% endfor %}

g1_0_baseline_original_code__edit/250M/config.json

@@ -0,0 +1,27 @@
+{
+  "architectures": [
+    "Olmo2ForCausalLM"
+  ],
+  "attention_bias": false,
+  "attention_dropout": 0.0,
+  "bos_token_id": 100257,
+  "dtype": "bfloat16",
+  "eos_token_id": 100257,
+  "hidden_act": "silu",
+  "hidden_size": 4096,
+  "initializer_range": 0.02,
+  "intermediate_size": 11008,
+  "max_position_embeddings": 4096,
+  "model_type": "olmo2",
+  "num_attention_heads": 32,
+  "num_hidden_layers": 32,
+  "num_key_value_heads": 32,
+  "pad_token_id": 100277,
+  "rms_norm_eps": 1e-06,
+  "rope_scaling": null,
+  "rope_theta": 500000,
+  "tie_word_embeddings": false,
+  "transformers_version": "4.57.6",
+  "use_cache": false,
+  "vocab_size": 100352
+}

g1_0_baseline_original_code__edit/250M/generation_config.json

@@ -0,0 +1,9 @@
+{
+  "_from_model_config": true,
+  "bos_token_id": 100257,
+  "eos_token_id": [
+    100257
+  ],
+  "pad_token_id": 100277,
+  "transformers_version": "4.57.6"
+}

g1_0_baseline_original_code__edit/250M/model.safetensors.index.json

@@ -0,0 +1,363 @@
+{
+  "metadata": {
+    "total_parameters": 7298617344,
+    "total_size": 14597234688
+  },
+  "weight_map": {
+    "lm_head.weight": "model-00003-of-00003.safetensors",
+    "model.embed_tokens.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.post_feedforward_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.self_attn.k_norm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.self_attn.q_norm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.post_feedforward_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.self_attn.k_norm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.self_attn.q_norm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.10.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.10.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.10.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.10.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.10.post_feedforward_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.10.self_attn.k_norm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.10.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.10.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.10.self_attn.q_norm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.10.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.10.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.11.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.11.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.11.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.11.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.11.post_feedforward_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.11.self_attn.k_norm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.11.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.11.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.11.self_attn.q_norm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.11.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.11.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.post_feedforward_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.self_attn.k_norm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.self_attn.q_norm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.post_feedforward_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.self_attn.k_norm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.self_attn.q_norm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.post_feedforward_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.self_attn.k_norm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.self_attn.q_norm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.post_feedforward_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.self_attn.k_norm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.self_attn.q_norm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.post_feedforward_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.self_attn.k_norm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.self_attn.q_norm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.post_feedforward_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.self_attn.k_norm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.self_attn.q_norm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.post_feedforward_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.self_attn.k_norm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.self_attn.q_norm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.post_feedforward_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.self_attn.k_norm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.self_attn.q_norm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.2.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.2.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.2.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.2.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.2.post_feedforward_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.2.self_attn.k_norm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.2.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.2.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.2.self_attn.q_norm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.2.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.2.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.20.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.20.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.20.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.20.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.20.post_feedforward_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.20.self_attn.k_norm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.20.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.20.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.20.self_attn.q_norm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.20.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.20.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.post_feedforward_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.self_attn.k_norm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.self_attn.q_norm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.22.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.22.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.22.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.22.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.22.post_feedforward_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.22.self_attn.k_norm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.22.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.22.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.22.self_attn.q_norm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.22.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.22.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.23.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.23.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.23.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.23.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.23.post_feedforward_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.23.self_attn.k_norm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.23.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.23.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.23.self_attn.q_norm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.23.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.23.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.post_feedforward_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.self_attn.k_norm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.self_attn.q_norm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.post_feedforward_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.self_attn.k_norm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.self_attn.q_norm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.post_feedforward_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.self_attn.k_norm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.self_attn.q_norm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.post_feedforward_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.self_attn.k_norm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.self_attn.q_norm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.28.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.28.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.28.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.28.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.28.post_feedforward_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.28.self_attn.k_norm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.28.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.28.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.28.self_attn.q_norm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.28.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.28.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.29.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.29.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.29.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.29.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.29.post_feedforward_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.29.self_attn.k_norm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.29.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.29.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.29.self_attn.q_norm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.29.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.29.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.3.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.3.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.3.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.3.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.3.post_feedforward_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.3.self_attn.k_norm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.3.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.3.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.3.self_attn.q_norm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.3.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.3.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.30.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.30.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.30.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.30.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.30.post_feedforward_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.30.self_attn.k_norm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.30.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.30.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.30.self_attn.q_norm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.30.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.30.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.31.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.31.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.31.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.31.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.31.post_feedforward_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.31.self_attn.k_norm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.31.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.31.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.31.self_attn.q_norm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.31.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.31.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.4.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.4.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.4.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.4.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.4.post_feedforward_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.4.self_attn.k_norm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.4.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.4.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.4.self_attn.q_norm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.4.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.4.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.post_feedforward_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.self_attn.k_norm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.self_attn.q_norm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.post_feedforward_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.self_attn.k_norm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.self_attn.q_norm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.post_feedforward_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.self_attn.k_norm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.self_attn.q_norm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.post_feedforward_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.self_attn.k_norm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.self_attn.q_norm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.post_feedforward_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.self_attn.k_norm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.self_attn.q_norm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.norm.weight": "model-00003-of-00003.safetensors"
+  }
+}

g1_0_baseline_original_code__edit/250M/special_tokens_map.json

@@ -0,0 +1,30 @@
+{
+  "bos_token": {
+    "content": "<|endoftext|>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "eos_token": {
+    "content": "<|endoftext|>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "pad_token": {
+    "content": "<|pad|>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "unk_token": {
+    "content": "<|endoftext|>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  }
+}

g1_0_baseline_original_code__edit/250M/tokenizer_config.json

@@ -0,0 +1,191 @@
+{
+  "add_prefix_space": false,
+  "added_tokens_decoder": {
+    "100256": {
+      "content": "<|extra_id_0|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "100257": {
+      "content": "<|endoftext|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "100258": {
+      "content": "<|fim_prefix|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "100259": {
+      "content": "<|fim_middle|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "100260": {
+      "content": "<|fim_suffix|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "100261": {
+      "content": "|||PHONE_NUMBER|||",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "100262": {
+      "content": "|||EMAIL_ADDRESS|||",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "100263": {
+      "content": "|||IP_ADDRESS|||",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "100264": {
+      "content": "<|im_start|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "100265": {
+      "content": "<|im_end|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "100266": {
+      "content": "<|extra_id_1|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "100267": {
+      "content": "<|extra_id_2|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "100268": {
+      "content": "<|extra_id_3|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "100269": {
+      "content": "<|extra_id_4|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "100270": {
+      "content": "<|extra_id_5|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "100271": {
+      "content": "<|extra_id_6|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "100272": {
+      "content": "<|extra_id_7|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "100273": {
+      "content": "<|extra_id_8|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "100274": {
+      "content": "<|extra_id_9|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "100275": {
+      "content": "<|extra_id_10|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "100276": {
+      "content": "<|endofprompt|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "100277": {
+      "content": "<|pad|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    }
+  },
+  "bos_token": "<|endoftext|>",
+  "clean_up_tokenization_spaces": false,
+  "eos_token": "<|endoftext|>",
+  "extra_special_tokens": {},
+  "model_max_length": 1000000000000000019884624838656,
+  "pad_token": "<|pad|>",
+  "padding_side": "right",
+  "split_special_tokens": false,
+  "tokenizer_class": "GPT2Tokenizer",
+  "unk_token": "<|endoftext|>"
+}

g1_0_baseline_original_code__edit/250M/train_results.json

@@ -0,0 +1,8 @@
+{
+    "epoch": 0.23494835838441513,
+    "total_flos": 1.1366756130249245e+19,
+    "train_loss": 0.08313362047960282,
+    "train_runtime": 2289.8517,
+    "train_samples_per_second": 14.663,
+    "train_steps_per_second": 1.222
+}

g1_0_baseline_original_code__edit/250M/trainer_log.jsonl

@@ -0,0 +1,26 @@
+{"current_steps": 2550, "total_steps": 2798, "loss": 0.9353, "lr": 1.9980979278401095e-05, "epoch": 0.21412377193719037, "percentage": 91.14, "elapsed_time": "0:00:55", "remaining_time": "0:00:05"}
+{"current_steps": 2560, "total_steps": 2798, "loss": 0.949, "lr": 1.982924955463327e-05, "epoch": 0.2149634730036107, "percentage": 91.49, "elapsed_time": "0:02:24", "remaining_time": "0:00:13"}
+{"current_steps": 2570, "total_steps": 2798, "loss": 0.9345, "lr": 1.9528108298328235e-05, "epoch": 0.21580317407003108, "percentage": 91.85, "elapsed_time": "0:03:53", "remaining_time": "0:00:20"}
+{"current_steps": 2580, "total_steps": 2798, "loss": 0.9186, "lr": 1.9082156473635994e-05, "epoch": 0.21664287513645142, "percentage": 92.21, "elapsed_time": "0:05:22", "remaining_time": "0:00:27"}
+{"current_steps": 2590, "total_steps": 2798, "loss": 0.9305, "lr": 1.8498207522139434e-05, "epoch": 0.21748257620287179, "percentage": 92.57, "elapsed_time": "0:06:51", "remaining_time": "0:00:33"}
+{"current_steps": 2600, "total_steps": 2798, "loss": 0.9257, "lr": 1.7785183264196977e-05, "epoch": 0.21832227726929213, "percentage": 92.92, "elapsed_time": "0:08:19", "remaining_time": "0:00:38"}
+{"current_steps": 2610, "total_steps": 2798, "loss": 0.9085, "lr": 1.6953977587577846e-05, "epoch": 0.2191619783357125, "percentage": 93.28, "elapsed_time": "0:09:48", "remaining_time": "0:00:42"}
+{"current_steps": 2620, "total_steps": 2798, "loss": 0.9627, "lr": 1.601729000601297e-05, "epoch": 0.22000167940213283, "percentage": 93.64, "elapsed_time": "0:11:17", "remaining_time": "0:00:46"}
+{"current_steps": 2630, "total_steps": 2798, "loss": 0.9898, "lr": 1.4989431630624763e-05, "epoch": 0.2208413804685532, "percentage": 94.0, "elapsed_time": "0:12:46", "remaining_time": "0:00:48"}
+{"current_steps": 2640, "total_steps": 2798, "loss": 0.9656, "lr": 1.3886106518686746e-05, "epoch": 0.22168108153497354, "percentage": 94.35, "elapsed_time": "0:14:15", "remaining_time": "0:00:51"}
+{"current_steps": 2650, "total_steps": 2798, "loss": 0.8703, "lr": 1.2724171740359612e-05, "epoch": 0.2225207826013939, "percentage": 94.71, "elapsed_time": "0:15:44", "remaining_time": "0:00:52"}
+{"current_steps": 2660, "total_steps": 2798, "loss": 0.9159, "lr": 1.1521379829206218e-05, "epoch": 0.22336048366781425, "percentage": 95.07, "elapsed_time": "0:17:12", "remaining_time": "0:00:53"}
+{"current_steps": 2670, "total_steps": 2798, "loss": 0.8652, "lr": 1.029610755143595e-05, "epoch": 0.22420018473423461, "percentage": 95.43, "elapsed_time": "0:18:41", "remaining_time": "0:00:53"}
+{"current_steps": 2680, "total_steps": 2798, "loss": 0.9115, "lr": 9.067075137857348e-06, "epoch": 0.22503988580065495, "percentage": 95.78, "elapsed_time": "0:20:10", "remaining_time": "0:00:53"}
+{"current_steps": 2690, "total_steps": 2798, "loss": 0.8543, "lr": 7.853060268232525e-06, "epoch": 0.22587958686707532, "percentage": 96.14, "elapsed_time": "0:21:39", "remaining_time": "0:00:52"}
+{"current_steps": 2700, "total_steps": 2798, "loss": 0.8857, "lr": 6.672611177902054e-06, "epoch": 0.2267192879334957, "percentage": 96.5, "elapsed_time": "0:23:07", "remaining_time": "0:00:50"}
+{"current_steps": 2710, "total_steps": 2798, "loss": 0.9259, "lr": 5.543763269959312e-06, "epoch": 0.22755898899991603, "percentage": 96.85, "elapsed_time": "0:24:52", "remaining_time": "0:00:48"}
+{"current_steps": 2720, "total_steps": 2798, "loss": 0.8609, "lr": 4.483763562693867e-06, "epoch": 0.2283986900663364, "percentage": 97.21, "elapsed_time": "0:26:21", "remaining_time": "0:00:45"}
+{"current_steps": 2730, "total_steps": 2798, "loss": 0.9164, "lr": 3.508807182313187e-06, "epoch": 0.22923839113275674, "percentage": 97.57, "elapsed_time": "0:27:50", "remaining_time": "0:00:41"}
+{"current_steps": 2740, "total_steps": 2798, "loss": 0.9371, "lr": 2.6337899269187294e-06, "epoch": 0.2300780921991771, "percentage": 97.93, "elapsed_time": "0:29:19", "remaining_time": "0:00:37"}
+{"current_steps": 2750, "total_steps": 2798, "loss": 0.8998, "lr": 1.8720806821693126e-06, "epoch": 0.23091779326559744, "percentage": 98.28, "elapsed_time": "0:30:47", "remaining_time": "0:00:32"}
+{"current_steps": 2760, "total_steps": 2798, "loss": 0.8957, "lr": 1.2353171657619887e-06, "epoch": 0.2317574943320178, "percentage": 98.64, "elapsed_time": "0:32:16", "remaining_time": "0:00:26"}
+{"current_steps": 2770, "total_steps": 2798, "loss": 0.8876, "lr": 7.332281214330356e-07, "epoch": 0.23259719539843815, "percentage": 99.0, "elapsed_time": "0:33:45", "remaining_time": "0:00:20"}
+{"current_steps": 2780, "total_steps": 2798, "loss": 0.9128, "lr": 3.73484679074808e-07, "epoch": 0.23343689646485852, "percentage": 99.36, "elapsed_time": "0:35:13", "remaining_time": "0:00:13"}
+{"current_steps": 2790, "total_steps": 2798, "loss": 0.9019, "lr": 1.61583151951702e-07, "epoch": 0.23427659753127886, "percentage": 99.71, "elapsed_time": "0:36:42", "remaining_time": "0:00:06"}
+{"current_steps": 2798, "total_steps": 2798, "epoch": 0.23494835838441513, "percentage": 100.0, "elapsed_time": "0:38:09", "remaining_time": "0:00:00"}

g1_0_baseline_original_code__edit/250M/trainer_state.json

@@ -0,0 +1,1996 @@
+{
+  "best_global_step": null,
+  "best_metric": null,
+  "best_model_checkpoint": null,
+  "epoch": 0.23494835838441513,
+  "eval_steps": 500,
+  "global_step": 2798,
+  "is_hyper_param_search": false,
+  "is_local_process_zero": true,
+  "is_world_process_zero": true,
+  "log_history": [
+    {
+      "epoch": 0.0008397010664203543,
+      "grad_norm": 5.668807506561279,
+      "learning_rate": 1.1061797752808989e-06,
+      "loss": 2.1204,
+      "step": 10
+    },
+    {
+      "epoch": 0.0016794021328407087,
+      "grad_norm": 3.580942153930664,
+      "learning_rate": 2.2241573033707866e-06,
+      "loss": 2.0377,
+      "step": 20
+    },
+    {
+      "epoch": 0.002519103199261063,
+      "grad_norm": 1.8832168579101562,
+      "learning_rate": 3.342134831460675e-06,
+      "loss": 1.7534,
+      "step": 30
+    },
+    {
+      "epoch": 0.0033588042656814174,
+      "grad_norm": 1.5660103559494019,
+      "learning_rate": 4.4601123595505624e-06,
+      "loss": 1.5411,
+      "step": 40
+    },
+    {
+      "epoch": 0.004198505332101771,
+      "grad_norm": 0.9797653555870056,
+      "learning_rate": 5.5780898876404495e-06,
+      "loss": 1.2893,
+      "step": 50
+    },
+    {
+      "epoch": 0.005038206398522126,
+      "grad_norm": 1.1221965551376343,
+      "learning_rate": 6.6960674157303374e-06,
+      "loss": 1.2156,
+      "step": 60
+    },
+    {
+      "epoch": 0.00587790746494248,
+      "grad_norm": 0.6183356046676636,
+      "learning_rate": 7.814044943820226e-06,
+      "loss": 1.1003,
+      "step": 70
+    },
+    {
+      "epoch": 0.006717608531362835,
+      "grad_norm": 0.6748894453048706,
+      "learning_rate": 8.932022471910112e-06,
+      "loss": 1.1552,
+      "step": 80
+    },
+    {
+      "epoch": 0.007557309597783189,
+      "grad_norm": 0.6007770299911499,
+      "learning_rate": 1.005e-05,
+      "loss": 1.0217,
+      "step": 90
+    },
+    {
+      "epoch": 0.008397010664203543,
+      "grad_norm": 0.5850606560707092,
+      "learning_rate": 1.1167977528089888e-05,
+      "loss": 1.0789,
+      "step": 100
+    },
+    {
+      "epoch": 0.009236711730623898,
+      "grad_norm": 0.5961630940437317,
+      "learning_rate": 1.2285955056179776e-05,
+      "loss": 1.0721,
+      "step": 110
+    },
+    {
+      "epoch": 0.010076412797044252,
+      "grad_norm": 0.6326676607131958,
+      "learning_rate": 1.3403932584269664e-05,
+      "loss": 1.0648,
+      "step": 120
+    },
+    {
+      "epoch": 0.010916113863464607,
+      "grad_norm": 0.602372944355011,
+      "learning_rate": 1.4521910112359552e-05,
+      "loss": 1.0706,
+      "step": 130
+    },
+    {
+      "epoch": 0.01175581492988496,
+      "grad_norm": 0.5172364711761475,
+      "learning_rate": 1.563988764044944e-05,
+      "loss": 1.082,
+      "step": 140
+    },
+    {
+      "epoch": 0.012595515996305316,
+      "grad_norm": 0.5634326934814453,
+      "learning_rate": 1.6757865168539328e-05,
+      "loss": 1.0723,
+      "step": 150
+    },
+    {
+      "epoch": 0.01343521706272567,
+      "grad_norm": 0.592811107635498,
+      "learning_rate": 1.7875842696629214e-05,
+      "loss": 1.0995,
+      "step": 160
+    },
+    {
+      "epoch": 0.014274918129146025,
+      "grad_norm": 0.5542990565299988,
+      "learning_rate": 1.89938202247191e-05,
+      "loss": 1.1087,
+      "step": 170
+    },
+    {
+      "epoch": 0.015114619195566378,
+      "grad_norm": 0.6526479125022888,
+      "learning_rate": 2e-05,
+      "loss": 1.0882,
+      "step": 180
+    },
+    {
+      "epoch": 0.015954320261986732,
+      "grad_norm": 1.4111237525939941,
+      "learning_rate": 2e-05,
+      "loss": 1.0556,
+      "step": 190
+    },
+    {
+      "epoch": 0.016794021328407086,
+      "grad_norm": 0.5873704552650452,
+      "learning_rate": 2e-05,
+      "loss": 1.0444,
+      "step": 200
+    },
+    {
+      "epoch": 0.017633722394827443,
+      "grad_norm": 0.6317474246025085,
+      "learning_rate": 2e-05,
+      "loss": 1.0822,
+      "step": 210
+    },
+    {
+      "epoch": 0.018473423461247796,
+      "grad_norm": 0.8979406952857971,
+      "learning_rate": 2e-05,
+      "loss": 1.0702,
+      "step": 220
+    },
+    {
+      "epoch": 0.01931312452766815,
+      "grad_norm": 0.4911442995071411,
+      "learning_rate": 2e-05,
+      "loss": 1.0149,
+      "step": 230
+    },
+    {
+      "epoch": 0.020152825594088503,
+      "grad_norm": 0.5504240393638611,
+      "learning_rate": 2e-05,
+      "loss": 1.076,
+      "step": 240
+    },
+    {
+      "epoch": 0.02099252666050886,
+      "grad_norm": 0.5281411409378052,
+      "learning_rate": 2e-05,
+      "loss": 1.0851,
+      "step": 250
+    },
+    {
+      "epoch": 0.021832227726929214,
+      "grad_norm": 0.5294110178947449,
+      "learning_rate": 2e-05,
+      "loss": 1.0858,
+      "step": 260
+    },
+    {
+      "epoch": 0.022671928793349568,
+      "grad_norm": 0.649016797542572,
+      "learning_rate": 2e-05,
+      "loss": 1.0559,
+      "step": 270
+    },
+    {
+      "epoch": 0.02351162985976992,
+      "grad_norm": 0.5234949588775635,
+      "learning_rate": 2e-05,
+      "loss": 1.0851,
+      "step": 280
+    },
+    {
+      "epoch": 0.024351330926190275,
+      "grad_norm": 0.5594491958618164,
+      "learning_rate": 2e-05,
+      "loss": 1.0611,
+      "step": 290
+    },
+    {
+      "epoch": 0.02519103199261063,
+      "grad_norm": 0.49003034830093384,
+      "learning_rate": 2e-05,
+      "loss": 1.0656,
+      "step": 300
+    },
+    {
+      "epoch": 0.026030733059030985,
+      "grad_norm": 0.5069130063056946,
+      "learning_rate": 2e-05,
+      "loss": 1.0262,
+      "step": 310
+    },
+    {
+      "epoch": 0.02687043412545134,
+      "grad_norm": 0.47297877073287964,
+      "learning_rate": 2e-05,
+      "loss": 1.0137,
+      "step": 320
+    },
+    {
+      "epoch": 0.027710135191871692,
+      "grad_norm": 0.743406355381012,
+      "learning_rate": 2e-05,
+      "loss": 1.0958,
+      "step": 330
+    },
+    {
+      "epoch": 0.02854983625829205,
+      "grad_norm": 0.7573960423469543,
+      "learning_rate": 2e-05,
+      "loss": 1.0409,
+      "step": 340
+    },
+    {
+      "epoch": 0.029389537324712403,
+      "grad_norm": 0.6635141968727112,
+      "learning_rate": 2e-05,
+      "loss": 1.0382,
+      "step": 350
+    },
+    {
+      "epoch": 0.030229238391132757,
+      "grad_norm": 0.5210668444633484,
+      "learning_rate": 2e-05,
+      "loss": 1.0299,
+      "step": 360
+    },
+    {
+      "epoch": 0.03106893945755311,
+      "grad_norm": 0.4993533790111542,
+      "learning_rate": 2e-05,
+      "loss": 1.0115,
+      "step": 370
+    },
+    {
+      "epoch": 0.031908640523973464,
+      "grad_norm": 0.554338812828064,
+      "learning_rate": 2e-05,
+      "loss": 1.0602,
+      "step": 380
+    },
+    {
+      "epoch": 0.03274834159039382,
+      "grad_norm": 0.4986264407634735,
+      "learning_rate": 2e-05,
+      "loss": 1.0648,
+      "step": 390
+    },
+    {
+      "epoch": 0.03358804265681417,
+      "grad_norm": 0.6211596727371216,
+      "learning_rate": 2e-05,
+      "loss": 1.0759,
+      "step": 400
+    },
+    {
+      "epoch": 0.03442774372323453,
+      "grad_norm": 0.5103005170822144,
+      "learning_rate": 2e-05,
+      "loss": 1.0676,
+      "step": 410
+    },
+    {
+      "epoch": 0.035267444789654885,
+      "grad_norm": 0.5187060236930847,
+      "learning_rate": 2e-05,
+      "loss": 1.1182,
+      "step": 420
+    },
+    {
+      "epoch": 0.03610714585607524,
+      "grad_norm": 0.5082318782806396,
+      "learning_rate": 2e-05,
+      "loss": 1.0266,
+      "step": 430
+    },
+    {
+      "epoch": 0.03694684692249559,
+      "grad_norm": 0.8675495386123657,
+      "learning_rate": 2e-05,
+      "loss": 1.097,
+      "step": 440
+    },
+    {
+      "epoch": 0.037786547988915946,
+      "grad_norm": 0.515365481376648,
+      "learning_rate": 2e-05,
+      "loss": 0.9767,
+      "step": 450
+    },
+    {
+      "epoch": 0.0386262490553363,
+      "grad_norm": 0.7490212321281433,
+      "learning_rate": 2e-05,
+      "loss": 1.0692,
+      "step": 460
+    },
+    {
+      "epoch": 0.03946595012175665,
+      "grad_norm": 0.5549370050430298,
+      "learning_rate": 2e-05,
+      "loss": 0.9803,
+      "step": 470
+    },
+    {
+      "epoch": 0.04030565118817701,
+      "grad_norm": 0.48201850056648254,
+      "learning_rate": 2e-05,
+      "loss": 1.0323,
+      "step": 480
+    },
+    {
+      "epoch": 0.04114535225459736,
+      "grad_norm": 0.5301784873008728,
+      "learning_rate": 2e-05,
+      "loss": 1.0324,
+      "step": 490
+    },
+    {
+      "epoch": 0.04198505332101772,
+      "grad_norm": 0.5167050957679749,
+      "learning_rate": 2e-05,
+      "loss": 1.0686,
+      "step": 500
+    },
+    {
+      "epoch": 0.042824754387438074,
+      "grad_norm": 0.5225110650062561,
+      "learning_rate": 2e-05,
+      "loss": 1.0385,
+      "step": 510
+    },
+    {
+      "epoch": 0.04366445545385843,
+      "grad_norm": 0.5310977101325989,
+      "learning_rate": 2e-05,
+      "loss": 0.9866,
+      "step": 520
+    },
+    {
+      "epoch": 0.04450415652027878,
+      "grad_norm": 0.5529776215553284,
+      "learning_rate": 2e-05,
+      "loss": 1.0253,
+      "step": 530
+    },
+    {
+      "epoch": 0.045343857586699135,
+      "grad_norm": 0.5400099158287048,
+      "learning_rate": 2e-05,
+      "loss": 1.055,
+      "step": 540
+    },
+    {
+      "epoch": 0.04618355865311949,
+      "grad_norm": 0.9615269899368286,
+      "learning_rate": 2e-05,
+      "loss": 1.0941,
+      "step": 550
+    },
+    {
+      "epoch": 0.04702325971953984,
+      "grad_norm": 0.508506715297699,
+      "learning_rate": 2e-05,
+      "loss": 1.0047,
+      "step": 560
+    },
+    {
+      "epoch": 0.047862960785960196,
+      "grad_norm": 0.5063767433166504,
+      "learning_rate": 2e-05,
+      "loss": 1.071,
+      "step": 570
+    },
+    {
+      "epoch": 0.04870266185238055,
+      "grad_norm": 1.4059205055236816,
+      "learning_rate": 2e-05,
+      "loss": 1.036,
+      "step": 580
+    },
+    {
+      "epoch": 0.04954236291880091,
+      "grad_norm": 0.516341507434845,
+      "learning_rate": 2e-05,
+      "loss": 0.9713,
+      "step": 590
+    },
+    {
+      "epoch": 0.05038206398522126,
+      "grad_norm": 0.47716212272644043,
+      "learning_rate": 2e-05,
+      "loss": 0.9969,
+      "step": 600
+    },
+    {
+      "epoch": 0.05122176505164162,
+      "grad_norm": 0.5038254857063293,
+      "learning_rate": 2e-05,
+      "loss": 0.9859,
+      "step": 610
+    },
+    {
+      "epoch": 0.05206146611806197,
+      "grad_norm": 0.48962506651878357,
+      "learning_rate": 2e-05,
+      "loss": 1.0219,
+      "step": 620
+    },
+    {
+      "epoch": 0.052901167184482324,
+      "grad_norm": 0.5235401391983032,
+      "learning_rate": 2e-05,
+      "loss": 1.0495,
+      "step": 630
+    },
+    {
+      "epoch": 0.05374086825090268,
+      "grad_norm": 0.5100108981132507,
+      "learning_rate": 2e-05,
+      "loss": 1.122,
+      "step": 640
+    },
+    {
+      "epoch": 0.05458056931732303,
+      "grad_norm": 0.5479400157928467,
+      "learning_rate": 2e-05,
+      "loss": 1.0515,
+      "step": 650
+    },
+    {
+      "epoch": 0.055420270383743385,
+      "grad_norm": 0.5148715972900391,
+      "learning_rate": 2e-05,
+      "loss": 1.01,
+      "step": 660
+    },
+    {
+      "epoch": 0.05625997145016374,
+      "grad_norm": 0.5130622386932373,
+      "learning_rate": 2e-05,
+      "loss": 1.0013,
+      "step": 670
+    },
+    {
+      "epoch": 0.0570996725165841,
+      "grad_norm": 0.5072581768035889,
+      "learning_rate": 2e-05,
+      "loss": 1.0026,
+      "step": 680
+    },
+    {
+      "epoch": 0.05793937358300445,
+      "grad_norm": 0.5298655033111572,
+      "learning_rate": 2e-05,
+      "loss": 1.0896,
+      "step": 690
+    },
+    {
+      "epoch": 0.058779074649424806,
+      "grad_norm": 0.43709930777549744,
+      "learning_rate": 2e-05,
+      "loss": 1.0101,
+      "step": 700
+    },
+    {
+      "epoch": 0.05961877571584516,
+      "grad_norm": 0.48056909441947937,
+      "learning_rate": 2e-05,
+      "loss": 1.0414,
+      "step": 710
+    },
+    {
+      "epoch": 0.06045847678226551,
+      "grad_norm": 0.47372716665267944,
+      "learning_rate": 2e-05,
+      "loss": 1.0143,
+      "step": 720
+    },
+    {
+      "epoch": 0.06129817784868587,
+      "grad_norm": 0.5192368626594543,
+      "learning_rate": 2e-05,
+      "loss": 1.0052,
+      "step": 730
+    },
+    {
+      "epoch": 0.06213787891510622,
+      "grad_norm": 0.5142897367477417,
+      "learning_rate": 2e-05,
+      "loss": 0.9942,
+      "step": 740
+    },
+    {
+      "epoch": 0.06297757998152657,
+      "grad_norm": 0.5235256552696228,
+      "learning_rate": 2e-05,
+      "loss": 1.0316,
+      "step": 750
+    },
+    {
+      "epoch": 0.06381728104794693,
+      "grad_norm": 0.5314453840255737,
+      "learning_rate": 2e-05,
+      "loss": 1.041,
+      "step": 760
+    },
+    {
+      "epoch": 0.06465698211436728,
+      "grad_norm": 0.5321397185325623,
+      "learning_rate": 2e-05,
+      "loss": 1.0239,
+      "step": 770
+    },
+    {
+      "epoch": 0.06549668318078763,
+      "grad_norm": 0.5020036101341248,
+      "learning_rate": 2e-05,
+      "loss": 1.0601,
+      "step": 780
+    },
+    {
+      "epoch": 0.06633638424720799,
+      "grad_norm": 0.5363470911979675,
+      "learning_rate": 2e-05,
+      "loss": 1.0054,
+      "step": 790
+    },
+    {
+      "epoch": 0.06717608531362834,
+      "grad_norm": 0.47058793902397156,
+      "learning_rate": 2e-05,
+      "loss": 1.0528,
+      "step": 800
+    },
+    {
+      "epoch": 0.06801578638004871,
+      "grad_norm": 0.5474422574043274,
+      "learning_rate": 2e-05,
+      "loss": 0.9921,
+      "step": 810
+    },
+    {
+      "epoch": 0.06885548744646906,
+      "grad_norm": 0.48051396012306213,
+      "learning_rate": 2e-05,
+      "loss": 0.9717,
+      "step": 820
+    },
+    {
+      "epoch": 0.06969518851288942,
+      "grad_norm": 0.5143232941627502,
+      "learning_rate": 2e-05,
+      "loss": 1.0737,
+      "step": 830
+    },
+    {
+      "epoch": 0.07053488957930977,
+      "grad_norm": 0.5245751142501831,
+      "learning_rate": 2e-05,
+      "loss": 1.0081,
+      "step": 840
+    },
+    {
+      "epoch": 0.07137459064573012,
+      "grad_norm": 0.46237924695014954,
+      "learning_rate": 2e-05,
+      "loss": 1.0306,
+      "step": 850
+    },
+    {
+      "epoch": 0.07221429171215048,
+      "grad_norm": 0.48196980357170105,
+      "learning_rate": 2e-05,
+      "loss": 1.0197,
+      "step": 860
+    },
+    {
+      "epoch": 0.07305399277857083,
+      "grad_norm": 0.8797135949134827,
+      "learning_rate": 2e-05,
+      "loss": 1.0063,
+      "step": 870
+    },
+    {
+      "epoch": 0.07389369384499118,
+      "grad_norm": 0.476870059967041,
+      "learning_rate": 2e-05,
+      "loss": 0.9671,
+      "step": 880
+    },
+    {
+      "epoch": 0.07473339491141154,
+      "grad_norm": 0.4863646328449249,
+      "learning_rate": 2e-05,
+      "loss": 0.9929,
+      "step": 890
+    },
+    {
+      "epoch": 0.07557309597783189,
+      "grad_norm": 0.48433804512023926,
+      "learning_rate": 2e-05,
+      "loss": 0.9765,
+      "step": 900
+    },
+    {
+      "epoch": 0.07641279704425225,
+      "grad_norm": 0.5103152990341187,
+      "learning_rate": 2e-05,
+      "loss": 1.0255,
+      "step": 910
+    },
+    {
+      "epoch": 0.0772524981106726,
+      "grad_norm": 0.4950040876865387,
+      "learning_rate": 2e-05,
+      "loss": 0.9897,
+      "step": 920
+    },
+    {
+      "epoch": 0.07809219917709295,
+      "grad_norm": 0.6963955163955688,
+      "learning_rate": 2e-05,
+      "loss": 0.9603,
+      "step": 930
+    },
+    {
+      "epoch": 0.0789319002435133,
+      "grad_norm": 0.532052755355835,
+      "learning_rate": 2e-05,
+      "loss": 1.0265,
+      "step": 940
+    },
+    {
+      "epoch": 0.07977160130993366,
+      "grad_norm": 0.4793180227279663,
+      "learning_rate": 2e-05,
+      "loss": 1.0437,
+      "step": 950
+    },
+    {
+      "epoch": 0.08061130237635401,
+      "grad_norm": 0.5289978384971619,
+      "learning_rate": 2e-05,
+      "loss": 0.9697,
+      "step": 960
+    },
+    {
+      "epoch": 0.08145100344277437,
+      "grad_norm": 0.5345343947410583,
+      "learning_rate": 2e-05,
+      "loss": 1.0023,
+      "step": 970
+    },
+    {
+      "epoch": 0.08229070450919472,
+      "grad_norm": 0.5331494212150574,
+      "learning_rate": 2e-05,
+      "loss": 0.9736,
+      "step": 980
+    },
+    {
+      "epoch": 0.08313040557561509,
+      "grad_norm": 0.5220827460289001,
+      "learning_rate": 2e-05,
+      "loss": 1.0074,
+      "step": 990
+    },
+    {
+      "epoch": 0.08397010664203544,
+      "grad_norm": 0.4963169991970062,
+      "learning_rate": 2e-05,
+      "loss": 1.0102,
+      "step": 1000
+    },
+    {
+      "epoch": 0.0848098077084558,
+      "grad_norm": 0.5005528926849365,
+      "learning_rate": 2e-05,
+      "loss": 0.9515,
+      "step": 1010
+    },
+    {
+      "epoch": 0.08564950877487615,
+      "grad_norm": 0.5249394178390503,
+      "learning_rate": 2e-05,
+      "loss": 1.067,
+      "step": 1020
+    },
+    {
+      "epoch": 0.0864892098412965,
+      "grad_norm": 0.5880621075630188,
+      "learning_rate": 2e-05,
+      "loss": 1.0018,
+      "step": 1030
+    },
+    {
+      "epoch": 0.08732891090771686,
+      "grad_norm": 0.48338451981544495,
+      "learning_rate": 2e-05,
+      "loss": 0.9992,
+      "step": 1040
+    },
+    {
+      "epoch": 0.08816861197413721,
+      "grad_norm": 0.5206268429756165,
+      "learning_rate": 2e-05,
+      "loss": 0.9833,
+      "step": 1050
+    },
+    {
+      "epoch": 0.08900831304055756,
+      "grad_norm": 0.49556973576545715,
+      "learning_rate": 2e-05,
+      "loss": 1.0337,
+      "step": 1060
+    },
+    {
+      "epoch": 0.08984801410697792,
+      "grad_norm": 0.49861377477645874,
+      "learning_rate": 2e-05,
+      "loss": 1.0504,
+      "step": 1070
+    },
+    {
+      "epoch": 0.09068771517339827,
+      "grad_norm": 0.5358458161354065,
+      "learning_rate": 2e-05,
+      "loss": 1.0343,
+      "step": 1080
+    },
+    {
+      "epoch": 0.09152741623981862,
+      "grad_norm": 0.4752683639526367,
+      "learning_rate": 2e-05,
+      "loss": 1.0416,
+      "step": 1090
+    },
+    {
+      "epoch": 0.09236711730623898,
+      "grad_norm": 0.4865322411060333,
+      "learning_rate": 2e-05,
+      "loss": 1.0123,
+      "step": 1100
+    },
+    {
+      "epoch": 0.09320681837265933,
+      "grad_norm": 0.5330458283424377,
+      "learning_rate": 2e-05,
+      "loss": 1.0148,
+      "step": 1110
+    },
+    {
+      "epoch": 0.09404651943907968,
+      "grad_norm": 0.5267438292503357,
+      "learning_rate": 2e-05,
+      "loss": 1.0177,
+      "step": 1120
+    },
+    {
+      "epoch": 0.09488622050550004,
+      "grad_norm": 0.5309559106826782,
+      "learning_rate": 2e-05,
+      "loss": 0.9725,
+      "step": 1130
+    },
+    {
+      "epoch": 0.09572592157192039,
+      "grad_norm": 0.48289164900779724,
+      "learning_rate": 2e-05,
+      "loss": 0.9703,
+      "step": 1140
+    },
+    {
+      "epoch": 0.09656562263834075,
+      "grad_norm": 0.4635581970214844,
+      "learning_rate": 2e-05,
+      "loss": 0.9609,
+      "step": 1150
+    },
+    {
+      "epoch": 0.0974053237047611,
+      "grad_norm": 0.5546231865882874,
+      "learning_rate": 2e-05,
+      "loss": 1.0781,
+      "step": 1160
+    },
+    {
+      "epoch": 0.09824502477118147,
+      "grad_norm": 0.45515337586402893,
+      "learning_rate": 2e-05,
+      "loss": 0.9659,
+      "step": 1170
+    },
+    {
+      "epoch": 0.09908472583760182,
+      "grad_norm": 0.7485978603363037,
+      "learning_rate": 2e-05,
+      "loss": 1.0496,
+      "step": 1180
+    },
+    {
+      "epoch": 0.09992442690402217,
+      "grad_norm": 0.44106537103652954,
+      "learning_rate": 2e-05,
+      "loss": 1.0104,
+      "step": 1190
+    },
+    {
+      "epoch": 0.10076412797044253,
+      "grad_norm": 0.5454259514808655,
+      "learning_rate": 2e-05,
+      "loss": 0.9924,
+      "step": 1200
+    },
+    {
+      "epoch": 0.10160382903686288,
+      "grad_norm": 0.5239803791046143,
+      "learning_rate": 2e-05,
+      "loss": 0.9876,
+      "step": 1210
+    },
+    {
+      "epoch": 0.10244353010328323,
+      "grad_norm": 0.5559198260307312,
+      "learning_rate": 2e-05,
+      "loss": 1.0101,
+      "step": 1220
+    },
+    {
+      "epoch": 0.10328323116970359,
+      "grad_norm": 0.6441607475280762,
+      "learning_rate": 2e-05,
+      "loss": 0.9951,
+      "step": 1230
+    },
+    {
+      "epoch": 0.10412293223612394,
+      "grad_norm": 0.4984257221221924,
+      "learning_rate": 2e-05,
+      "loss": 1.0129,
+      "step": 1240
+    },
+    {
+      "epoch": 0.1049626333025443,
+      "grad_norm": 0.47933027148246765,
+      "learning_rate": 2e-05,
+      "loss": 0.9967,
+      "step": 1250
+    },
+    {
+      "epoch": 0.10580233436896465,
+      "grad_norm": 0.49340662360191345,
+      "learning_rate": 2e-05,
+      "loss": 0.9645,
+      "step": 1260
+    },
+    {
+      "epoch": 0.106642035435385,
+      "grad_norm": 0.4745008051395416,
+      "learning_rate": 2e-05,
+      "loss": 0.9875,
+      "step": 1270
+    },
+    {
+      "epoch": 0.10748173650180536,
+      "grad_norm": 0.569976806640625,
+      "learning_rate": 2e-05,
+      "loss": 1.0185,
+      "step": 1280
+    },
+    {
+      "epoch": 0.10832143756822571,
+      "grad_norm": 0.44903650879859924,
+      "learning_rate": 2e-05,
+      "loss": 0.9481,
+      "step": 1290
+    },
+    {
+      "epoch": 0.10916113863464606,
+      "grad_norm": 0.48480257391929626,
+      "learning_rate": 2e-05,
+      "loss": 0.9925,
+      "step": 1300
+    },
+    {
+      "epoch": 0.11000083970106642,
+      "grad_norm": 0.5419672727584839,
+      "learning_rate": 2e-05,
+      "loss": 0.9571,
+      "step": 1310
+    },
+    {
+      "epoch": 0.11084054076748677,
+      "grad_norm": 0.502124011516571,
+      "learning_rate": 2e-05,
+      "loss": 0.9847,
+      "step": 1320
+    },
+    {
+      "epoch": 0.11168024183390712,
+      "grad_norm": 0.5213430523872375,
+      "learning_rate": 2e-05,
+      "loss": 1.0044,
+      "step": 1330
+    },
+    {
+      "epoch": 0.11251994290032748,
+      "grad_norm": 0.5590944886207581,
+      "learning_rate": 2e-05,
+      "loss": 0.9712,
+      "step": 1340
+    },
+    {
+      "epoch": 0.11335964396674784,
+      "grad_norm": 0.9190903306007385,
+      "learning_rate": 2e-05,
+      "loss": 1.0349,
+      "step": 1350
+    },
+    {
+      "epoch": 0.1141993450331682,
+      "grad_norm": 0.4655185043811798,
+      "learning_rate": 2e-05,
+      "loss": 0.907,
+      "step": 1360
+    },
+    {
+      "epoch": 0.11503904609958855,
+      "grad_norm": 0.43830716609954834,
+      "learning_rate": 2e-05,
+      "loss": 0.9866,
+      "step": 1370
+    },
+    {
+      "epoch": 0.1158787471660089,
+      "grad_norm": 0.46941494941711426,
+      "learning_rate": 2e-05,
+      "loss": 0.9884,
+      "step": 1380
+    },
+    {
+      "epoch": 0.11671844823242926,
+      "grad_norm": 0.6129980683326721,
+      "learning_rate": 2e-05,
+      "loss": 1.0102,
+      "step": 1390
+    },
+    {
+      "epoch": 0.11755814929884961,
+      "grad_norm": 0.4963046610355377,
+      "learning_rate": 2e-05,
+      "loss": 0.9826,
+      "step": 1400
+    },
+    {
+      "epoch": 0.11839785036526997,
+      "grad_norm": 0.49613454937934875,
+      "learning_rate": 2e-05,
+      "loss": 1.0176,
+      "step": 1410
+    },
+    {
+      "epoch": 0.11923755143169032,
+      "grad_norm": 0.5326101779937744,
+      "learning_rate": 2e-05,
+      "loss": 0.9313,
+      "step": 1420
+    },
+    {
+      "epoch": 0.12007725249811067,
+      "grad_norm": 0.48534440994262695,
+      "learning_rate": 2e-05,
+      "loss": 1.0199,
+      "step": 1430
+    },
+    {
+      "epoch": 0.12091695356453103,
+      "grad_norm": 0.4562290608882904,
+      "learning_rate": 2e-05,
+      "loss": 1.022,
+      "step": 1440
+    },
+    {
+      "epoch": 0.12175665463095138,
+      "grad_norm": 1.3759952783584595,
+      "learning_rate": 2e-05,
+      "loss": 0.9604,
+      "step": 1450
+    },
+    {
+      "epoch": 0.12259635569737173,
+      "grad_norm": 0.4703039228916168,
+      "learning_rate": 2e-05,
+      "loss": 0.9843,
+      "step": 1460
+    },
+    {
+      "epoch": 0.12343605676379209,
+      "grad_norm": 0.4847295880317688,
+      "learning_rate": 2e-05,
+      "loss": 0.9822,
+      "step": 1470
+    },
+    {
+      "epoch": 0.12427575783021244,
+      "grad_norm": 0.4986240267753601,
+      "learning_rate": 2e-05,
+      "loss": 0.9697,
+      "step": 1480
+    },
+    {
+      "epoch": 0.1251154588966328,
+      "grad_norm": 0.49270740151405334,
+      "learning_rate": 2e-05,
+      "loss": 0.9673,
+      "step": 1490
+    },
+    {
+      "epoch": 0.12595515996305315,
+      "grad_norm": 0.4328170418739319,
+      "learning_rate": 2e-05,
+      "loss": 0.9471,
+      "step": 1500
+    },
+    {
+      "epoch": 0.12679486102947352,
+      "grad_norm": 0.47089821100234985,
+      "learning_rate": 2e-05,
+      "loss": 0.9195,
+      "step": 1510
+    },
+    {
+      "epoch": 0.12763456209589386,
+      "grad_norm": 0.45632755756378174,
+      "learning_rate": 2e-05,
+      "loss": 0.9079,
+      "step": 1520
+    },
+    {
+      "epoch": 0.12847426316231422,
+      "grad_norm": 0.48846668004989624,
+      "learning_rate": 2e-05,
+      "loss": 0.9452,
+      "step": 1530
+    },
+    {
+      "epoch": 0.12931396422873456,
+      "grad_norm": 0.46305525302886963,
+      "learning_rate": 2e-05,
+      "loss": 0.9962,
+      "step": 1540
+    },
+    {
+      "epoch": 0.13015366529515493,
+      "grad_norm": 0.4676452577114105,
+      "learning_rate": 2e-05,
+      "loss": 0.9582,
+      "step": 1550
+    },
+    {
+      "epoch": 0.13099336636157527,
+      "grad_norm": 0.6805684566497803,
+      "learning_rate": 2e-05,
+      "loss": 1.0018,
+      "step": 1560
+    },
+    {
+      "epoch": 0.13183306742799564,
+      "grad_norm": 0.5028660297393799,
+      "learning_rate": 2e-05,
+      "loss": 0.933,
+      "step": 1570
+    },
+    {
+      "epoch": 0.13267276849441598,
+      "grad_norm": 0.9517537951469421,
+      "learning_rate": 2e-05,
+      "loss": 0.9456,
+      "step": 1580
+    },
+    {
+      "epoch": 0.13351246956083634,
+      "grad_norm": 0.49718722701072693,
+      "learning_rate": 2e-05,
+      "loss": 0.964,
+      "step": 1590
+    },
+    {
+      "epoch": 0.13435217062725668,
+      "grad_norm": 0.4775601327419281,
+      "learning_rate": 2e-05,
+      "loss": 0.9496,
+      "step": 1600
+    },
+    {
+      "epoch": 0.13519187169367705,
+      "grad_norm": 0.5251036882400513,
+      "learning_rate": 2e-05,
+      "loss": 0.9465,
+      "step": 1610
+    },
+    {
+      "epoch": 0.13603157276009742,
+      "grad_norm": 0.4998883306980133,
+      "learning_rate": 2e-05,
+      "loss": 1.0076,
+      "step": 1620
+    },
+    {
+      "epoch": 0.13687127382651776,
+      "grad_norm": 0.4440484046936035,
+      "learning_rate": 2e-05,
+      "loss": 0.9921,
+      "step": 1630
+    },
+    {
+      "epoch": 0.13771097489293813,
+      "grad_norm": 0.47233253717422485,
+      "learning_rate": 2e-05,
+      "loss": 0.9507,
+      "step": 1640
+    },
+    {
+      "epoch": 0.13855067595935847,
+      "grad_norm": 0.5169973969459534,
+      "learning_rate": 2e-05,
+      "loss": 0.9693,
+      "step": 1650
+    },
+    {
+      "epoch": 0.13939037702577883,
+      "grad_norm": 0.5089414715766907,
+      "learning_rate": 2e-05,
+      "loss": 0.9597,
+      "step": 1660
+    },
+    {
+      "epoch": 0.14023007809219917,
+      "grad_norm": 0.509657084941864,
+      "learning_rate": 2e-05,
+      "loss": 0.9271,
+      "step": 1670
+    },
+    {
+      "epoch": 0.14106977915861954,
+      "grad_norm": 0.6538721323013306,
+      "learning_rate": 2e-05,
+      "loss": 0.9264,
+      "step": 1680
+    },
+    {
+      "epoch": 0.14190948022503988,
+      "grad_norm": 0.500372588634491,
+      "learning_rate": 2e-05,
+      "loss": 1.0109,
+      "step": 1690
+    },
+    {
+      "epoch": 0.14274918129146025,
+      "grad_norm": 0.5145060420036316,
+      "learning_rate": 2e-05,
+      "loss": 1.0252,
+      "step": 1700
+    },
+    {
+      "epoch": 0.1435888823578806,
+      "grad_norm": 0.4911811649799347,
+      "learning_rate": 2e-05,
+      "loss": 1.0003,
+      "step": 1710
+    },
+    {
+      "epoch": 0.14442858342430095,
+      "grad_norm": 0.6961483955383301,
+      "learning_rate": 2e-05,
+      "loss": 0.9963,
+      "step": 1720
+    },
+    {
+      "epoch": 0.1452682844907213,
+      "grad_norm": 0.49579185247421265,
+      "learning_rate": 2e-05,
+      "loss": 0.9595,
+      "step": 1730
+    },
+    {
+      "epoch": 0.14610798555714166,
+      "grad_norm": 0.5168765783309937,
+      "learning_rate": 2e-05,
+      "loss": 0.9119,
+      "step": 1740
+    },
+    {
+      "epoch": 0.146947686623562,
+      "grad_norm": 0.5454058647155762,
+      "learning_rate": 2e-05,
+      "loss": 0.9757,
+      "step": 1750
+    },
+    {
+      "epoch": 0.14778738768998237,
+      "grad_norm": 0.5128822922706604,
+      "learning_rate": 2e-05,
+      "loss": 0.9439,
+      "step": 1760
+    },
+    {
+      "epoch": 0.1486270887564027,
+      "grad_norm": 0.4754522442817688,
+      "learning_rate": 2e-05,
+      "loss": 1.0083,
+      "step": 1770
+    },
+    {
+      "epoch": 0.14946678982282308,
+      "grad_norm": 0.4661597013473511,
+      "learning_rate": 2e-05,
+      "loss": 0.9544,
+      "step": 1780
+    },
+    {
+      "epoch": 0.15030649088924342,
+      "grad_norm": 0.48215746879577637,
+      "learning_rate": 2e-05,
+      "loss": 0.9397,
+      "step": 1790
+    },
+    {
+      "epoch": 0.15114619195566378,
+      "grad_norm": 0.6749794483184814,
+      "learning_rate": 2e-05,
+      "loss": 0.9572,
+      "step": 1800
+    },
+    {
+      "epoch": 0.15198589302208415,
+      "grad_norm": 0.5560160279273987,
+      "learning_rate": 2e-05,
+      "loss": 0.9433,
+      "step": 1810
+    },
+    {
+      "epoch": 0.1528255940885045,
+      "grad_norm": 0.45195090770721436,
+      "learning_rate": 2e-05,
+      "loss": 0.92,
+      "step": 1820
+    },
+    {
+      "epoch": 0.15366529515492486,
+      "grad_norm": 0.4917123317718506,
+      "learning_rate": 2e-05,
+      "loss": 0.949,
+      "step": 1830
+    },
+    {
+      "epoch": 0.1545049962213452,
+      "grad_norm": 0.4223521947860718,
+      "learning_rate": 2e-05,
+      "loss": 0.933,
+      "step": 1840
+    },
+    {
+      "epoch": 0.15534469728776557,
+      "grad_norm": 0.5012931227684021,
+      "learning_rate": 2e-05,
+      "loss": 0.9113,
+      "step": 1850
+    },
+    {
+      "epoch": 0.1561843983541859,
+      "grad_norm": 0.5609027743339539,
+      "learning_rate": 2e-05,
+      "loss": 0.9313,
+      "step": 1860
+    },
+    {
+      "epoch": 0.15702409942060627,
+      "grad_norm": 0.8982095718383789,
+      "learning_rate": 2e-05,
+      "loss": 0.9312,
+      "step": 1870
+    },
+    {
+      "epoch": 0.1578638004870266,
+      "grad_norm": 0.48062509298324585,
+      "learning_rate": 2e-05,
+      "loss": 0.9603,
+      "step": 1880
+    },
+    {
+      "epoch": 0.15870350155344698,
+      "grad_norm": 0.4441513121128082,
+      "learning_rate": 2e-05,
+      "loss": 0.9869,
+      "step": 1890
+    },
+    {
+      "epoch": 0.15954320261986732,
+      "grad_norm": 1.625954031944275,
+      "learning_rate": 2e-05,
+      "loss": 0.8942,
+      "step": 1900
+    },
+    {
+      "epoch": 0.1603829036862877,
+      "grad_norm": 0.5065054297447205,
+      "learning_rate": 2e-05,
+      "loss": 1.0136,
+      "step": 1910
+    },
+    {
+      "epoch": 0.16122260475270803,
+      "grad_norm": 0.5414623022079468,
+      "learning_rate": 2e-05,
+      "loss": 0.9623,
+      "step": 1920
+    },
+    {
+      "epoch": 0.1620623058191284,
+      "grad_norm": 0.4631492495536804,
+      "learning_rate": 2e-05,
+      "loss": 0.9574,
+      "step": 1930
+    },
+    {
+      "epoch": 0.16290200688554873,
+      "grad_norm": 0.6957469582557678,
+      "learning_rate": 2e-05,
+      "loss": 0.9149,
+      "step": 1940
+    },
+    {
+      "epoch": 0.1637417079519691,
+      "grad_norm": 0.551231324672699,
+      "learning_rate": 2e-05,
+      "loss": 0.9918,
+      "step": 1950
+    },
+    {
+      "epoch": 0.16458140901838944,
+      "grad_norm": 0.5242336988449097,
+      "learning_rate": 2e-05,
+      "loss": 0.9842,
+      "step": 1960
+    },
+    {
+      "epoch": 0.1654211100848098,
+      "grad_norm": 0.4801785945892334,
+      "learning_rate": 2e-05,
+      "loss": 0.9296,
+      "step": 1970
+    },
+    {
+      "epoch": 0.16626081115123018,
+      "grad_norm": 0.5458486676216125,
+      "learning_rate": 2e-05,
+      "loss": 0.909,
+      "step": 1980
+    },
+    {
+      "epoch": 0.16710051221765052,
+      "grad_norm": 0.4695436656475067,
+      "learning_rate": 2e-05,
+      "loss": 0.9716,
+      "step": 1990
+    },
+    {
+      "epoch": 0.16794021328407088,
+      "grad_norm": 0.45850250124931335,
+      "learning_rate": 2e-05,
+      "loss": 0.946,
+      "step": 2000
+    },
+    {
+      "epoch": 0.16877991435049122,
+      "grad_norm": 0.520365834236145,
+      "learning_rate": 2e-05,
+      "loss": 0.8999,
+      "step": 2010
+    },
+    {
+      "epoch": 0.1696196154169116,
+      "grad_norm": 0.4501103460788727,
+      "learning_rate": 2e-05,
+      "loss": 0.9102,
+      "step": 2020
+    },
+    {
+      "epoch": 0.17045931648333193,
+      "grad_norm": 0.45634540915489197,
+      "learning_rate": 2e-05,
+      "loss": 0.9477,
+      "step": 2030
+    },
+    {
+      "epoch": 0.1712990175497523,
+      "grad_norm": 0.5216748714447021,
+      "learning_rate": 2e-05,
+      "loss": 0.9625,
+      "step": 2040
+    },
+    {
+      "epoch": 0.17213871861617264,
+      "grad_norm": 0.5178518891334534,
+      "learning_rate": 2e-05,
+      "loss": 0.9047,
+      "step": 2050
+    },
+    {
+      "epoch": 0.172978419682593,
+      "grad_norm": 0.49203771352767944,
+      "learning_rate": 2e-05,
+      "loss": 0.9061,
+      "step": 2060
+    },
+    {
+      "epoch": 0.17381812074901334,
+      "grad_norm": 0.6264152526855469,
+      "learning_rate": 2e-05,
+      "loss": 1.0226,
+      "step": 2070
+    },
+    {
+      "epoch": 0.1746578218154337,
+      "grad_norm": 0.43532794713974,
+      "learning_rate": 2e-05,
+      "loss": 0.9101,
+      "step": 2080
+    },
+    {
+      "epoch": 0.17549752288185405,
+      "grad_norm": 0.49513307213783264,
+      "learning_rate": 2e-05,
+      "loss": 1.0229,
+      "step": 2090
+    },
+    {
+      "epoch": 0.17633722394827442,
+      "grad_norm": 0.9167293310165405,
+      "learning_rate": 2e-05,
+      "loss": 0.9497,
+      "step": 2100
+    },
+    {
+      "epoch": 0.17717692501469476,
+      "grad_norm": 0.4796413481235504,
+      "learning_rate": 2e-05,
+      "loss": 1.053,
+      "step": 2110
+    },
+    {
+      "epoch": 0.17801662608111513,
+      "grad_norm": 0.45403558015823364,
+      "learning_rate": 2e-05,
+      "loss": 0.931,
+      "step": 2120
+    },
+    {
+      "epoch": 0.17885632714753547,
+      "grad_norm": 0.49153926968574524,
+      "learning_rate": 2e-05,
+      "loss": 0.9776,
+      "step": 2130
+    },
+    {
+      "epoch": 0.17969602821395583,
+      "grad_norm": 0.4547460675239563,
+      "learning_rate": 2e-05,
+      "loss": 1.0066,
+      "step": 2140
+    },
+    {
+      "epoch": 0.18053572928037617,
+      "grad_norm": 0.5737180113792419,
+      "learning_rate": 2e-05,
+      "loss": 0.9392,
+      "step": 2150
+    },
+    {
+      "epoch": 0.18137543034679654,
+      "grad_norm": 0.45654165744781494,
+      "learning_rate": 2e-05,
+      "loss": 0.9507,
+      "step": 2160
+    },
+    {
+      "epoch": 0.1822151314132169,
+      "grad_norm": 0.5037363767623901,
+      "learning_rate": 2e-05,
+      "loss": 0.9561,
+      "step": 2170
+    },
+    {
+      "epoch": 0.18305483247963725,
+      "grad_norm": 0.480439156293869,
+      "learning_rate": 2e-05,
+      "loss": 0.9459,
+      "step": 2180
+    },
+    {
+      "epoch": 0.18389453354605761,
+      "grad_norm": 0.47951242327690125,
+      "learning_rate": 2e-05,
+      "loss": 0.995,
+      "step": 2190
+    },
+    {
+      "epoch": 0.18473423461247795,
+      "grad_norm": 0.45319557189941406,
+      "learning_rate": 2e-05,
+      "loss": 0.9632,
+      "step": 2200
+    },
+    {
+      "epoch": 0.18557393567889832,
+      "grad_norm": 0.4815351665019989,
+      "learning_rate": 2e-05,
+      "loss": 0.9138,
+      "step": 2210
+    },
+    {
+      "epoch": 0.18641363674531866,
+      "grad_norm": 0.6240055561065674,
+      "learning_rate": 2e-05,
+      "loss": 0.9472,
+      "step": 2220
+    },
+    {
+      "epoch": 0.18725333781173903,
+      "grad_norm": 0.4974870979785919,
+      "learning_rate": 2e-05,
+      "loss": 0.8905,
+      "step": 2230
+    },
+    {
+      "epoch": 0.18809303887815937,
+      "grad_norm": 0.49529772996902466,
+      "learning_rate": 2e-05,
+      "loss": 0.912,
+      "step": 2240
+    },
+    {
+      "epoch": 0.18893273994457974,
+      "grad_norm": 0.47586724162101746,
+      "learning_rate": 2e-05,
+      "loss": 0.9509,
+      "step": 2250
+    },
+    {
+      "epoch": 0.18977244101100008,
+      "grad_norm": 0.45849329233169556,
+      "learning_rate": 2e-05,
+      "loss": 0.9058,
+      "step": 2260
+    },
+    {
+      "epoch": 0.19061214207742044,
+      "grad_norm": 0.5080339312553406,
+      "learning_rate": 2e-05,
+      "loss": 0.9455,
+      "step": 2270
+    },
+    {
+      "epoch": 0.19145184314384078,
+      "grad_norm": 0.5078399181365967,
+      "learning_rate": 2e-05,
+      "loss": 0.9504,
+      "step": 2280
+    },
+    {
+      "epoch": 0.19229154421026115,
+      "grad_norm": 0.5137534141540527,
+      "learning_rate": 2e-05,
+      "loss": 0.9379,
+      "step": 2290
+    },
+    {
+      "epoch": 0.1931312452766815,
+      "grad_norm": 0.523794412612915,
+      "learning_rate": 2e-05,
+      "loss": 0.9181,
+      "step": 2300
+    },
+    {
+      "epoch": 0.19397094634310186,
+      "grad_norm": 0.4992673099040985,
+      "learning_rate": 2e-05,
+      "loss": 0.8763,
+      "step": 2310
+    },
+    {
+      "epoch": 0.1948106474095222,
+      "grad_norm": 0.4604392647743225,
+      "learning_rate": 2e-05,
+      "loss": 0.9369,
+      "step": 2320
+    },
+    {
+      "epoch": 0.19565034847594256,
+      "grad_norm": 0.479475200176239,
+      "learning_rate": 2e-05,
+      "loss": 0.9176,
+      "step": 2330
+    },
+    {
+      "epoch": 0.19649004954236293,
+      "grad_norm": 0.47156229615211487,
+      "learning_rate": 2e-05,
+      "loss": 0.9327,
+      "step": 2340
+    },
+    {
+      "epoch": 0.19732975060878327,
+      "grad_norm": 0.5441238880157471,
+      "learning_rate": 2e-05,
+      "loss": 0.992,
+      "step": 2350
+    },
+    {
+      "epoch": 0.19816945167520364,
+      "grad_norm": 0.4867047369480133,
+      "learning_rate": 2e-05,
+      "loss": 0.9237,
+      "step": 2360
+    },
+    {
+      "epoch": 0.19900915274162398,
+      "grad_norm": 0.42399755120277405,
+      "learning_rate": 2e-05,
+      "loss": 0.9195,
+      "step": 2370
+    },
+    {
+      "epoch": 0.19984885380804435,
+      "grad_norm": 0.45596733689308167,
+      "learning_rate": 2e-05,
+      "loss": 0.8518,
+      "step": 2380
+    },
+    {
+      "epoch": 0.2006885548744647,
+      "grad_norm": 0.4902798533439636,
+      "learning_rate": 2e-05,
+      "loss": 0.955,
+      "step": 2390
+    },
+    {
+      "epoch": 0.20152825594088505,
+      "grad_norm": 0.5207685232162476,
+      "learning_rate": 2e-05,
+      "loss": 0.8705,
+      "step": 2400
+    },
+    {
+      "epoch": 0.2023679570073054,
+      "grad_norm": 1.1840689182281494,
+      "learning_rate": 2e-05,
+      "loss": 0.9338,
+      "step": 2410
+    },
+    {
+      "epoch": 0.20320765807372576,
+      "grad_norm": 0.5640371441841125,
+      "learning_rate": 2e-05,
+      "loss": 0.9348,
+      "step": 2420
+    },
+    {
+      "epoch": 0.2040473591401461,
+      "grad_norm": 0.4559391140937805,
+      "learning_rate": 2e-05,
+      "loss": 0.9464,
+      "step": 2430
+    },
+    {
+      "epoch": 0.20488706020656647,
+      "grad_norm": 0.5070319771766663,
+      "learning_rate": 2e-05,
+      "loss": 0.9117,
+      "step": 2440
+    },
+    {
+      "epoch": 0.2057267612729868,
+      "grad_norm": 0.4451445937156677,
+      "learning_rate": 2e-05,
+      "loss": 0.9582,
+      "step": 2450
+    },
+    {
+      "epoch": 0.20656646233940718,
+      "grad_norm": 0.44806957244873047,
+      "learning_rate": 2e-05,
+      "loss": 0.9391,
+      "step": 2460
+    },
+    {
+      "epoch": 0.20740616340582751,
+      "grad_norm": 0.4711221158504486,
+      "learning_rate": 2e-05,
+      "loss": 0.8628,
+      "step": 2470
+    },
+    {
+      "epoch": 0.20824586447224788,
+      "grad_norm": 0.5185308456420898,
+      "learning_rate": 2e-05,
+      "loss": 0.91,
+      "step": 2480
+    },
+    {
+      "epoch": 0.20908556553866822,
+      "grad_norm": 0.4656069278717041,
+      "learning_rate": 2e-05,
+      "loss": 0.9087,
+      "step": 2490
+    },
+    {
+      "epoch": 0.2099252666050886,
+      "grad_norm": 1.3008824586868286,
+      "learning_rate": 2e-05,
+      "loss": 0.8874,
+      "step": 2500
+    },
+    {
+      "epoch": 0.21076496767150893,
+      "grad_norm": 0.5072458982467651,
+      "learning_rate": 2e-05,
+      "loss": 0.8753,
+      "step": 2510
+    },
+    {
+      "epoch": 0.2116046687379293,
+      "grad_norm": 0.5111760497093201,
+      "learning_rate": 2e-05,
+      "loss": 0.9099,
+      "step": 2520
+    },
+    {
+      "epoch": 0.21244436980434966,
+      "grad_norm": 0.5030725598335266,
+      "learning_rate": 2e-05,
+      "loss": 0.917,
+      "step": 2530
+    },
+    {
+      "epoch": 0.21328407087077,
+      "grad_norm": 0.49836936593055725,
+      "learning_rate": 2e-05,
+      "loss": 0.9084,
+      "step": 2540
+    },
+    {
+      "epoch": 0.21412377193719037,
+      "grad_norm": 0.5667709708213806,
+      "learning_rate": 1.9980979278401095e-05,
+      "loss": 0.9353,
+      "step": 2550
+    },
+    {
+      "epoch": 0.2149634730036107,
+      "grad_norm": 0.6131989359855652,
+      "learning_rate": 1.982924955463327e-05,
+      "loss": 0.949,
+      "step": 2560
+    },
+    {
+      "epoch": 0.21580317407003108,
+      "grad_norm": 0.5101274847984314,
+      "learning_rate": 1.9528108298328235e-05,
+      "loss": 0.9345,
+      "step": 2570
+    },
+    {
+      "epoch": 0.21664287513645142,
+      "grad_norm": 0.6203171014785767,
+      "learning_rate": 1.9082156473635994e-05,
+      "loss": 0.9186,
+      "step": 2580
+    },
+    {
+      "epoch": 0.21748257620287179,
+      "grad_norm": 0.486038476228714,
+      "learning_rate": 1.8498207522139434e-05,
+      "loss": 0.9305,
+      "step": 2590
+    },
+    {
+      "epoch": 0.21832227726929213,
+      "grad_norm": 0.5271502733230591,
+      "learning_rate": 1.7785183264196977e-05,
+      "loss": 0.9257,
+      "step": 2600
+    },
+    {
+      "epoch": 0.2191619783357125,
+      "grad_norm": 0.47834500670433044,
+      "learning_rate": 1.6953977587577846e-05,
+      "loss": 0.9085,
+      "step": 2610
+    },
+    {
+      "epoch": 0.22000167940213283,
+      "grad_norm": 0.4812767803668976,
+      "learning_rate": 1.601729000601297e-05,
+      "loss": 0.9627,
+      "step": 2620
+    },
+    {
+      "epoch": 0.2208413804685532,
+      "grad_norm": 0.5123939514160156,
+      "learning_rate": 1.4989431630624763e-05,
+      "loss": 0.9898,
+      "step": 2630
+    },
+    {
+      "epoch": 0.22168108153497354,
+      "grad_norm": 0.5261260271072388,
+      "learning_rate": 1.3886106518686746e-05,
+      "loss": 0.9656,
+      "step": 2640
+    },
+    {
+      "epoch": 0.2225207826013939,
+      "grad_norm": 0.43107956647872925,
+      "learning_rate": 1.2724171740359612e-05,
+      "loss": 0.8703,
+      "step": 2650
+    },
+    {
+      "epoch": 0.22336048366781425,
+      "grad_norm": 0.4265885651111603,
+      "learning_rate": 1.1521379829206218e-05,
+      "loss": 0.9159,
+      "step": 2660
+    },
+    {
+      "epoch": 0.22420018473423461,
+      "grad_norm": 0.43352046608924866,
+      "learning_rate": 1.029610755143595e-05,
+      "loss": 0.8652,
+      "step": 2670
+    },
+    {
+      "epoch": 0.22503988580065495,
+      "grad_norm": 0.4556196331977844,
+      "learning_rate": 9.067075137857348e-06,
+      "loss": 0.9115,
+      "step": 2680
+    },
+    {
+      "epoch": 0.22587958686707532,
+      "grad_norm": 0.6424864530563354,
+      "learning_rate": 7.853060268232525e-06,
+      "loss": 0.8543,
+      "step": 2690
+    },
+    {
+      "epoch": 0.2267192879334957,
+      "grad_norm": 0.4368049204349518,
+      "learning_rate": 6.672611177902054e-06,
+      "loss": 0.8857,
+      "step": 2700
+    },
+    {
+      "epoch": 0.22755898899991603,
+      "grad_norm": 0.4711613655090332,
+      "learning_rate": 5.543763269959312e-06,
+      "loss": 0.9259,
+      "step": 2710
+    },
+    {
+      "epoch": 0.2283986900663364,
+      "grad_norm": 0.45124053955078125,
+      "learning_rate": 4.483763562693867e-06,
+      "loss": 0.8609,
+      "step": 2720
+    },
+    {
+      "epoch": 0.22923839113275674,
+      "grad_norm": 0.4743526577949524,
+      "learning_rate": 3.508807182313187e-06,
+      "loss": 0.9164,
+      "step": 2730
+    },
+    {
+      "epoch": 0.2300780921991771,
+      "grad_norm": 0.46023938059806824,
+      "learning_rate": 2.6337899269187294e-06,
+      "loss": 0.9371,
+      "step": 2740
+    },
+    {
+      "epoch": 0.23091779326559744,
+      "grad_norm": 0.4640144109725952,
+      "learning_rate": 1.8720806821693126e-06,
+      "loss": 0.8998,
+      "step": 2750
+    },
+    {
+      "epoch": 0.2317574943320178,
+      "grad_norm": 0.45083093643188477,
+      "learning_rate": 1.2353171657619887e-06,
+      "loss": 0.8957,
+      "step": 2760
+    },
+    {
+      "epoch": 0.23259719539843815,
+      "grad_norm": 0.4911569356918335,
+      "learning_rate": 7.332281214330356e-07,
+      "loss": 0.8876,
+      "step": 2770
+    },
+    {
+      "epoch": 0.23343689646485852,
+      "grad_norm": 0.46846678853034973,
+      "learning_rate": 3.73484679074808e-07,
+      "loss": 0.9128,
+      "step": 2780
+    },
+    {
+      "epoch": 0.23427659753127886,
+      "grad_norm": 0.43356049060821533,
+      "learning_rate": 1.61583151951702e-07,
+      "loss": 0.9019,
+      "step": 2790
+    },
+    {
+      "epoch": 0.23494835838441513,
+      "step": 2798,
+      "total_flos": 1.1366756130249245e+19,
+      "train_loss": 0.08313362047960282,
+      "train_runtime": 2289.8517,
+      "train_samples_per_second": 14.663,
+      "train_steps_per_second": 1.222
+    }
+  ],
+  "logging_steps": 10,
+  "max_steps": 2798,
+  "num_input_tokens_seen": 0,
+  "num_train_epochs": 1,
+  "save_steps": 159,
+  "stateful_callbacks": {
+    "TrainerControl": {
+      "args": {
+        "should_epoch_stop": false,
+        "should_evaluate": false,
+        "should_log": false,
+        "should_save": true,
+        "should_training_stop": true
+      },
+      "attributes": {}
+    }
+  },
+  "total_flos": 1.1366756130249245e+19,
+  "train_batch_size": 1,
+  "trial_name": null,
+  "trial_params": null
+}

g1_0_baseline_original_code__edit/500M/all_results.json

@@ -0,0 +1,8 @@
+{
+    "epoch": 0.46998068687547234,
+    "total_flos": 2.2737574718397743e+19,
+    "train_loss": 0.02770314997001186,
+    "train_runtime": 1763.7483,
+    "train_samples_per_second": 38.08,
+    "train_steps_per_second": 3.173
+}

g1_0_baseline_original_code__edit/500M/chat_template.jinja

@@ -0,0 +1 @@
+{% if messages[0]['role'] == 'system' %}{% set loop_messages = messages[1:] %}{% set system_message = messages[0]['content'] %}{% else %}{% set loop_messages = messages %}{% endif %}{% if system_message is defined %}{{ system_message }}{% endif %}{% for message in loop_messages %}{% set content = message['content'] %}{% if message['role'] == 'user' %}{{ content }}{% elif message['role'] == 'assistant' %}{{ content + '<|endoftext|>' }}{% endif %}{% endfor %}

g1_0_baseline_original_code__edit/500M/config.json

@@ -0,0 +1,27 @@
+{
+  "architectures": [
+    "Olmo2ForCausalLM"
+  ],
+  "attention_bias": false,
+  "attention_dropout": 0.0,
+  "bos_token_id": 100257,
+  "dtype": "bfloat16",
+  "eos_token_id": 100257,
+  "hidden_act": "silu",
+  "hidden_size": 4096,
+  "initializer_range": 0.02,
+  "intermediate_size": 11008,
+  "max_position_embeddings": 4096,
+  "model_type": "olmo2",
+  "num_attention_heads": 32,
+  "num_hidden_layers": 32,
+  "num_key_value_heads": 32,
+  "pad_token_id": 100277,
+  "rms_norm_eps": 1e-06,
+  "rope_scaling": null,
+  "rope_theta": 500000,
+  "tie_word_embeddings": false,
+  "transformers_version": "4.57.6",
+  "use_cache": false,
+  "vocab_size": 100352
+}

g1_0_baseline_original_code__edit/500M/generation_config.json

@@ -0,0 +1,9 @@
+{
+  "_from_model_config": true,
+  "bos_token_id": 100257,
+  "eos_token_id": [
+    100257
+  ],
+  "pad_token_id": 100277,
+  "transformers_version": "4.57.6"
+}

g1_0_baseline_original_code__edit/500M/model.safetensors.index.json

@@ -0,0 +1,363 @@
+{
+  "metadata": {
+    "total_parameters": 7298617344,
+    "total_size": 14597234688
+  },
+  "weight_map": {
+    "lm_head.weight": "model-00003-of-00003.safetensors",
+    "model.embed_tokens.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.post_feedforward_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.self_attn.k_norm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.self_attn.q_norm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.post_feedforward_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.self_attn.k_norm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.self_attn.q_norm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.10.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.10.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.10.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.10.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.10.post_feedforward_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.10.self_attn.k_norm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.10.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.10.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.10.self_attn.q_norm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.10.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.10.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.11.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.11.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.11.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.11.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.11.post_feedforward_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.11.self_attn.k_norm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.11.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.11.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.11.self_attn.q_norm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.11.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.11.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.post_feedforward_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.self_attn.k_norm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.self_attn.q_norm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.post_feedforward_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.self_attn.k_norm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.self_attn.q_norm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.post_feedforward_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.self_attn.k_norm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.self_attn.q_norm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.post_feedforward_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.self_attn.k_norm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.self_attn.q_norm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.post_feedforward_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.self_attn.k_norm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.self_attn.q_norm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.post_feedforward_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.self_attn.k_norm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.self_attn.q_norm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.post_feedforward_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.self_attn.k_norm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.self_attn.q_norm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.post_feedforward_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.self_attn.k_norm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.self_attn.q_norm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.2.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.2.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.2.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.2.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.2.post_feedforward_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.2.self_attn.k_norm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.2.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.2.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.2.self_attn.q_norm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.2.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.2.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.20.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.20.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.20.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.20.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.20.post_feedforward_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.20.self_attn.k_norm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.20.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.20.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.20.self_attn.q_norm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.20.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.20.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.post_feedforward_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.self_attn.k_norm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.self_attn.q_norm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.22.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.22.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.22.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.22.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.22.post_feedforward_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.22.self_attn.k_norm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.22.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.22.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.22.self_attn.q_norm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.22.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.22.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.23.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.23.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.23.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.23.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.23.post_feedforward_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.23.self_attn.k_norm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.23.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.23.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.23.self_attn.q_norm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.23.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.23.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.post_feedforward_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.self_attn.k_norm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.self_attn.q_norm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.post_feedforward_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.self_attn.k_norm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.self_attn.q_norm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.post_feedforward_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.self_attn.k_norm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.self_attn.q_norm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.post_feedforward_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.self_attn.k_norm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.self_attn.q_norm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.28.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.28.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.28.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.28.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.28.post_feedforward_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.28.self_attn.k_norm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.28.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.28.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.28.self_attn.q_norm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.28.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.28.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.29.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.29.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.29.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.29.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.29.post_feedforward_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.29.self_attn.k_norm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.29.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.29.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.29.self_attn.q_norm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.29.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.29.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.3.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.3.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.3.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.3.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.3.post_feedforward_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.3.self_attn.k_norm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.3.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.3.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.3.self_attn.q_norm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.3.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.3.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.30.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.30.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.30.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.30.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.30.post_feedforward_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.30.self_attn.k_norm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.30.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.30.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.30.self_attn.q_norm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.30.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.30.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.31.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.31.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.31.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.31.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.31.post_feedforward_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.31.self_attn.k_norm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.31.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.31.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.31.self_attn.q_norm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.31.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.31.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.4.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.4.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.4.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.4.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.4.post_feedforward_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.4.self_attn.k_norm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.4.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.4.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.4.self_attn.q_norm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.4.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.4.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.post_feedforward_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.self_attn.k_norm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.self_attn.q_norm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.post_feedforward_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.self_attn.k_norm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.self_attn.q_norm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.post_feedforward_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.self_attn.k_norm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.self_attn.q_norm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.post_feedforward_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.self_attn.k_norm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.self_attn.q_norm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.post_feedforward_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.self_attn.k_norm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.self_attn.q_norm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.norm.weight": "model-00003-of-00003.safetensors"
+  }
+}

g1_0_baseline_original_code__edit/500M/special_tokens_map.json

@@ -0,0 +1,30 @@
+{
+  "bos_token": {
+    "content": "<|endoftext|>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "eos_token": {
+    "content": "<|endoftext|>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "pad_token": {
+    "content": "<|pad|>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "unk_token": {
+    "content": "<|endoftext|>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  }
+}

g1_0_baseline_original_code__edit/500M/tokenizer_config.json

@@ -0,0 +1,191 @@
+{
+  "add_prefix_space": false,
+  "added_tokens_decoder": {
+    "100256": {
+      "content": "<|extra_id_0|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "100257": {
+      "content": "<|endoftext|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "100258": {
+      "content": "<|fim_prefix|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "100259": {
+      "content": "<|fim_middle|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "100260": {
+      "content": "<|fim_suffix|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "100261": {
+      "content": "|||PHONE_NUMBER|||",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "100262": {
+      "content": "|||EMAIL_ADDRESS|||",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "100263": {
+      "content": "|||IP_ADDRESS|||",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "100264": {
+      "content": "<|im_start|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "100265": {
+      "content": "<|im_end|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "100266": {
+      "content": "<|extra_id_1|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "100267": {
+      "content": "<|extra_id_2|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "100268": {
+      "content": "<|extra_id_3|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "100269": {
+      "content": "<|extra_id_4|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "100270": {
+      "content": "<|extra_id_5|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "100271": {
+      "content": "<|extra_id_6|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "100272": {
+      "content": "<|extra_id_7|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "100273": {
+      "content": "<|extra_id_8|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "100274": {
+      "content": "<|extra_id_9|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "100275": {
+      "content": "<|extra_id_10|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "100276": {
+      "content": "<|endofprompt|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "100277": {
+      "content": "<|pad|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    }
+  },
+  "bos_token": "<|endoftext|>",
+  "clean_up_tokenization_spaces": false,
+  "eos_token": "<|endoftext|>",
+  "extra_special_tokens": {},
+  "model_max_length": 1000000000000000019884624838656,
+  "pad_token": "<|pad|>",
+  "padding_side": "right",
+  "split_special_tokens": false,
+  "tokenizer_class": "GPT2Tokenizer",
+  "unk_token": "<|endoftext|>"
+}

g1_0_baseline_original_code__edit/500M/train_results.json

@@ -0,0 +1,8 @@
+{
+    "epoch": 0.46998068687547234,
+    "total_flos": 2.2737574718397743e+19,
+    "train_loss": 0.02770314997001186,
+    "train_runtime": 1763.7483,
+    "train_samples_per_second": 38.08,
+    "train_steps_per_second": 3.173
+}

g1_0_baseline_original_code__edit/500M/trainer_log.jsonl

@@ -0,0 +1,55 @@
+{"current_steps": 5090, "total_steps": 5597, "loss": 0.8271, "lr": 1.9999810479633703e-05, "epoch": 0.42740784280796035, "percentage": 90.94, "elapsed_time": "0:00:20", "remaining_time": "0:00:02"}
+{"current_steps": 5100, "total_steps": 5597, "loss": 0.8462, "lr": 1.997707677020786e-05, "epoch": 0.42824754387438074, "percentage": 91.12, "elapsed_time": "0:01:49", "remaining_time": "0:00:10"}
+{"current_steps": 5110, "total_steps": 5597, "loss": 0.8597, "lr": 1.9916538196124637e-05, "epoch": 0.4290872449408011, "percentage": 91.3, "elapsed_time": "0:03:17", "remaining_time": "0:00:18"}
+{"current_steps": 5120, "total_steps": 5597, "loss": 0.8261, "lr": 1.9818425303863402e-05, "epoch": 0.4299269460072214, "percentage": 91.48, "elapsed_time": "0:04:46", "remaining_time": "0:00:26"}
+{"current_steps": 5130, "total_steps": 5597, "loss": 0.8084, "lr": 1.9683111732582887e-05, "epoch": 0.43076664707364176, "percentage": 91.66, "elapsed_time": "0:06:14", "remaining_time": "0:00:34"}
+{"current_steps": 5140, "total_steps": 5597, "loss": 0.8461, "lr": 1.951111279120706e-05, "epoch": 0.43160634814006216, "percentage": 91.83, "elapsed_time": "0:07:43", "remaining_time": "0:00:41"}
+{"current_steps": 5150, "total_steps": 5597, "loss": 0.8514, "lr": 1.9303083495996028e-05, "epoch": 0.4324460492064825, "percentage": 92.01, "elapsed_time": "0:09:11", "remaining_time": "0:00:47"}
+{"current_steps": 5160, "total_steps": 5597, "loss": 0.7843, "lr": 1.9059816076075482e-05, "epoch": 0.43328575027290284, "percentage": 92.19, "elapsed_time": "0:10:39", "remaining_time": "0:00:54"}
+{"current_steps": 5170, "total_steps": 5597, "loss": 0.8607, "lr": 1.8782236956424142e-05, "epoch": 0.4341254513393232, "percentage": 92.37, "elapsed_time": "0:12:08", "remaining_time": "0:01:00"}
+{"current_steps": 5180, "total_steps": 5597, "loss": 0.8366, "lr": 1.8471403229808897e-05, "epoch": 0.43496515240574357, "percentage": 92.55, "elapsed_time": "0:13:36", "remaining_time": "0:01:05"}
+{"current_steps": 5190, "total_steps": 5597, "loss": 0.7797, "lr": 1.8128498631103333e-05, "epoch": 0.4358048534721639, "percentage": 92.73, "elapsed_time": "0:15:05", "remaining_time": "0:01:10"}
+{"current_steps": 5200, "total_steps": 5597, "loss": 0.8761, "lr": 1.7754829029320507e-05, "epoch": 0.43664455453858425, "percentage": 92.91, "elapsed_time": "0:16:33", "remaining_time": "0:01:15"}
+{"current_steps": 5210, "total_steps": 5597, "loss": 0.8728, "lr": 1.7351817454527473e-05, "epoch": 0.43748425560500465, "percentage": 93.09, "elapsed_time": "0:18:01", "remaining_time": "0:01:20"}
+{"current_steps": 5220, "total_steps": 5597, "loss": 0.8455, "lr": 1.6920998678580345e-05, "epoch": 0.438323956671425, "percentage": 93.26, "elapsed_time": "0:19:30", "remaining_time": "0:01:24"}
+{"current_steps": 5230, "total_steps": 5597, "loss": 0.802, "lr": 1.6464013370317854e-05, "epoch": 0.4391636577378453, "percentage": 93.44, "elapsed_time": "0:20:58", "remaining_time": "0:01:28"}
+{"current_steps": 5240, "total_steps": 5597, "loss": 0.7881, "lr": 1.5982601847471915e-05, "epoch": 0.44000335880426567, "percentage": 93.62, "elapsed_time": "0:22:27", "remaining_time": "0:01:31"}
+{"current_steps": 5250, "total_steps": 5597, "loss": 0.8326, "lr": 1.5478597449089553e-05, "epoch": 0.44084305987068606, "percentage": 93.8, "elapsed_time": "0:24:34", "remaining_time": "0:01:37"}
+{"current_steps": 5260, "total_steps": 5597, "loss": 0.7809, "lr": 1.4953919553705725e-05, "epoch": 0.4416827609371064, "percentage": 93.98, "elapsed_time": "0:26:03", "remaining_time": "0:01:40"}
+{"current_steps": 5270, "total_steps": 5597, "loss": 0.814, "lr": 1.441056626985559e-05, "epoch": 0.44252246200352674, "percentage": 94.16, "elapsed_time": "0:27:31", "remaining_time": "0:01:42"}
+{"current_steps": 5280, "total_steps": 5597, "loss": 0.8284, "lr": 1.3850606826762668e-05, "epoch": 0.4433621630699471, "percentage": 94.34, "elapsed_time": "0:29:00", "remaining_time": "0:01:44"}
+{"current_steps": 5290, "total_steps": 5597, "loss": 0.8609, "lr": 1.3276173694181128e-05, "epoch": 0.4442018641363675, "percentage": 94.51, "elapsed_time": "0:30:28", "remaining_time": "0:01:46"}
+{"current_steps": 5300, "total_steps": 5597, "loss": 0.8025, "lr": 1.2689454461401822e-05, "epoch": 0.4450415652027878, "percentage": 94.69, "elapsed_time": "0:31:57", "remaining_time": "0:01:47"}
+{"current_steps": 5310, "total_steps": 5597, "loss": 0.8427, "lr": 1.2092683506348939e-05, "epoch": 0.44588126626920815, "percentage": 94.87, "elapsed_time": "0:33:25", "remaining_time": "0:01:48"}
+{"current_steps": 5320, "total_steps": 5597, "loss": 0.8682, "lr": 1.14881334864935e-05, "epoch": 0.4467209673356285, "percentage": 95.05, "elapsed_time": "0:34:54", "remaining_time": "0:01:49"}
+{"current_steps": 5330, "total_steps": 5597, "loss": 0.8136, "lr": 1.0878106683988481e-05, "epoch": 0.4475606684020489, "percentage": 95.23, "elapsed_time": "0:36:22", "remaining_time": "0:01:49"}
+{"current_steps": 5340, "total_steps": 5597, "loss": 0.8311, "lr": 1.026492623798551e-05, "epoch": 0.44840036946846923, "percentage": 95.41, "elapsed_time": "0:37:51", "remaining_time": "0:01:49"}
+{"current_steps": 5350, "total_steps": 5597, "loss": 0.8376, "lr": 9.650927297522743e-06, "epoch": 0.44924007053488957, "percentage": 95.59, "elapsed_time": "0:39:19", "remaining_time": "0:01:48"}
+{"current_steps": 5360, "total_steps": 5597, "loss": 0.8677, "lr": 9.038448128676012e-06, "epoch": 0.4500797716013099, "percentage": 95.77, "elapsed_time": "0:40:48", "remaining_time": "0:01:48"}
+{"current_steps": 5370, "total_steps": 5597, "loss": 0.841, "lr": 8.42982120983949e-06, "epoch": 0.4509194726677303, "percentage": 95.94, "elapsed_time": "0:42:16", "remaining_time": "0:01:47"}
+{"current_steps": 5380, "total_steps": 5597, "loss": 0.7853, "lr": 7.827364349047303e-06, "epoch": 0.45175917373415064, "percentage": 96.12, "elapsed_time": "0:43:44", "remaining_time": "0:01:45"}
+{"current_steps": 5390, "total_steps": 5597, "loss": 0.7908, "lr": 7.233371857163706e-06, "epoch": 0.452598874800571, "percentage": 96.3, "elapsed_time": "0:45:13", "remaining_time": "0:01:44"}
+{"current_steps": 5400, "total_steps": 5597, "loss": 0.8142, "lr": 6.650105810556492e-06, "epoch": 0.4534385758669914, "percentage": 96.48, "elapsed_time": "0:46:41", "remaining_time": "0:01:42"}
+{"current_steps": 5410, "total_steps": 5597, "loss": 0.8145, "lr": 6.079787436527681e-06, "epoch": 0.4542782769334117, "percentage": 96.66, "elapsed_time": "0:48:47", "remaining_time": "0:01:41"}
+{"current_steps": 5420, "total_steps": 5597, "loss": 0.766, "lr": 5.524588654307961e-06, "epoch": 0.45511797799983206, "percentage": 96.84, "elapsed_time": "0:50:15", "remaining_time": "0:01:38"}
+{"current_steps": 5430, "total_steps": 5597, "loss": 0.8903, "lr": 4.986623803828985e-06, "epoch": 0.4559576790662524, "percentage": 97.02, "elapsed_time": "0:51:43", "remaining_time": "0:01:35"}
+{"current_steps": 5410, "total_steps": 5597, "loss": 0.7964, "lr": 6.079787436527681e-06, "epoch": 0.4542782769334117, "percentage": 96.66, "elapsed_time": "0:00:38", "remaining_time": "0:00:01"}
+{"current_steps": 5420, "total_steps": 5597, "loss": 0.766, "lr": 5.524588654307961e-06, "epoch": 0.45511797799983206, "percentage": 96.84, "elapsed_time": "0:02:06", "remaining_time": "0:00:04"}
+{"current_steps": 5430, "total_steps": 5597, "loss": 0.8903, "lr": 4.986623803828985e-06, "epoch": 0.4559576790662524, "percentage": 97.02, "elapsed_time": "0:03:35", "remaining_time": "0:00:06"}
+{"current_steps": 5440, "total_steps": 5597, "loss": 0.9261, "lr": 4.467941593772423e-06, "epoch": 0.4567973801326728, "percentage": 97.19, "elapsed_time": "0:05:03", "remaining_time": "0:00:08"}
+{"current_steps": 5450, "total_steps": 5597, "loss": 0.758, "lr": 3.9705172995597606e-06, "epoch": 0.45763708119909313, "percentage": 97.37, "elapsed_time": "0:06:31", "remaining_time": "0:00:10"}
+{"current_steps": 5460, "total_steps": 5597, "loss": 0.8045, "lr": 3.4962452409948346e-06, "epoch": 0.45847678226551347, "percentage": 97.55, "elapsed_time": "0:07:59", "remaining_time": "0:00:12"}
+{"current_steps": 5470, "total_steps": 5597, "loss": 0.8011, "lr": 3.046931568206211e-06, "epoch": 0.4593164833319338, "percentage": 97.73, "elapsed_time": "0:09:27", "remaining_time": "0:00:13"}
+{"current_steps": 5480, "total_steps": 5597, "loss": 0.8599, "lr": 2.624287383362404e-06, "epoch": 0.4601561843983542, "percentage": 97.91, "elapsed_time": "0:10:56", "remaining_time": "0:00:14"}
+{"current_steps": 5490, "total_steps": 5597, "loss": 0.8061, "lr": 2.2299222243541845e-06, "epoch": 0.46099588546477455, "percentage": 98.09, "elapsed_time": "0:12:24", "remaining_time": "0:00:14"}
+{"current_steps": 5500, "total_steps": 5597, "loss": 0.8021, "lr": 1.8653379352598246e-06, "epoch": 0.4618355865311949, "percentage": 98.27, "elapsed_time": "0:13:52", "remaining_time": "0:00:14"}
+{"current_steps": 5510, "total_steps": 5597, "loss": 0.756, "lr": 1.5319229469361075e-06, "epoch": 0.4626752875976152, "percentage": 98.45, "elapsed_time": "0:15:20", "remaining_time": "0:00:14"}
+{"current_steps": 5520, "total_steps": 5597, "loss": 0.8761, "lr": 1.2309469895160917e-06, "epoch": 0.4635149886640356, "percentage": 98.62, "elapsed_time": "0:16:48", "remaining_time": "0:00:14"}
+{"current_steps": 5530, "total_steps": 5597, "loss": 0.7795, "lr": 9.635562569497974e-07, "epoch": 0.46435468973045596, "percentage": 98.8, "elapsed_time": "0:18:17", "remaining_time": "0:00:13"}
+{"current_steps": 5540, "total_steps": 5597, "loss": 0.7897, "lr": 7.307690420024867e-07, "epoch": 0.4651943907968763, "percentage": 98.98, "elapsed_time": "0:19:45", "remaining_time": "0:00:12"}
+{"current_steps": 5550, "total_steps": 5597, "loss": 0.801, "lr": 5.334718583335942e-07, "epoch": 0.46603409186329664, "percentage": 99.16, "elapsed_time": "0:21:13", "remaining_time": "0:00:10"}
+{"current_steps": 5560, "total_steps": 5597, "loss": 0.8479, "lr": 3.7241606442442763e-07, "epoch": 0.46687379292971704, "percentage": 99.34, "elapsed_time": "0:22:41", "remaining_time": "0:00:09"}
+{"current_steps": 5570, "total_steps": 5597, "loss": 0.7874, "lr": 2.482150022115879e-07, "epoch": 0.4677134939961374, "percentage": 99.52, "elapsed_time": "0:24:47", "remaining_time": "0:00:07"}
+{"current_steps": 5580, "total_steps": 5597, "loss": 0.786, "lr": 1.6134166132293174e-07, "epoch": 0.4685531950625577, "percentage": 99.7, "elapsed_time": "0:26:15", "remaining_time": "0:00:04"}
+{"current_steps": 5590, "total_steps": 5597, "loss": 0.7645, "lr": 1.1212687781127357e-07, "epoch": 0.4693928961289781, "percentage": 99.87, "elapsed_time": "0:27:43", "remaining_time": "0:00:02"}
+{"current_steps": 5597, "total_steps": 5597, "epoch": 0.46998068687547234, "percentage": 100.0, "elapsed_time": "0:29:23", "remaining_time": "0:00:00"}

g1_0_baseline_original_code__edit/500M/trainer_state.json

@@ -0,0 +1,3956 @@
+{
+  "best_global_step": null,
+  "best_metric": null,
+  "best_model_checkpoint": null,
+  "epoch": 0.46998068687547234,
+  "eval_steps": 500,
+  "global_step": 5597,
+  "is_hyper_param_search": false,
+  "is_local_process_zero": true,
+  "is_world_process_zero": true,
+  "log_history": [
+    {
+      "epoch": 0.0008397010664203543,
+      "grad_norm": 5.668807506561279,
+      "learning_rate": 1.1061797752808989e-06,
+      "loss": 2.1204,
+      "step": 10
+    },
+    {
+      "epoch": 0.0016794021328407087,
+      "grad_norm": 3.580942153930664,
+      "learning_rate": 2.2241573033707866e-06,
+      "loss": 2.0377,
+      "step": 20
+    },
+    {
+      "epoch": 0.002519103199261063,
+      "grad_norm": 1.8832168579101562,
+      "learning_rate": 3.342134831460675e-06,
+      "loss": 1.7534,
+      "step": 30
+    },
+    {
+      "epoch": 0.0033588042656814174,
+      "grad_norm": 1.5660103559494019,
+      "learning_rate": 4.4601123595505624e-06,
+      "loss": 1.5411,
+      "step": 40
+    },
+    {
+      "epoch": 0.004198505332101771,
+      "grad_norm": 0.9797653555870056,
+      "learning_rate": 5.5780898876404495e-06,
+      "loss": 1.2893,
+      "step": 50
+    },
+    {
+      "epoch": 0.005038206398522126,
+      "grad_norm": 1.1221965551376343,
+      "learning_rate": 6.6960674157303374e-06,
+      "loss": 1.2156,
+      "step": 60
+    },
+    {
+      "epoch": 0.00587790746494248,
+      "grad_norm": 0.6183356046676636,
+      "learning_rate": 7.814044943820226e-06,
+      "loss": 1.1003,
+      "step": 70
+    },
+    {
+      "epoch": 0.006717608531362835,
+      "grad_norm": 0.6748894453048706,
+      "learning_rate": 8.932022471910112e-06,
+      "loss": 1.1552,
+      "step": 80
+    },
+    {
+      "epoch": 0.007557309597783189,
+      "grad_norm": 0.6007770299911499,
+      "learning_rate": 1.005e-05,
+      "loss": 1.0217,
+      "step": 90
+    },
+    {
+      "epoch": 0.008397010664203543,
+      "grad_norm": 0.5850606560707092,
+      "learning_rate": 1.1167977528089888e-05,
+      "loss": 1.0789,
+      "step": 100
+    },
+    {
+      "epoch": 0.009236711730623898,
+      "grad_norm": 0.5961630940437317,
+      "learning_rate": 1.2285955056179776e-05,
+      "loss": 1.0721,
+      "step": 110
+    },
+    {
+      "epoch": 0.010076412797044252,
+      "grad_norm": 0.6326676607131958,
+      "learning_rate": 1.3403932584269664e-05,
+      "loss": 1.0648,
+      "step": 120
+    },
+    {
+      "epoch": 0.010916113863464607,
+      "grad_norm": 0.602372944355011,
+      "learning_rate": 1.4521910112359552e-05,
+      "loss": 1.0706,
+      "step": 130
+    },
+    {
+      "epoch": 0.01175581492988496,
+      "grad_norm": 0.5172364711761475,
+      "learning_rate": 1.563988764044944e-05,
+      "loss": 1.082,
+      "step": 140
+    },
+    {
+      "epoch": 0.012595515996305316,
+      "grad_norm": 0.5634326934814453,
+      "learning_rate": 1.6757865168539328e-05,
+      "loss": 1.0723,
+      "step": 150
+    },
+    {
+      "epoch": 0.01343521706272567,
+      "grad_norm": 0.592811107635498,
+      "learning_rate": 1.7875842696629214e-05,
+      "loss": 1.0995,
+      "step": 160
+    },
+    {
+      "epoch": 0.014274918129146025,
+      "grad_norm": 0.5542990565299988,
+      "learning_rate": 1.89938202247191e-05,
+      "loss": 1.1087,
+      "step": 170
+    },
+    {
+      "epoch": 0.015114619195566378,
+      "grad_norm": 0.6526479125022888,
+      "learning_rate": 2e-05,
+      "loss": 1.0882,
+      "step": 180
+    },
+    {
+      "epoch": 0.015954320261986732,
+      "grad_norm": 1.4111237525939941,
+      "learning_rate": 2e-05,
+      "loss": 1.0556,
+      "step": 190
+    },
+    {
+      "epoch": 0.016794021328407086,
+      "grad_norm": 0.5873704552650452,
+      "learning_rate": 2e-05,
+      "loss": 1.0444,
+      "step": 200
+    },
+    {
+      "epoch": 0.017633722394827443,
+      "grad_norm": 0.6317474246025085,
+      "learning_rate": 2e-05,
+      "loss": 1.0822,
+      "step": 210
+    },
+    {
+      "epoch": 0.018473423461247796,
+      "grad_norm": 0.8979406952857971,
+      "learning_rate": 2e-05,
+      "loss": 1.0702,
+      "step": 220
+    },
+    {
+      "epoch": 0.01931312452766815,
+      "grad_norm": 0.4911442995071411,
+      "learning_rate": 2e-05,
+      "loss": 1.0149,
+      "step": 230
+    },
+    {
+      "epoch": 0.020152825594088503,
+      "grad_norm": 0.5504240393638611,
+      "learning_rate": 2e-05,
+      "loss": 1.076,
+      "step": 240
+    },
+    {
+      "epoch": 0.02099252666050886,
+      "grad_norm": 0.5281411409378052,
+      "learning_rate": 2e-05,
+      "loss": 1.0851,
+      "step": 250
+    },
+    {
+      "epoch": 0.021832227726929214,
+      "grad_norm": 0.5294110178947449,
+      "learning_rate": 2e-05,
+      "loss": 1.0858,
+      "step": 260
+    },
+    {
+      "epoch": 0.022671928793349568,
+      "grad_norm": 0.649016797542572,
+      "learning_rate": 2e-05,
+      "loss": 1.0559,
+      "step": 270
+    },
+    {
+      "epoch": 0.02351162985976992,
+      "grad_norm": 0.5234949588775635,
+      "learning_rate": 2e-05,
+      "loss": 1.0851,
+      "step": 280
+    },
+    {
+      "epoch": 0.024351330926190275,
+      "grad_norm": 0.5594491958618164,
+      "learning_rate": 2e-05,
+      "loss": 1.0611,
+      "step": 290
+    },
+    {
+      "epoch": 0.02519103199261063,
+      "grad_norm": 0.49003034830093384,
+      "learning_rate": 2e-05,
+      "loss": 1.0656,
+      "step": 300
+    },
+    {
+      "epoch": 0.026030733059030985,
+      "grad_norm": 0.5069130063056946,
+      "learning_rate": 2e-05,
+      "loss": 1.0262,
+      "step": 310
+    },
+    {
+      "epoch": 0.02687043412545134,
+      "grad_norm": 0.47297877073287964,
+      "learning_rate": 2e-05,
+      "loss": 1.0137,
+      "step": 320
+    },
+    {
+      "epoch": 0.027710135191871692,
+      "grad_norm": 0.743406355381012,
+      "learning_rate": 2e-05,
+      "loss": 1.0958,
+      "step": 330
+    },
+    {
+      "epoch": 0.02854983625829205,
+      "grad_norm": 0.7573960423469543,
+      "learning_rate": 2e-05,
+      "loss": 1.0409,
+      "step": 340
+    },
+    {
+      "epoch": 0.029389537324712403,
+      "grad_norm": 0.6635141968727112,
+      "learning_rate": 2e-05,
+      "loss": 1.0382,
+      "step": 350
+    },
+    {
+      "epoch": 0.030229238391132757,
+      "grad_norm": 0.5210668444633484,
+      "learning_rate": 2e-05,
+      "loss": 1.0299,
+      "step": 360
+    },
+    {
+      "epoch": 0.03106893945755311,
+      "grad_norm": 0.4993533790111542,
+      "learning_rate": 2e-05,
+      "loss": 1.0115,
+      "step": 370
+    },
+    {
+      "epoch": 0.031908640523973464,
+      "grad_norm": 0.554338812828064,
+      "learning_rate": 2e-05,
+      "loss": 1.0602,
+      "step": 380
+    },
+    {
+      "epoch": 0.03274834159039382,
+      "grad_norm": 0.4986264407634735,
+      "learning_rate": 2e-05,
+      "loss": 1.0648,
+      "step": 390
+    },
+    {
+      "epoch": 0.03358804265681417,
+      "grad_norm": 0.6211596727371216,
+      "learning_rate": 2e-05,
+      "loss": 1.0759,
+      "step": 400
+    },
+    {
+      "epoch": 0.03442774372323453,
+      "grad_norm": 0.5103005170822144,
+      "learning_rate": 2e-05,
+      "loss": 1.0676,
+      "step": 410
+    },
+    {
+      "epoch": 0.035267444789654885,
+      "grad_norm": 0.5187060236930847,
+      "learning_rate": 2e-05,
+      "loss": 1.1182,
+      "step": 420
+    },
+    {
+      "epoch": 0.03610714585607524,
+      "grad_norm": 0.5082318782806396,
+      "learning_rate": 2e-05,
+      "loss": 1.0266,
+      "step": 430
+    },
+    {
+      "epoch": 0.03694684692249559,
+      "grad_norm": 0.8675495386123657,
+      "learning_rate": 2e-05,
+      "loss": 1.097,
+      "step": 440
+    },
+    {
+      "epoch": 0.037786547988915946,
+      "grad_norm": 0.515365481376648,
+      "learning_rate": 2e-05,
+      "loss": 0.9767,
+      "step": 450
+    },
+    {
+      "epoch": 0.0386262490553363,
+      "grad_norm": 0.7490212321281433,
+      "learning_rate": 2e-05,
+      "loss": 1.0692,
+      "step": 460
+    },
+    {
+      "epoch": 0.03946595012175665,
+      "grad_norm": 0.5549370050430298,
+      "learning_rate": 2e-05,
+      "loss": 0.9803,
+      "step": 470
+    },
+    {
+      "epoch": 0.04030565118817701,
+      "grad_norm": 0.48201850056648254,
+      "learning_rate": 2e-05,
+      "loss": 1.0323,
+      "step": 480
+    },
+    {
+      "epoch": 0.04114535225459736,
+      "grad_norm": 0.5301784873008728,
+      "learning_rate": 2e-05,
+      "loss": 1.0324,
+      "step": 490
+    },
+    {
+      "epoch": 0.04198505332101772,
+      "grad_norm": 0.5167050957679749,
+      "learning_rate": 2e-05,
+      "loss": 1.0686,
+      "step": 500
+    },
+    {
+      "epoch": 0.042824754387438074,
+      "grad_norm": 0.5225110650062561,
+      "learning_rate": 2e-05,
+      "loss": 1.0385,
+      "step": 510
+    },
+    {
+      "epoch": 0.04366445545385843,
+      "grad_norm": 0.5310977101325989,
+      "learning_rate": 2e-05,
+      "loss": 0.9866,
+      "step": 520
+    },
+    {
+      "epoch": 0.04450415652027878,
+      "grad_norm": 0.5529776215553284,
+      "learning_rate": 2e-05,
+      "loss": 1.0253,
+      "step": 530
+    },
+    {
+      "epoch": 0.045343857586699135,
+      "grad_norm": 0.5400099158287048,
+      "learning_rate": 2e-05,
+      "loss": 1.055,
+      "step": 540
+    },
+    {
+      "epoch": 0.04618355865311949,
+      "grad_norm": 0.9615269899368286,
+      "learning_rate": 2e-05,
+      "loss": 1.0941,
+      "step": 550
+    },
+    {
+      "epoch": 0.04702325971953984,
+      "grad_norm": 0.508506715297699,
+      "learning_rate": 2e-05,
+      "loss": 1.0047,
+      "step": 560
+    },
+    {
+      "epoch": 0.047862960785960196,
+      "grad_norm": 0.5063767433166504,
+      "learning_rate": 2e-05,
+      "loss": 1.071,
+      "step": 570
+    },
+    {
+      "epoch": 0.04870266185238055,
+      "grad_norm": 1.4059205055236816,
+      "learning_rate": 2e-05,
+      "loss": 1.036,
+      "step": 580
+    },
+    {
+      "epoch": 0.04954236291880091,
+      "grad_norm": 0.516341507434845,
+      "learning_rate": 2e-05,
+      "loss": 0.9713,
+      "step": 590
+    },
+    {
+      "epoch": 0.05038206398522126,
+      "grad_norm": 0.47716212272644043,
+      "learning_rate": 2e-05,
+      "loss": 0.9969,
+      "step": 600
+    },
+    {
+      "epoch": 0.05122176505164162,
+      "grad_norm": 0.5038254857063293,
+      "learning_rate": 2e-05,
+      "loss": 0.9859,
+      "step": 610
+    },
+    {
+      "epoch": 0.05206146611806197,
+      "grad_norm": 0.48962506651878357,
+      "learning_rate": 2e-05,
+      "loss": 1.0219,
+      "step": 620
+    },
+    {
+      "epoch": 0.052901167184482324,
+      "grad_norm": 0.5235401391983032,
+      "learning_rate": 2e-05,
+      "loss": 1.0495,
+      "step": 630
+    },
+    {
+      "epoch": 0.05374086825090268,
+      "grad_norm": 0.5100108981132507,
+      "learning_rate": 2e-05,
+      "loss": 1.122,
+      "step": 640
+    },
+    {
+      "epoch": 0.05458056931732303,
+      "grad_norm": 0.5479400157928467,
+      "learning_rate": 2e-05,
+      "loss": 1.0515,
+      "step": 650
+    },
+    {
+      "epoch": 0.055420270383743385,
+      "grad_norm": 0.5148715972900391,
+      "learning_rate": 2e-05,
+      "loss": 1.01,
+      "step": 660
+    },
+    {
+      "epoch": 0.05625997145016374,
+      "grad_norm": 0.5130622386932373,
+      "learning_rate": 2e-05,
+      "loss": 1.0013,
+      "step": 670
+    },
+    {
+      "epoch": 0.0570996725165841,
+      "grad_norm": 0.5072581768035889,
+      "learning_rate": 2e-05,
+      "loss": 1.0026,
+      "step": 680
+    },
+    {
+      "epoch": 0.05793937358300445,
+      "grad_norm": 0.5298655033111572,
+      "learning_rate": 2e-05,
+      "loss": 1.0896,
+      "step": 690
+    },
+    {
+      "epoch": 0.058779074649424806,
+      "grad_norm": 0.43709930777549744,
+      "learning_rate": 2e-05,
+      "loss": 1.0101,
+      "step": 700
+    },
+    {
+      "epoch": 0.05961877571584516,
+      "grad_norm": 0.48056909441947937,
+      "learning_rate": 2e-05,
+      "loss": 1.0414,
+      "step": 710
+    },
+    {
+      "epoch": 0.06045847678226551,
+      "grad_norm": 0.47372716665267944,
+      "learning_rate": 2e-05,
+      "loss": 1.0143,
+      "step": 720
+    },
+    {
+      "epoch": 0.06129817784868587,
+      "grad_norm": 0.5192368626594543,
+      "learning_rate": 2e-05,
+      "loss": 1.0052,
+      "step": 730
+    },
+    {
+      "epoch": 0.06213787891510622,
+      "grad_norm": 0.5142897367477417,
+      "learning_rate": 2e-05,
+      "loss": 0.9942,
+      "step": 740
+    },
+    {
+      "epoch": 0.06297757998152657,
+      "grad_norm": 0.5235256552696228,
+      "learning_rate": 2e-05,
+      "loss": 1.0316,
+      "step": 750
+    },
+    {
+      "epoch": 0.06381728104794693,
+      "grad_norm": 0.5314453840255737,
+      "learning_rate": 2e-05,
+      "loss": 1.041,
+      "step": 760
+    },
+    {
+      "epoch": 0.06465698211436728,
+      "grad_norm": 0.5321397185325623,
+      "learning_rate": 2e-05,
+      "loss": 1.0239,
+      "step": 770
+    },
+    {
+      "epoch": 0.06549668318078763,
+      "grad_norm": 0.5020036101341248,
+      "learning_rate": 2e-05,
+      "loss": 1.0601,
+      "step": 780
+    },
+    {
+      "epoch": 0.06633638424720799,
+      "grad_norm": 0.5363470911979675,
+      "learning_rate": 2e-05,
+      "loss": 1.0054,
+      "step": 790
+    },
+    {
+      "epoch": 0.06717608531362834,
+      "grad_norm": 0.47058793902397156,
+      "learning_rate": 2e-05,
+      "loss": 1.0528,
+      "step": 800
+    },
+    {
+      "epoch": 0.06801578638004871,
+      "grad_norm": 0.5474422574043274,
+      "learning_rate": 2e-05,
+      "loss": 0.9921,
+      "step": 810
+    },
+    {
+      "epoch": 0.06885548744646906,
+      "grad_norm": 0.48051396012306213,
+      "learning_rate": 2e-05,
+      "loss": 0.9717,
+      "step": 820
+    },
+    {
+      "epoch": 0.06969518851288942,
+      "grad_norm": 0.5143232941627502,
+      "learning_rate": 2e-05,
+      "loss": 1.0737,
+      "step": 830
+    },
+    {
+      "epoch": 0.07053488957930977,
+      "grad_norm": 0.5245751142501831,
+      "learning_rate": 2e-05,
+      "loss": 1.0081,
+      "step": 840
+    },
+    {
+      "epoch": 0.07137459064573012,
+      "grad_norm": 0.46237924695014954,
+      "learning_rate": 2e-05,
+      "loss": 1.0306,
+      "step": 850
+    },
+    {
+      "epoch": 0.07221429171215048,
+      "grad_norm": 0.48196980357170105,
+      "learning_rate": 2e-05,
+      "loss": 1.0197,
+      "step": 860
+    },
+    {
+      "epoch": 0.07305399277857083,
+      "grad_norm": 0.8797135949134827,
+      "learning_rate": 2e-05,
+      "loss": 1.0063,
+      "step": 870
+    },
+    {
+      "epoch": 0.07389369384499118,
+      "grad_norm": 0.476870059967041,
+      "learning_rate": 2e-05,
+      "loss": 0.9671,
+      "step": 880
+    },
+    {
+      "epoch": 0.07473339491141154,
+      "grad_norm": 0.4863646328449249,
+      "learning_rate": 2e-05,
+      "loss": 0.9929,
+      "step": 890
+    },
+    {
+      "epoch": 0.07557309597783189,
+      "grad_norm": 0.48433804512023926,
+      "learning_rate": 2e-05,
+      "loss": 0.9765,
+      "step": 900
+    },
+    {
+      "epoch": 0.07641279704425225,
+      "grad_norm": 0.5103152990341187,
+      "learning_rate": 2e-05,
+      "loss": 1.0255,
+      "step": 910
+    },
+    {
+      "epoch": 0.0772524981106726,
+      "grad_norm": 0.4950040876865387,
+      "learning_rate": 2e-05,
+      "loss": 0.9897,
+      "step": 920
+    },
+    {
+      "epoch": 0.07809219917709295,
+      "grad_norm": 0.6963955163955688,
+      "learning_rate": 2e-05,
+      "loss": 0.9603,
+      "step": 930
+    },
+    {
+      "epoch": 0.0789319002435133,
+      "grad_norm": 0.532052755355835,
+      "learning_rate": 2e-05,
+      "loss": 1.0265,
+      "step": 940
+    },
+    {
+      "epoch": 0.07977160130993366,
+      "grad_norm": 0.4793180227279663,
+      "learning_rate": 2e-05,
+      "loss": 1.0437,
+      "step": 950
+    },
+    {
+      "epoch": 0.08061130237635401,
+      "grad_norm": 0.5289978384971619,
+      "learning_rate": 2e-05,
+      "loss": 0.9697,
+      "step": 960
+    },
+    {
+      "epoch": 0.08145100344277437,
+      "grad_norm": 0.5345343947410583,
+      "learning_rate": 2e-05,
+      "loss": 1.0023,
+      "step": 970
+    },
+    {
+      "epoch": 0.08229070450919472,
+      "grad_norm": 0.5331494212150574,
+      "learning_rate": 2e-05,
+      "loss": 0.9736,
+      "step": 980
+    },
+    {
+      "epoch": 0.08313040557561509,
+      "grad_norm": 0.5220827460289001,
+      "learning_rate": 2e-05,
+      "loss": 1.0074,
+      "step": 990
+    },
+    {
+      "epoch": 0.08397010664203544,
+      "grad_norm": 0.4963169991970062,
+      "learning_rate": 2e-05,
+      "loss": 1.0102,
+      "step": 1000
+    },
+    {
+      "epoch": 0.0848098077084558,
+      "grad_norm": 0.5005528926849365,
+      "learning_rate": 2e-05,
+      "loss": 0.9515,
+      "step": 1010
+    },
+    {
+      "epoch": 0.08564950877487615,
+      "grad_norm": 0.5249394178390503,
+      "learning_rate": 2e-05,
+      "loss": 1.067,
+      "step": 1020
+    },
+    {
+      "epoch": 0.0864892098412965,
+      "grad_norm": 0.5880621075630188,
+      "learning_rate": 2e-05,
+      "loss": 1.0018,
+      "step": 1030
+    },
+    {
+      "epoch": 0.08732891090771686,
+      "grad_norm": 0.48338451981544495,
+      "learning_rate": 2e-05,
+      "loss": 0.9992,
+      "step": 1040
+    },
+    {
+      "epoch": 0.08816861197413721,
+      "grad_norm": 0.5206268429756165,
+      "learning_rate": 2e-05,
+      "loss": 0.9833,
+      "step": 1050
+    },
+    {
+      "epoch": 0.08900831304055756,
+      "grad_norm": 0.49556973576545715,
+      "learning_rate": 2e-05,
+      "loss": 1.0337,
+      "step": 1060
+    },
+    {
+      "epoch": 0.08984801410697792,
+      "grad_norm": 0.49861377477645874,
+      "learning_rate": 2e-05,
+      "loss": 1.0504,
+      "step": 1070
+    },
+    {
+      "epoch": 0.09068771517339827,
+      "grad_norm": 0.5358458161354065,
+      "learning_rate": 2e-05,
+      "loss": 1.0343,
+      "step": 1080
+    },
+    {
+      "epoch": 0.09152741623981862,
+      "grad_norm": 0.4752683639526367,
+      "learning_rate": 2e-05,
+      "loss": 1.0416,
+      "step": 1090
+    },
+    {
+      "epoch": 0.09236711730623898,
+      "grad_norm": 0.4865322411060333,
+      "learning_rate": 2e-05,
+      "loss": 1.0123,
+      "step": 1100
+    },
+    {
+      "epoch": 0.09320681837265933,
+      "grad_norm": 0.5330458283424377,
+      "learning_rate": 2e-05,
+      "loss": 1.0148,
+      "step": 1110
+    },
+    {
+      "epoch": 0.09404651943907968,
+      "grad_norm": 0.5267438292503357,
+      "learning_rate": 2e-05,
+      "loss": 1.0177,
+      "step": 1120
+    },
+    {
+      "epoch": 0.09488622050550004,
+      "grad_norm": 0.5309559106826782,
+      "learning_rate": 2e-05,
+      "loss": 0.9725,
+      "step": 1130
+    },
+    {
+      "epoch": 0.09572592157192039,
+      "grad_norm": 0.48289164900779724,
+      "learning_rate": 2e-05,
+      "loss": 0.9703,
+      "step": 1140
+    },
+    {
+      "epoch": 0.09656562263834075,
+      "grad_norm": 0.4635581970214844,
+      "learning_rate": 2e-05,
+      "loss": 0.9609,
+      "step": 1150
+    },
+    {
+      "epoch": 0.0974053237047611,
+      "grad_norm": 0.5546231865882874,
+      "learning_rate": 2e-05,
+      "loss": 1.0781,
+      "step": 1160
+    },
+    {
+      "epoch": 0.09824502477118147,
+      "grad_norm": 0.45515337586402893,
+      "learning_rate": 2e-05,
+      "loss": 0.9659,
+      "step": 1170
+    },
+    {
+      "epoch": 0.09908472583760182,
+      "grad_norm": 0.7485978603363037,
+      "learning_rate": 2e-05,
+      "loss": 1.0496,
+      "step": 1180
+    },
+    {
+      "epoch": 0.09992442690402217,
+      "grad_norm": 0.44106537103652954,
+      "learning_rate": 2e-05,
+      "loss": 1.0104,
+      "step": 1190
+    },
+    {
+      "epoch": 0.10076412797044253,
+      "grad_norm": 0.5454259514808655,
+      "learning_rate": 2e-05,
+      "loss": 0.9924,
+      "step": 1200
+    },
+    {
+      "epoch": 0.10160382903686288,
+      "grad_norm": 0.5239803791046143,
+      "learning_rate": 2e-05,
+      "loss": 0.9876,
+      "step": 1210
+    },
+    {
+      "epoch": 0.10244353010328323,
+      "grad_norm": 0.5559198260307312,
+      "learning_rate": 2e-05,
+      "loss": 1.0101,
+      "step": 1220
+    },
+    {
+      "epoch": 0.10328323116970359,
+      "grad_norm": 0.6441607475280762,
+      "learning_rate": 2e-05,
+      "loss": 0.9951,
+      "step": 1230
+    },
+    {
+      "epoch": 0.10412293223612394,
+      "grad_norm": 0.4984257221221924,
+      "learning_rate": 2e-05,
+      "loss": 1.0129,
+      "step": 1240
+    },
+    {
+      "epoch": 0.1049626333025443,
+      "grad_norm": 0.47933027148246765,
+      "learning_rate": 2e-05,
+      "loss": 0.9967,
+      "step": 1250
+    },
+    {
+      "epoch": 0.10580233436896465,
+      "grad_norm": 0.49340662360191345,
+      "learning_rate": 2e-05,
+      "loss": 0.9645,
+      "step": 1260
+    },
+    {
+      "epoch": 0.106642035435385,
+      "grad_norm": 0.4745008051395416,
+      "learning_rate": 2e-05,
+      "loss": 0.9875,
+      "step": 1270
+    },
+    {
+      "epoch": 0.10748173650180536,
+      "grad_norm": 0.569976806640625,
+      "learning_rate": 2e-05,
+      "loss": 1.0185,
+      "step": 1280
+    },
+    {
+      "epoch": 0.10832143756822571,
+      "grad_norm": 0.44903650879859924,
+      "learning_rate": 2e-05,
+      "loss": 0.9481,
+      "step": 1290
+    },
+    {
+      "epoch": 0.10916113863464606,
+      "grad_norm": 0.48480257391929626,
+      "learning_rate": 2e-05,
+      "loss": 0.9925,
+      "step": 1300
+    },
+    {
+      "epoch": 0.11000083970106642,
+      "grad_norm": 0.5419672727584839,
+      "learning_rate": 2e-05,
+      "loss": 0.9571,
+      "step": 1310
+    },
+    {
+      "epoch": 0.11084054076748677,
+      "grad_norm": 0.502124011516571,
+      "learning_rate": 2e-05,
+      "loss": 0.9847,
+      "step": 1320
+    },
+    {
+      "epoch": 0.11168024183390712,
+      "grad_norm": 0.5213430523872375,
+      "learning_rate": 2e-05,
+      "loss": 1.0044,
+      "step": 1330
+    },
+    {
+      "epoch": 0.11251994290032748,
+      "grad_norm": 0.5590944886207581,
+      "learning_rate": 2e-05,
+      "loss": 0.9712,
+      "step": 1340
+    },
+    {
+      "epoch": 0.11335964396674784,
+      "grad_norm": 0.9190903306007385,
+      "learning_rate": 2e-05,
+      "loss": 1.0349,
+      "step": 1350
+    },
+    {
+      "epoch": 0.1141993450331682,
+      "grad_norm": 0.4655185043811798,
+      "learning_rate": 2e-05,
+      "loss": 0.907,
+      "step": 1360
+    },
+    {
+      "epoch": 0.11503904609958855,
+      "grad_norm": 0.43830716609954834,
+      "learning_rate": 2e-05,
+      "loss": 0.9866,
+      "step": 1370
+    },
+    {
+      "epoch": 0.1158787471660089,
+      "grad_norm": 0.46941494941711426,
+      "learning_rate": 2e-05,
+      "loss": 0.9884,
+      "step": 1380
+    },
+    {
+      "epoch": 0.11671844823242926,
+      "grad_norm": 0.6129980683326721,
+      "learning_rate": 2e-05,
+      "loss": 1.0102,
+      "step": 1390
+    },
+    {
+      "epoch": 0.11755814929884961,
+      "grad_norm": 0.4963046610355377,
+      "learning_rate": 2e-05,
+      "loss": 0.9826,
+      "step": 1400
+    },
+    {
+      "epoch": 0.11839785036526997,
+      "grad_norm": 0.49613454937934875,
+      "learning_rate": 2e-05,
+      "loss": 1.0176,
+      "step": 1410
+    },
+    {
+      "epoch": 0.11923755143169032,
+      "grad_norm": 0.5326101779937744,
+      "learning_rate": 2e-05,
+      "loss": 0.9313,
+      "step": 1420
+    },
+    {
+      "epoch": 0.12007725249811067,
+      "grad_norm": 0.48534440994262695,
+      "learning_rate": 2e-05,
+      "loss": 1.0199,
+      "step": 1430
+    },
+    {
+      "epoch": 0.12091695356453103,
+      "grad_norm": 0.4562290608882904,
+      "learning_rate": 2e-05,
+      "loss": 1.022,
+      "step": 1440
+    },
+    {
+      "epoch": 0.12175665463095138,
+      "grad_norm": 1.3759952783584595,
+      "learning_rate": 2e-05,
+      "loss": 0.9604,
+      "step": 1450
+    },
+    {
+      "epoch": 0.12259635569737173,
+      "grad_norm": 0.4703039228916168,
+      "learning_rate": 2e-05,
+      "loss": 0.9843,
+      "step": 1460
+    },
+    {
+      "epoch": 0.12343605676379209,
+      "grad_norm": 0.4847295880317688,
+      "learning_rate": 2e-05,
+      "loss": 0.9822,
+      "step": 1470
+    },
+    {
+      "epoch": 0.12427575783021244,
+      "grad_norm": 0.4986240267753601,
+      "learning_rate": 2e-05,
+      "loss": 0.9697,
+      "step": 1480
+    },
+    {
+      "epoch": 0.1251154588966328,
+      "grad_norm": 0.49270740151405334,
+      "learning_rate": 2e-05,
+      "loss": 0.9673,
+      "step": 1490
+    },
+    {
+      "epoch": 0.12595515996305315,
+      "grad_norm": 0.4328170418739319,
+      "learning_rate": 2e-05,
+      "loss": 0.9471,
+      "step": 1500
+    },
+    {
+      "epoch": 0.12679486102947352,
+      "grad_norm": 0.47089821100234985,
+      "learning_rate": 2e-05,
+      "loss": 0.9195,
+      "step": 1510
+    },
+    {
+      "epoch": 0.12763456209589386,
+      "grad_norm": 0.45632755756378174,
+      "learning_rate": 2e-05,
+      "loss": 0.9079,
+      "step": 1520
+    },
+    {
+      "epoch": 0.12847426316231422,
+      "grad_norm": 0.48846668004989624,
+      "learning_rate": 2e-05,
+      "loss": 0.9452,
+      "step": 1530
+    },
+    {
+      "epoch": 0.12931396422873456,
+      "grad_norm": 0.46305525302886963,
+      "learning_rate": 2e-05,
+      "loss": 0.9962,
+      "step": 1540
+    },
+    {
+      "epoch": 0.13015366529515493,
+      "grad_norm": 0.4676452577114105,
+      "learning_rate": 2e-05,
+      "loss": 0.9582,
+      "step": 1550
+    },
+    {
+      "epoch": 0.13099336636157527,
+      "grad_norm": 0.6805684566497803,
+      "learning_rate": 2e-05,
+      "loss": 1.0018,
+      "step": 1560
+    },
+    {
+      "epoch": 0.13183306742799564,
+      "grad_norm": 0.5028660297393799,
+      "learning_rate": 2e-05,
+      "loss": 0.933,
+      "step": 1570
+    },
+    {
+      "epoch": 0.13267276849441598,
+      "grad_norm": 0.9517537951469421,
+      "learning_rate": 2e-05,
+      "loss": 0.9456,
+      "step": 1580
+    },
+    {
+      "epoch": 0.13351246956083634,
+      "grad_norm": 0.49718722701072693,
+      "learning_rate": 2e-05,
+      "loss": 0.964,
+      "step": 1590
+    },
+    {
+      "epoch": 0.13435217062725668,
+      "grad_norm": 0.4775601327419281,
+      "learning_rate": 2e-05,
+      "loss": 0.9496,
+      "step": 1600
+    },
+    {
+      "epoch": 0.13519187169367705,
+      "grad_norm": 0.5251036882400513,
+      "learning_rate": 2e-05,
+      "loss": 0.9465,
+      "step": 1610
+    },
+    {
+      "epoch": 0.13603157276009742,
+      "grad_norm": 0.4998883306980133,
+      "learning_rate": 2e-05,
+      "loss": 1.0076,
+      "step": 1620
+    },
+    {
+      "epoch": 0.13687127382651776,
+      "grad_norm": 0.4440484046936035,
+      "learning_rate": 2e-05,
+      "loss": 0.9921,
+      "step": 1630
+    },
+    {
+      "epoch": 0.13771097489293813,
+      "grad_norm": 0.47233253717422485,
+      "learning_rate": 2e-05,
+      "loss": 0.9507,
+      "step": 1640
+    },
+    {
+      "epoch": 0.13855067595935847,
+      "grad_norm": 0.5169973969459534,
+      "learning_rate": 2e-05,
+      "loss": 0.9693,
+      "step": 1650
+    },
+    {
+      "epoch": 0.13939037702577883,
+      "grad_norm": 0.5089414715766907,
+      "learning_rate": 2e-05,
+      "loss": 0.9597,
+      "step": 1660
+    },
+    {
+      "epoch": 0.14023007809219917,
+      "grad_norm": 0.509657084941864,
+      "learning_rate": 2e-05,
+      "loss": 0.9271,
+      "step": 1670
+    },
+    {
+      "epoch": 0.14106977915861954,
+      "grad_norm": 0.6538721323013306,
+      "learning_rate": 2e-05,
+      "loss": 0.9264,
+      "step": 1680
+    },
+    {
+      "epoch": 0.14190948022503988,
+      "grad_norm": 0.500372588634491,
+      "learning_rate": 2e-05,
+      "loss": 1.0109,
+      "step": 1690
+    },
+    {
+      "epoch": 0.14274918129146025,
+      "grad_norm": 0.5145060420036316,
+      "learning_rate": 2e-05,
+      "loss": 1.0252,
+      "step": 1700
+    },
+    {
+      "epoch": 0.1435888823578806,
+      "grad_norm": 0.4911811649799347,
+      "learning_rate": 2e-05,
+      "loss": 1.0003,
+      "step": 1710
+    },
+    {
+      "epoch": 0.14442858342430095,
+      "grad_norm": 0.6961483955383301,
+      "learning_rate": 2e-05,
+      "loss": 0.9963,
+      "step": 1720
+    },
+    {
+      "epoch": 0.1452682844907213,
+      "grad_norm": 0.49579185247421265,
+      "learning_rate": 2e-05,
+      "loss": 0.9595,
+      "step": 1730
+    },
+    {
+      "epoch": 0.14610798555714166,
+      "grad_norm": 0.5168765783309937,
+      "learning_rate": 2e-05,
+      "loss": 0.9119,
+      "step": 1740
+    },
+    {
+      "epoch": 0.146947686623562,
+      "grad_norm": 0.5454058647155762,
+      "learning_rate": 2e-05,
+      "loss": 0.9757,
+      "step": 1750
+    },
+    {
+      "epoch": 0.14778738768998237,
+      "grad_norm": 0.5128822922706604,
+      "learning_rate": 2e-05,
+      "loss": 0.9439,
+      "step": 1760
+    },
+    {
+      "epoch": 0.1486270887564027,
+      "grad_norm": 0.4754522442817688,
+      "learning_rate": 2e-05,
+      "loss": 1.0083,
+      "step": 1770
+    },
+    {
+      "epoch": 0.14946678982282308,
+      "grad_norm": 0.4661597013473511,
+      "learning_rate": 2e-05,
+      "loss": 0.9544,
+      "step": 1780
+    },
+    {
+      "epoch": 0.15030649088924342,
+      "grad_norm": 0.48215746879577637,
+      "learning_rate": 2e-05,
+      "loss": 0.9397,
+      "step": 1790
+    },
+    {
+      "epoch": 0.15114619195566378,
+      "grad_norm": 0.6749794483184814,
+      "learning_rate": 2e-05,
+      "loss": 0.9572,
+      "step": 1800
+    },
+    {
+      "epoch": 0.15198589302208415,
+      "grad_norm": 0.5560160279273987,
+      "learning_rate": 2e-05,
+      "loss": 0.9433,
+      "step": 1810
+    },
+    {
+      "epoch": 0.1528255940885045,
+      "grad_norm": 0.45195090770721436,
+      "learning_rate": 2e-05,
+      "loss": 0.92,
+      "step": 1820
+    },
+    {
+      "epoch": 0.15366529515492486,
+      "grad_norm": 0.4917123317718506,
+      "learning_rate": 2e-05,
+      "loss": 0.949,
+      "step": 1830
+    },
+    {
+      "epoch": 0.1545049962213452,
+      "grad_norm": 0.4223521947860718,
+      "learning_rate": 2e-05,
+      "loss": 0.933,
+      "step": 1840
+    },
+    {
+      "epoch": 0.15534469728776557,
+      "grad_norm": 0.5012931227684021,
+      "learning_rate": 2e-05,
+      "loss": 0.9113,
+      "step": 1850
+    },
+    {
+      "epoch": 0.1561843983541859,
+      "grad_norm": 0.5609027743339539,
+      "learning_rate": 2e-05,
+      "loss": 0.9313,
+      "step": 1860
+    },
+    {
+      "epoch": 0.15702409942060627,
+      "grad_norm": 0.8982095718383789,
+      "learning_rate": 2e-05,
+      "loss": 0.9312,
+      "step": 1870
+    },
+    {
+      "epoch": 0.1578638004870266,
+      "grad_norm": 0.48062509298324585,
+      "learning_rate": 2e-05,
+      "loss": 0.9603,
+      "step": 1880
+    },
+    {
+      "epoch": 0.15870350155344698,
+      "grad_norm": 0.4441513121128082,
+      "learning_rate": 2e-05,
+      "loss": 0.9869,
+      "step": 1890
+    },
+    {
+      "epoch": 0.15954320261986732,
+      "grad_norm": 1.625954031944275,
+      "learning_rate": 2e-05,
+      "loss": 0.8942,
+      "step": 1900
+    },
+    {
+      "epoch": 0.1603829036862877,
+      "grad_norm": 0.5065054297447205,
+      "learning_rate": 2e-05,
+      "loss": 1.0136,
+      "step": 1910
+    },
+    {
+      "epoch": 0.16122260475270803,
+      "grad_norm": 0.5414623022079468,
+      "learning_rate": 2e-05,
+      "loss": 0.9623,
+      "step": 1920
+    },
+    {
+      "epoch": 0.1620623058191284,
+      "grad_norm": 0.4631492495536804,
+      "learning_rate": 2e-05,
+      "loss": 0.9574,
+      "step": 1930
+    },
+    {
+      "epoch": 0.16290200688554873,
+      "grad_norm": 0.6957469582557678,
+      "learning_rate": 2e-05,
+      "loss": 0.9149,
+      "step": 1940
+    },
+    {
+      "epoch": 0.1637417079519691,
+      "grad_norm": 0.551231324672699,
+      "learning_rate": 2e-05,
+      "loss": 0.9918,
+      "step": 1950
+    },
+    {
+      "epoch": 0.16458140901838944,
+      "grad_norm": 0.5242336988449097,
+      "learning_rate": 2e-05,
+      "loss": 0.9842,
+      "step": 1960
+    },
+    {
+      "epoch": 0.1654211100848098,
+      "grad_norm": 0.4801785945892334,
+      "learning_rate": 2e-05,
+      "loss": 0.9296,
+      "step": 1970
+    },
+    {
+      "epoch": 0.16626081115123018,
+      "grad_norm": 0.5458486676216125,
+      "learning_rate": 2e-05,
+      "loss": 0.909,
+      "step": 1980
+    },
+    {
+      "epoch": 0.16710051221765052,
+      "grad_norm": 0.4695436656475067,
+      "learning_rate": 2e-05,
+      "loss": 0.9716,
+      "step": 1990
+    },
+    {
+      "epoch": 0.16794021328407088,
+      "grad_norm": 0.45850250124931335,
+      "learning_rate": 2e-05,
+      "loss": 0.946,
+      "step": 2000
+    },
+    {
+      "epoch": 0.16877991435049122,
+      "grad_norm": 0.520365834236145,
+      "learning_rate": 2e-05,
+      "loss": 0.8999,
+      "step": 2010
+    },
+    {
+      "epoch": 0.1696196154169116,
+      "grad_norm": 0.4501103460788727,
+      "learning_rate": 2e-05,
+      "loss": 0.9102,
+      "step": 2020
+    },
+    {
+      "epoch": 0.17045931648333193,
+      "grad_norm": 0.45634540915489197,
+      "learning_rate": 2e-05,
+      "loss": 0.9477,
+      "step": 2030
+    },
+    {
+      "epoch": 0.1712990175497523,
+      "grad_norm": 0.5216748714447021,
+      "learning_rate": 2e-05,
+      "loss": 0.9625,
+      "step": 2040
+    },
+    {
+      "epoch": 0.17213871861617264,
+      "grad_norm": 0.5178518891334534,
+      "learning_rate": 2e-05,
+      "loss": 0.9047,
+      "step": 2050
+    },
+    {
+      "epoch": 0.172978419682593,
+      "grad_norm": 0.49203771352767944,
+      "learning_rate": 2e-05,
+      "loss": 0.9061,
+      "step": 2060
+    },
+    {
+      "epoch": 0.17381812074901334,
+      "grad_norm": 0.6264152526855469,
+      "learning_rate": 2e-05,
+      "loss": 1.0226,
+      "step": 2070
+    },
+    {
+      "epoch": 0.1746578218154337,
+      "grad_norm": 0.43532794713974,
+      "learning_rate": 2e-05,
+      "loss": 0.9101,
+      "step": 2080
+    },
+    {
+      "epoch": 0.17549752288185405,
+      "grad_norm": 0.49513307213783264,
+      "learning_rate": 2e-05,
+      "loss": 1.0229,
+      "step": 2090
+    },
+    {
+      "epoch": 0.17633722394827442,
+      "grad_norm": 0.9167293310165405,
+      "learning_rate": 2e-05,
+      "loss": 0.9497,
+      "step": 2100
+    },
+    {
+      "epoch": 0.17717692501469476,
+      "grad_norm": 0.4796413481235504,
+      "learning_rate": 2e-05,
+      "loss": 1.053,
+      "step": 2110
+    },
+    {
+      "epoch": 0.17801662608111513,
+      "grad_norm": 0.45403558015823364,
+      "learning_rate": 2e-05,
+      "loss": 0.931,
+      "step": 2120
+    },
+    {
+      "epoch": 0.17885632714753547,
+      "grad_norm": 0.49153926968574524,
+      "learning_rate": 2e-05,
+      "loss": 0.9776,
+      "step": 2130
+    },
+    {
+      "epoch": 0.17969602821395583,
+      "grad_norm": 0.4547460675239563,
+      "learning_rate": 2e-05,
+      "loss": 1.0066,
+      "step": 2140
+    },
+    {
+      "epoch": 0.18053572928037617,
+      "grad_norm": 0.5737180113792419,
+      "learning_rate": 2e-05,
+      "loss": 0.9392,
+      "step": 2150
+    },
+    {
+      "epoch": 0.18137543034679654,
+      "grad_norm": 0.45654165744781494,
+      "learning_rate": 2e-05,
+      "loss": 0.9507,
+      "step": 2160
+    },
+    {
+      "epoch": 0.1822151314132169,
+      "grad_norm": 0.5037363767623901,
+      "learning_rate": 2e-05,
+      "loss": 0.9561,
+      "step": 2170
+    },
+    {
+      "epoch": 0.18305483247963725,
+      "grad_norm": 0.480439156293869,
+      "learning_rate": 2e-05,
+      "loss": 0.9459,
+      "step": 2180
+    },
+    {
+      "epoch": 0.18389453354605761,
+      "grad_norm": 0.47951242327690125,
+      "learning_rate": 2e-05,
+      "loss": 0.995,
+      "step": 2190
+    },
+    {
+      "epoch": 0.18473423461247795,
+      "grad_norm": 0.45319557189941406,
+      "learning_rate": 2e-05,
+      "loss": 0.9632,
+      "step": 2200
+    },
+    {
+      "epoch": 0.18557393567889832,
+      "grad_norm": 0.4815351665019989,
+      "learning_rate": 2e-05,
+      "loss": 0.9138,
+      "step": 2210
+    },
+    {
+      "epoch": 0.18641363674531866,
+      "grad_norm": 0.6240055561065674,
+      "learning_rate": 2e-05,
+      "loss": 0.9472,
+      "step": 2220
+    },
+    {
+      "epoch": 0.18725333781173903,
+      "grad_norm": 0.4974870979785919,
+      "learning_rate": 2e-05,
+      "loss": 0.8905,
+      "step": 2230
+    },
+    {
+      "epoch": 0.18809303887815937,
+      "grad_norm": 0.49529772996902466,
+      "learning_rate": 2e-05,
+      "loss": 0.912,
+      "step": 2240
+    },
+    {
+      "epoch": 0.18893273994457974,
+      "grad_norm": 0.47586724162101746,
+      "learning_rate": 2e-05,
+      "loss": 0.9509,
+      "step": 2250
+    },
+    {
+      "epoch": 0.18977244101100008,
+      "grad_norm": 0.45849329233169556,
+      "learning_rate": 2e-05,
+      "loss": 0.9058,
+      "step": 2260
+    },
+    {
+      "epoch": 0.19061214207742044,
+      "grad_norm": 0.5080339312553406,
+      "learning_rate": 2e-05,
+      "loss": 0.9455,
+      "step": 2270
+    },
+    {
+      "epoch": 0.19145184314384078,
+      "grad_norm": 0.5078399181365967,
+      "learning_rate": 2e-05,
+      "loss": 0.9504,
+      "step": 2280
+    },
+    {
+      "epoch": 0.19229154421026115,
+      "grad_norm": 0.5137534141540527,
+      "learning_rate": 2e-05,
+      "loss": 0.9379,
+      "step": 2290
+    },
+    {
+      "epoch": 0.1931312452766815,
+      "grad_norm": 0.523794412612915,
+      "learning_rate": 2e-05,
+      "loss": 0.9181,
+      "step": 2300
+    },
+    {
+      "epoch": 0.19397094634310186,
+      "grad_norm": 0.4992673099040985,
+      "learning_rate": 2e-05,
+      "loss": 0.8763,
+      "step": 2310
+    },
+    {
+      "epoch": 0.1948106474095222,
+      "grad_norm": 0.4604392647743225,
+      "learning_rate": 2e-05,
+      "loss": 0.9369,
+      "step": 2320
+    },
+    {
+      "epoch": 0.19565034847594256,
+      "grad_norm": 0.479475200176239,
+      "learning_rate": 2e-05,
+      "loss": 0.9176,
+      "step": 2330
+    },
+    {
+      "epoch": 0.19649004954236293,
+      "grad_norm": 0.47156229615211487,
+      "learning_rate": 2e-05,
+      "loss": 0.9327,
+      "step": 2340
+    },
+    {
+      "epoch": 0.19732975060878327,
+      "grad_norm": 0.5441238880157471,
+      "learning_rate": 2e-05,
+      "loss": 0.992,
+      "step": 2350
+    },
+    {
+      "epoch": 0.19816945167520364,
+      "grad_norm": 0.4867047369480133,
+      "learning_rate": 2e-05,
+      "loss": 0.9237,
+      "step": 2360
+    },
+    {
+      "epoch": 0.19900915274162398,
+      "grad_norm": 0.42399755120277405,
+      "learning_rate": 2e-05,
+      "loss": 0.9195,
+      "step": 2370
+    },
+    {
+      "epoch": 0.19984885380804435,
+      "grad_norm": 0.45596733689308167,
+      "learning_rate": 2e-05,
+      "loss": 0.8518,
+      "step": 2380
+    },
+    {
+      "epoch": 0.2006885548744647,
+      "grad_norm": 0.4902798533439636,
+      "learning_rate": 2e-05,
+      "loss": 0.955,
+      "step": 2390
+    },
+    {
+      "epoch": 0.20152825594088505,
+      "grad_norm": 0.5207685232162476,
+      "learning_rate": 2e-05,
+      "loss": 0.8705,
+      "step": 2400
+    },
+    {
+      "epoch": 0.2023679570073054,
+      "grad_norm": 1.1840689182281494,
+      "learning_rate": 2e-05,
+      "loss": 0.9338,
+      "step": 2410
+    },
+    {
+      "epoch": 0.20320765807372576,
+      "grad_norm": 0.5640371441841125,
+      "learning_rate": 2e-05,
+      "loss": 0.9348,
+      "step": 2420
+    },
+    {
+      "epoch": 0.2040473591401461,
+      "grad_norm": 0.4559391140937805,
+      "learning_rate": 2e-05,
+      "loss": 0.9464,
+      "step": 2430
+    },
+    {
+      "epoch": 0.20488706020656647,
+      "grad_norm": 0.5070319771766663,
+      "learning_rate": 2e-05,
+      "loss": 0.9117,
+      "step": 2440
+    },
+    {
+      "epoch": 0.2057267612729868,
+      "grad_norm": 0.4451445937156677,
+      "learning_rate": 2e-05,
+      "loss": 0.9582,
+      "step": 2450
+    },
+    {
+      "epoch": 0.20656646233940718,
+      "grad_norm": 0.44806957244873047,
+      "learning_rate": 2e-05,
+      "loss": 0.9391,
+      "step": 2460
+    },
+    {
+      "epoch": 0.20740616340582751,
+      "grad_norm": 0.4711221158504486,
+      "learning_rate": 2e-05,
+      "loss": 0.8628,
+      "step": 2470
+    },
+    {
+      "epoch": 0.20824586447224788,
+      "grad_norm": 0.5185308456420898,
+      "learning_rate": 2e-05,
+      "loss": 0.91,
+      "step": 2480
+    },
+    {
+      "epoch": 0.20908556553866822,
+      "grad_norm": 0.4656069278717041,
+      "learning_rate": 2e-05,
+      "loss": 0.9087,
+      "step": 2490
+    },
+    {
+      "epoch": 0.2099252666050886,
+      "grad_norm": 1.3008824586868286,
+      "learning_rate": 2e-05,
+      "loss": 0.8874,
+      "step": 2500
+    },
+    {
+      "epoch": 0.21076496767150893,
+      "grad_norm": 0.5072458982467651,
+      "learning_rate": 2e-05,
+      "loss": 0.8753,
+      "step": 2510
+    },
+    {
+      "epoch": 0.2116046687379293,
+      "grad_norm": 0.5111760497093201,
+      "learning_rate": 2e-05,
+      "loss": 0.9099,
+      "step": 2520
+    },
+    {
+      "epoch": 0.21244436980434966,
+      "grad_norm": 0.5030725598335266,
+      "learning_rate": 2e-05,
+      "loss": 0.917,
+      "step": 2530
+    },
+    {
+      "epoch": 0.21328407087077,
+      "grad_norm": 0.49836936593055725,
+      "learning_rate": 2e-05,
+      "loss": 0.9084,
+      "step": 2540
+    },
+    {
+      "epoch": 0.21412377193719037,
+      "grad_norm": 0.5167638659477234,
+      "learning_rate": 2e-05,
+      "loss": 0.9352,
+      "step": 2550
+    },
+    {
+      "epoch": 0.2149634730036107,
+      "grad_norm": 0.6232368350028992,
+      "learning_rate": 2e-05,
+      "loss": 0.9489,
+      "step": 2560
+    },
+    {
+      "epoch": 0.21580317407003108,
+      "grad_norm": 0.4973466694355011,
+      "learning_rate": 2e-05,
+      "loss": 0.9344,
+      "step": 2570
+    },
+    {
+      "epoch": 0.21664287513645142,
+      "grad_norm": 0.6005841493606567,
+      "learning_rate": 2e-05,
+      "loss": 0.9189,
+      "step": 2580
+    },
+    {
+      "epoch": 0.21748257620287179,
+      "grad_norm": 0.4937240183353424,
+      "learning_rate": 2e-05,
+      "loss": 0.9311,
+      "step": 2590
+    },
+    {
+      "epoch": 0.21832227726929213,
+      "grad_norm": 0.5319569110870361,
+      "learning_rate": 2e-05,
+      "loss": 0.9266,
+      "step": 2600
+    },
+    {
+      "epoch": 0.2191619783357125,
+      "grad_norm": 0.5008692145347595,
+      "learning_rate": 2e-05,
+      "loss": 0.9103,
+      "step": 2610
+    },
+    {
+      "epoch": 0.22000167940213283,
+      "grad_norm": 0.5848636031150818,
+      "learning_rate": 2e-05,
+      "loss": 0.9648,
+      "step": 2620
+    },
+    {
+      "epoch": 0.2208413804685532,
+      "grad_norm": 0.5213149785995483,
+      "learning_rate": 2e-05,
+      "loss": 0.9927,
+      "step": 2630
+    },
+    {
+      "epoch": 0.22168108153497354,
+      "grad_norm": 0.5254894495010376,
+      "learning_rate": 2e-05,
+      "loss": 0.97,
+      "step": 2640
+    },
+    {
+      "epoch": 0.2225207826013939,
+      "grad_norm": 0.4357723295688629,
+      "learning_rate": 2e-05,
+      "loss": 0.8742,
+      "step": 2650
+    },
+    {
+      "epoch": 0.22336048366781425,
+      "grad_norm": 0.4424242377281189,
+      "learning_rate": 2e-05,
+      "loss": 0.9214,
+      "step": 2660
+    },
+    {
+      "epoch": 0.22420018473423461,
+      "grad_norm": 0.47744500637054443,
+      "learning_rate": 2e-05,
+      "loss": 0.8714,
+      "step": 2670
+    },
+    {
+      "epoch": 0.22503988580065495,
+      "grad_norm": 0.48441562056541443,
+      "learning_rate": 2e-05,
+      "loss": 0.9186,
+      "step": 2680
+    },
+    {
+      "epoch": 0.22587958686707532,
+      "grad_norm": 0.5078462958335876,
+      "learning_rate": 2e-05,
+      "loss": 0.8622,
+      "step": 2690
+    },
+    {
+      "epoch": 0.2267192879334957,
+      "grad_norm": 0.4458276331424713,
+      "learning_rate": 2e-05,
+      "loss": 0.8928,
+      "step": 2700
+    },
+    {
+      "epoch": 0.22755898899991603,
+      "grad_norm": 0.44709163904190063,
+      "learning_rate": 2e-05,
+      "loss": 0.9354,
+      "step": 2710
+    },
+    {
+      "epoch": 0.2283986900663364,
+      "grad_norm": 0.49451521039009094,
+      "learning_rate": 2e-05,
+      "loss": 0.8733,
+      "step": 2720
+    },
+    {
+      "epoch": 0.22923839113275674,
+      "grad_norm": 0.4975358843803406,
+      "learning_rate": 2e-05,
+      "loss": 0.9298,
+      "step": 2730
+    },
+    {
+      "epoch": 0.2300780921991771,
+      "grad_norm": 0.8602306842803955,
+      "learning_rate": 2e-05,
+      "loss": 0.9485,
+      "step": 2740
+    },
+    {
+      "epoch": 0.23091779326559744,
+      "grad_norm": 0.481277197599411,
+      "learning_rate": 2e-05,
+      "loss": 0.9119,
+      "step": 2750
+    },
+    {
+      "epoch": 0.2317574943320178,
+      "grad_norm": 0.47115132212638855,
+      "learning_rate": 2e-05,
+      "loss": 0.9078,
+      "step": 2760
+    },
+    {
+      "epoch": 0.23259719539843815,
+      "grad_norm": 0.4939975142478943,
+      "learning_rate": 2e-05,
+      "loss": 0.896,
+      "step": 2770
+    },
+    {
+      "epoch": 0.23343689646485852,
+      "grad_norm": 0.4805504083633423,
+      "learning_rate": 2e-05,
+      "loss": 0.9236,
+      "step": 2780
+    },
+    {
+      "epoch": 0.23427659753127886,
+      "grad_norm": 0.5339744687080383,
+      "learning_rate": 2e-05,
+      "loss": 0.912,
+      "step": 2790
+    },
+    {
+      "epoch": 0.23511629859769922,
+      "grad_norm": 0.5092911720275879,
+      "learning_rate": 2e-05,
+      "loss": 0.9462,
+      "step": 2800
+    },
+    {
+      "epoch": 0.23595599966411956,
+      "grad_norm": 0.44436362385749817,
+      "learning_rate": 2e-05,
+      "loss": 0.9103,
+      "step": 2810
+    },
+    {
+      "epoch": 0.23679570073053993,
+      "grad_norm": 0.47075724601745605,
+      "learning_rate": 2e-05,
+      "loss": 0.939,
+      "step": 2820
+    },
+    {
+      "epoch": 0.23763540179696027,
+      "grad_norm": 0.4800730049610138,
+      "learning_rate": 2e-05,
+      "loss": 0.9019,
+      "step": 2830
+    },
+    {
+      "epoch": 0.23847510286338064,
+      "grad_norm": 0.4602144658565521,
+      "learning_rate": 2e-05,
+      "loss": 0.8817,
+      "step": 2840
+    },
+    {
+      "epoch": 0.23931480392980098,
+      "grad_norm": 0.490458607673645,
+      "learning_rate": 2e-05,
+      "loss": 1.0127,
+      "step": 2850
+    },
+    {
+      "epoch": 0.24015450499622135,
+      "grad_norm": 0.47498565912246704,
+      "learning_rate": 2e-05,
+      "loss": 0.995,
+      "step": 2860
+    },
+    {
+      "epoch": 0.24099420606264169,
+      "grad_norm": 0.4692985415458679,
+      "learning_rate": 2e-05,
+      "loss": 0.9004,
+      "step": 2870
+    },
+    {
+      "epoch": 0.24183390712906205,
+      "grad_norm": 0.46756234765052795,
+      "learning_rate": 2e-05,
+      "loss": 0.9189,
+      "step": 2880
+    },
+    {
+      "epoch": 0.24267360819548242,
+      "grad_norm": 0.47283902764320374,
+      "learning_rate": 2e-05,
+      "loss": 0.9111,
+      "step": 2890
+    },
+    {
+      "epoch": 0.24351330926190276,
+      "grad_norm": 0.48731324076652527,
+      "learning_rate": 2e-05,
+      "loss": 0.8653,
+      "step": 2900
+    },
+    {
+      "epoch": 0.24435301032832313,
+      "grad_norm": 0.48551827669143677,
+      "learning_rate": 2e-05,
+      "loss": 0.888,
+      "step": 2910
+    },
+    {
+      "epoch": 0.24519271139474347,
+      "grad_norm": 0.468881219625473,
+      "learning_rate": 2e-05,
+      "loss": 0.9883,
+      "step": 2920
+    },
+    {
+      "epoch": 0.24603241246116384,
+      "grad_norm": 0.479223370552063,
+      "learning_rate": 2e-05,
+      "loss": 0.8993,
+      "step": 2930
+    },
+    {
+      "epoch": 0.24687211352758417,
+      "grad_norm": 0.47568804025650024,
+      "learning_rate": 2e-05,
+      "loss": 0.8931,
+      "step": 2940
+    },
+    {
+      "epoch": 0.24771181459400454,
+      "grad_norm": 0.4757213890552521,
+      "learning_rate": 2e-05,
+      "loss": 0.8602,
+      "step": 2950
+    },
+    {
+      "epoch": 0.24855151566042488,
+      "grad_norm": 0.5283371806144714,
+      "learning_rate": 2e-05,
+      "loss": 0.8601,
+      "step": 2960
+    },
+    {
+      "epoch": 0.24939121672684525,
+      "grad_norm": 0.44496625661849976,
+      "learning_rate": 2e-05,
+      "loss": 0.9017,
+      "step": 2970
+    },
+    {
+      "epoch": 0.2502309177932656,
+      "grad_norm": 0.5315223336219788,
+      "learning_rate": 2e-05,
+      "loss": 0.8748,
+      "step": 2980
+    },
+    {
+      "epoch": 0.25107061885968596,
+      "grad_norm": 0.4719948172569275,
+      "learning_rate": 2e-05,
+      "loss": 0.9148,
+      "step": 2990
+    },
+    {
+      "epoch": 0.2519103199261063,
+      "grad_norm": 0.43613871932029724,
+      "learning_rate": 2e-05,
+      "loss": 0.8421,
+      "step": 3000
+    },
+    {
+      "epoch": 0.25275002099252664,
+      "grad_norm": 0.4245431423187256,
+      "learning_rate": 2e-05,
+      "loss": 0.9678,
+      "step": 3010
+    },
+    {
+      "epoch": 0.25358972205894703,
+      "grad_norm": 0.4249719977378845,
+      "learning_rate": 2e-05,
+      "loss": 0.9147,
+      "step": 3020
+    },
+    {
+      "epoch": 0.25442942312536737,
+      "grad_norm": 0.4567497968673706,
+      "learning_rate": 2e-05,
+      "loss": 0.8738,
+      "step": 3030
+    },
+    {
+      "epoch": 0.2552691241917877,
+      "grad_norm": 0.9555009007453918,
+      "learning_rate": 2e-05,
+      "loss": 0.8908,
+      "step": 3040
+    },
+    {
+      "epoch": 0.25610882525820805,
+      "grad_norm": 0.7387804388999939,
+      "learning_rate": 2e-05,
+      "loss": 0.9529,
+      "step": 3050
+    },
+    {
+      "epoch": 0.25694852632462845,
+      "grad_norm": 0.5330400466918945,
+      "learning_rate": 2e-05,
+      "loss": 0.8635,
+      "step": 3060
+    },
+    {
+      "epoch": 0.2577882273910488,
+      "grad_norm": 0.4954015612602234,
+      "learning_rate": 2e-05,
+      "loss": 0.8641,
+      "step": 3070
+    },
+    {
+      "epoch": 0.2586279284574691,
+      "grad_norm": 0.4634343683719635,
+      "learning_rate": 2e-05,
+      "loss": 0.9088,
+      "step": 3080
+    },
+    {
+      "epoch": 0.2594676295238895,
+      "grad_norm": 0.5557254552841187,
+      "learning_rate": 2e-05,
+      "loss": 0.9411,
+      "step": 3090
+    },
+    {
+      "epoch": 0.26030733059030986,
+      "grad_norm": 0.4372376501560211,
+      "learning_rate": 2e-05,
+      "loss": 0.8865,
+      "step": 3100
+    },
+    {
+      "epoch": 0.2611470316567302,
+      "grad_norm": 0.5180758237838745,
+      "learning_rate": 2e-05,
+      "loss": 0.9007,
+      "step": 3110
+    },
+    {
+      "epoch": 0.26198673272315054,
+      "grad_norm": 0.4825594425201416,
+      "learning_rate": 2e-05,
+      "loss": 0.913,
+      "step": 3120
+    },
+    {
+      "epoch": 0.26282643378957093,
+      "grad_norm": 0.531485915184021,
+      "learning_rate": 2e-05,
+      "loss": 0.8433,
+      "step": 3130
+    },
+    {
+      "epoch": 0.2636661348559913,
+      "grad_norm": 0.4625248312950134,
+      "learning_rate": 2e-05,
+      "loss": 0.8805,
+      "step": 3140
+    },
+    {
+      "epoch": 0.2645058359224116,
+      "grad_norm": 0.5101214051246643,
+      "learning_rate": 2e-05,
+      "loss": 0.936,
+      "step": 3150
+    },
+    {
+      "epoch": 0.26534553698883195,
+      "grad_norm": 0.5552709102630615,
+      "learning_rate": 2e-05,
+      "loss": 0.9973,
+      "step": 3160
+    },
+    {
+      "epoch": 0.26618523805525235,
+      "grad_norm": 0.4541288912296295,
+      "learning_rate": 2e-05,
+      "loss": 0.9449,
+      "step": 3170
+    },
+    {
+      "epoch": 0.2670249391216727,
+      "grad_norm": 0.6081951856613159,
+      "learning_rate": 2e-05,
+      "loss": 0.875,
+      "step": 3180
+    },
+    {
+      "epoch": 0.26786464018809303,
+      "grad_norm": 0.4702683389186859,
+      "learning_rate": 2e-05,
+      "loss": 0.8547,
+      "step": 3190
+    },
+    {
+      "epoch": 0.26870434125451337,
+      "grad_norm": 0.5702682137489319,
+      "learning_rate": 2e-05,
+      "loss": 0.9052,
+      "step": 3200
+    },
+    {
+      "epoch": 0.26954404232093376,
+      "grad_norm": 0.465741366147995,
+      "learning_rate": 2e-05,
+      "loss": 0.9133,
+      "step": 3210
+    },
+    {
+      "epoch": 0.2703837433873541,
+      "grad_norm": 0.4537079334259033,
+      "learning_rate": 2e-05,
+      "loss": 0.907,
+      "step": 3220
+    },
+    {
+      "epoch": 0.27122344445377444,
+      "grad_norm": 0.5163260698318481,
+      "learning_rate": 2e-05,
+      "loss": 0.881,
+      "step": 3230
+    },
+    {
+      "epoch": 0.27206314552019484,
+      "grad_norm": 0.4609355926513672,
+      "learning_rate": 2e-05,
+      "loss": 0.9118,
+      "step": 3240
+    },
+    {
+      "epoch": 0.2729028465866152,
+      "grad_norm": 0.5809486508369446,
+      "learning_rate": 2e-05,
+      "loss": 0.9267,
+      "step": 3250
+    },
+    {
+      "epoch": 0.2737425476530355,
+      "grad_norm": 0.5758808851242065,
+      "learning_rate": 2e-05,
+      "loss": 0.8266,
+      "step": 3260
+    },
+    {
+      "epoch": 0.27458224871945586,
+      "grad_norm": 0.4374668002128601,
+      "learning_rate": 2e-05,
+      "loss": 0.8668,
+      "step": 3270
+    },
+    {
+      "epoch": 0.27542194978587625,
+      "grad_norm": 0.7117616534233093,
+      "learning_rate": 2e-05,
+      "loss": 0.941,
+      "step": 3280
+    },
+    {
+      "epoch": 0.2762616508522966,
+      "grad_norm": 0.43848082423210144,
+      "learning_rate": 2e-05,
+      "loss": 0.8833,
+      "step": 3290
+    },
+    {
+      "epoch": 0.27710135191871693,
+      "grad_norm": 0.4706605076789856,
+      "learning_rate": 2e-05,
+      "loss": 0.8103,
+      "step": 3300
+    },
+    {
+      "epoch": 0.27794105298513727,
+      "grad_norm": 0.5562008619308472,
+      "learning_rate": 2e-05,
+      "loss": 0.8108,
+      "step": 3310
+    },
+    {
+      "epoch": 0.27878075405155767,
+      "grad_norm": 0.47418683767318726,
+      "learning_rate": 2e-05,
+      "loss": 0.8061,
+      "step": 3320
+    },
+    {
+      "epoch": 0.279620455117978,
+      "grad_norm": 0.5074454545974731,
+      "learning_rate": 2e-05,
+      "loss": 0.8998,
+      "step": 3330
+    },
+    {
+      "epoch": 0.28046015618439835,
+      "grad_norm": 0.44265636801719666,
+      "learning_rate": 2e-05,
+      "loss": 0.9648,
+      "step": 3340
+    },
+    {
+      "epoch": 0.2812998572508187,
+      "grad_norm": 0.5261510610580444,
+      "learning_rate": 2e-05,
+      "loss": 0.942,
+      "step": 3350
+    },
+    {
+      "epoch": 0.2821395583172391,
+      "grad_norm": 0.5490853786468506,
+      "learning_rate": 2e-05,
+      "loss": 0.9293,
+      "step": 3360
+    },
+    {
+      "epoch": 0.2829792593836594,
+      "grad_norm": 0.5360944867134094,
+      "learning_rate": 2e-05,
+      "loss": 0.8687,
+      "step": 3370
+    },
+    {
+      "epoch": 0.28381896045007976,
+      "grad_norm": 0.4663020670413971,
+      "learning_rate": 2e-05,
+      "loss": 0.8435,
+      "step": 3380
+    },
+    {
+      "epoch": 0.2846586615165001,
+      "grad_norm": 0.5098922252655029,
+      "learning_rate": 2e-05,
+      "loss": 0.9456,
+      "step": 3390
+    },
+    {
+      "epoch": 0.2854983625829205,
+      "grad_norm": 0.5601612329483032,
+      "learning_rate": 2e-05,
+      "loss": 0.8523,
+      "step": 3400
+    },
+    {
+      "epoch": 0.28633806364934083,
+      "grad_norm": 0.4711262881755829,
+      "learning_rate": 2e-05,
+      "loss": 0.8916,
+      "step": 3410
+    },
+    {
+      "epoch": 0.2871777647157612,
+      "grad_norm": 0.45892903208732605,
+      "learning_rate": 2e-05,
+      "loss": 0.8955,
+      "step": 3420
+    },
+    {
+      "epoch": 0.28801746578218157,
+      "grad_norm": 0.486325740814209,
+      "learning_rate": 2e-05,
+      "loss": 0.8729,
+      "step": 3430
+    },
+    {
+      "epoch": 0.2888571668486019,
+      "grad_norm": 0.43561339378356934,
+      "learning_rate": 2e-05,
+      "loss": 0.925,
+      "step": 3440
+    },
+    {
+      "epoch": 0.28969686791502225,
+      "grad_norm": 0.479550302028656,
+      "learning_rate": 2e-05,
+      "loss": 0.8622,
+      "step": 3450
+    },
+    {
+      "epoch": 0.2905365689814426,
+      "grad_norm": 0.4714283049106598,
+      "learning_rate": 2e-05,
+      "loss": 0.8821,
+      "step": 3460
+    },
+    {
+      "epoch": 0.291376270047863,
+      "grad_norm": 0.4764579236507416,
+      "learning_rate": 2e-05,
+      "loss": 0.8862,
+      "step": 3470
+    },
+    {
+      "epoch": 0.2922159711142833,
+      "grad_norm": 0.5598776936531067,
+      "learning_rate": 2e-05,
+      "loss": 0.8991,
+      "step": 3480
+    },
+    {
+      "epoch": 0.29305567218070366,
+      "grad_norm": 0.8285524845123291,
+      "learning_rate": 2e-05,
+      "loss": 0.8632,
+      "step": 3490
+    },
+    {
+      "epoch": 0.293895373247124,
+      "grad_norm": 0.4616228938102722,
+      "learning_rate": 2e-05,
+      "loss": 0.8195,
+      "step": 3500
+    },
+    {
+      "epoch": 0.2947350743135444,
+      "grad_norm": 0.4887092411518097,
+      "learning_rate": 2e-05,
+      "loss": 0.8703,
+      "step": 3510
+    },
+    {
+      "epoch": 0.29557477537996474,
+      "grad_norm": 0.5094156861305237,
+      "learning_rate": 2e-05,
+      "loss": 0.8484,
+      "step": 3520
+    },
+    {
+      "epoch": 0.2964144764463851,
+      "grad_norm": 0.47875961661338806,
+      "learning_rate": 2e-05,
+      "loss": 0.8795,
+      "step": 3530
+    },
+    {
+      "epoch": 0.2972541775128054,
+      "grad_norm": 0.5190816521644592,
+      "learning_rate": 2e-05,
+      "loss": 0.9053,
+      "step": 3540
+    },
+    {
+      "epoch": 0.2980938785792258,
+      "grad_norm": 0.5685892701148987,
+      "learning_rate": 2e-05,
+      "loss": 0.8638,
+      "step": 3550
+    },
+    {
+      "epoch": 0.29893357964564615,
+      "grad_norm": 0.4755197763442993,
+      "learning_rate": 2e-05,
+      "loss": 0.8933,
+      "step": 3560
+    },
+    {
+      "epoch": 0.2997732807120665,
+      "grad_norm": 0.4564337432384491,
+      "learning_rate": 2e-05,
+      "loss": 0.9014,
+      "step": 3570
+    },
+    {
+      "epoch": 0.30061298177848683,
+      "grad_norm": 0.4828345477581024,
+      "learning_rate": 2e-05,
+      "loss": 0.8652,
+      "step": 3580
+    },
+    {
+      "epoch": 0.3014526828449072,
+      "grad_norm": 0.7579197287559509,
+      "learning_rate": 2e-05,
+      "loss": 0.8544,
+      "step": 3590
+    },
+    {
+      "epoch": 0.30229238391132757,
+      "grad_norm": 0.5159316658973694,
+      "learning_rate": 2e-05,
+      "loss": 0.9056,
+      "step": 3600
+    },
+    {
+      "epoch": 0.3031320849777479,
+      "grad_norm": 0.6052481532096863,
+      "learning_rate": 2e-05,
+      "loss": 0.9495,
+      "step": 3610
+    },
+    {
+      "epoch": 0.3039717860441683,
+      "grad_norm": 0.45634427666664124,
+      "learning_rate": 2e-05,
+      "loss": 0.9264,
+      "step": 3620
+    },
+    {
+      "epoch": 0.30481148711058864,
+      "grad_norm": 0.4991096258163452,
+      "learning_rate": 2e-05,
+      "loss": 0.9691,
+      "step": 3630
+    },
+    {
+      "epoch": 0.305651188177009,
+      "grad_norm": 0.47846415638923645,
+      "learning_rate": 2e-05,
+      "loss": 0.8726,
+      "step": 3640
+    },
+    {
+      "epoch": 0.3064908892434293,
+      "grad_norm": 0.47555962204933167,
+      "learning_rate": 2e-05,
+      "loss": 0.8383,
+      "step": 3650
+    },
+    {
+      "epoch": 0.3073305903098497,
+      "grad_norm": 0.42626917362213135,
+      "learning_rate": 2e-05,
+      "loss": 0.853,
+      "step": 3660
+    },
+    {
+      "epoch": 0.30817029137627006,
+      "grad_norm": 0.4851888120174408,
+      "learning_rate": 2e-05,
+      "loss": 0.8785,
+      "step": 3670
+    },
+    {
+      "epoch": 0.3090099924426904,
+      "grad_norm": 0.47288089990615845,
+      "learning_rate": 2e-05,
+      "loss": 0.8503,
+      "step": 3680
+    },
+    {
+      "epoch": 0.30984969350911074,
+      "grad_norm": 0.791861891746521,
+      "learning_rate": 2e-05,
+      "loss": 0.9196,
+      "step": 3690
+    },
+    {
+      "epoch": 0.31068939457553113,
+      "grad_norm": 0.44296127557754517,
+      "learning_rate": 2e-05,
+      "loss": 0.8381,
+      "step": 3700
+    },
+    {
+      "epoch": 0.31152909564195147,
+      "grad_norm": 0.5468772053718567,
+      "learning_rate": 2e-05,
+      "loss": 0.9147,
+      "step": 3710
+    },
+    {
+      "epoch": 0.3123687967083718,
+      "grad_norm": 0.507901132106781,
+      "learning_rate": 2e-05,
+      "loss": 0.944,
+      "step": 3720
+    },
+    {
+      "epoch": 0.31320849777479215,
+      "grad_norm": 0.4485001862049103,
+      "learning_rate": 2e-05,
+      "loss": 0.8927,
+      "step": 3730
+    },
+    {
+      "epoch": 0.31404819884121254,
+      "grad_norm": 0.46642786264419556,
+      "learning_rate": 2e-05,
+      "loss": 0.9019,
+      "step": 3740
+    },
+    {
+      "epoch": 0.3148878999076329,
+      "grad_norm": 0.5192999839782715,
+      "learning_rate": 2e-05,
+      "loss": 0.8688,
+      "step": 3750
+    },
+    {
+      "epoch": 0.3157276009740532,
+      "grad_norm": 0.4995100498199463,
+      "learning_rate": 2e-05,
+      "loss": 0.8326,
+      "step": 3760
+    },
+    {
+      "epoch": 0.31656730204047356,
+      "grad_norm": 0.4528861343860626,
+      "learning_rate": 2e-05,
+      "loss": 0.8884,
+      "step": 3770
+    },
+    {
+      "epoch": 0.31740700310689396,
+      "grad_norm": 0.5414633750915527,
+      "learning_rate": 2e-05,
+      "loss": 0.8932,
+      "step": 3780
+    },
+    {
+      "epoch": 0.3182467041733143,
+      "grad_norm": 0.5236119627952576,
+      "learning_rate": 2e-05,
+      "loss": 0.919,
+      "step": 3790
+    },
+    {
+      "epoch": 0.31908640523973464,
+      "grad_norm": 0.4783165156841278,
+      "learning_rate": 2e-05,
+      "loss": 0.8834,
+      "step": 3800
+    },
+    {
+      "epoch": 0.31992610630615503,
+      "grad_norm": 0.532555341720581,
+      "learning_rate": 2e-05,
+      "loss": 0.9094,
+      "step": 3810
+    },
+    {
+      "epoch": 0.3207658073725754,
+      "grad_norm": 0.4301692247390747,
+      "learning_rate": 2e-05,
+      "loss": 0.8845,
+      "step": 3820
+    },
+    {
+      "epoch": 0.3216055084389957,
+      "grad_norm": 0.4719502925872803,
+      "learning_rate": 2e-05,
+      "loss": 0.8896,
+      "step": 3830
+    },
+    {
+      "epoch": 0.32244520950541605,
+      "grad_norm": 0.5248209238052368,
+      "learning_rate": 2e-05,
+      "loss": 0.9158,
+      "step": 3840
+    },
+    {
+      "epoch": 0.32328491057183645,
+      "grad_norm": 0.4534075856208801,
+      "learning_rate": 2e-05,
+      "loss": 0.8622,
+      "step": 3850
+    },
+    {
+      "epoch": 0.3241246116382568,
+      "grad_norm": 0.935385525226593,
+      "learning_rate": 2e-05,
+      "loss": 0.905,
+      "step": 3860
+    },
+    {
+      "epoch": 0.3249643127046771,
+      "grad_norm": 0.44329261779785156,
+      "learning_rate": 2e-05,
+      "loss": 0.9345,
+      "step": 3870
+    },
+    {
+      "epoch": 0.32580401377109747,
+      "grad_norm": 0.5373802781105042,
+      "learning_rate": 2e-05,
+      "loss": 0.829,
+      "step": 3880
+    },
+    {
+      "epoch": 0.32664371483751786,
+      "grad_norm": 0.4774850308895111,
+      "learning_rate": 2e-05,
+      "loss": 0.8675,
+      "step": 3890
+    },
+    {
+      "epoch": 0.3274834159039382,
+      "grad_norm": 0.4673742353916168,
+      "learning_rate": 2e-05,
+      "loss": 0.8475,
+      "step": 3900
+    },
+    {
+      "epoch": 0.32832311697035854,
+      "grad_norm": 0.48385006189346313,
+      "learning_rate": 2e-05,
+      "loss": 0.8811,
+      "step": 3910
+    },
+    {
+      "epoch": 0.3291628180367789,
+      "grad_norm": 0.48337429761886597,
+      "learning_rate": 2e-05,
+      "loss": 0.8506,
+      "step": 3920
+    },
+    {
+      "epoch": 0.3300025191031993,
+      "grad_norm": 0.5003883242607117,
+      "learning_rate": 2e-05,
+      "loss": 0.9028,
+      "step": 3930
+    },
+    {
+      "epoch": 0.3308422201696196,
+      "grad_norm": 0.4657689332962036,
+      "learning_rate": 2e-05,
+      "loss": 0.893,
+      "step": 3940
+    },
+    {
+      "epoch": 0.33168192123603996,
+      "grad_norm": 0.46615493297576904,
+      "learning_rate": 2e-05,
+      "loss": 0.8496,
+      "step": 3950
+    },
+    {
+      "epoch": 0.33252162230246035,
+      "grad_norm": 0.724042534828186,
+      "learning_rate": 2e-05,
+      "loss": 0.8666,
+      "step": 3960
+    },
+    {
+      "epoch": 0.3333613233688807,
+      "grad_norm": 0.48005014657974243,
+      "learning_rate": 2e-05,
+      "loss": 0.9155,
+      "step": 3970
+    },
+    {
+      "epoch": 0.33420102443530103,
+      "grad_norm": 0.5058119893074036,
+      "learning_rate": 2e-05,
+      "loss": 0.9489,
+      "step": 3980
+    },
+    {
+      "epoch": 0.33504072550172137,
+      "grad_norm": 0.47297027707099915,
+      "learning_rate": 2e-05,
+      "loss": 0.8757,
+      "step": 3990
+    },
+    {
+      "epoch": 0.33588042656814177,
+      "grad_norm": 0.4306064248085022,
+      "learning_rate": 2e-05,
+      "loss": 0.873,
+      "step": 4000
+    },
+    {
+      "epoch": 0.3367201276345621,
+      "grad_norm": 0.4857984781265259,
+      "learning_rate": 2e-05,
+      "loss": 0.8761,
+      "step": 4010
+    },
+    {
+      "epoch": 0.33755982870098244,
+      "grad_norm": 0.4788801670074463,
+      "learning_rate": 2e-05,
+      "loss": 0.8145,
+      "step": 4020
+    },
+    {
+      "epoch": 0.3383995297674028,
+      "grad_norm": 1.068161964416504,
+      "learning_rate": 2e-05,
+      "loss": 0.8323,
+      "step": 4030
+    },
+    {
+      "epoch": 0.3392392308338232,
+      "grad_norm": 0.5176240801811218,
+      "learning_rate": 2e-05,
+      "loss": 0.8625,
+      "step": 4040
+    },
+    {
+      "epoch": 0.3400789319002435,
+      "grad_norm": 0.5025022029876709,
+      "learning_rate": 2e-05,
+      "loss": 0.8327,
+      "step": 4050
+    },
+    {
+      "epoch": 0.34091863296666386,
+      "grad_norm": 0.41971564292907715,
+      "learning_rate": 2e-05,
+      "loss": 0.8911,
+      "step": 4060
+    },
+    {
+      "epoch": 0.3417583340330842,
+      "grad_norm": 0.47302740812301636,
+      "learning_rate": 2e-05,
+      "loss": 0.9044,
+      "step": 4070
+    },
+    {
+      "epoch": 0.3425980350995046,
+      "grad_norm": 0.44141674041748047,
+      "learning_rate": 2e-05,
+      "loss": 0.8326,
+      "step": 4080
+    },
+    {
+      "epoch": 0.34343773616592493,
+      "grad_norm": 0.4380483031272888,
+      "learning_rate": 2e-05,
+      "loss": 0.8784,
+      "step": 4090
+    },
+    {
+      "epoch": 0.3442774372323453,
+      "grad_norm": 0.46832966804504395,
+      "learning_rate": 2e-05,
+      "loss": 0.9281,
+      "step": 4100
+    },
+    {
+      "epoch": 0.3451171382987656,
+      "grad_norm": 0.5256158709526062,
+      "learning_rate": 2e-05,
+      "loss": 0.8433,
+      "step": 4110
+    },
+    {
+      "epoch": 0.345956839365186,
+      "grad_norm": 0.54636549949646,
+      "learning_rate": 2e-05,
+      "loss": 0.8506,
+      "step": 4120
+    },
+    {
+      "epoch": 0.34679654043160635,
+      "grad_norm": 0.44160526990890503,
+      "learning_rate": 2e-05,
+      "loss": 0.8812,
+      "step": 4130
+    },
+    {
+      "epoch": 0.3476362414980267,
+      "grad_norm": 0.5728407502174377,
+      "learning_rate": 2e-05,
+      "loss": 0.8891,
+      "step": 4140
+    },
+    {
+      "epoch": 0.3484759425644471,
+      "grad_norm": 0.4904657006263733,
+      "learning_rate": 2e-05,
+      "loss": 0.8447,
+      "step": 4150
+    },
+    {
+      "epoch": 0.3493156436308674,
+      "grad_norm": 0.48392900824546814,
+      "learning_rate": 2e-05,
+      "loss": 0.8579,
+      "step": 4160
+    },
+    {
+      "epoch": 0.35015534469728776,
+      "grad_norm": 0.46334561705589294,
+      "learning_rate": 2e-05,
+      "loss": 0.8386,
+      "step": 4170
+    },
+    {
+      "epoch": 0.3509950457637081,
+      "grad_norm": 0.5467081069946289,
+      "learning_rate": 2e-05,
+      "loss": 0.8444,
+      "step": 4180
+    },
+    {
+      "epoch": 0.3518347468301285,
+      "grad_norm": 0.4506303071975708,
+      "learning_rate": 2e-05,
+      "loss": 0.893,
+      "step": 4190
+    },
+    {
+      "epoch": 0.35267444789654884,
+      "grad_norm": 0.5566191077232361,
+      "learning_rate": 2e-05,
+      "loss": 0.8839,
+      "step": 4200
+    },
+    {
+      "epoch": 0.3535141489629692,
+      "grad_norm": 0.5077056884765625,
+      "learning_rate": 2e-05,
+      "loss": 0.8792,
+      "step": 4210
+    },
+    {
+      "epoch": 0.3543538500293895,
+      "grad_norm": 0.49789318442344666,
+      "learning_rate": 2e-05,
+      "loss": 0.833,
+      "step": 4220
+    },
+    {
+      "epoch": 0.3551935510958099,
+      "grad_norm": 0.5057146549224854,
+      "learning_rate": 2e-05,
+      "loss": 0.807,
+      "step": 4230
+    },
+    {
+      "epoch": 0.35603325216223025,
+      "grad_norm": 0.529106080532074,
+      "learning_rate": 2e-05,
+      "loss": 0.8125,
+      "step": 4240
+    },
+    {
+      "epoch": 0.3568729532286506,
+      "grad_norm": 0.543289840221405,
+      "learning_rate": 2e-05,
+      "loss": 0.8511,
+      "step": 4250
+    },
+    {
+      "epoch": 0.35771265429507093,
+      "grad_norm": 0.4791346788406372,
+      "learning_rate": 2e-05,
+      "loss": 0.8781,
+      "step": 4260
+    },
+    {
+      "epoch": 0.3585523553614913,
+      "grad_norm": 0.4720413386821747,
+      "learning_rate": 2e-05,
+      "loss": 0.8857,
+      "step": 4270
+    },
+    {
+      "epoch": 0.35939205642791167,
+      "grad_norm": 0.4704269468784332,
+      "learning_rate": 2e-05,
+      "loss": 0.8302,
+      "step": 4280
+    },
+    {
+      "epoch": 0.360231757494332,
+      "grad_norm": 0.5099755525588989,
+      "learning_rate": 2e-05,
+      "loss": 0.8137,
+      "step": 4290
+    },
+    {
+      "epoch": 0.36107145856075235,
+      "grad_norm": 0.5124514102935791,
+      "learning_rate": 2e-05,
+      "loss": 0.8525,
+      "step": 4300
+    },
+    {
+      "epoch": 0.36191115962717274,
+      "grad_norm": 0.564842939376831,
+      "learning_rate": 2e-05,
+      "loss": 0.9213,
+      "step": 4310
+    },
+    {
+      "epoch": 0.3627508606935931,
+      "grad_norm": 0.47409090399742126,
+      "learning_rate": 2e-05,
+      "loss": 0.8365,
+      "step": 4320
+    },
+    {
+      "epoch": 0.3635905617600134,
+      "grad_norm": 0.46713560819625854,
+      "learning_rate": 2e-05,
+      "loss": 0.8685,
+      "step": 4330
+    },
+    {
+      "epoch": 0.3644302628264338,
+      "grad_norm": 1.4942078590393066,
+      "learning_rate": 2e-05,
+      "loss": 0.8232,
+      "step": 4340
+    },
+    {
+      "epoch": 0.36526996389285415,
+      "grad_norm": 0.45779988169670105,
+      "learning_rate": 2e-05,
+      "loss": 0.9612,
+      "step": 4350
+    },
+    {
+      "epoch": 0.3661096649592745,
+      "grad_norm": 0.4784534275531769,
+      "learning_rate": 2e-05,
+      "loss": 0.8855,
+      "step": 4360
+    },
+    {
+      "epoch": 0.36694936602569483,
+      "grad_norm": 0.4833981692790985,
+      "learning_rate": 2e-05,
+      "loss": 0.8303,
+      "step": 4370
+    },
+    {
+      "epoch": 0.36778906709211523,
+      "grad_norm": 0.465872198343277,
+      "learning_rate": 2e-05,
+      "loss": 0.8396,
+      "step": 4380
+    },
+    {
+      "epoch": 0.36862876815853557,
+      "grad_norm": 0.4918821454048157,
+      "learning_rate": 2e-05,
+      "loss": 0.9396,
+      "step": 4390
+    },
+    {
+      "epoch": 0.3694684692249559,
+      "grad_norm": 0.42498672008514404,
+      "learning_rate": 2e-05,
+      "loss": 0.8429,
+      "step": 4400
+    },
+    {
+      "epoch": 0.37030817029137625,
+      "grad_norm": 0.46347522735595703,
+      "learning_rate": 2e-05,
+      "loss": 0.8252,
+      "step": 4410
+    },
+    {
+      "epoch": 0.37114787135779664,
+      "grad_norm": 0.6583391427993774,
+      "learning_rate": 2e-05,
+      "loss": 0.8414,
+      "step": 4420
+    },
+    {
+      "epoch": 0.371987572424217,
+      "grad_norm": 0.6269922256469727,
+      "learning_rate": 2e-05,
+      "loss": 0.8644,
+      "step": 4430
+    },
+    {
+      "epoch": 0.3728272734906373,
+      "grad_norm": 0.5062037110328674,
+      "learning_rate": 2e-05,
+      "loss": 0.8425,
+      "step": 4440
+    },
+    {
+      "epoch": 0.37366697455705766,
+      "grad_norm": 0.5079794526100159,
+      "learning_rate": 2e-05,
+      "loss": 0.8571,
+      "step": 4450
+    },
+    {
+      "epoch": 0.37450667562347806,
+      "grad_norm": 0.48571908473968506,
+      "learning_rate": 2e-05,
+      "loss": 0.8119,
+      "step": 4460
+    },
+    {
+      "epoch": 0.3753463766898984,
+      "grad_norm": 0.45302456617355347,
+      "learning_rate": 2e-05,
+      "loss": 0.8526,
+      "step": 4470
+    },
+    {
+      "epoch": 0.37618607775631874,
+      "grad_norm": 0.47922539710998535,
+      "learning_rate": 2e-05,
+      "loss": 0.8522,
+      "step": 4480
+    },
+    {
+      "epoch": 0.3770257788227391,
+      "grad_norm": 0.45087531208992004,
+      "learning_rate": 2e-05,
+      "loss": 0.9179,
+      "step": 4490
+    },
+    {
+      "epoch": 0.3778654798891595,
+      "grad_norm": 0.47525715827941895,
+      "learning_rate": 2e-05,
+      "loss": 0.8519,
+      "step": 4500
+    },
+    {
+      "epoch": 0.3787051809555798,
+      "grad_norm": 0.48206281661987305,
+      "learning_rate": 2e-05,
+      "loss": 0.8651,
+      "step": 4510
+    },
+    {
+      "epoch": 0.37954488202200015,
+      "grad_norm": 0.5328836441040039,
+      "learning_rate": 2e-05,
+      "loss": 0.8583,
+      "step": 4520
+    },
+    {
+      "epoch": 0.38038458308842055,
+      "grad_norm": 0.49187734723091125,
+      "learning_rate": 2e-05,
+      "loss": 0.8583,
+      "step": 4530
+    },
+    {
+      "epoch": 0.3812242841548409,
+      "grad_norm": 0.5241971611976624,
+      "learning_rate": 2e-05,
+      "loss": 0.8563,
+      "step": 4540
+    },
+    {
+      "epoch": 0.3820639852212612,
+      "grad_norm": 0.520952045917511,
+      "learning_rate": 2e-05,
+      "loss": 0.8473,
+      "step": 4550
+    },
+    {
+      "epoch": 0.38290368628768157,
+      "grad_norm": 0.4690222442150116,
+      "learning_rate": 2e-05,
+      "loss": 0.8372,
+      "step": 4560
+    },
+    {
+      "epoch": 0.38374338735410196,
+      "grad_norm": 0.5850653648376465,
+      "learning_rate": 2e-05,
+      "loss": 0.8437,
+      "step": 4570
+    },
+    {
+      "epoch": 0.3845830884205223,
+      "grad_norm": 0.5002773404121399,
+      "learning_rate": 2e-05,
+      "loss": 0.8658,
+      "step": 4580
+    },
+    {
+      "epoch": 0.38542278948694264,
+      "grad_norm": 0.4715169072151184,
+      "learning_rate": 2e-05,
+      "loss": 0.813,
+      "step": 4590
+    },
+    {
+      "epoch": 0.386262490553363,
+      "grad_norm": 0.4478045105934143,
+      "learning_rate": 2e-05,
+      "loss": 0.9109,
+      "step": 4600
+    },
+    {
+      "epoch": 0.3871021916197834,
+      "grad_norm": 0.44789138436317444,
+      "learning_rate": 2e-05,
+      "loss": 0.8246,
+      "step": 4610
+    },
+    {
+      "epoch": 0.3879418926862037,
+      "grad_norm": 0.5138265490531921,
+      "learning_rate": 2e-05,
+      "loss": 0.8694,
+      "step": 4620
+    },
+    {
+      "epoch": 0.38878159375262406,
+      "grad_norm": 0.48978182673454285,
+      "learning_rate": 2e-05,
+      "loss": 0.8246,
+      "step": 4630
+    },
+    {
+      "epoch": 0.3896212948190444,
+      "grad_norm": 0.4671839773654938,
+      "learning_rate": 2e-05,
+      "loss": 0.8686,
+      "step": 4640
+    },
+    {
+      "epoch": 0.3904609958854648,
+      "grad_norm": 0.44903305172920227,
+      "learning_rate": 2e-05,
+      "loss": 0.8219,
+      "step": 4650
+    },
+    {
+      "epoch": 0.39130069695188513,
+      "grad_norm": 0.483102411031723,
+      "learning_rate": 2e-05,
+      "loss": 0.9721,
+      "step": 4660
+    },
+    {
+      "epoch": 0.39214039801830547,
+      "grad_norm": 0.47924095392227173,
+      "learning_rate": 2e-05,
+      "loss": 0.8647,
+      "step": 4670
+    },
+    {
+      "epoch": 0.39298009908472586,
+      "grad_norm": 0.484160840511322,
+      "learning_rate": 2e-05,
+      "loss": 0.838,
+      "step": 4680
+    },
+    {
+      "epoch": 0.3938198001511462,
+      "grad_norm": 0.49385976791381836,
+      "learning_rate": 2e-05,
+      "loss": 0.8098,
+      "step": 4690
+    },
+    {
+      "epoch": 0.39465950121756654,
+      "grad_norm": 0.4838296175003052,
+      "learning_rate": 2e-05,
+      "loss": 0.8258,
+      "step": 4700
+    },
+    {
+      "epoch": 0.3954992022839869,
+      "grad_norm": 0.4659811556339264,
+      "learning_rate": 2e-05,
+      "loss": 0.9348,
+      "step": 4710
+    },
+    {
+      "epoch": 0.3963389033504073,
+      "grad_norm": 0.44028934836387634,
+      "learning_rate": 2e-05,
+      "loss": 0.8326,
+      "step": 4720
+    },
+    {
+      "epoch": 0.3971786044168276,
+      "grad_norm": 0.4840226173400879,
+      "learning_rate": 2e-05,
+      "loss": 0.7522,
+      "step": 4730
+    },
+    {
+      "epoch": 0.39801830548324796,
+      "grad_norm": 0.5117940902709961,
+      "learning_rate": 2e-05,
+      "loss": 0.9238,
+      "step": 4740
+    },
+    {
+      "epoch": 0.3988580065496683,
+      "grad_norm": 0.4852059781551361,
+      "learning_rate": 2e-05,
+      "loss": 0.8824,
+      "step": 4750
+    },
+    {
+      "epoch": 0.3996977076160887,
+      "grad_norm": 0.4467109143733978,
+      "learning_rate": 2e-05,
+      "loss": 0.7785,
+      "step": 4760
+    },
+    {
+      "epoch": 0.40053740868250903,
+      "grad_norm": 0.8021454811096191,
+      "learning_rate": 2e-05,
+      "loss": 0.8602,
+      "step": 4770
+    },
+    {
+      "epoch": 0.4013771097489294,
+      "grad_norm": 0.6500348448753357,
+      "learning_rate": 2e-05,
+      "loss": 0.886,
+      "step": 4780
+    },
+    {
+      "epoch": 0.4022168108153497,
+      "grad_norm": 0.4327065348625183,
+      "learning_rate": 2e-05,
+      "loss": 0.853,
+      "step": 4790
+    },
+    {
+      "epoch": 0.4030565118817701,
+      "grad_norm": 0.4886786639690399,
+      "learning_rate": 2e-05,
+      "loss": 0.881,
+      "step": 4800
+    },
+    {
+      "epoch": 0.40389621294819045,
+      "grad_norm": 0.4518788158893585,
+      "learning_rate": 2e-05,
+      "loss": 0.8407,
+      "step": 4810
+    },
+    {
+      "epoch": 0.4047359140146108,
+      "grad_norm": 0.5445396900177002,
+      "learning_rate": 2e-05,
+      "loss": 0.8717,
+      "step": 4820
+    },
+    {
+      "epoch": 0.4055756150810311,
+      "grad_norm": 0.5282107591629028,
+      "learning_rate": 2e-05,
+      "loss": 0.9409,
+      "step": 4830
+    },
+    {
+      "epoch": 0.4064153161474515,
+      "grad_norm": 0.47181981801986694,
+      "learning_rate": 2e-05,
+      "loss": 0.8867,
+      "step": 4840
+    },
+    {
+      "epoch": 0.40725501721387186,
+      "grad_norm": 0.49117928743362427,
+      "learning_rate": 2e-05,
+      "loss": 0.8416,
+      "step": 4850
+    },
+    {
+      "epoch": 0.4080947182802922,
+      "grad_norm": 0.5328199863433838,
+      "learning_rate": 2e-05,
+      "loss": 0.8686,
+      "step": 4860
+    },
+    {
+      "epoch": 0.4089344193467126,
+      "grad_norm": 0.46452075242996216,
+      "learning_rate": 2e-05,
+      "loss": 0.811,
+      "step": 4870
+    },
+    {
+      "epoch": 0.40977412041313294,
+      "grad_norm": 0.5018109083175659,
+      "learning_rate": 2e-05,
+      "loss": 0.8255,
+      "step": 4880
+    },
+    {
+      "epoch": 0.4106138214795533,
+      "grad_norm": 0.4541451036930084,
+      "learning_rate": 2e-05,
+      "loss": 0.8971,
+      "step": 4890
+    },
+    {
+      "epoch": 0.4114535225459736,
+      "grad_norm": 0.5086500644683838,
+      "learning_rate": 2e-05,
+      "loss": 0.856,
+      "step": 4900
+    },
+    {
+      "epoch": 0.412293223612394,
+      "grad_norm": 0.5232313275337219,
+      "learning_rate": 2e-05,
+      "loss": 0.8566,
+      "step": 4910
+    },
+    {
+      "epoch": 0.41313292467881435,
+      "grad_norm": 0.4457840621471405,
+      "learning_rate": 2e-05,
+      "loss": 0.8254,
+      "step": 4920
+    },
+    {
+      "epoch": 0.4139726257452347,
+      "grad_norm": 0.6703500151634216,
+      "learning_rate": 2e-05,
+      "loss": 0.8428,
+      "step": 4930
+    },
+    {
+      "epoch": 0.41481232681165503,
+      "grad_norm": 0.5483640432357788,
+      "learning_rate": 2e-05,
+      "loss": 0.8359,
+      "step": 4940
+    },
+    {
+      "epoch": 0.4156520278780754,
+      "grad_norm": 0.5065145492553711,
+      "learning_rate": 2e-05,
+      "loss": 0.8146,
+      "step": 4950
+    },
+    {
+      "epoch": 0.41649172894449576,
+      "grad_norm": 0.44994375109672546,
+      "learning_rate": 2e-05,
+      "loss": 0.774,
+      "step": 4960
+    },
+    {
+      "epoch": 0.4173314300109161,
+      "grad_norm": 0.4791608154773712,
+      "learning_rate": 2e-05,
+      "loss": 0.9011,
+      "step": 4970
+    },
+    {
+      "epoch": 0.41817113107733644,
+      "grad_norm": 0.5078067183494568,
+      "learning_rate": 2e-05,
+      "loss": 0.8275,
+      "step": 4980
+    },
+    {
+      "epoch": 0.41901083214375684,
+      "grad_norm": 0.45128104090690613,
+      "learning_rate": 2e-05,
+      "loss": 0.8261,
+      "step": 4990
+    },
+    {
+      "epoch": 0.4198505332101772,
+      "grad_norm": 0.5237569212913513,
+      "learning_rate": 2e-05,
+      "loss": 0.8118,
+      "step": 5000
+    },
+    {
+      "epoch": 0.4206902342765975,
+      "grad_norm": 0.5107096433639526,
+      "learning_rate": 2e-05,
+      "loss": 0.8605,
+      "step": 5010
+    },
+    {
+      "epoch": 0.42152993534301786,
+      "grad_norm": 0.4750487506389618,
+      "learning_rate": 2e-05,
+      "loss": 0.8298,
+      "step": 5020
+    },
+    {
+      "epoch": 0.42236963640943825,
+      "grad_norm": 0.4311191737651825,
+      "learning_rate": 2e-05,
+      "loss": 0.8581,
+      "step": 5030
+    },
+    {
+      "epoch": 0.4232093374758586,
+      "grad_norm": 0.41115230321884155,
+      "learning_rate": 2e-05,
+      "loss": 0.8684,
+      "step": 5040
+    },
+    {
+      "epoch": 0.42404903854227893,
+      "grad_norm": 0.46505263447761536,
+      "learning_rate": 2e-05,
+      "loss": 0.867,
+      "step": 5050
+    },
+    {
+      "epoch": 0.42488873960869933,
+      "grad_norm": 0.4513060748577118,
+      "learning_rate": 2e-05,
+      "loss": 0.82,
+      "step": 5060
+    },
+    {
+      "epoch": 0.42572844067511967,
+      "grad_norm": 0.4570145010948181,
+      "learning_rate": 2e-05,
+      "loss": 0.8157,
+      "step": 5070
+    },
+    {
+      "epoch": 0.42656814174154,
+      "grad_norm": 0.5044353604316711,
+      "learning_rate": 2e-05,
+      "loss": 0.8439,
+      "step": 5080
+    },
+    {
+      "epoch": 0.42740784280796035,
+      "grad_norm": 0.45760828256607056,
+      "learning_rate": 1.9999810479633703e-05,
+      "loss": 0.8271,
+      "step": 5090
+    },
+    {
+      "epoch": 0.42824754387438074,
+      "grad_norm": 0.4821126163005829,
+      "learning_rate": 1.997707677020786e-05,
+      "loss": 0.8462,
+      "step": 5100
+    },
+    {
+      "epoch": 0.4290872449408011,
+      "grad_norm": 0.477150559425354,
+      "learning_rate": 1.9916538196124637e-05,
+      "loss": 0.8597,
+      "step": 5110
+    },
+    {
+      "epoch": 0.4299269460072214,
+      "grad_norm": 0.4903765916824341,
+      "learning_rate": 1.9818425303863402e-05,
+      "loss": 0.8261,
+      "step": 5120
+    },
+    {
+      "epoch": 0.43076664707364176,
+      "grad_norm": 0.4558253586292267,
+      "learning_rate": 1.9683111732582887e-05,
+      "loss": 0.8084,
+      "step": 5130
+    },
+    {
+      "epoch": 0.43160634814006216,
+      "grad_norm": 0.4683677852153778,
+      "learning_rate": 1.951111279120706e-05,
+      "loss": 0.8461,
+      "step": 5140
+    },
+    {
+      "epoch": 0.4324460492064825,
+      "grad_norm": 0.6132014989852905,
+      "learning_rate": 1.9303083495996028e-05,
+      "loss": 0.8514,
+      "step": 5150
+    },
+    {
+      "epoch": 0.43328575027290284,
+      "grad_norm": 0.4441957473754883,
+      "learning_rate": 1.9059816076075482e-05,
+      "loss": 0.7843,
+      "step": 5160
+    },
+    {
+      "epoch": 0.4341254513393232,
+      "grad_norm": 0.48467063903808594,
+      "learning_rate": 1.8782236956424142e-05,
+      "loss": 0.8607,
+      "step": 5170
+    },
+    {
+      "epoch": 0.43496515240574357,
+      "grad_norm": 0.4647083580493927,
+      "learning_rate": 1.8471403229808897e-05,
+      "loss": 0.8366,
+      "step": 5180
+    },
+    {
+      "epoch": 0.4358048534721639,
+      "grad_norm": 0.5834803581237793,
+      "learning_rate": 1.8128498631103333e-05,
+      "loss": 0.7797,
+      "step": 5190
+    },
+    {
+      "epoch": 0.43664455453858425,
+      "grad_norm": 0.46700340509414673,
+      "learning_rate": 1.7754829029320507e-05,
+      "loss": 0.8761,
+      "step": 5200
+    },
+    {
+      "epoch": 0.43748425560500465,
+      "grad_norm": 0.48982134461402893,
+      "learning_rate": 1.7351817454527473e-05,
+      "loss": 0.8728,
+      "step": 5210
+    },
+    {
+      "epoch": 0.438323956671425,
+      "grad_norm": 0.5032446980476379,
+      "learning_rate": 1.6920998678580345e-05,
+      "loss": 0.8455,
+      "step": 5220
+    },
+    {
+      "epoch": 0.4391636577378453,
+      "grad_norm": 0.4976018965244293,
+      "learning_rate": 1.6464013370317854e-05,
+      "loss": 0.802,
+      "step": 5230
+    },
+    {
+      "epoch": 0.44000335880426567,
+      "grad_norm": 0.47935158014297485,
+      "learning_rate": 1.5982601847471915e-05,
+      "loss": 0.7881,
+      "step": 5240
+    },
+    {
+      "epoch": 0.44084305987068606,
+      "grad_norm": 0.4816844165325165,
+      "learning_rate": 1.5478597449089553e-05,
+      "loss": 0.8326,
+      "step": 5250
+    },
+    {
+      "epoch": 0.4416827609371064,
+      "grad_norm": 0.5442057251930237,
+      "learning_rate": 1.4953919553705725e-05,
+      "loss": 0.7809,
+      "step": 5260
+    },
+    {
+      "epoch": 0.44252246200352674,
+      "grad_norm": 0.46870455145835876,
+      "learning_rate": 1.441056626985559e-05,
+      "loss": 0.814,
+      "step": 5270
+    },
+    {
+      "epoch": 0.4433621630699471,
+      "grad_norm": 0.45419636368751526,
+      "learning_rate": 1.3850606826762668e-05,
+      "loss": 0.8284,
+      "step": 5280
+    },
+    {
+      "epoch": 0.4442018641363675,
+      "grad_norm": 0.48418232798576355,
+      "learning_rate": 1.3276173694181128e-05,
+      "loss": 0.8609,
+      "step": 5290
+    },
+    {
+      "epoch": 0.4450415652027878,
+      "grad_norm": 0.536445677280426,
+      "learning_rate": 1.2689454461401822e-05,
+      "loss": 0.8025,
+      "step": 5300
+    },
+    {
+      "epoch": 0.44588126626920815,
+      "grad_norm": 0.49737629294395447,
+      "learning_rate": 1.2092683506348939e-05,
+      "loss": 0.8427,
+      "step": 5310
+    },
+    {
+      "epoch": 0.4467209673356285,
+      "grad_norm": 1.0030624866485596,
+      "learning_rate": 1.14881334864935e-05,
+      "loss": 0.8682,
+      "step": 5320
+    },
+    {
+      "epoch": 0.4475606684020489,
+      "grad_norm": 0.4441104531288147,
+      "learning_rate": 1.0878106683988481e-05,
+      "loss": 0.8136,
+      "step": 5330
+    },
+    {
+      "epoch": 0.44840036946846923,
+      "grad_norm": 0.4298957288265228,
+      "learning_rate": 1.026492623798551e-05,
+      "loss": 0.8311,
+      "step": 5340
+    },
+    {
+      "epoch": 0.44924007053488957,
+      "grad_norm": 0.5332731008529663,
+      "learning_rate": 9.650927297522743e-06,
+      "loss": 0.8376,
+      "step": 5350
+    },
+    {
+      "epoch": 0.4500797716013099,
+      "grad_norm": 0.4805138111114502,
+      "learning_rate": 9.038448128676012e-06,
+      "loss": 0.8677,
+      "step": 5360
+    },
+    {
+      "epoch": 0.4509194726677303,
+      "grad_norm": 0.4591268301010132,
+      "learning_rate": 8.42982120983949e-06,
+      "loss": 0.841,
+      "step": 5370
+    },
+    {
+      "epoch": 0.45175917373415064,
+      "grad_norm": 0.44623425602912903,
+      "learning_rate": 7.827364349047303e-06,
+      "loss": 0.7853,
+      "step": 5380
+    },
+    {
+      "epoch": 0.452598874800571,
+      "grad_norm": 0.4689062535762787,
+      "learning_rate": 7.233371857163706e-06,
+      "loss": 0.7908,
+      "step": 5390
+    },
+    {
+      "epoch": 0.4534385758669914,
+      "grad_norm": 0.5083698034286499,
+      "learning_rate": 6.650105810556492e-06,
+      "loss": 0.8142,
+      "step": 5400
+    },
+    {
+      "epoch": 0.4542782769334117,
+      "grad_norm": 0.507642388343811,
+      "learning_rate": 6.079787436527681e-06,
+      "loss": 0.7964,
+      "step": 5410
+    },
+    {
+      "epoch": 0.45511797799983206,
+      "grad_norm": 0.47586846351623535,
+      "learning_rate": 5.524588654307961e-06,
+      "loss": 0.766,
+      "step": 5420
+    },
+    {
+      "epoch": 0.4559576790662524,
+      "grad_norm": 0.47993049025535583,
+      "learning_rate": 4.986623803828985e-06,
+      "loss": 0.8903,
+      "step": 5430
+    },
+    {
+      "epoch": 0.4567973801326728,
+      "grad_norm": 0.46089205145835876,
+      "learning_rate": 4.467941593772423e-06,
+      "loss": 0.9261,
+      "step": 5440
+    },
+    {
+      "epoch": 0.45763708119909313,
+      "grad_norm": 0.4603840112686157,
+      "learning_rate": 3.9705172995597606e-06,
+      "loss": 0.758,
+      "step": 5450
+    },
+    {
+      "epoch": 0.45847678226551347,
+      "grad_norm": 0.44591331481933594,
+      "learning_rate": 3.4962452409948346e-06,
+      "loss": 0.8045,
+      "step": 5460
+    },
+    {
+      "epoch": 0.4593164833319338,
+      "grad_norm": 0.45714354515075684,
+      "learning_rate": 3.046931568206211e-06,
+      "loss": 0.8011,
+      "step": 5470
+    },
+    {
+      "epoch": 0.4601561843983542,
+      "grad_norm": 0.4888162314891815,
+      "learning_rate": 2.624287383362404e-06,
+      "loss": 0.8599,
+      "step": 5480
+    },
+    {
+      "epoch": 0.46099588546477455,
+      "grad_norm": 0.5092567801475525,
+      "learning_rate": 2.2299222243541845e-06,
+      "loss": 0.8061,
+      "step": 5490
+    },
+    {
+      "epoch": 0.4618355865311949,
+      "grad_norm": 0.454471230506897,
+      "learning_rate": 1.8653379352598246e-06,
+      "loss": 0.8021,
+      "step": 5500
+    },
+    {
+      "epoch": 0.4626752875976152,
+      "grad_norm": 0.4467802047729492,
+      "learning_rate": 1.5319229469361075e-06,
+      "loss": 0.756,
+      "step": 5510
+    },
+    {
+      "epoch": 0.4635149886640356,
+      "grad_norm": 0.4715058207511902,
+      "learning_rate": 1.2309469895160917e-06,
+      "loss": 0.8761,
+      "step": 5520
+    },
+    {
+      "epoch": 0.46435468973045596,
+      "grad_norm": 0.4278869330883026,
+      "learning_rate": 9.635562569497974e-07,
+      "loss": 0.7795,
+      "step": 5530
+    },
+    {
+      "epoch": 0.4651943907968763,
+      "grad_norm": 0.48767799139022827,
+      "learning_rate": 7.307690420024867e-07,
+      "loss": 0.7897,
+      "step": 5540
+    },
+    {
+      "epoch": 0.46603409186329664,
+      "grad_norm": 0.41573023796081543,
+      "learning_rate": 5.334718583335942e-07,
+      "loss": 0.801,
+      "step": 5550
+    },
+    {
+      "epoch": 0.46687379292971704,
+      "grad_norm": 0.45790743827819824,
+      "learning_rate": 3.7241606442442763e-07,
+      "loss": 0.8479,
+      "step": 5560
+    },
+    {
+      "epoch": 0.4677134939961374,
+      "grad_norm": 0.47034752368927,
+      "learning_rate": 2.482150022115879e-07,
+      "loss": 0.7874,
+      "step": 5570
+    },
+    {
+      "epoch": 0.4685531950625577,
+      "grad_norm": 0.42278674244880676,
+      "learning_rate": 1.6134166132293174e-07,
+      "loss": 0.786,
+      "step": 5580
+    },
+    {
+      "epoch": 0.4693928961289781,
+      "grad_norm": 0.4075852930545807,
+      "learning_rate": 1.1212687781127357e-07,
+      "loss": 0.7645,
+      "step": 5590
+    },
+    {
+      "epoch": 0.46998068687547234,
+      "step": 5597,
+      "total_flos": 2.2737574718397743e+19,
+      "train_loss": 0.02770314997001186,
+      "train_runtime": 1763.7483,
+      "train_samples_per_second": 38.08,
+      "train_steps_per_second": 3.173
+    }
+  ],
+  "logging_steps": 10,
+  "max_steps": 5597,
+  "num_input_tokens_seen": 0,
+  "num_train_epochs": 1,
+  "save_steps": 159,
+  "stateful_callbacks": {
+    "TrainerControl": {
+      "args": {
+        "should_epoch_stop": false,
+        "should_evaluate": false,
+        "should_log": false,
+        "should_save": true,
+        "should_training_stop": true
+      },
+      "attributes": {}
+    }
+  },
+  "total_flos": 2.2737574718397743e+19,
+  "train_batch_size": 1,
+  "trial_name": null,
+  "trial_params": null
+}

g1_cleanup_code/1000M/all_results.json

@@ -0,0 +1,8 @@
+{
+    "epoch": 0.9951106765045782,
+    "total_flos": 4.547514943786923e+19,
+    "train_loss": 0.00443073098140251,
+    "train_runtime": 609.4553,
+    "train_samples_per_second": 220.407,
+    "train_steps_per_second": 18.367
+}

g1_cleanup_code/1000M/chat_template.jinja

@@ -0,0 +1 @@
+{% if messages[0]['role'] == 'system' %}{% set loop_messages = messages[1:] %}{% set system_message = messages[0]['content'] %}{% else %}{% set loop_messages = messages %}{% endif %}{% if system_message is defined %}{{ system_message }}{% endif %}{% for message in loop_messages %}{% set content = message['content'] %}{% if message['role'] == 'user' %}{{ content }}{% elif message['role'] == 'assistant' %}{{ content + '<|endoftext|>' }}{% endif %}{% endfor %}

g1_cleanup_code/1000M/config.json

@@ -0,0 +1,27 @@
+{
+  "architectures": [
+    "Olmo2ForCausalLM"
+  ],
+  "attention_bias": false,
+  "attention_dropout": 0.0,
+  "bos_token_id": 100257,
+  "dtype": "bfloat16",
+  "eos_token_id": 100257,
+  "hidden_act": "silu",
+  "hidden_size": 4096,
+  "initializer_range": 0.02,
+  "intermediate_size": 11008,
+  "max_position_embeddings": 4096,
+  "model_type": "olmo2",
+  "num_attention_heads": 32,
+  "num_hidden_layers": 32,
+  "num_key_value_heads": 32,
+  "pad_token_id": 100277,
+  "rms_norm_eps": 1e-06,
+  "rope_scaling": null,
+  "rope_theta": 500000,
+  "tie_word_embeddings": false,
+  "transformers_version": "4.57.6",
+  "use_cache": false,
+  "vocab_size": 100352
+}

g1_cleanup_code/1000M/generation_config.json

@@ -0,0 +1,9 @@
+{
+  "_from_model_config": true,
+  "bos_token_id": 100257,
+  "eos_token_id": [
+    100257
+  ],
+  "pad_token_id": 100277,
+  "transformers_version": "4.57.6"
+}

g1_cleanup_code/1000M/model.safetensors.index.json

@@ -0,0 +1,363 @@
+{
+  "metadata": {
+    "total_parameters": 7298617344,
+    "total_size": 14597234688
+  },
+  "weight_map": {
+    "lm_head.weight": "model-00003-of-00003.safetensors",
+    "model.embed_tokens.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.post_feedforward_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.self_attn.k_norm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.self_attn.q_norm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.post_feedforward_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.self_attn.k_norm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.self_attn.q_norm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.10.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.10.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.10.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.10.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.10.post_feedforward_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.10.self_attn.k_norm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.10.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.10.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.10.self_attn.q_norm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.10.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.10.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.11.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.11.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.11.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.11.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.11.post_feedforward_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.11.self_attn.k_norm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.11.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.11.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.11.self_attn.q_norm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.11.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.11.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.post_feedforward_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.self_attn.k_norm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.self_attn.q_norm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.post_feedforward_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.self_attn.k_norm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.self_attn.q_norm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.post_feedforward_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.self_attn.k_norm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.self_attn.q_norm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.post_feedforward_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.self_attn.k_norm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.self_attn.q_norm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.post_feedforward_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.self_attn.k_norm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.self_attn.q_norm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.post_feedforward_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.self_attn.k_norm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.self_attn.q_norm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.post_feedforward_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.self_attn.k_norm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.self_attn.q_norm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.post_feedforward_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.self_attn.k_norm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.self_attn.q_norm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.2.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.2.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.2.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.2.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.2.post_feedforward_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.2.self_attn.k_norm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.2.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.2.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.2.self_attn.q_norm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.2.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.2.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.20.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.20.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.20.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.20.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.20.post_feedforward_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.20.self_attn.k_norm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.20.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.20.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.20.self_attn.q_norm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.20.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.20.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.post_feedforward_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.self_attn.k_norm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.self_attn.q_norm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.22.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.22.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.22.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.22.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.22.post_feedforward_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.22.self_attn.k_norm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.22.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.22.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.22.self_attn.q_norm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.22.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.22.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.23.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.23.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.23.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.23.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.23.post_feedforward_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.23.self_attn.k_norm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.23.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.23.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.23.self_attn.q_norm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.23.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.23.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.post_feedforward_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.self_attn.k_norm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.self_attn.q_norm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.post_feedforward_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.self_attn.k_norm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.self_attn.q_norm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.post_feedforward_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.self_attn.k_norm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.self_attn.q_norm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.post_feedforward_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.self_attn.k_norm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.self_attn.q_norm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.28.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.28.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.28.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.28.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.28.post_feedforward_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.28.self_attn.k_norm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.28.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.28.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.28.self_attn.q_norm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.28.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.28.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.29.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.29.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.29.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.29.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.29.post_feedforward_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.29.self_attn.k_norm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.29.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.29.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.29.self_attn.q_norm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.29.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.29.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.3.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.3.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.3.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.3.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.3.post_feedforward_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.3.self_attn.k_norm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.3.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.3.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.3.self_attn.q_norm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.3.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.3.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.30.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.30.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.30.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.30.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.30.post_feedforward_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.30.self_attn.k_norm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.30.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.30.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.30.self_attn.q_norm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.30.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.30.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.31.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.31.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.31.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.31.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.31.post_feedforward_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.31.self_attn.k_norm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.31.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.31.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.31.self_attn.q_norm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.31.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.31.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.4.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.4.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.4.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.4.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.4.post_feedforward_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.4.self_attn.k_norm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.4.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.4.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.4.self_attn.q_norm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.4.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.4.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.post_feedforward_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.self_attn.k_norm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.self_attn.q_norm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.post_feedforward_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.self_attn.k_norm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.self_attn.q_norm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.post_feedforward_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.self_attn.k_norm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.self_attn.q_norm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.post_feedforward_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.self_attn.k_norm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.self_attn.q_norm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.post_feedforward_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.self_attn.k_norm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.self_attn.q_norm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.norm.weight": "model-00003-of-00003.safetensors"
+  }
+}

g1_cleanup_code/1000M/special_tokens_map.json

@@ -0,0 +1,30 @@
+{
+  "bos_token": {
+    "content": "<|endoftext|>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "eos_token": {
+    "content": "<|endoftext|>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "pad_token": {
+    "content": "<|pad|>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "unk_token": {
+    "content": "<|endoftext|>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  }
+}