diff --git a/posttrain_models/8b_cold_start_sft_46k_multi_32k_timelens_1epoch_grpo_format_tiou_f1_cacc_multi_10k/global_step_260/added_tokens.json b/posttrain_models/8b_cold_start_sft_46k_multi_32k_timelens_1epoch_grpo_format_tiou_f1_cacc_multi_10k/global_step_260/added_tokens.json new file mode 100644 index 0000000000000000000000000000000000000000..b54f9135e44c1e81047e8d05cb027af8bc039eed --- /dev/null +++ b/posttrain_models/8b_cold_start_sft_46k_multi_32k_timelens_1epoch_grpo_format_tiou_f1_cacc_multi_10k/global_step_260/added_tokens.json @@ -0,0 +1,28 @@ +{ + "": 151668, + "": 151658, + "": 151666, + "": 151667, + "": 151657, + "": 151665, + "<|box_end|>": 151649, + "<|box_start|>": 151648, + "<|endoftext|>": 151643, + "<|file_sep|>": 151664, + "<|fim_middle|>": 151660, + "<|fim_pad|>": 151662, + "<|fim_prefix|>": 151659, + "<|fim_suffix|>": 151661, + "<|im_end|>": 151645, + "<|im_start|>": 151644, + "<|image_pad|>": 151655, + "<|object_ref_end|>": 151647, + "<|object_ref_start|>": 151646, + "<|quad_end|>": 151651, + "<|quad_start|>": 151650, + "<|repo_name|>": 151663, + "<|video_pad|>": 151656, + "<|vision_end|>": 151653, + "<|vision_pad|>": 151654, + "<|vision_start|>": 151652 +} diff --git a/posttrain_models/8b_cold_start_sft_46k_multi_32k_timelens_1epoch_grpo_format_tiou_f1_cacc_multi_10k/global_step_260/chat_template.jinja b/posttrain_models/8b_cold_start_sft_46k_multi_32k_timelens_1epoch_grpo_format_tiou_f1_cacc_multi_10k/global_step_260/chat_template.jinja new file mode 100644 index 0000000000000000000000000000000000000000..124386803f142761528f710e77ae483f5f8c4fc4 --- /dev/null +++ b/posttrain_models/8b_cold_start_sft_46k_multi_32k_timelens_1epoch_grpo_format_tiou_f1_cacc_multi_10k/global_step_260/chat_template.jinja @@ -0,0 +1,120 @@ +{%- if tools %} + {{- '<|im_start|>system\n' }} + {%- if messages[0].role == 'system' %} + {%- if messages[0].content is string %} + {{- messages[0].content }} + {%- else %} + {%- for content in messages[0].content %} + {%- if 'text' in content %} + {{- content.text }} + {%- endif %} + {%- endfor %} + {%- endif %} + {{- '\n\n' }} + {%- endif %} + {{- "# Tools\n\nYou may call one or more functions to assist with the user query.\n\nYou are provided with function signatures within XML tags:\n" }} + {%- for tool in tools %} + {{- "\n" }} + {{- tool | tojson }} + {%- endfor %} + {{- "\n\n\nFor each function call, return a json object with function name and arguments within XML tags:\n\n{\"name\": , \"arguments\": }\n<|im_end|>\n" }} +{%- else %} + {%- if messages[0].role == 'system' %} + {{- '<|im_start|>system\n' }} + {%- if messages[0].content is string %} + {{- messages[0].content }} + {%- else %} + {%- for content in messages[0].content %} + {%- if 'text' in content %} + {{- content.text }} + {%- endif %} + {%- endfor %} + {%- endif %} + {{- '<|im_end|>\n' }} + {%- endif %} +{%- endif %} +{%- set image_count = namespace(value=0) %} +{%- set video_count = namespace(value=0) %} +{%- for message in messages %} + {%- if message.role == "user" %} + {{- '<|im_start|>' + message.role + '\n' }} + {%- if message.content is string %} + {{- message.content }} + {%- else %} + {%- for content in message.content %} + {%- if content.type == 'image' or 'image' in content or 'image_url' in content %} + {%- set image_count.value = image_count.value + 1 %} + {%- if add_vision_id %}Picture {{ image_count.value }}: {% endif -%} + <|vision_start|><|image_pad|><|vision_end|> + {%- elif content.type == 'video' or 'video' in content %} + {%- set video_count.value = video_count.value + 1 %} + {%- if add_vision_id %}Video {{ video_count.value }}: {% endif -%} + <|vision_start|><|video_pad|><|vision_end|> + {%- elif 'text' in content %} + {{- content.text }} + {%- endif %} + {%- endfor %} + {%- endif %} + {{- '<|im_end|>\n' }} + {%- elif message.role == "assistant" %} + {{- '<|im_start|>' + message.role + '\n' }} + {%- if message.content is string %} + {{- message.content }} + {%- else %} + {%- for content_item in message.content %} + {%- if 'text' in content_item %} + {{- content_item.text }} + {%- endif %} + {%- endfor %} + {%- endif %} + {%- if message.tool_calls %} + {%- for tool_call in message.tool_calls %} + {%- if (loop.first and message.content) or (not loop.first) %} + {{- '\n' }} + {%- endif %} + {%- if tool_call.function %} + {%- set tool_call = tool_call.function %} + {%- endif %} + {{- '\n{"name": "' }} + {{- tool_call.name }} + {{- '", "arguments": ' }} + {%- if tool_call.arguments is string %} + {{- tool_call.arguments }} + {%- else %} + {{- tool_call.arguments | tojson }} + {%- endif %} + {{- '}\n' }} + {%- endfor %} + {%- endif %} + {{- '<|im_end|>\n' }} + {%- elif message.role == "tool" %} + {%- if loop.first or (messages[loop.index0 - 1].role != "tool") %} + {{- '<|im_start|>user' }} + {%- endif %} + {{- '\n\n' }} + {%- if message.content is string %} + {{- message.content }} + {%- else %} + {%- for content in message.content %} + {%- if content.type == 'image' or 'image' in content or 'image_url' in content %} + {%- set image_count.value = image_count.value + 1 %} + {%- if add_vision_id %}Picture {{ image_count.value }}: {% endif -%} + <|vision_start|><|image_pad|><|vision_end|> + {%- elif content.type == 'video' or 'video' in content %} + {%- set video_count.value = video_count.value + 1 %} + {%- if add_vision_id %}Video {{ video_count.value }}: {% endif -%} + <|vision_start|><|video_pad|><|vision_end|> + {%- elif 'text' in content %} + {{- content.text }} + {%- endif %} + {%- endfor %} + {%- endif %} + {{- '\n' }} + {%- if loop.last or (messages[loop.index0 + 1].role != "tool") %} + {{- '<|im_end|>\n' }} + {%- endif %} + {%- endif %} +{%- endfor %} +{%- if add_generation_prompt %} + {{- '<|im_start|>assistant\n' }} +{%- endif %} diff --git a/posttrain_models/8b_cold_start_sft_46k_multi_32k_timelens_1epoch_grpo_format_tiou_f1_cacc_multi_10k/global_step_260/config.json b/posttrain_models/8b_cold_start_sft_46k_multi_32k_timelens_1epoch_grpo_format_tiou_f1_cacc_multi_10k/global_step_260/config.json new file mode 100644 index 0000000000000000000000000000000000000000..db69009e04e5bed06b6959b1b3f36d034048a1df --- /dev/null +++ b/posttrain_models/8b_cold_start_sft_46k_multi_32k_timelens_1epoch_grpo_format_tiou_f1_cacc_multi_10k/global_step_260/config.json @@ -0,0 +1,70 @@ +{ + "architectures": [ + "Qwen3VLForConditionalGeneration" + ], + "dtype": "float32", + "eos_token_id": 151645, + "hidden_size": 4096, + "image_token_id": 151655, + "model_type": "qwen3_vl", + "pad_token_id": 151643, + "text_config": { + "attention_bias": false, + "attention_dropout": 0.0, + "bos_token_id": 151643, + "dtype": "float32", + "eos_token_id": 151645, + "head_dim": 128, + "hidden_act": "silu", + "hidden_size": 4096, + "initializer_range": 0.02, + "intermediate_size": 12288, + "max_position_embeddings": 262144, + "model_type": "qwen3_vl_text", + "num_attention_heads": 32, + "num_hidden_layers": 36, + "num_key_value_heads": 8, + "pad_token_id": 151643, + "rms_norm_eps": 1e-06, + "rope_theta": 5000000, + "use_cache": false, + "vocab_size": 151936, + "rope_scaling": { + "mrope_interleaved": true, + "mrope_section": [ + 24, + 20, + 20 + ], + "rope_theta": 5000000, + "rope_type": "default" + } + }, + "tie_word_embeddings": false, + "transformers_version": "5.0.0.dev0", + "video_token_id": 151656, + "vision_config": { + "deepstack_visual_indexes": [ + 8, + 16, + 24 + ], + "depth": 27, + "dtype": "float32", + "hidden_act": "gelu_pytorch_tanh", + "hidden_size": 1152, + "in_channels": 3, + "initializer_range": 0.02, + "intermediate_size": 4304, + "model_type": "qwen3_vl", + "num_heads": 16, + "num_position_embeddings": 2304, + "out_hidden_size": 4096, + "pad_token_id": 151643, + "patch_size": 16, + "spatial_merge_size": 2, + "temporal_patch_size": 2 + }, + "vision_end_token_id": 151653, + "vision_start_token_id": 151652 +} \ No newline at end of file diff --git a/posttrain_models/8b_cold_start_sft_46k_multi_32k_timelens_1epoch_grpo_format_tiou_f1_cacc_multi_10k/global_step_260/generation_config.json b/posttrain_models/8b_cold_start_sft_46k_multi_32k_timelens_1epoch_grpo_format_tiou_f1_cacc_multi_10k/global_step_260/generation_config.json new file mode 100644 index 0000000000000000000000000000000000000000..8458f29aec2209e3f63ca59bca95814310a80de1 --- /dev/null +++ b/posttrain_models/8b_cold_start_sft_46k_multi_32k_timelens_1epoch_grpo_format_tiou_f1_cacc_multi_10k/global_step_260/generation_config.json @@ -0,0 +1,13 @@ +{ + "bos_token_id": 151643, + "do_sample": true, + "eos_token_id": [ + 151645, + 151643 + ], + "pad_token_id": 151643, + "temperature": 0.7, + "top_k": 20, + "top_p": 0.8, + "transformers_version": "5.0.0.dev0" +} diff --git a/posttrain_models/8b_cold_start_sft_46k_multi_32k_timelens_1epoch_grpo_format_tiou_f1_cacc_multi_10k/global_step_260/model.safetensors.index.json b/posttrain_models/8b_cold_start_sft_46k_multi_32k_timelens_1epoch_grpo_format_tiou_f1_cacc_multi_10k/global_step_260/model.safetensors.index.json new file mode 100644 index 0000000000000000000000000000000000000000..0dd23f0fb8c578a01697fc727d7fde1dca6e2f41 --- /dev/null +++ b/posttrain_models/8b_cold_start_sft_46k_multi_32k_timelens_1epoch_grpo_format_tiou_f1_cacc_multi_10k/global_step_260/model.safetensors.index.json @@ -0,0 +1,758 @@ +{ + "metadata": { + "total_parameters": 8767123696, + "total_size": 17534247392 + }, + "weight_map": { + "lm_head.weight": "model-00001-of-00004.safetensors", + "model.language_model.embed_tokens.weight": "model-00002-of-00004.safetensors", + "model.language_model.layers.0.input_layernorm.weight": "model-00001-of-00004.safetensors", + "model.language_model.layers.0.mlp.down_proj.weight": "model-00003-of-00004.safetensors", + "model.language_model.layers.0.mlp.gate_proj.weight": "model-00001-of-00004.safetensors", + "model.language_model.layers.0.mlp.up_proj.weight": "model-00004-of-00004.safetensors", + "model.language_model.layers.0.post_attention_layernorm.weight": "model-00003-of-00004.safetensors", + "model.language_model.layers.0.self_attn.k_norm.weight": "model-00003-of-00004.safetensors", + "model.language_model.layers.0.self_attn.k_proj.weight": "model-00002-of-00004.safetensors", + "model.language_model.layers.0.self_attn.o_proj.weight": "model-00003-of-00004.safetensors", + "model.language_model.layers.0.self_attn.q_norm.weight": "model-00003-of-00004.safetensors", + "model.language_model.layers.0.self_attn.q_proj.weight": "model-00002-of-00004.safetensors", + "model.language_model.layers.0.self_attn.v_proj.weight": "model-00001-of-00004.safetensors", + "model.language_model.layers.1.input_layernorm.weight": "model-00004-of-00004.safetensors", + "model.language_model.layers.1.mlp.down_proj.weight": "model-00003-of-00004.safetensors", + "model.language_model.layers.1.mlp.gate_proj.weight": "model-00001-of-00004.safetensors", + "model.language_model.layers.1.mlp.up_proj.weight": "model-00003-of-00004.safetensors", + "model.language_model.layers.1.post_attention_layernorm.weight": "model-00002-of-00004.safetensors", + "model.language_model.layers.1.self_attn.k_norm.weight": "model-00001-of-00004.safetensors", + "model.language_model.layers.1.self_attn.k_proj.weight": "model-00001-of-00004.safetensors", + "model.language_model.layers.1.self_attn.o_proj.weight": "model-00002-of-00004.safetensors", + "model.language_model.layers.1.self_attn.q_norm.weight": "model-00003-of-00004.safetensors", + "model.language_model.layers.1.self_attn.q_proj.weight": "model-00002-of-00004.safetensors", + "model.language_model.layers.1.self_attn.v_proj.weight": "model-00004-of-00004.safetensors", + "model.language_model.layers.10.input_layernorm.weight": "model-00002-of-00004.safetensors", + "model.language_model.layers.10.mlp.down_proj.weight": "model-00002-of-00004.safetensors", + "model.language_model.layers.10.mlp.gate_proj.weight": "model-00001-of-00004.safetensors", + "model.language_model.layers.10.mlp.up_proj.weight": "model-00001-of-00004.safetensors", + "model.language_model.layers.10.post_attention_layernorm.weight": "model-00004-of-00004.safetensors", + "model.language_model.layers.10.self_attn.k_norm.weight": "model-00003-of-00004.safetensors", + "model.language_model.layers.10.self_attn.k_proj.weight": "model-00003-of-00004.safetensors", + "model.language_model.layers.10.self_attn.o_proj.weight": "model-00004-of-00004.safetensors", + "model.language_model.layers.10.self_attn.q_norm.weight": "model-00004-of-00004.safetensors", + "model.language_model.layers.10.self_attn.q_proj.weight": "model-00003-of-00004.safetensors", + "model.language_model.layers.10.self_attn.v_proj.weight": "model-00004-of-00004.safetensors", + "model.language_model.layers.11.input_layernorm.weight": "model-00001-of-00004.safetensors", + "model.language_model.layers.11.mlp.down_proj.weight": "model-00003-of-00004.safetensors", + "model.language_model.layers.11.mlp.gate_proj.weight": "model-00004-of-00004.safetensors", + "model.language_model.layers.11.mlp.up_proj.weight": "model-00003-of-00004.safetensors", + "model.language_model.layers.11.post_attention_layernorm.weight": "model-00002-of-00004.safetensors", + "model.language_model.layers.11.self_attn.k_norm.weight": "model-00002-of-00004.safetensors", + "model.language_model.layers.11.self_attn.k_proj.weight": "model-00003-of-00004.safetensors", + "model.language_model.layers.11.self_attn.o_proj.weight": "model-00001-of-00004.safetensors", + "model.language_model.layers.11.self_attn.q_norm.weight": "model-00001-of-00004.safetensors", + "model.language_model.layers.11.self_attn.q_proj.weight": "model-00004-of-00004.safetensors", + "model.language_model.layers.11.self_attn.v_proj.weight": "model-00004-of-00004.safetensors", + "model.language_model.layers.12.input_layernorm.weight": "model-00002-of-00004.safetensors", + "model.language_model.layers.12.mlp.down_proj.weight": "model-00003-of-00004.safetensors", + "model.language_model.layers.12.mlp.gate_proj.weight": "model-00004-of-00004.safetensors", + "model.language_model.layers.12.mlp.up_proj.weight": "model-00001-of-00004.safetensors", + "model.language_model.layers.12.post_attention_layernorm.weight": "model-00001-of-00004.safetensors", + "model.language_model.layers.12.self_attn.k_norm.weight": "model-00004-of-00004.safetensors", + "model.language_model.layers.12.self_attn.k_proj.weight": "model-00001-of-00004.safetensors", + "model.language_model.layers.12.self_attn.o_proj.weight": "model-00001-of-00004.safetensors", + "model.language_model.layers.12.self_attn.q_norm.weight": "model-00003-of-00004.safetensors", + "model.language_model.layers.12.self_attn.q_proj.weight": "model-00002-of-00004.safetensors", + "model.language_model.layers.12.self_attn.v_proj.weight": "model-00001-of-00004.safetensors", + "model.language_model.layers.13.input_layernorm.weight": "model-00001-of-00004.safetensors", + "model.language_model.layers.13.mlp.down_proj.weight": "model-00002-of-00004.safetensors", + "model.language_model.layers.13.mlp.gate_proj.weight": "model-00002-of-00004.safetensors", + "model.language_model.layers.13.mlp.up_proj.weight": "model-00002-of-00004.safetensors", + "model.language_model.layers.13.post_attention_layernorm.weight": "model-00004-of-00004.safetensors", + "model.language_model.layers.13.self_attn.k_norm.weight": "model-00001-of-00004.safetensors", + "model.language_model.layers.13.self_attn.k_proj.weight": "model-00004-of-00004.safetensors", + "model.language_model.layers.13.self_attn.o_proj.weight": "model-00001-of-00004.safetensors", + "model.language_model.layers.13.self_attn.q_norm.weight": "model-00001-of-00004.safetensors", + "model.language_model.layers.13.self_attn.q_proj.weight": "model-00004-of-00004.safetensors", + "model.language_model.layers.13.self_attn.v_proj.weight": "model-00002-of-00004.safetensors", + "model.language_model.layers.14.input_layernorm.weight": "model-00001-of-00004.safetensors", + "model.language_model.layers.14.mlp.down_proj.weight": "model-00001-of-00004.safetensors", + "model.language_model.layers.14.mlp.gate_proj.weight": "model-00003-of-00004.safetensors", + "model.language_model.layers.14.mlp.up_proj.weight": "model-00003-of-00004.safetensors", + "model.language_model.layers.14.post_attention_layernorm.weight": "model-00002-of-00004.safetensors", + "model.language_model.layers.14.self_attn.k_norm.weight": "model-00002-of-00004.safetensors", + "model.language_model.layers.14.self_attn.k_proj.weight": "model-00003-of-00004.safetensors", + "model.language_model.layers.14.self_attn.o_proj.weight": "model-00002-of-00004.safetensors", + "model.language_model.layers.14.self_attn.q_norm.weight": "model-00004-of-00004.safetensors", + "model.language_model.layers.14.self_attn.q_proj.weight": "model-00003-of-00004.safetensors", + "model.language_model.layers.14.self_attn.v_proj.weight": "model-00001-of-00004.safetensors", + "model.language_model.layers.15.input_layernorm.weight": "model-00003-of-00004.safetensors", + "model.language_model.layers.15.mlp.down_proj.weight": "model-00004-of-00004.safetensors", + "model.language_model.layers.15.mlp.gate_proj.weight": "model-00003-of-00004.safetensors", + "model.language_model.layers.15.mlp.up_proj.weight": "model-00003-of-00004.safetensors", + "model.language_model.layers.15.post_attention_layernorm.weight": "model-00002-of-00004.safetensors", + "model.language_model.layers.15.self_attn.k_norm.weight": "model-00003-of-00004.safetensors", + "model.language_model.layers.15.self_attn.k_proj.weight": "model-00003-of-00004.safetensors", + "model.language_model.layers.15.self_attn.o_proj.weight": "model-00001-of-00004.safetensors", + "model.language_model.layers.15.self_attn.q_norm.weight": "model-00001-of-00004.safetensors", + "model.language_model.layers.15.self_attn.q_proj.weight": "model-00003-of-00004.safetensors", + "model.language_model.layers.15.self_attn.v_proj.weight": "model-00003-of-00004.safetensors", + "model.language_model.layers.16.input_layernorm.weight": "model-00003-of-00004.safetensors", + "model.language_model.layers.16.mlp.down_proj.weight": "model-00004-of-00004.safetensors", + "model.language_model.layers.16.mlp.gate_proj.weight": "model-00002-of-00004.safetensors", + "model.language_model.layers.16.mlp.up_proj.weight": "model-00001-of-00004.safetensors", + "model.language_model.layers.16.post_attention_layernorm.weight": "model-00002-of-00004.safetensors", + "model.language_model.layers.16.self_attn.k_norm.weight": "model-00003-of-00004.safetensors", + "model.language_model.layers.16.self_attn.k_proj.weight": "model-00001-of-00004.safetensors", + "model.language_model.layers.16.self_attn.o_proj.weight": "model-00004-of-00004.safetensors", + "model.language_model.layers.16.self_attn.q_norm.weight": "model-00002-of-00004.safetensors", + "model.language_model.layers.16.self_attn.q_proj.weight": "model-00002-of-00004.safetensors", + "model.language_model.layers.16.self_attn.v_proj.weight": "model-00001-of-00004.safetensors", + "model.language_model.layers.17.input_layernorm.weight": "model-00001-of-00004.safetensors", + "model.language_model.layers.17.mlp.down_proj.weight": "model-00004-of-00004.safetensors", + "model.language_model.layers.17.mlp.gate_proj.weight": "model-00003-of-00004.safetensors", + "model.language_model.layers.17.mlp.up_proj.weight": "model-00004-of-00004.safetensors", + "model.language_model.layers.17.post_attention_layernorm.weight": "model-00004-of-00004.safetensors", + "model.language_model.layers.17.self_attn.k_norm.weight": "model-00003-of-00004.safetensors", + "model.language_model.layers.17.self_attn.k_proj.weight": "model-00003-of-00004.safetensors", + "model.language_model.layers.17.self_attn.o_proj.weight": "model-00003-of-00004.safetensors", + "model.language_model.layers.17.self_attn.q_norm.weight": "model-00002-of-00004.safetensors", + "model.language_model.layers.17.self_attn.q_proj.weight": "model-00001-of-00004.safetensors", + "model.language_model.layers.17.self_attn.v_proj.weight": "model-00003-of-00004.safetensors", + "model.language_model.layers.18.input_layernorm.weight": "model-00003-of-00004.safetensors", + "model.language_model.layers.18.mlp.down_proj.weight": "model-00001-of-00004.safetensors", + "model.language_model.layers.18.mlp.gate_proj.weight": "model-00001-of-00004.safetensors", + "model.language_model.layers.18.mlp.up_proj.weight": "model-00002-of-00004.safetensors", + "model.language_model.layers.18.post_attention_layernorm.weight": "model-00001-of-00004.safetensors", + "model.language_model.layers.18.self_attn.k_norm.weight": "model-00003-of-00004.safetensors", + "model.language_model.layers.18.self_attn.k_proj.weight": "model-00003-of-00004.safetensors", + "model.language_model.layers.18.self_attn.o_proj.weight": "model-00001-of-00004.safetensors", + "model.language_model.layers.18.self_attn.q_norm.weight": "model-00002-of-00004.safetensors", + "model.language_model.layers.18.self_attn.q_proj.weight": "model-00002-of-00004.safetensors", + "model.language_model.layers.18.self_attn.v_proj.weight": "model-00004-of-00004.safetensors", + "model.language_model.layers.19.input_layernorm.weight": "model-00003-of-00004.safetensors", + "model.language_model.layers.19.mlp.down_proj.weight": "model-00004-of-00004.safetensors", + "model.language_model.layers.19.mlp.gate_proj.weight": "model-00003-of-00004.safetensors", + "model.language_model.layers.19.mlp.up_proj.weight": "model-00003-of-00004.safetensors", + "model.language_model.layers.19.post_attention_layernorm.weight": "model-00003-of-00004.safetensors", + "model.language_model.layers.19.self_attn.k_norm.weight": "model-00002-of-00004.safetensors", + "model.language_model.layers.19.self_attn.k_proj.weight": "model-00002-of-00004.safetensors", + "model.language_model.layers.19.self_attn.o_proj.weight": "model-00003-of-00004.safetensors", + "model.language_model.layers.19.self_attn.q_norm.weight": "model-00003-of-00004.safetensors", + "model.language_model.layers.19.self_attn.q_proj.weight": "model-00004-of-00004.safetensors", + "model.language_model.layers.19.self_attn.v_proj.weight": "model-00004-of-00004.safetensors", + "model.language_model.layers.2.input_layernorm.weight": "model-00004-of-00004.safetensors", + "model.language_model.layers.2.mlp.down_proj.weight": "model-00003-of-00004.safetensors", + "model.language_model.layers.2.mlp.gate_proj.weight": "model-00001-of-00004.safetensors", + "model.language_model.layers.2.mlp.up_proj.weight": "model-00002-of-00004.safetensors", + "model.language_model.layers.2.post_attention_layernorm.weight": "model-00002-of-00004.safetensors", + "model.language_model.layers.2.self_attn.k_norm.weight": "model-00003-of-00004.safetensors", + "model.language_model.layers.2.self_attn.k_proj.weight": "model-00001-of-00004.safetensors", + "model.language_model.layers.2.self_attn.o_proj.weight": "model-00002-of-00004.safetensors", + "model.language_model.layers.2.self_attn.q_norm.weight": "model-00003-of-00004.safetensors", + "model.language_model.layers.2.self_attn.q_proj.weight": "model-00003-of-00004.safetensors", + "model.language_model.layers.2.self_attn.v_proj.weight": "model-00002-of-00004.safetensors", + "model.language_model.layers.20.input_layernorm.weight": "model-00002-of-00004.safetensors", + "model.language_model.layers.20.mlp.down_proj.weight": "model-00004-of-00004.safetensors", + "model.language_model.layers.20.mlp.gate_proj.weight": "model-00002-of-00004.safetensors", + "model.language_model.layers.20.mlp.up_proj.weight": "model-00003-of-00004.safetensors", + "model.language_model.layers.20.post_attention_layernorm.weight": "model-00003-of-00004.safetensors", + "model.language_model.layers.20.self_attn.k_norm.weight": "model-00001-of-00004.safetensors", + "model.language_model.layers.20.self_attn.k_proj.weight": "model-00003-of-00004.safetensors", + "model.language_model.layers.20.self_attn.o_proj.weight": "model-00004-of-00004.safetensors", + "model.language_model.layers.20.self_attn.q_norm.weight": "model-00004-of-00004.safetensors", + "model.language_model.layers.20.self_attn.q_proj.weight": "model-00001-of-00004.safetensors", + "model.language_model.layers.20.self_attn.v_proj.weight": "model-00004-of-00004.safetensors", + "model.language_model.layers.21.input_layernorm.weight": "model-00002-of-00004.safetensors", + "model.language_model.layers.21.mlp.down_proj.weight": "model-00003-of-00004.safetensors", + "model.language_model.layers.21.mlp.gate_proj.weight": "model-00001-of-00004.safetensors", + "model.language_model.layers.21.mlp.up_proj.weight": "model-00004-of-00004.safetensors", + "model.language_model.layers.21.post_attention_layernorm.weight": "model-00003-of-00004.safetensors", + "model.language_model.layers.21.self_attn.k_norm.weight": "model-00002-of-00004.safetensors", + "model.language_model.layers.21.self_attn.k_proj.weight": "model-00001-of-00004.safetensors", + "model.language_model.layers.21.self_attn.o_proj.weight": "model-00001-of-00004.safetensors", + "model.language_model.layers.21.self_attn.q_norm.weight": "model-00003-of-00004.safetensors", + "model.language_model.layers.21.self_attn.q_proj.weight": "model-00004-of-00004.safetensors", + "model.language_model.layers.21.self_attn.v_proj.weight": "model-00004-of-00004.safetensors", + "model.language_model.layers.22.input_layernorm.weight": "model-00003-of-00004.safetensors", + "model.language_model.layers.22.mlp.down_proj.weight": "model-00001-of-00004.safetensors", + "model.language_model.layers.22.mlp.gate_proj.weight": "model-00003-of-00004.safetensors", + "model.language_model.layers.22.mlp.up_proj.weight": "model-00003-of-00004.safetensors", + "model.language_model.layers.22.post_attention_layernorm.weight": "model-00001-of-00004.safetensors", + "model.language_model.layers.22.self_attn.k_norm.weight": "model-00004-of-00004.safetensors", + "model.language_model.layers.22.self_attn.k_proj.weight": "model-00003-of-00004.safetensors", + "model.language_model.layers.22.self_attn.o_proj.weight": "model-00004-of-00004.safetensors", + "model.language_model.layers.22.self_attn.q_norm.weight": "model-00002-of-00004.safetensors", + "model.language_model.layers.22.self_attn.q_proj.weight": "model-00003-of-00004.safetensors", + "model.language_model.layers.22.self_attn.v_proj.weight": "model-00002-of-00004.safetensors", + "model.language_model.layers.23.input_layernorm.weight": "model-00003-of-00004.safetensors", + "model.language_model.layers.23.mlp.down_proj.weight": "model-00004-of-00004.safetensors", + "model.language_model.layers.23.mlp.gate_proj.weight": "model-00002-of-00004.safetensors", + "model.language_model.layers.23.mlp.up_proj.weight": "model-00002-of-00004.safetensors", + "model.language_model.layers.23.post_attention_layernorm.weight": "model-00002-of-00004.safetensors", + "model.language_model.layers.23.self_attn.k_norm.weight": "model-00003-of-00004.safetensors", + "model.language_model.layers.23.self_attn.k_proj.weight": "model-00001-of-00004.safetensors", + "model.language_model.layers.23.self_attn.o_proj.weight": "model-00004-of-00004.safetensors", + "model.language_model.layers.23.self_attn.q_norm.weight": "model-00002-of-00004.safetensors", + "model.language_model.layers.23.self_attn.q_proj.weight": "model-00002-of-00004.safetensors", + "model.language_model.layers.23.self_attn.v_proj.weight": "model-00001-of-00004.safetensors", + "model.language_model.layers.24.input_layernorm.weight": "model-00002-of-00004.safetensors", + "model.language_model.layers.24.mlp.down_proj.weight": "model-00001-of-00004.safetensors", + "model.language_model.layers.24.mlp.gate_proj.weight": "model-00001-of-00004.safetensors", + "model.language_model.layers.24.mlp.up_proj.weight": "model-00002-of-00004.safetensors", + "model.language_model.layers.24.post_attention_layernorm.weight": "model-00002-of-00004.safetensors", + "model.language_model.layers.24.self_attn.k_norm.weight": "model-00003-of-00004.safetensors", + "model.language_model.layers.24.self_attn.k_proj.weight": "model-00003-of-00004.safetensors", + "model.language_model.layers.24.self_attn.o_proj.weight": "model-00003-of-00004.safetensors", + "model.language_model.layers.24.self_attn.q_norm.weight": "model-00004-of-00004.safetensors", + "model.language_model.layers.24.self_attn.q_proj.weight": "model-00001-of-00004.safetensors", + "model.language_model.layers.24.self_attn.v_proj.weight": "model-00001-of-00004.safetensors", + "model.language_model.layers.25.input_layernorm.weight": "model-00002-of-00004.safetensors", + "model.language_model.layers.25.mlp.down_proj.weight": "model-00002-of-00004.safetensors", + "model.language_model.layers.25.mlp.gate_proj.weight": "model-00003-of-00004.safetensors", + "model.language_model.layers.25.mlp.up_proj.weight": "model-00001-of-00004.safetensors", + "model.language_model.layers.25.post_attention_layernorm.weight": "model-00001-of-00004.safetensors", + "model.language_model.layers.25.self_attn.k_norm.weight": "model-00002-of-00004.safetensors", + "model.language_model.layers.25.self_attn.k_proj.weight": "model-00003-of-00004.safetensors", + "model.language_model.layers.25.self_attn.o_proj.weight": "model-00004-of-00004.safetensors", + "model.language_model.layers.25.self_attn.q_norm.weight": "model-00001-of-00004.safetensors", + "model.language_model.layers.25.self_attn.q_proj.weight": "model-00002-of-00004.safetensors", + "model.language_model.layers.25.self_attn.v_proj.weight": "model-00001-of-00004.safetensors", + "model.language_model.layers.26.input_layernorm.weight": "model-00002-of-00004.safetensors", + "model.language_model.layers.26.mlp.down_proj.weight": "model-00001-of-00004.safetensors", + "model.language_model.layers.26.mlp.gate_proj.weight": "model-00004-of-00004.safetensors", + "model.language_model.layers.26.mlp.up_proj.weight": "model-00002-of-00004.safetensors", + "model.language_model.layers.26.post_attention_layernorm.weight": "model-00002-of-00004.safetensors", + "model.language_model.layers.26.self_attn.k_norm.weight": "model-00003-of-00004.safetensors", + "model.language_model.layers.26.self_attn.k_proj.weight": "model-00003-of-00004.safetensors", + "model.language_model.layers.26.self_attn.o_proj.weight": "model-00001-of-00004.safetensors", + "model.language_model.layers.26.self_attn.q_norm.weight": "model-00003-of-00004.safetensors", + "model.language_model.layers.26.self_attn.q_proj.weight": "model-00003-of-00004.safetensors", + "model.language_model.layers.26.self_attn.v_proj.weight": "model-00001-of-00004.safetensors", + "model.language_model.layers.27.input_layernorm.weight": "model-00004-of-00004.safetensors", + "model.language_model.layers.27.mlp.down_proj.weight": "model-00003-of-00004.safetensors", + "model.language_model.layers.27.mlp.gate_proj.weight": "model-00003-of-00004.safetensors", + "model.language_model.layers.27.mlp.up_proj.weight": "model-00002-of-00004.safetensors", + "model.language_model.layers.27.post_attention_layernorm.weight": "model-00003-of-00004.safetensors", + "model.language_model.layers.27.self_attn.k_norm.weight": "model-00001-of-00004.safetensors", + "model.language_model.layers.27.self_attn.k_proj.weight": "model-00001-of-00004.safetensors", + "model.language_model.layers.27.self_attn.o_proj.weight": "model-00002-of-00004.safetensors", + "model.language_model.layers.27.self_attn.q_norm.weight": "model-00003-of-00004.safetensors", + "model.language_model.layers.27.self_attn.q_proj.weight": "model-00002-of-00004.safetensors", + "model.language_model.layers.27.self_attn.v_proj.weight": "model-00003-of-00004.safetensors", + "model.language_model.layers.28.input_layernorm.weight": "model-00002-of-00004.safetensors", + "model.language_model.layers.28.mlp.down_proj.weight": "model-00003-of-00004.safetensors", + "model.language_model.layers.28.mlp.gate_proj.weight": "model-00001-of-00004.safetensors", + "model.language_model.layers.28.mlp.up_proj.weight": "model-00003-of-00004.safetensors", + "model.language_model.layers.28.post_attention_layernorm.weight": "model-00003-of-00004.safetensors", + "model.language_model.layers.28.self_attn.k_norm.weight": "model-00003-of-00004.safetensors", + "model.language_model.layers.28.self_attn.k_proj.weight": "model-00001-of-00004.safetensors", + "model.language_model.layers.28.self_attn.o_proj.weight": "model-00001-of-00004.safetensors", + "model.language_model.layers.28.self_attn.q_norm.weight": "model-00003-of-00004.safetensors", + "model.language_model.layers.28.self_attn.q_proj.weight": "model-00003-of-00004.safetensors", + "model.language_model.layers.28.self_attn.v_proj.weight": "model-00001-of-00004.safetensors", + "model.language_model.layers.29.input_layernorm.weight": "model-00002-of-00004.safetensors", + "model.language_model.layers.29.mlp.down_proj.weight": "model-00001-of-00004.safetensors", + "model.language_model.layers.29.mlp.gate_proj.weight": "model-00001-of-00004.safetensors", + "model.language_model.layers.29.mlp.up_proj.weight": "model-00003-of-00004.safetensors", + "model.language_model.layers.29.post_attention_layernorm.weight": "model-00002-of-00004.safetensors", + "model.language_model.layers.29.self_attn.k_norm.weight": "model-00003-of-00004.safetensors", + "model.language_model.layers.29.self_attn.k_proj.weight": "model-00001-of-00004.safetensors", + "model.language_model.layers.29.self_attn.o_proj.weight": "model-00003-of-00004.safetensors", + "model.language_model.layers.29.self_attn.q_norm.weight": "model-00004-of-00004.safetensors", + "model.language_model.layers.29.self_attn.q_proj.weight": "model-00004-of-00004.safetensors", + "model.language_model.layers.29.self_attn.v_proj.weight": "model-00001-of-00004.safetensors", + "model.language_model.layers.3.input_layernorm.weight": "model-00004-of-00004.safetensors", + "model.language_model.layers.3.mlp.down_proj.weight": "model-00002-of-00004.safetensors", + "model.language_model.layers.3.mlp.gate_proj.weight": "model-00004-of-00004.safetensors", + "model.language_model.layers.3.mlp.up_proj.weight": "model-00004-of-00004.safetensors", + "model.language_model.layers.3.post_attention_layernorm.weight": "model-00003-of-00004.safetensors", + "model.language_model.layers.3.self_attn.k_norm.weight": "model-00004-of-00004.safetensors", + "model.language_model.layers.3.self_attn.k_proj.weight": "model-00001-of-00004.safetensors", + "model.language_model.layers.3.self_attn.o_proj.weight": "model-00001-of-00004.safetensors", + "model.language_model.layers.3.self_attn.q_norm.weight": "model-00002-of-00004.safetensors", + "model.language_model.layers.3.self_attn.q_proj.weight": "model-00002-of-00004.safetensors", + "model.language_model.layers.3.self_attn.v_proj.weight": "model-00004-of-00004.safetensors", + "model.language_model.layers.30.input_layernorm.weight": "model-00004-of-00004.safetensors", + "model.language_model.layers.30.mlp.down_proj.weight": "model-00003-of-00004.safetensors", + "model.language_model.layers.30.mlp.gate_proj.weight": "model-00004-of-00004.safetensors", + "model.language_model.layers.30.mlp.up_proj.weight": "model-00004-of-00004.safetensors", + "model.language_model.layers.30.post_attention_layernorm.weight": "model-00004-of-00004.safetensors", + "model.language_model.layers.30.self_attn.k_norm.weight": "model-00004-of-00004.safetensors", + "model.language_model.layers.30.self_attn.k_proj.weight": "model-00004-of-00004.safetensors", + "model.language_model.layers.30.self_attn.o_proj.weight": "model-00003-of-00004.safetensors", + "model.language_model.layers.30.self_attn.q_norm.weight": "model-00003-of-00004.safetensors", + "model.language_model.layers.30.self_attn.q_proj.weight": "model-00003-of-00004.safetensors", + "model.language_model.layers.30.self_attn.v_proj.weight": "model-00003-of-00004.safetensors", + "model.language_model.layers.31.input_layernorm.weight": "model-00002-of-00004.safetensors", + "model.language_model.layers.31.mlp.down_proj.weight": "model-00004-of-00004.safetensors", + "model.language_model.layers.31.mlp.gate_proj.weight": "model-00003-of-00004.safetensors", + "model.language_model.layers.31.mlp.up_proj.weight": "model-00001-of-00004.safetensors", + "model.language_model.layers.31.post_attention_layernorm.weight": "model-00002-of-00004.safetensors", + "model.language_model.layers.31.self_attn.k_norm.weight": "model-00001-of-00004.safetensors", + "model.language_model.layers.31.self_attn.k_proj.weight": "model-00004-of-00004.safetensors", + "model.language_model.layers.31.self_attn.o_proj.weight": "model-00004-of-00004.safetensors", + "model.language_model.layers.31.self_attn.q_norm.weight": "model-00002-of-00004.safetensors", + "model.language_model.layers.31.self_attn.q_proj.weight": "model-00003-of-00004.safetensors", + "model.language_model.layers.31.self_attn.v_proj.weight": "model-00003-of-00004.safetensors", + "model.language_model.layers.32.input_layernorm.weight": "model-00002-of-00004.safetensors", + "model.language_model.layers.32.mlp.down_proj.weight": "model-00003-of-00004.safetensors", + "model.language_model.layers.32.mlp.gate_proj.weight": "model-00001-of-00004.safetensors", + "model.language_model.layers.32.mlp.up_proj.weight": "model-00001-of-00004.safetensors", + "model.language_model.layers.32.post_attention_layernorm.weight": "model-00002-of-00004.safetensors", + "model.language_model.layers.32.self_attn.k_norm.weight": "model-00002-of-00004.safetensors", + "model.language_model.layers.32.self_attn.k_proj.weight": "model-00001-of-00004.safetensors", + "model.language_model.layers.32.self_attn.o_proj.weight": "model-00002-of-00004.safetensors", + "model.language_model.layers.32.self_attn.q_norm.weight": "model-00002-of-00004.safetensors", + "model.language_model.layers.32.self_attn.q_proj.weight": "model-00003-of-00004.safetensors", + "model.language_model.layers.32.self_attn.v_proj.weight": "model-00001-of-00004.safetensors", + "model.language_model.layers.33.input_layernorm.weight": "model-00002-of-00004.safetensors", + "model.language_model.layers.33.mlp.down_proj.weight": "model-00003-of-00004.safetensors", + "model.language_model.layers.33.mlp.gate_proj.weight": "model-00003-of-00004.safetensors", + "model.language_model.layers.33.mlp.up_proj.weight": "model-00002-of-00004.safetensors", + "model.language_model.layers.33.post_attention_layernorm.weight": "model-00003-of-00004.safetensors", + "model.language_model.layers.33.self_attn.k_norm.weight": "model-00003-of-00004.safetensors", + "model.language_model.layers.33.self_attn.k_proj.weight": "model-00001-of-00004.safetensors", + "model.language_model.layers.33.self_attn.o_proj.weight": "model-00001-of-00004.safetensors", + "model.language_model.layers.33.self_attn.q_norm.weight": "model-00003-of-00004.safetensors", + "model.language_model.layers.33.self_attn.q_proj.weight": "model-00002-of-00004.safetensors", + "model.language_model.layers.33.self_attn.v_proj.weight": "model-00001-of-00004.safetensors", + "model.language_model.layers.34.input_layernorm.weight": "model-00004-of-00004.safetensors", + "model.language_model.layers.34.mlp.down_proj.weight": "model-00002-of-00004.safetensors", + "model.language_model.layers.34.mlp.gate_proj.weight": "model-00002-of-00004.safetensors", + "model.language_model.layers.34.mlp.up_proj.weight": "model-00003-of-00004.safetensors", + "model.language_model.layers.34.post_attention_layernorm.weight": "model-00002-of-00004.safetensors", + "model.language_model.layers.34.self_attn.k_norm.weight": "model-00001-of-00004.safetensors", + "model.language_model.layers.34.self_attn.k_proj.weight": "model-00003-of-00004.safetensors", + "model.language_model.layers.34.self_attn.o_proj.weight": "model-00003-of-00004.safetensors", + "model.language_model.layers.34.self_attn.q_norm.weight": "model-00003-of-00004.safetensors", + "model.language_model.layers.34.self_attn.q_proj.weight": "model-00003-of-00004.safetensors", + "model.language_model.layers.34.self_attn.v_proj.weight": "model-00002-of-00004.safetensors", + "model.language_model.layers.35.input_layernorm.weight": "model-00004-of-00004.safetensors", + "model.language_model.layers.35.mlp.down_proj.weight": "model-00002-of-00004.safetensors", + "model.language_model.layers.35.mlp.gate_proj.weight": "model-00003-of-00004.safetensors", + "model.language_model.layers.35.mlp.up_proj.weight": "model-00002-of-00004.safetensors", + "model.language_model.layers.35.post_attention_layernorm.weight": "model-00003-of-00004.safetensors", + "model.language_model.layers.35.self_attn.k_norm.weight": "model-00001-of-00004.safetensors", + "model.language_model.layers.35.self_attn.k_proj.weight": "model-00002-of-00004.safetensors", + "model.language_model.layers.35.self_attn.o_proj.weight": "model-00001-of-00004.safetensors", + "model.language_model.layers.35.self_attn.q_norm.weight": "model-00004-of-00004.safetensors", + "model.language_model.layers.35.self_attn.q_proj.weight": "model-00001-of-00004.safetensors", + "model.language_model.layers.35.self_attn.v_proj.weight": "model-00004-of-00004.safetensors", + "model.language_model.layers.4.input_layernorm.weight": "model-00002-of-00004.safetensors", + "model.language_model.layers.4.mlp.down_proj.weight": "model-00004-of-00004.safetensors", + "model.language_model.layers.4.mlp.gate_proj.weight": "model-00004-of-00004.safetensors", + "model.language_model.layers.4.mlp.up_proj.weight": "model-00003-of-00004.safetensors", + "model.language_model.layers.4.post_attention_layernorm.weight": "model-00003-of-00004.safetensors", + "model.language_model.layers.4.self_attn.k_norm.weight": "model-00001-of-00004.safetensors", + "model.language_model.layers.4.self_attn.k_proj.weight": "model-00002-of-00004.safetensors", + "model.language_model.layers.4.self_attn.o_proj.weight": "model-00002-of-00004.safetensors", + "model.language_model.layers.4.self_attn.q_norm.weight": "model-00003-of-00004.safetensors", + "model.language_model.layers.4.self_attn.q_proj.weight": "model-00002-of-00004.safetensors", + "model.language_model.layers.4.self_attn.v_proj.weight": "model-00003-of-00004.safetensors", + "model.language_model.layers.5.input_layernorm.weight": "model-00001-of-00004.safetensors", + "model.language_model.layers.5.mlp.down_proj.weight": "model-00002-of-00004.safetensors", + "model.language_model.layers.5.mlp.gate_proj.weight": "model-00002-of-00004.safetensors", + "model.language_model.layers.5.mlp.up_proj.weight": "model-00003-of-00004.safetensors", + "model.language_model.layers.5.post_attention_layernorm.weight": "model-00004-of-00004.safetensors", + "model.language_model.layers.5.self_attn.k_norm.weight": "model-00001-of-00004.safetensors", + "model.language_model.layers.5.self_attn.k_proj.weight": "model-00002-of-00004.safetensors", + "model.language_model.layers.5.self_attn.o_proj.weight": "model-00003-of-00004.safetensors", + "model.language_model.layers.5.self_attn.q_norm.weight": "model-00002-of-00004.safetensors", + "model.language_model.layers.5.self_attn.q_proj.weight": "model-00002-of-00004.safetensors", + "model.language_model.layers.5.self_attn.v_proj.weight": "model-00003-of-00004.safetensors", + "model.language_model.layers.6.input_layernorm.weight": "model-00003-of-00004.safetensors", + "model.language_model.layers.6.mlp.down_proj.weight": "model-00001-of-00004.safetensors", + "model.language_model.layers.6.mlp.gate_proj.weight": "model-00003-of-00004.safetensors", + "model.language_model.layers.6.mlp.up_proj.weight": "model-00002-of-00004.safetensors", + "model.language_model.layers.6.post_attention_layernorm.weight": "model-00001-of-00004.safetensors", + "model.language_model.layers.6.self_attn.k_norm.weight": "model-00002-of-00004.safetensors", + "model.language_model.layers.6.self_attn.k_proj.weight": "model-00003-of-00004.safetensors", + "model.language_model.layers.6.self_attn.o_proj.weight": "model-00002-of-00004.safetensors", + "model.language_model.layers.6.self_attn.q_norm.weight": "model-00004-of-00004.safetensors", + "model.language_model.layers.6.self_attn.q_proj.weight": "model-00004-of-00004.safetensors", + "model.language_model.layers.6.self_attn.v_proj.weight": "model-00004-of-00004.safetensors", + "model.language_model.layers.7.input_layernorm.weight": "model-00002-of-00004.safetensors", + "model.language_model.layers.7.mlp.down_proj.weight": "model-00002-of-00004.safetensors", + "model.language_model.layers.7.mlp.gate_proj.weight": "model-00003-of-00004.safetensors", + "model.language_model.layers.7.mlp.up_proj.weight": "model-00004-of-00004.safetensors", + "model.language_model.layers.7.post_attention_layernorm.weight": "model-00001-of-00004.safetensors", + "model.language_model.layers.7.self_attn.k_norm.weight": "model-00001-of-00004.safetensors", + "model.language_model.layers.7.self_attn.k_proj.weight": "model-00004-of-00004.safetensors", + "model.language_model.layers.7.self_attn.o_proj.weight": "model-00003-of-00004.safetensors", + "model.language_model.layers.7.self_attn.q_norm.weight": "model-00002-of-00004.safetensors", + "model.language_model.layers.7.self_attn.q_proj.weight": "model-00004-of-00004.safetensors", + "model.language_model.layers.7.self_attn.v_proj.weight": "model-00003-of-00004.safetensors", + "model.language_model.layers.8.input_layernorm.weight": "model-00001-of-00004.safetensors", + "model.language_model.layers.8.mlp.down_proj.weight": "model-00001-of-00004.safetensors", + "model.language_model.layers.8.mlp.gate_proj.weight": "model-00004-of-00004.safetensors", + "model.language_model.layers.8.mlp.up_proj.weight": "model-00002-of-00004.safetensors", + "model.language_model.layers.8.post_attention_layernorm.weight": "model-00004-of-00004.safetensors", + "model.language_model.layers.8.self_attn.k_norm.weight": "model-00001-of-00004.safetensors", + "model.language_model.layers.8.self_attn.k_proj.weight": "model-00001-of-00004.safetensors", + "model.language_model.layers.8.self_attn.o_proj.weight": "model-00003-of-00004.safetensors", + "model.language_model.layers.8.self_attn.q_norm.weight": "model-00003-of-00004.safetensors", + "model.language_model.layers.8.self_attn.q_proj.weight": "model-00001-of-00004.safetensors", + "model.language_model.layers.8.self_attn.v_proj.weight": "model-00002-of-00004.safetensors", + "model.language_model.layers.9.input_layernorm.weight": "model-00003-of-00004.safetensors", + "model.language_model.layers.9.mlp.down_proj.weight": "model-00001-of-00004.safetensors", + "model.language_model.layers.9.mlp.gate_proj.weight": "model-00003-of-00004.safetensors", + "model.language_model.layers.9.mlp.up_proj.weight": "model-00003-of-00004.safetensors", + "model.language_model.layers.9.post_attention_layernorm.weight": "model-00001-of-00004.safetensors", + "model.language_model.layers.9.self_attn.k_norm.weight": "model-00003-of-00004.safetensors", + "model.language_model.layers.9.self_attn.k_proj.weight": "model-00001-of-00004.safetensors", + "model.language_model.layers.9.self_attn.o_proj.weight": "model-00004-of-00004.safetensors", + "model.language_model.layers.9.self_attn.q_norm.weight": "model-00004-of-00004.safetensors", + "model.language_model.layers.9.self_attn.q_proj.weight": "model-00004-of-00004.safetensors", + "model.language_model.layers.9.self_attn.v_proj.weight": "model-00003-of-00004.safetensors", + "model.language_model.norm.weight": "model-00004-of-00004.safetensors", + "model.visual.blocks.0.attn.proj.bias": "model-00003-of-00004.safetensors", + "model.visual.blocks.0.attn.proj.weight": "model-00001-of-00004.safetensors", + "model.visual.blocks.0.attn.qkv.bias": "model-00003-of-00004.safetensors", + "model.visual.blocks.0.attn.qkv.weight": "model-00001-of-00004.safetensors", + "model.visual.blocks.0.mlp.linear_fc1.bias": "model-00004-of-00004.safetensors", + "model.visual.blocks.0.mlp.linear_fc1.weight": "model-00003-of-00004.safetensors", + "model.visual.blocks.0.mlp.linear_fc2.bias": "model-00003-of-00004.safetensors", + "model.visual.blocks.0.mlp.linear_fc2.weight": "model-00002-of-00004.safetensors", + "model.visual.blocks.0.norm1.bias": "model-00001-of-00004.safetensors", + "model.visual.blocks.0.norm1.weight": "model-00002-of-00004.safetensors", + "model.visual.blocks.0.norm2.bias": "model-00003-of-00004.safetensors", + "model.visual.blocks.0.norm2.weight": "model-00003-of-00004.safetensors", + "model.visual.blocks.1.attn.proj.bias": "model-00003-of-00004.safetensors", + "model.visual.blocks.1.attn.proj.weight": "model-00001-of-00004.safetensors", + "model.visual.blocks.1.attn.qkv.bias": "model-00003-of-00004.safetensors", + "model.visual.blocks.1.attn.qkv.weight": "model-00003-of-00004.safetensors", + "model.visual.blocks.1.mlp.linear_fc1.bias": "model-00003-of-00004.safetensors", + "model.visual.blocks.1.mlp.linear_fc1.weight": "model-00004-of-00004.safetensors", + "model.visual.blocks.1.mlp.linear_fc2.bias": "model-00003-of-00004.safetensors", + "model.visual.blocks.1.mlp.linear_fc2.weight": "model-00001-of-00004.safetensors", + "model.visual.blocks.1.norm1.bias": "model-00001-of-00004.safetensors", + "model.visual.blocks.1.norm1.weight": "model-00002-of-00004.safetensors", + "model.visual.blocks.1.norm2.bias": "model-00001-of-00004.safetensors", + "model.visual.blocks.1.norm2.weight": "model-00003-of-00004.safetensors", + "model.visual.blocks.10.attn.proj.bias": "model-00001-of-00004.safetensors", + "model.visual.blocks.10.attn.proj.weight": "model-00004-of-00004.safetensors", + "model.visual.blocks.10.attn.qkv.bias": "model-00002-of-00004.safetensors", + "model.visual.blocks.10.attn.qkv.weight": "model-00003-of-00004.safetensors", + "model.visual.blocks.10.mlp.linear_fc1.bias": "model-00001-of-00004.safetensors", + "model.visual.blocks.10.mlp.linear_fc1.weight": "model-00002-of-00004.safetensors", + "model.visual.blocks.10.mlp.linear_fc2.bias": "model-00002-of-00004.safetensors", + "model.visual.blocks.10.mlp.linear_fc2.weight": "model-00004-of-00004.safetensors", + "model.visual.blocks.10.norm1.bias": "model-00004-of-00004.safetensors", + "model.visual.blocks.10.norm1.weight": "model-00001-of-00004.safetensors", + "model.visual.blocks.10.norm2.bias": "model-00001-of-00004.safetensors", + "model.visual.blocks.10.norm2.weight": "model-00001-of-00004.safetensors", + "model.visual.blocks.11.attn.proj.bias": "model-00004-of-00004.safetensors", + "model.visual.blocks.11.attn.proj.weight": "model-00002-of-00004.safetensors", + "model.visual.blocks.11.attn.qkv.bias": "model-00002-of-00004.safetensors", + "model.visual.blocks.11.attn.qkv.weight": "model-00003-of-00004.safetensors", + "model.visual.blocks.11.mlp.linear_fc1.bias": "model-00003-of-00004.safetensors", + "model.visual.blocks.11.mlp.linear_fc1.weight": "model-00004-of-00004.safetensors", + "model.visual.blocks.11.mlp.linear_fc2.bias": "model-00001-of-00004.safetensors", + "model.visual.blocks.11.mlp.linear_fc2.weight": "model-00001-of-00004.safetensors", + "model.visual.blocks.11.norm1.bias": "model-00002-of-00004.safetensors", + "model.visual.blocks.11.norm1.weight": "model-00003-of-00004.safetensors", + "model.visual.blocks.11.norm2.bias": "model-00004-of-00004.safetensors", + "model.visual.blocks.11.norm2.weight": "model-00002-of-00004.safetensors", + "model.visual.blocks.12.attn.proj.bias": "model-00003-of-00004.safetensors", + "model.visual.blocks.12.attn.proj.weight": "model-00002-of-00004.safetensors", + "model.visual.blocks.12.attn.qkv.bias": "model-00002-of-00004.safetensors", + "model.visual.blocks.12.attn.qkv.weight": "model-00001-of-00004.safetensors", + "model.visual.blocks.12.mlp.linear_fc1.bias": "model-00003-of-00004.safetensors", + "model.visual.blocks.12.mlp.linear_fc1.weight": "model-00003-of-00004.safetensors", + "model.visual.blocks.12.mlp.linear_fc2.bias": "model-00001-of-00004.safetensors", + "model.visual.blocks.12.mlp.linear_fc2.weight": "model-00001-of-00004.safetensors", + "model.visual.blocks.12.norm1.bias": "model-00003-of-00004.safetensors", + "model.visual.blocks.12.norm1.weight": "model-00002-of-00004.safetensors", + "model.visual.blocks.12.norm2.bias": "model-00002-of-00004.safetensors", + "model.visual.blocks.12.norm2.weight": "model-00003-of-00004.safetensors", + "model.visual.blocks.13.attn.proj.bias": "model-00002-of-00004.safetensors", + "model.visual.blocks.13.attn.proj.weight": "model-00002-of-00004.safetensors", + "model.visual.blocks.13.attn.qkv.bias": "model-00004-of-00004.safetensors", + "model.visual.blocks.13.attn.qkv.weight": "model-00003-of-00004.safetensors", + "model.visual.blocks.13.mlp.linear_fc1.bias": "model-00004-of-00004.safetensors", + "model.visual.blocks.13.mlp.linear_fc1.weight": "model-00004-of-00004.safetensors", + "model.visual.blocks.13.mlp.linear_fc2.bias": "model-00001-of-00004.safetensors", + "model.visual.blocks.13.mlp.linear_fc2.weight": "model-00004-of-00004.safetensors", + "model.visual.blocks.13.norm1.bias": "model-00001-of-00004.safetensors", + "model.visual.blocks.13.norm1.weight": "model-00003-of-00004.safetensors", + "model.visual.blocks.13.norm2.bias": "model-00004-of-00004.safetensors", + "model.visual.blocks.13.norm2.weight": "model-00002-of-00004.safetensors", + "model.visual.blocks.14.attn.proj.bias": "model-00002-of-00004.safetensors", + "model.visual.blocks.14.attn.proj.weight": "model-00002-of-00004.safetensors", + "model.visual.blocks.14.attn.qkv.bias": "model-00002-of-00004.safetensors", + "model.visual.blocks.14.attn.qkv.weight": "model-00003-of-00004.safetensors", + "model.visual.blocks.14.mlp.linear_fc1.bias": "model-00004-of-00004.safetensors", + "model.visual.blocks.14.mlp.linear_fc1.weight": "model-00003-of-00004.safetensors", + "model.visual.blocks.14.mlp.linear_fc2.bias": "model-00004-of-00004.safetensors", + "model.visual.blocks.14.mlp.linear_fc2.weight": "model-00002-of-00004.safetensors", + "model.visual.blocks.14.norm1.bias": "model-00003-of-00004.safetensors", + "model.visual.blocks.14.norm1.weight": "model-00004-of-00004.safetensors", + "model.visual.blocks.14.norm2.bias": "model-00001-of-00004.safetensors", + "model.visual.blocks.14.norm2.weight": "model-00003-of-00004.safetensors", + "model.visual.blocks.15.attn.proj.bias": "model-00003-of-00004.safetensors", + "model.visual.blocks.15.attn.proj.weight": "model-00001-of-00004.safetensors", + "model.visual.blocks.15.attn.qkv.bias": "model-00003-of-00004.safetensors", + "model.visual.blocks.15.attn.qkv.weight": "model-00002-of-00004.safetensors", + "model.visual.blocks.15.mlp.linear_fc1.bias": "model-00001-of-00004.safetensors", + "model.visual.blocks.15.mlp.linear_fc1.weight": "model-00002-of-00004.safetensors", + "model.visual.blocks.15.mlp.linear_fc2.bias": "model-00001-of-00004.safetensors", + "model.visual.blocks.15.mlp.linear_fc2.weight": "model-00002-of-00004.safetensors", + "model.visual.blocks.15.norm1.bias": "model-00003-of-00004.safetensors", + "model.visual.blocks.15.norm1.weight": "model-00004-of-00004.safetensors", + "model.visual.blocks.15.norm2.bias": "model-00003-of-00004.safetensors", + "model.visual.blocks.15.norm2.weight": "model-00004-of-00004.safetensors", + "model.visual.blocks.16.attn.proj.bias": "model-00004-of-00004.safetensors", + "model.visual.blocks.16.attn.proj.weight": "model-00003-of-00004.safetensors", + "model.visual.blocks.16.attn.qkv.bias": "model-00003-of-00004.safetensors", + "model.visual.blocks.16.attn.qkv.weight": "model-00003-of-00004.safetensors", + "model.visual.blocks.16.mlp.linear_fc1.bias": "model-00004-of-00004.safetensors", + "model.visual.blocks.16.mlp.linear_fc1.weight": "model-00002-of-00004.safetensors", + "model.visual.blocks.16.mlp.linear_fc2.bias": "model-00003-of-00004.safetensors", + "model.visual.blocks.16.mlp.linear_fc2.weight": "model-00003-of-00004.safetensors", + "model.visual.blocks.16.norm1.bias": "model-00003-of-00004.safetensors", + "model.visual.blocks.16.norm1.weight": "model-00003-of-00004.safetensors", + "model.visual.blocks.16.norm2.bias": "model-00004-of-00004.safetensors", + "model.visual.blocks.16.norm2.weight": "model-00003-of-00004.safetensors", + "model.visual.blocks.17.attn.proj.bias": "model-00004-of-00004.safetensors", + "model.visual.blocks.17.attn.proj.weight": "model-00001-of-00004.safetensors", + "model.visual.blocks.17.attn.qkv.bias": "model-00002-of-00004.safetensors", + "model.visual.blocks.17.attn.qkv.weight": "model-00001-of-00004.safetensors", + "model.visual.blocks.17.mlp.linear_fc1.bias": "model-00003-of-00004.safetensors", + "model.visual.blocks.17.mlp.linear_fc1.weight": "model-00003-of-00004.safetensors", + "model.visual.blocks.17.mlp.linear_fc2.bias": "model-00004-of-00004.safetensors", + "model.visual.blocks.17.mlp.linear_fc2.weight": "model-00002-of-00004.safetensors", + "model.visual.blocks.17.norm1.bias": "model-00002-of-00004.safetensors", + "model.visual.blocks.17.norm1.weight": "model-00003-of-00004.safetensors", + "model.visual.blocks.17.norm2.bias": "model-00001-of-00004.safetensors", + "model.visual.blocks.17.norm2.weight": "model-00003-of-00004.safetensors", + "model.visual.blocks.18.attn.proj.bias": "model-00002-of-00004.safetensors", + "model.visual.blocks.18.attn.proj.weight": "model-00004-of-00004.safetensors", + "model.visual.blocks.18.attn.qkv.bias": "model-00001-of-00004.safetensors", + "model.visual.blocks.18.attn.qkv.weight": "model-00004-of-00004.safetensors", + "model.visual.blocks.18.mlp.linear_fc1.bias": "model-00002-of-00004.safetensors", + "model.visual.blocks.18.mlp.linear_fc1.weight": "model-00001-of-00004.safetensors", + "model.visual.blocks.18.mlp.linear_fc2.bias": "model-00003-of-00004.safetensors", + "model.visual.blocks.18.mlp.linear_fc2.weight": "model-00002-of-00004.safetensors", + "model.visual.blocks.18.norm1.bias": "model-00002-of-00004.safetensors", + "model.visual.blocks.18.norm1.weight": "model-00001-of-00004.safetensors", + "model.visual.blocks.18.norm2.bias": "model-00002-of-00004.safetensors", + "model.visual.blocks.18.norm2.weight": "model-00003-of-00004.safetensors", + "model.visual.blocks.19.attn.proj.bias": "model-00002-of-00004.safetensors", + "model.visual.blocks.19.attn.proj.weight": "model-00002-of-00004.safetensors", + "model.visual.blocks.19.attn.qkv.bias": "model-00004-of-00004.safetensors", + "model.visual.blocks.19.attn.qkv.weight": "model-00002-of-00004.safetensors", + "model.visual.blocks.19.mlp.linear_fc1.bias": "model-00003-of-00004.safetensors", + "model.visual.blocks.19.mlp.linear_fc1.weight": "model-00004-of-00004.safetensors", + "model.visual.blocks.19.mlp.linear_fc2.bias": "model-00003-of-00004.safetensors", + "model.visual.blocks.19.mlp.linear_fc2.weight": "model-00004-of-00004.safetensors", + "model.visual.blocks.19.norm1.bias": "model-00002-of-00004.safetensors", + "model.visual.blocks.19.norm1.weight": "model-00001-of-00004.safetensors", + "model.visual.blocks.19.norm2.bias": "model-00003-of-00004.safetensors", + "model.visual.blocks.19.norm2.weight": "model-00002-of-00004.safetensors", + "model.visual.blocks.2.attn.proj.bias": "model-00001-of-00004.safetensors", + "model.visual.blocks.2.attn.proj.weight": "model-00001-of-00004.safetensors", + "model.visual.blocks.2.attn.qkv.bias": "model-00003-of-00004.safetensors", + "model.visual.blocks.2.attn.qkv.weight": "model-00002-of-00004.safetensors", + "model.visual.blocks.2.mlp.linear_fc1.bias": "model-00003-of-00004.safetensors", + "model.visual.blocks.2.mlp.linear_fc1.weight": "model-00001-of-00004.safetensors", + "model.visual.blocks.2.mlp.linear_fc2.bias": "model-00003-of-00004.safetensors", + "model.visual.blocks.2.mlp.linear_fc2.weight": "model-00004-of-00004.safetensors", + "model.visual.blocks.2.norm1.bias": "model-00004-of-00004.safetensors", + "model.visual.blocks.2.norm1.weight": "model-00001-of-00004.safetensors", + "model.visual.blocks.2.norm2.bias": "model-00003-of-00004.safetensors", + "model.visual.blocks.2.norm2.weight": "model-00002-of-00004.safetensors", + "model.visual.blocks.20.attn.proj.bias": "model-00003-of-00004.safetensors", + "model.visual.blocks.20.attn.proj.weight": "model-00002-of-00004.safetensors", + "model.visual.blocks.20.attn.qkv.bias": "model-00001-of-00004.safetensors", + "model.visual.blocks.20.attn.qkv.weight": "model-00002-of-00004.safetensors", + "model.visual.blocks.20.mlp.linear_fc1.bias": "model-00002-of-00004.safetensors", + "model.visual.blocks.20.mlp.linear_fc1.weight": "model-00004-of-00004.safetensors", + "model.visual.blocks.20.mlp.linear_fc2.bias": "model-00002-of-00004.safetensors", + "model.visual.blocks.20.mlp.linear_fc2.weight": "model-00004-of-00004.safetensors", + "model.visual.blocks.20.norm1.bias": "model-00003-of-00004.safetensors", + "model.visual.blocks.20.norm1.weight": "model-00001-of-00004.safetensors", + "model.visual.blocks.20.norm2.bias": "model-00001-of-00004.safetensors", + "model.visual.blocks.20.norm2.weight": "model-00002-of-00004.safetensors", + "model.visual.blocks.21.attn.proj.bias": "model-00002-of-00004.safetensors", + "model.visual.blocks.21.attn.proj.weight": "model-00001-of-00004.safetensors", + "model.visual.blocks.21.attn.qkv.bias": "model-00003-of-00004.safetensors", + "model.visual.blocks.21.attn.qkv.weight": "model-00002-of-00004.safetensors", + "model.visual.blocks.21.mlp.linear_fc1.bias": "model-00004-of-00004.safetensors", + "model.visual.blocks.21.mlp.linear_fc1.weight": "model-00002-of-00004.safetensors", + "model.visual.blocks.21.mlp.linear_fc2.bias": "model-00003-of-00004.safetensors", + "model.visual.blocks.21.mlp.linear_fc2.weight": "model-00004-of-00004.safetensors", + "model.visual.blocks.21.norm1.bias": "model-00003-of-00004.safetensors", + "model.visual.blocks.21.norm1.weight": "model-00004-of-00004.safetensors", + "model.visual.blocks.21.norm2.bias": "model-00004-of-00004.safetensors", + "model.visual.blocks.21.norm2.weight": "model-00004-of-00004.safetensors", + "model.visual.blocks.22.attn.proj.bias": "model-00001-of-00004.safetensors", + "model.visual.blocks.22.attn.proj.weight": "model-00002-of-00004.safetensors", + "model.visual.blocks.22.attn.qkv.bias": "model-00003-of-00004.safetensors", + "model.visual.blocks.22.attn.qkv.weight": "model-00003-of-00004.safetensors", + "model.visual.blocks.22.mlp.linear_fc1.bias": "model-00001-of-00004.safetensors", + "model.visual.blocks.22.mlp.linear_fc1.weight": "model-00003-of-00004.safetensors", + "model.visual.blocks.22.mlp.linear_fc2.bias": "model-00003-of-00004.safetensors", + "model.visual.blocks.22.mlp.linear_fc2.weight": "model-00001-of-00004.safetensors", + "model.visual.blocks.22.norm1.bias": "model-00004-of-00004.safetensors", + "model.visual.blocks.22.norm1.weight": "model-00002-of-00004.safetensors", + "model.visual.blocks.22.norm2.bias": "model-00001-of-00004.safetensors", + "model.visual.blocks.22.norm2.weight": "model-00004-of-00004.safetensors", + "model.visual.blocks.23.attn.proj.bias": "model-00003-of-00004.safetensors", + "model.visual.blocks.23.attn.proj.weight": "model-00001-of-00004.safetensors", + "model.visual.blocks.23.attn.qkv.bias": "model-00002-of-00004.safetensors", + "model.visual.blocks.23.attn.qkv.weight": "model-00003-of-00004.safetensors", + "model.visual.blocks.23.mlp.linear_fc1.bias": "model-00003-of-00004.safetensors", + "model.visual.blocks.23.mlp.linear_fc1.weight": "model-00003-of-00004.safetensors", + "model.visual.blocks.23.mlp.linear_fc2.bias": "model-00001-of-00004.safetensors", + "model.visual.blocks.23.mlp.linear_fc2.weight": "model-00004-of-00004.safetensors", + "model.visual.blocks.23.norm1.bias": "model-00002-of-00004.safetensors", + "model.visual.blocks.23.norm1.weight": "model-00003-of-00004.safetensors", + "model.visual.blocks.23.norm2.bias": "model-00001-of-00004.safetensors", + "model.visual.blocks.23.norm2.weight": "model-00003-of-00004.safetensors", + "model.visual.blocks.24.attn.proj.bias": "model-00002-of-00004.safetensors", + "model.visual.blocks.24.attn.proj.weight": "model-00001-of-00004.safetensors", + "model.visual.blocks.24.attn.qkv.bias": "model-00003-of-00004.safetensors", + "model.visual.blocks.24.attn.qkv.weight": "model-00004-of-00004.safetensors", + "model.visual.blocks.24.mlp.linear_fc1.bias": "model-00004-of-00004.safetensors", + "model.visual.blocks.24.mlp.linear_fc1.weight": "model-00002-of-00004.safetensors", + "model.visual.blocks.24.mlp.linear_fc2.bias": "model-00004-of-00004.safetensors", + "model.visual.blocks.24.mlp.linear_fc2.weight": "model-00002-of-00004.safetensors", + "model.visual.blocks.24.norm1.bias": "model-00002-of-00004.safetensors", + "model.visual.blocks.24.norm1.weight": "model-00003-of-00004.safetensors", + "model.visual.blocks.24.norm2.bias": "model-00002-of-00004.safetensors", + "model.visual.blocks.24.norm2.weight": "model-00002-of-00004.safetensors", + "model.visual.blocks.25.attn.proj.bias": "model-00003-of-00004.safetensors", + "model.visual.blocks.25.attn.proj.weight": "model-00002-of-00004.safetensors", + "model.visual.blocks.25.attn.qkv.bias": "model-00002-of-00004.safetensors", + "model.visual.blocks.25.attn.qkv.weight": "model-00001-of-00004.safetensors", + "model.visual.blocks.25.mlp.linear_fc1.bias": "model-00001-of-00004.safetensors", + "model.visual.blocks.25.mlp.linear_fc1.weight": "model-00003-of-00004.safetensors", + "model.visual.blocks.25.mlp.linear_fc2.bias": "model-00004-of-00004.safetensors", + "model.visual.blocks.25.mlp.linear_fc2.weight": "model-00002-of-00004.safetensors", + "model.visual.blocks.25.norm1.bias": "model-00003-of-00004.safetensors", + "model.visual.blocks.25.norm1.weight": "model-00003-of-00004.safetensors", + "model.visual.blocks.25.norm2.bias": "model-00003-of-00004.safetensors", + "model.visual.blocks.25.norm2.weight": "model-00001-of-00004.safetensors", + "model.visual.blocks.26.attn.proj.bias": "model-00001-of-00004.safetensors", + "model.visual.blocks.26.attn.proj.weight": "model-00003-of-00004.safetensors", + "model.visual.blocks.26.attn.qkv.bias": "model-00004-of-00004.safetensors", + "model.visual.blocks.26.attn.qkv.weight": "model-00002-of-00004.safetensors", + "model.visual.blocks.26.mlp.linear_fc1.bias": "model-00001-of-00004.safetensors", + "model.visual.blocks.26.mlp.linear_fc1.weight": "model-00003-of-00004.safetensors", + "model.visual.blocks.26.mlp.linear_fc2.bias": "model-00004-of-00004.safetensors", + "model.visual.blocks.26.mlp.linear_fc2.weight": "model-00002-of-00004.safetensors", + "model.visual.blocks.26.norm1.bias": "model-00003-of-00004.safetensors", + "model.visual.blocks.26.norm1.weight": "model-00001-of-00004.safetensors", + "model.visual.blocks.26.norm2.bias": "model-00001-of-00004.safetensors", + "model.visual.blocks.26.norm2.weight": "model-00004-of-00004.safetensors", + "model.visual.blocks.3.attn.proj.bias": "model-00003-of-00004.safetensors", + "model.visual.blocks.3.attn.proj.weight": "model-00004-of-00004.safetensors", + "model.visual.blocks.3.attn.qkv.bias": "model-00001-of-00004.safetensors", + "model.visual.blocks.3.attn.qkv.weight": "model-00003-of-00004.safetensors", + "model.visual.blocks.3.mlp.linear_fc1.bias": "model-00001-of-00004.safetensors", + "model.visual.blocks.3.mlp.linear_fc1.weight": "model-00003-of-00004.safetensors", + "model.visual.blocks.3.mlp.linear_fc2.bias": "model-00002-of-00004.safetensors", + "model.visual.blocks.3.mlp.linear_fc2.weight": "model-00002-of-00004.safetensors", + "model.visual.blocks.3.norm1.bias": "model-00002-of-00004.safetensors", + "model.visual.blocks.3.norm1.weight": "model-00002-of-00004.safetensors", + "model.visual.blocks.3.norm2.bias": "model-00001-of-00004.safetensors", + "model.visual.blocks.3.norm2.weight": "model-00001-of-00004.safetensors", + "model.visual.blocks.4.attn.proj.bias": "model-00003-of-00004.safetensors", + "model.visual.blocks.4.attn.proj.weight": "model-00004-of-00004.safetensors", + "model.visual.blocks.4.attn.qkv.bias": "model-00004-of-00004.safetensors", + "model.visual.blocks.4.attn.qkv.weight": "model-00003-of-00004.safetensors", + "model.visual.blocks.4.mlp.linear_fc1.bias": "model-00003-of-00004.safetensors", + "model.visual.blocks.4.mlp.linear_fc1.weight": "model-00003-of-00004.safetensors", + "model.visual.blocks.4.mlp.linear_fc2.bias": "model-00002-of-00004.safetensors", + "model.visual.blocks.4.mlp.linear_fc2.weight": "model-00002-of-00004.safetensors", + "model.visual.blocks.4.norm1.bias": "model-00003-of-00004.safetensors", + "model.visual.blocks.4.norm1.weight": "model-00002-of-00004.safetensors", + "model.visual.blocks.4.norm2.bias": "model-00001-of-00004.safetensors", + "model.visual.blocks.4.norm2.weight": "model-00002-of-00004.safetensors", + "model.visual.blocks.5.attn.proj.bias": "model-00001-of-00004.safetensors", + "model.visual.blocks.5.attn.proj.weight": "model-00004-of-00004.safetensors", + "model.visual.blocks.5.attn.qkv.bias": "model-00003-of-00004.safetensors", + "model.visual.blocks.5.attn.qkv.weight": "model-00003-of-00004.safetensors", + "model.visual.blocks.5.mlp.linear_fc1.bias": "model-00004-of-00004.safetensors", + "model.visual.blocks.5.mlp.linear_fc1.weight": "model-00003-of-00004.safetensors", + "model.visual.blocks.5.mlp.linear_fc2.bias": "model-00001-of-00004.safetensors", + "model.visual.blocks.5.mlp.linear_fc2.weight": "model-00003-of-00004.safetensors", + "model.visual.blocks.5.norm1.bias": "model-00003-of-00004.safetensors", + "model.visual.blocks.5.norm1.weight": "model-00002-of-00004.safetensors", + "model.visual.blocks.5.norm2.bias": "model-00002-of-00004.safetensors", + "model.visual.blocks.5.norm2.weight": "model-00002-of-00004.safetensors", + "model.visual.blocks.6.attn.proj.bias": "model-00001-of-00004.safetensors", + "model.visual.blocks.6.attn.proj.weight": "model-00001-of-00004.safetensors", + "model.visual.blocks.6.attn.qkv.bias": "model-00004-of-00004.safetensors", + "model.visual.blocks.6.attn.qkv.weight": "model-00002-of-00004.safetensors", + "model.visual.blocks.6.mlp.linear_fc1.bias": "model-00001-of-00004.safetensors", + "model.visual.blocks.6.mlp.linear_fc1.weight": "model-00003-of-00004.safetensors", + "model.visual.blocks.6.mlp.linear_fc2.bias": "model-00003-of-00004.safetensors", + "model.visual.blocks.6.mlp.linear_fc2.weight": "model-00003-of-00004.safetensors", + "model.visual.blocks.6.norm1.bias": "model-00002-of-00004.safetensors", + "model.visual.blocks.6.norm1.weight": "model-00003-of-00004.safetensors", + "model.visual.blocks.6.norm2.bias": "model-00003-of-00004.safetensors", + "model.visual.blocks.6.norm2.weight": "model-00003-of-00004.safetensors", + "model.visual.blocks.7.attn.proj.bias": "model-00003-of-00004.safetensors", + "model.visual.blocks.7.attn.proj.weight": "model-00003-of-00004.safetensors", + "model.visual.blocks.7.attn.qkv.bias": "model-00003-of-00004.safetensors", + "model.visual.blocks.7.attn.qkv.weight": "model-00002-of-00004.safetensors", + "model.visual.blocks.7.mlp.linear_fc1.bias": "model-00001-of-00004.safetensors", + "model.visual.blocks.7.mlp.linear_fc1.weight": "model-00001-of-00004.safetensors", + "model.visual.blocks.7.mlp.linear_fc2.bias": "model-00004-of-00004.safetensors", + "model.visual.blocks.7.mlp.linear_fc2.weight": "model-00001-of-00004.safetensors", + "model.visual.blocks.7.norm1.bias": "model-00002-of-00004.safetensors", + "model.visual.blocks.7.norm1.weight": "model-00004-of-00004.safetensors", + "model.visual.blocks.7.norm2.bias": "model-00003-of-00004.safetensors", + "model.visual.blocks.7.norm2.weight": "model-00001-of-00004.safetensors", + "model.visual.blocks.8.attn.proj.bias": "model-00001-of-00004.safetensors", + "model.visual.blocks.8.attn.proj.weight": "model-00001-of-00004.safetensors", + "model.visual.blocks.8.attn.qkv.bias": "model-00003-of-00004.safetensors", + "model.visual.blocks.8.attn.qkv.weight": "model-00003-of-00004.safetensors", + "model.visual.blocks.8.mlp.linear_fc1.bias": "model-00003-of-00004.safetensors", + "model.visual.blocks.8.mlp.linear_fc1.weight": "model-00002-of-00004.safetensors", + "model.visual.blocks.8.mlp.linear_fc2.bias": "model-00002-of-00004.safetensors", + "model.visual.blocks.8.mlp.linear_fc2.weight": "model-00003-of-00004.safetensors", + "model.visual.blocks.8.norm1.bias": "model-00001-of-00004.safetensors", + "model.visual.blocks.8.norm1.weight": "model-00002-of-00004.safetensors", + "model.visual.blocks.8.norm2.bias": "model-00002-of-00004.safetensors", + "model.visual.blocks.8.norm2.weight": "model-00003-of-00004.safetensors", + "model.visual.blocks.9.attn.proj.bias": "model-00001-of-00004.safetensors", + "model.visual.blocks.9.attn.proj.weight": "model-00003-of-00004.safetensors", + "model.visual.blocks.9.attn.qkv.bias": "model-00003-of-00004.safetensors", + "model.visual.blocks.9.attn.qkv.weight": "model-00002-of-00004.safetensors", + "model.visual.blocks.9.mlp.linear_fc1.bias": "model-00001-of-00004.safetensors", + "model.visual.blocks.9.mlp.linear_fc1.weight": "model-00003-of-00004.safetensors", + "model.visual.blocks.9.mlp.linear_fc2.bias": "model-00001-of-00004.safetensors", + "model.visual.blocks.9.mlp.linear_fc2.weight": "model-00002-of-00004.safetensors", + "model.visual.blocks.9.norm1.bias": "model-00004-of-00004.safetensors", + "model.visual.blocks.9.norm1.weight": "model-00002-of-00004.safetensors", + "model.visual.blocks.9.norm2.bias": "model-00002-of-00004.safetensors", + "model.visual.blocks.9.norm2.weight": "model-00004-of-00004.safetensors", + "model.visual.deepstack_merger_list.0.linear_fc1.bias": "model-00003-of-00004.safetensors", + "model.visual.deepstack_merger_list.0.linear_fc1.weight": "model-00003-of-00004.safetensors", + "model.visual.deepstack_merger_list.0.linear_fc2.bias": "model-00003-of-00004.safetensors", + "model.visual.deepstack_merger_list.0.linear_fc2.weight": "model-00003-of-00004.safetensors", + "model.visual.deepstack_merger_list.0.norm.bias": "model-00004-of-00004.safetensors", + "model.visual.deepstack_merger_list.0.norm.weight": "model-00003-of-00004.safetensors", + "model.visual.deepstack_merger_list.1.linear_fc1.bias": "model-00001-of-00004.safetensors", + "model.visual.deepstack_merger_list.1.linear_fc1.weight": "model-00001-of-00004.safetensors", + "model.visual.deepstack_merger_list.1.linear_fc2.bias": "model-00003-of-00004.safetensors", + "model.visual.deepstack_merger_list.1.linear_fc2.weight": "model-00002-of-00004.safetensors", + "model.visual.deepstack_merger_list.1.norm.bias": "model-00004-of-00004.safetensors", + "model.visual.deepstack_merger_list.1.norm.weight": "model-00001-of-00004.safetensors", + "model.visual.deepstack_merger_list.2.linear_fc1.bias": "model-00001-of-00004.safetensors", + "model.visual.deepstack_merger_list.2.linear_fc1.weight": "model-00002-of-00004.safetensors", + "model.visual.deepstack_merger_list.2.linear_fc2.bias": "model-00002-of-00004.safetensors", + "model.visual.deepstack_merger_list.2.linear_fc2.weight": "model-00002-of-00004.safetensors", + "model.visual.deepstack_merger_list.2.norm.bias": "model-00003-of-00004.safetensors", + "model.visual.deepstack_merger_list.2.norm.weight": "model-00003-of-00004.safetensors", + "model.visual.merger.linear_fc1.bias": "model-00004-of-00004.safetensors", + "model.visual.merger.linear_fc1.weight": "model-00002-of-00004.safetensors", + "model.visual.merger.linear_fc2.bias": "model-00002-of-00004.safetensors", + "model.visual.merger.linear_fc2.weight": "model-00001-of-00004.safetensors", + "model.visual.merger.norm.bias": "model-00002-of-00004.safetensors", + "model.visual.merger.norm.weight": "model-00002-of-00004.safetensors", + "model.visual.patch_embed.proj.bias": "model-00003-of-00004.safetensors", + "model.visual.patch_embed.proj.weight": "model-00001-of-00004.safetensors", + "model.visual.pos_embed.weight": "model-00003-of-00004.safetensors" + } +} diff --git a/posttrain_models/8b_cold_start_sft_46k_multi_32k_timelens_1epoch_grpo_format_tiou_f1_cacc_multi_10k/global_step_260/processor_config.json b/posttrain_models/8b_cold_start_sft_46k_multi_32k_timelens_1epoch_grpo_format_tiou_f1_cacc_multi_10k/global_step_260/processor_config.json new file mode 100644 index 0000000000000000000000000000000000000000..a4fe483a685ede297970ea7a13145d5aabe63a1c --- /dev/null +++ b/posttrain_models/8b_cold_start_sft_46k_multi_32k_timelens_1epoch_grpo_format_tiou_f1_cacc_multi_10k/global_step_260/processor_config.json @@ -0,0 +1,85 @@ +{ + "image_processor": { + "crop_size": null, + "data_format": "channels_first", + "device": null, + "disable_grouping": null, + "do_center_crop": null, + "do_convert_rgb": true, + "do_normalize": true, + "do_pad": null, + "do_rescale": true, + "do_resize": true, + "image_mean": [ + 0.5, + 0.5, + 0.5 + ], + "image_processor_type": "Qwen2VLImageProcessorFast", + "image_seq_length": null, + "image_std": [ + 0.5, + 0.5, + 0.5 + ], + "input_data_format": null, + "max_pixels": null, + "merge_size": 2, + "min_pixels": null, + "pad_size": null, + "patch_size": 16, + "processor_class": "Qwen3VLProcessor", + "resample": 3, + "rescale_factor": 0.00392156862745098, + "return_tensors": null, + "size": { + "longest_edge": 16777216, + "shortest_edge": 65536 + }, + "temporal_patch_size": 2 + }, + "processor_class": "Qwen3VLProcessor", + "video_processor": { + "crop_size": null, + "data_format": "channels_first", + "default_to_square": true, + "device": null, + "do_center_crop": null, + "do_convert_rgb": true, + "do_normalize": true, + "do_pad": null, + "do_rescale": true, + "do_resize": true, + "do_sample_frames": true, + "fps": 2, + "image_mean": [ + 0.5, + 0.5, + 0.5 + ], + "image_std": [ + 0.5, + 0.5, + 0.5 + ], + "input_data_format": null, + "max_frames": 768, + "merge_size": 2, + "min_frames": 4, + "num_frames": null, + "pad_size": null, + "patch_size": 16, + "processor_class": "Qwen3VLProcessor", + "resample": 3, + "rescale_factor": 0.00392156862745098, + "return_metadata": false, + "return_tensors": null, + "size": { + "longest_edge": 25165824, + "shortest_edge": 4096 + }, + "temporal_patch_size": 2, + "video_metadata": null, + "video_processor_type": "Qwen3VLVideoProcessor" + } +} diff --git a/posttrain_models/8b_cold_start_sft_46k_multi_32k_timelens_1epoch_grpo_format_tiou_f1_cacc_multi_10k/global_step_260/special_tokens_map.json b/posttrain_models/8b_cold_start_sft_46k_multi_32k_timelens_1epoch_grpo_format_tiou_f1_cacc_multi_10k/global_step_260/special_tokens_map.json new file mode 100644 index 0000000000000000000000000000000000000000..ac23c0aaa2434523c494330aeb79c58395378103 --- /dev/null +++ b/posttrain_models/8b_cold_start_sft_46k_multi_32k_timelens_1epoch_grpo_format_tiou_f1_cacc_multi_10k/global_step_260/special_tokens_map.json @@ -0,0 +1,31 @@ +{ + "additional_special_tokens": [ + "<|im_start|>", + "<|im_end|>", + "<|object_ref_start|>", + "<|object_ref_end|>", + "<|box_start|>", + "<|box_end|>", + "<|quad_start|>", + "<|quad_end|>", + "<|vision_start|>", + "<|vision_end|>", + "<|vision_pad|>", + "<|image_pad|>", + "<|video_pad|>" + ], + "eos_token": { + "content": "<|im_end|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false + }, + "pad_token": { + "content": "<|endoftext|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false + } +} diff --git a/posttrain_models/8b_cold_start_sft_46k_multi_32k_timelens_1epoch_grpo_format_tiou_f1_cacc_multi_10k/global_step_260/tokenizer_config.json b/posttrain_models/8b_cold_start_sft_46k_multi_32k_timelens_1epoch_grpo_format_tiou_f1_cacc_multi_10k/global_step_260/tokenizer_config.json new file mode 100644 index 0000000000000000000000000000000000000000..fec7f182135c384a08ea2fc9635566ca79c45afc --- /dev/null +++ b/posttrain_models/8b_cold_start_sft_46k_multi_32k_timelens_1epoch_grpo_format_tiou_f1_cacc_multi_10k/global_step_260/tokenizer_config.json @@ -0,0 +1,240 @@ +{ + "add_bos_token": false, + "add_prefix_space": false, + "added_tokens_decoder": { + "151643": { + "content": "<|endoftext|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "151644": { + "content": "<|im_start|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "151645": { + "content": "<|im_end|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "151646": { + "content": "<|object_ref_start|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "151647": { + "content": "<|object_ref_end|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "151648": { + "content": "<|box_start|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "151649": { + "content": "<|box_end|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "151650": { + "content": "<|quad_start|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "151651": { + "content": "<|quad_end|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "151652": { + "content": "<|vision_start|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "151653": { + "content": "<|vision_end|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "151654": { + "content": "<|vision_pad|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "151655": { + "content": "<|image_pad|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "151656": { + "content": "<|video_pad|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "151657": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "151658": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "151659": { + "content": "<|fim_prefix|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "151660": { + "content": "<|fim_middle|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "151661": { + "content": "<|fim_suffix|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "151662": { + "content": "<|fim_pad|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "151663": { + "content": "<|repo_name|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "151664": { + "content": "<|file_sep|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "151665": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "151666": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "151667": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "151668": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + } + }, + "additional_special_tokens": [ + "<|im_start|>", + "<|im_end|>", + "<|object_ref_start|>", + "<|object_ref_end|>", + "<|box_start|>", + "<|box_end|>", + "<|quad_start|>", + "<|quad_end|>", + "<|vision_start|>", + "<|vision_end|>", + "<|vision_pad|>", + "<|image_pad|>", + "<|video_pad|>" + ], + "bos_token": null, + "clean_up_tokenization_spaces": false, + "eos_token": "<|im_end|>", + "errors": "replace", + "extra_special_tokens": {}, + "model_max_length": 262144, + "pad_token": "<|endoftext|>", + "processor_class": "Qwen3VLProcessor", + "split_special_tokens": false, + "tokenizer_class": "Qwen2Tokenizer", + "unk_token": null +} diff --git a/posttrain_models/8b_cold_start_sft_46k_multi_32k_timelens_1epoch_grpo_format_tiou_f1_cacc_multi_10k/global_step_280/added_tokens.json b/posttrain_models/8b_cold_start_sft_46k_multi_32k_timelens_1epoch_grpo_format_tiou_f1_cacc_multi_10k/global_step_280/added_tokens.json new file mode 100644 index 0000000000000000000000000000000000000000..b54f9135e44c1e81047e8d05cb027af8bc039eed --- /dev/null +++ b/posttrain_models/8b_cold_start_sft_46k_multi_32k_timelens_1epoch_grpo_format_tiou_f1_cacc_multi_10k/global_step_280/added_tokens.json @@ -0,0 +1,28 @@ +{ + "": 151668, + "": 151658, + "": 151666, + "": 151667, + "": 151657, + "": 151665, + "<|box_end|>": 151649, + "<|box_start|>": 151648, + "<|endoftext|>": 151643, + "<|file_sep|>": 151664, + "<|fim_middle|>": 151660, + "<|fim_pad|>": 151662, + "<|fim_prefix|>": 151659, + "<|fim_suffix|>": 151661, + "<|im_end|>": 151645, + "<|im_start|>": 151644, + "<|image_pad|>": 151655, + "<|object_ref_end|>": 151647, + "<|object_ref_start|>": 151646, + "<|quad_end|>": 151651, + "<|quad_start|>": 151650, + "<|repo_name|>": 151663, + "<|video_pad|>": 151656, + "<|vision_end|>": 151653, + "<|vision_pad|>": 151654, + "<|vision_start|>": 151652 +} diff --git a/posttrain_models/8b_cold_start_sft_46k_multi_32k_timelens_1epoch_grpo_format_tiou_f1_cacc_multi_10k/global_step_280/chat_template.jinja b/posttrain_models/8b_cold_start_sft_46k_multi_32k_timelens_1epoch_grpo_format_tiou_f1_cacc_multi_10k/global_step_280/chat_template.jinja new file mode 100644 index 0000000000000000000000000000000000000000..124386803f142761528f710e77ae483f5f8c4fc4 --- /dev/null +++ b/posttrain_models/8b_cold_start_sft_46k_multi_32k_timelens_1epoch_grpo_format_tiou_f1_cacc_multi_10k/global_step_280/chat_template.jinja @@ -0,0 +1,120 @@ +{%- if tools %} + {{- '<|im_start|>system\n' }} + {%- if messages[0].role == 'system' %} + {%- if messages[0].content is string %} + {{- messages[0].content }} + {%- else %} + {%- for content in messages[0].content %} + {%- if 'text' in content %} + {{- content.text }} + {%- endif %} + {%- endfor %} + {%- endif %} + {{- '\n\n' }} + {%- endif %} + {{- "# Tools\n\nYou may call one or more functions to assist with the user query.\n\nYou are provided with function signatures within XML tags:\n" }} + {%- for tool in tools %} + {{- "\n" }} + {{- tool | tojson }} + {%- endfor %} + {{- "\n\n\nFor each function call, return a json object with function name and arguments within XML tags:\n\n{\"name\": , \"arguments\": }\n<|im_end|>\n" }} +{%- else %} + {%- if messages[0].role == 'system' %} + {{- '<|im_start|>system\n' }} + {%- if messages[0].content is string %} + {{- messages[0].content }} + {%- else %} + {%- for content in messages[0].content %} + {%- if 'text' in content %} + {{- content.text }} + {%- endif %} + {%- endfor %} + {%- endif %} + {{- '<|im_end|>\n' }} + {%- endif %} +{%- endif %} +{%- set image_count = namespace(value=0) %} +{%- set video_count = namespace(value=0) %} +{%- for message in messages %} + {%- if message.role == "user" %} + {{- '<|im_start|>' + message.role + '\n' }} + {%- if message.content is string %} + {{- message.content }} + {%- else %} + {%- for content in message.content %} + {%- if content.type == 'image' or 'image' in content or 'image_url' in content %} + {%- set image_count.value = image_count.value + 1 %} + {%- if add_vision_id %}Picture {{ image_count.value }}: {% endif -%} + <|vision_start|><|image_pad|><|vision_end|> + {%- elif content.type == 'video' or 'video' in content %} + {%- set video_count.value = video_count.value + 1 %} + {%- if add_vision_id %}Video {{ video_count.value }}: {% endif -%} + <|vision_start|><|video_pad|><|vision_end|> + {%- elif 'text' in content %} + {{- content.text }} + {%- endif %} + {%- endfor %} + {%- endif %} + {{- '<|im_end|>\n' }} + {%- elif message.role == "assistant" %} + {{- '<|im_start|>' + message.role + '\n' }} + {%- if message.content is string %} + {{- message.content }} + {%- else %} + {%- for content_item in message.content %} + {%- if 'text' in content_item %} + {{- content_item.text }} + {%- endif %} + {%- endfor %} + {%- endif %} + {%- if message.tool_calls %} + {%- for tool_call in message.tool_calls %} + {%- if (loop.first and message.content) or (not loop.first) %} + {{- '\n' }} + {%- endif %} + {%- if tool_call.function %} + {%- set tool_call = tool_call.function %} + {%- endif %} + {{- '\n{"name": "' }} + {{- tool_call.name }} + {{- '", "arguments": ' }} + {%- if tool_call.arguments is string %} + {{- tool_call.arguments }} + {%- else %} + {{- tool_call.arguments | tojson }} + {%- endif %} + {{- '}\n' }} + {%- endfor %} + {%- endif %} + {{- '<|im_end|>\n' }} + {%- elif message.role == "tool" %} + {%- if loop.first or (messages[loop.index0 - 1].role != "tool") %} + {{- '<|im_start|>user' }} + {%- endif %} + {{- '\n\n' }} + {%- if message.content is string %} + {{- message.content }} + {%- else %} + {%- for content in message.content %} + {%- if content.type == 'image' or 'image' in content or 'image_url' in content %} + {%- set image_count.value = image_count.value + 1 %} + {%- if add_vision_id %}Picture {{ image_count.value }}: {% endif -%} + <|vision_start|><|image_pad|><|vision_end|> + {%- elif content.type == 'video' or 'video' in content %} + {%- set video_count.value = video_count.value + 1 %} + {%- if add_vision_id %}Video {{ video_count.value }}: {% endif -%} + <|vision_start|><|video_pad|><|vision_end|> + {%- elif 'text' in content %} + {{- content.text }} + {%- endif %} + {%- endfor %} + {%- endif %} + {{- '\n' }} + {%- if loop.last or (messages[loop.index0 + 1].role != "tool") %} + {{- '<|im_end|>\n' }} + {%- endif %} + {%- endif %} +{%- endfor %} +{%- if add_generation_prompt %} + {{- '<|im_start|>assistant\n' }} +{%- endif %} diff --git a/posttrain_models/8b_cold_start_sft_46k_multi_32k_timelens_1epoch_grpo_format_tiou_f1_cacc_multi_10k/global_step_280/config.json b/posttrain_models/8b_cold_start_sft_46k_multi_32k_timelens_1epoch_grpo_format_tiou_f1_cacc_multi_10k/global_step_280/config.json new file mode 100644 index 0000000000000000000000000000000000000000..db69009e04e5bed06b6959b1b3f36d034048a1df --- /dev/null +++ b/posttrain_models/8b_cold_start_sft_46k_multi_32k_timelens_1epoch_grpo_format_tiou_f1_cacc_multi_10k/global_step_280/config.json @@ -0,0 +1,70 @@ +{ + "architectures": [ + "Qwen3VLForConditionalGeneration" + ], + "dtype": "float32", + "eos_token_id": 151645, + "hidden_size": 4096, + "image_token_id": 151655, + "model_type": "qwen3_vl", + "pad_token_id": 151643, + "text_config": { + "attention_bias": false, + "attention_dropout": 0.0, + "bos_token_id": 151643, + "dtype": "float32", + "eos_token_id": 151645, + "head_dim": 128, + "hidden_act": "silu", + "hidden_size": 4096, + "initializer_range": 0.02, + "intermediate_size": 12288, + "max_position_embeddings": 262144, + "model_type": "qwen3_vl_text", + "num_attention_heads": 32, + "num_hidden_layers": 36, + "num_key_value_heads": 8, + "pad_token_id": 151643, + "rms_norm_eps": 1e-06, + "rope_theta": 5000000, + "use_cache": false, + "vocab_size": 151936, + "rope_scaling": { + "mrope_interleaved": true, + "mrope_section": [ + 24, + 20, + 20 + ], + "rope_theta": 5000000, + "rope_type": "default" + } + }, + "tie_word_embeddings": false, + "transformers_version": "5.0.0.dev0", + "video_token_id": 151656, + "vision_config": { + "deepstack_visual_indexes": [ + 8, + 16, + 24 + ], + "depth": 27, + "dtype": "float32", + "hidden_act": "gelu_pytorch_tanh", + "hidden_size": 1152, + "in_channels": 3, + "initializer_range": 0.02, + "intermediate_size": 4304, + "model_type": "qwen3_vl", + "num_heads": 16, + "num_position_embeddings": 2304, + "out_hidden_size": 4096, + "pad_token_id": 151643, + "patch_size": 16, + "spatial_merge_size": 2, + "temporal_patch_size": 2 + }, + "vision_end_token_id": 151653, + "vision_start_token_id": 151652 +} \ No newline at end of file diff --git a/posttrain_models/8b_cold_start_sft_46k_multi_32k_timelens_1epoch_grpo_format_tiou_f1_cacc_multi_10k/global_step_280/generation_config.json b/posttrain_models/8b_cold_start_sft_46k_multi_32k_timelens_1epoch_grpo_format_tiou_f1_cacc_multi_10k/global_step_280/generation_config.json new file mode 100644 index 0000000000000000000000000000000000000000..8458f29aec2209e3f63ca59bca95814310a80de1 --- /dev/null +++ b/posttrain_models/8b_cold_start_sft_46k_multi_32k_timelens_1epoch_grpo_format_tiou_f1_cacc_multi_10k/global_step_280/generation_config.json @@ -0,0 +1,13 @@ +{ + "bos_token_id": 151643, + "do_sample": true, + "eos_token_id": [ + 151645, + 151643 + ], + "pad_token_id": 151643, + "temperature": 0.7, + "top_k": 20, + "top_p": 0.8, + "transformers_version": "5.0.0.dev0" +} diff --git a/posttrain_models/8b_cold_start_sft_46k_multi_32k_timelens_1epoch_grpo_format_tiou_f1_cacc_multi_10k/global_step_280/model.safetensors.index.json b/posttrain_models/8b_cold_start_sft_46k_multi_32k_timelens_1epoch_grpo_format_tiou_f1_cacc_multi_10k/global_step_280/model.safetensors.index.json new file mode 100644 index 0000000000000000000000000000000000000000..75c5ec2c46daad7a1d81610404645d9c9d97e191 --- /dev/null +++ b/posttrain_models/8b_cold_start_sft_46k_multi_32k_timelens_1epoch_grpo_format_tiou_f1_cacc_multi_10k/global_step_280/model.safetensors.index.json @@ -0,0 +1,758 @@ +{ + "metadata": { + "total_parameters": 8767123696, + "total_size": 17534247392 + }, + "weight_map": { + "lm_head.weight": "model-00001-of-00004.safetensors", + "model.language_model.embed_tokens.weight": "model-00003-of-00004.safetensors", + "model.language_model.layers.0.input_layernorm.weight": "model-00004-of-00004.safetensors", + "model.language_model.layers.0.mlp.down_proj.weight": "model-00001-of-00004.safetensors", + "model.language_model.layers.0.mlp.gate_proj.weight": "model-00001-of-00004.safetensors", + "model.language_model.layers.0.mlp.up_proj.weight": "model-00003-of-00004.safetensors", + "model.language_model.layers.0.post_attention_layernorm.weight": "model-00003-of-00004.safetensors", + "model.language_model.layers.0.self_attn.k_norm.weight": "model-00002-of-00004.safetensors", + "model.language_model.layers.0.self_attn.k_proj.weight": "model-00002-of-00004.safetensors", + "model.language_model.layers.0.self_attn.o_proj.weight": "model-00004-of-00004.safetensors", + "model.language_model.layers.0.self_attn.q_norm.weight": "model-00004-of-00004.safetensors", + "model.language_model.layers.0.self_attn.q_proj.weight": "model-00001-of-00004.safetensors", + "model.language_model.layers.0.self_attn.v_proj.weight": "model-00002-of-00004.safetensors", + "model.language_model.layers.1.input_layernorm.weight": "model-00003-of-00004.safetensors", + "model.language_model.layers.1.mlp.down_proj.weight": "model-00004-of-00004.safetensors", + "model.language_model.layers.1.mlp.gate_proj.weight": "model-00002-of-00004.safetensors", + "model.language_model.layers.1.mlp.up_proj.weight": "model-00001-of-00004.safetensors", + "model.language_model.layers.1.post_attention_layernorm.weight": "model-00001-of-00004.safetensors", + "model.language_model.layers.1.self_attn.k_norm.weight": "model-00002-of-00004.safetensors", + "model.language_model.layers.1.self_attn.k_proj.weight": "model-00001-of-00004.safetensors", + "model.language_model.layers.1.self_attn.o_proj.weight": "model-00001-of-00004.safetensors", + "model.language_model.layers.1.self_attn.q_norm.weight": "model-00003-of-00004.safetensors", + "model.language_model.layers.1.self_attn.q_proj.weight": "model-00004-of-00004.safetensors", + "model.language_model.layers.1.self_attn.v_proj.weight": "model-00002-of-00004.safetensors", + "model.language_model.layers.10.input_layernorm.weight": "model-00001-of-00004.safetensors", + "model.language_model.layers.10.mlp.down_proj.weight": "model-00002-of-00004.safetensors", + "model.language_model.layers.10.mlp.gate_proj.weight": "model-00002-of-00004.safetensors", + "model.language_model.layers.10.mlp.up_proj.weight": "model-00002-of-00004.safetensors", + "model.language_model.layers.10.post_attention_layernorm.weight": "model-00001-of-00004.safetensors", + "model.language_model.layers.10.self_attn.k_norm.weight": "model-00002-of-00004.safetensors", + "model.language_model.layers.10.self_attn.k_proj.weight": "model-00002-of-00004.safetensors", + "model.language_model.layers.10.self_attn.o_proj.weight": "model-00002-of-00004.safetensors", + "model.language_model.layers.10.self_attn.q_norm.weight": "model-00002-of-00004.safetensors", + "model.language_model.layers.10.self_attn.q_proj.weight": "model-00003-of-00004.safetensors", + "model.language_model.layers.10.self_attn.v_proj.weight": "model-00002-of-00004.safetensors", + "model.language_model.layers.11.input_layernorm.weight": "model-00003-of-00004.safetensors", + "model.language_model.layers.11.mlp.down_proj.weight": "model-00003-of-00004.safetensors", + "model.language_model.layers.11.mlp.gate_proj.weight": "model-00002-of-00004.safetensors", + "model.language_model.layers.11.mlp.up_proj.weight": "model-00002-of-00004.safetensors", + "model.language_model.layers.11.post_attention_layernorm.weight": "model-00002-of-00004.safetensors", + "model.language_model.layers.11.self_attn.k_norm.weight": "model-00001-of-00004.safetensors", + "model.language_model.layers.11.self_attn.k_proj.weight": "model-00004-of-00004.safetensors", + "model.language_model.layers.11.self_attn.o_proj.weight": "model-00004-of-00004.safetensors", + "model.language_model.layers.11.self_attn.q_norm.weight": "model-00001-of-00004.safetensors", + "model.language_model.layers.11.self_attn.q_proj.weight": "model-00003-of-00004.safetensors", + "model.language_model.layers.11.self_attn.v_proj.weight": "model-00001-of-00004.safetensors", + "model.language_model.layers.12.input_layernorm.weight": "model-00003-of-00004.safetensors", + "model.language_model.layers.12.mlp.down_proj.weight": "model-00002-of-00004.safetensors", + "model.language_model.layers.12.mlp.gate_proj.weight": "model-00003-of-00004.safetensors", + "model.language_model.layers.12.mlp.up_proj.weight": "model-00003-of-00004.safetensors", + "model.language_model.layers.12.post_attention_layernorm.weight": "model-00003-of-00004.safetensors", + "model.language_model.layers.12.self_attn.k_norm.weight": "model-00001-of-00004.safetensors", + "model.language_model.layers.12.self_attn.k_proj.weight": "model-00002-of-00004.safetensors", + "model.language_model.layers.12.self_attn.o_proj.weight": "model-00002-of-00004.safetensors", + "model.language_model.layers.12.self_attn.q_norm.weight": "model-00003-of-00004.safetensors", + "model.language_model.layers.12.self_attn.q_proj.weight": "model-00003-of-00004.safetensors", + "model.language_model.layers.12.self_attn.v_proj.weight": "model-00002-of-00004.safetensors", + "model.language_model.layers.13.input_layernorm.weight": "model-00003-of-00004.safetensors", + "model.language_model.layers.13.mlp.down_proj.weight": "model-00002-of-00004.safetensors", + "model.language_model.layers.13.mlp.gate_proj.weight": "model-00002-of-00004.safetensors", + "model.language_model.layers.13.mlp.up_proj.weight": "model-00002-of-00004.safetensors", + "model.language_model.layers.13.post_attention_layernorm.weight": "model-00002-of-00004.safetensors", + "model.language_model.layers.13.self_attn.k_norm.weight": "model-00001-of-00004.safetensors", + "model.language_model.layers.13.self_attn.k_proj.weight": "model-00001-of-00004.safetensors", + "model.language_model.layers.13.self_attn.o_proj.weight": "model-00003-of-00004.safetensors", + "model.language_model.layers.13.self_attn.q_norm.weight": "model-00001-of-00004.safetensors", + "model.language_model.layers.13.self_attn.q_proj.weight": "model-00001-of-00004.safetensors", + "model.language_model.layers.13.self_attn.v_proj.weight": "model-00002-of-00004.safetensors", + "model.language_model.layers.14.input_layernorm.weight": "model-00003-of-00004.safetensors", + "model.language_model.layers.14.mlp.down_proj.weight": "model-00004-of-00004.safetensors", + "model.language_model.layers.14.mlp.gate_proj.weight": "model-00001-of-00004.safetensors", + "model.language_model.layers.14.mlp.up_proj.weight": "model-00002-of-00004.safetensors", + "model.language_model.layers.14.post_attention_layernorm.weight": "model-00004-of-00004.safetensors", + "model.language_model.layers.14.self_attn.k_norm.weight": "model-00002-of-00004.safetensors", + "model.language_model.layers.14.self_attn.k_proj.weight": "model-00002-of-00004.safetensors", + "model.language_model.layers.14.self_attn.o_proj.weight": "model-00003-of-00004.safetensors", + "model.language_model.layers.14.self_attn.q_norm.weight": "model-00001-of-00004.safetensors", + "model.language_model.layers.14.self_attn.q_proj.weight": "model-00001-of-00004.safetensors", + "model.language_model.layers.14.self_attn.v_proj.weight": "model-00003-of-00004.safetensors", + "model.language_model.layers.15.input_layernorm.weight": "model-00001-of-00004.safetensors", + "model.language_model.layers.15.mlp.down_proj.weight": "model-00001-of-00004.safetensors", + "model.language_model.layers.15.mlp.gate_proj.weight": "model-00002-of-00004.safetensors", + "model.language_model.layers.15.mlp.up_proj.weight": "model-00003-of-00004.safetensors", + "model.language_model.layers.15.post_attention_layernorm.weight": "model-00002-of-00004.safetensors", + "model.language_model.layers.15.self_attn.k_norm.weight": "model-00004-of-00004.safetensors", + "model.language_model.layers.15.self_attn.k_proj.weight": "model-00002-of-00004.safetensors", + "model.language_model.layers.15.self_attn.o_proj.weight": "model-00002-of-00004.safetensors", + "model.language_model.layers.15.self_attn.q_norm.weight": "model-00003-of-00004.safetensors", + "model.language_model.layers.15.self_attn.q_proj.weight": "model-00002-of-00004.safetensors", + "model.language_model.layers.15.self_attn.v_proj.weight": "model-00002-of-00004.safetensors", + "model.language_model.layers.16.input_layernorm.weight": "model-00002-of-00004.safetensors", + "model.language_model.layers.16.mlp.down_proj.weight": "model-00002-of-00004.safetensors", + "model.language_model.layers.16.mlp.gate_proj.weight": "model-00002-of-00004.safetensors", + "model.language_model.layers.16.mlp.up_proj.weight": "model-00001-of-00004.safetensors", + "model.language_model.layers.16.post_attention_layernorm.weight": "model-00003-of-00004.safetensors", + "model.language_model.layers.16.self_attn.k_norm.weight": "model-00003-of-00004.safetensors", + "model.language_model.layers.16.self_attn.k_proj.weight": "model-00002-of-00004.safetensors", + "model.language_model.layers.16.self_attn.o_proj.weight": "model-00002-of-00004.safetensors", + "model.language_model.layers.16.self_attn.q_norm.weight": "model-00002-of-00004.safetensors", + "model.language_model.layers.16.self_attn.q_proj.weight": "model-00001-of-00004.safetensors", + "model.language_model.layers.16.self_attn.v_proj.weight": "model-00003-of-00004.safetensors", + "model.language_model.layers.17.input_layernorm.weight": "model-00003-of-00004.safetensors", + "model.language_model.layers.17.mlp.down_proj.weight": "model-00001-of-00004.safetensors", + "model.language_model.layers.17.mlp.gate_proj.weight": "model-00004-of-00004.safetensors", + "model.language_model.layers.17.mlp.up_proj.weight": "model-00003-of-00004.safetensors", + "model.language_model.layers.17.post_attention_layernorm.weight": "model-00001-of-00004.safetensors", + "model.language_model.layers.17.self_attn.k_norm.weight": "model-00004-of-00004.safetensors", + "model.language_model.layers.17.self_attn.k_proj.weight": "model-00003-of-00004.safetensors", + "model.language_model.layers.17.self_attn.o_proj.weight": "model-00004-of-00004.safetensors", + "model.language_model.layers.17.self_attn.q_norm.weight": "model-00003-of-00004.safetensors", + "model.language_model.layers.17.self_attn.q_proj.weight": "model-00001-of-00004.safetensors", + "model.language_model.layers.17.self_attn.v_proj.weight": "model-00001-of-00004.safetensors", + "model.language_model.layers.18.input_layernorm.weight": "model-00003-of-00004.safetensors", + "model.language_model.layers.18.mlp.down_proj.weight": "model-00003-of-00004.safetensors", + "model.language_model.layers.18.mlp.gate_proj.weight": "model-00003-of-00004.safetensors", + "model.language_model.layers.18.mlp.up_proj.weight": "model-00001-of-00004.safetensors", + "model.language_model.layers.18.post_attention_layernorm.weight": "model-00002-of-00004.safetensors", + "model.language_model.layers.18.self_attn.k_norm.weight": "model-00004-of-00004.safetensors", + "model.language_model.layers.18.self_attn.k_proj.weight": "model-00002-of-00004.safetensors", + "model.language_model.layers.18.self_attn.o_proj.weight": "model-00002-of-00004.safetensors", + "model.language_model.layers.18.self_attn.q_norm.weight": "model-00002-of-00004.safetensors", + "model.language_model.layers.18.self_attn.q_proj.weight": "model-00004-of-00004.safetensors", + "model.language_model.layers.18.self_attn.v_proj.weight": "model-00002-of-00004.safetensors", + "model.language_model.layers.19.input_layernorm.weight": "model-00003-of-00004.safetensors", + "model.language_model.layers.19.mlp.down_proj.weight": "model-00003-of-00004.safetensors", + "model.language_model.layers.19.mlp.gate_proj.weight": "model-00003-of-00004.safetensors", + "model.language_model.layers.19.mlp.up_proj.weight": "model-00002-of-00004.safetensors", + "model.language_model.layers.19.post_attention_layernorm.weight": "model-00004-of-00004.safetensors", + "model.language_model.layers.19.self_attn.k_norm.weight": "model-00001-of-00004.safetensors", + "model.language_model.layers.19.self_attn.k_proj.weight": "model-00002-of-00004.safetensors", + "model.language_model.layers.19.self_attn.o_proj.weight": "model-00004-of-00004.safetensors", + "model.language_model.layers.19.self_attn.q_norm.weight": "model-00002-of-00004.safetensors", + "model.language_model.layers.19.self_attn.q_proj.weight": "model-00003-of-00004.safetensors", + "model.language_model.layers.19.self_attn.v_proj.weight": "model-00003-of-00004.safetensors", + "model.language_model.layers.2.input_layernorm.weight": "model-00001-of-00004.safetensors", + "model.language_model.layers.2.mlp.down_proj.weight": "model-00001-of-00004.safetensors", + "model.language_model.layers.2.mlp.gate_proj.weight": "model-00001-of-00004.safetensors", + "model.language_model.layers.2.mlp.up_proj.weight": "model-00004-of-00004.safetensors", + "model.language_model.layers.2.post_attention_layernorm.weight": "model-00003-of-00004.safetensors", + "model.language_model.layers.2.self_attn.k_norm.weight": "model-00004-of-00004.safetensors", + "model.language_model.layers.2.self_attn.k_proj.weight": "model-00002-of-00004.safetensors", + "model.language_model.layers.2.self_attn.o_proj.weight": "model-00002-of-00004.safetensors", + "model.language_model.layers.2.self_attn.q_norm.weight": "model-00002-of-00004.safetensors", + "model.language_model.layers.2.self_attn.q_proj.weight": "model-00004-of-00004.safetensors", + "model.language_model.layers.2.self_attn.v_proj.weight": "model-00002-of-00004.safetensors", + "model.language_model.layers.20.input_layernorm.weight": "model-00003-of-00004.safetensors", + "model.language_model.layers.20.mlp.down_proj.weight": "model-00002-of-00004.safetensors", + "model.language_model.layers.20.mlp.gate_proj.weight": "model-00001-of-00004.safetensors", + "model.language_model.layers.20.mlp.up_proj.weight": "model-00002-of-00004.safetensors", + "model.language_model.layers.20.post_attention_layernorm.weight": "model-00004-of-00004.safetensors", + "model.language_model.layers.20.self_attn.k_norm.weight": "model-00001-of-00004.safetensors", + "model.language_model.layers.20.self_attn.k_proj.weight": "model-00002-of-00004.safetensors", + "model.language_model.layers.20.self_attn.o_proj.weight": "model-00003-of-00004.safetensors", + "model.language_model.layers.20.self_attn.q_norm.weight": "model-00003-of-00004.safetensors", + "model.language_model.layers.20.self_attn.q_proj.weight": "model-00003-of-00004.safetensors", + "model.language_model.layers.20.self_attn.v_proj.weight": "model-00003-of-00004.safetensors", + "model.language_model.layers.21.input_layernorm.weight": "model-00003-of-00004.safetensors", + "model.language_model.layers.21.mlp.down_proj.weight": "model-00004-of-00004.safetensors", + "model.language_model.layers.21.mlp.gate_proj.weight": "model-00004-of-00004.safetensors", + "model.language_model.layers.21.mlp.up_proj.weight": "model-00001-of-00004.safetensors", + "model.language_model.layers.21.post_attention_layernorm.weight": "model-00002-of-00004.safetensors", + "model.language_model.layers.21.self_attn.k_norm.weight": "model-00003-of-00004.safetensors", + "model.language_model.layers.21.self_attn.k_proj.weight": "model-00004-of-00004.safetensors", + "model.language_model.layers.21.self_attn.o_proj.weight": "model-00002-of-00004.safetensors", + "model.language_model.layers.21.self_attn.q_norm.weight": "model-00001-of-00004.safetensors", + "model.language_model.layers.21.self_attn.q_proj.weight": "model-00003-of-00004.safetensors", + "model.language_model.layers.21.self_attn.v_proj.weight": "model-00002-of-00004.safetensors", + "model.language_model.layers.22.input_layernorm.weight": "model-00003-of-00004.safetensors", + "model.language_model.layers.22.mlp.down_proj.weight": "model-00003-of-00004.safetensors", + "model.language_model.layers.22.mlp.gate_proj.weight": "model-00002-of-00004.safetensors", + "model.language_model.layers.22.mlp.up_proj.weight": "model-00003-of-00004.safetensors", + "model.language_model.layers.22.post_attention_layernorm.weight": "model-00002-of-00004.safetensors", + "model.language_model.layers.22.self_attn.k_norm.weight": "model-00001-of-00004.safetensors", + "model.language_model.layers.22.self_attn.k_proj.weight": "model-00002-of-00004.safetensors", + "model.language_model.layers.22.self_attn.o_proj.weight": "model-00001-of-00004.safetensors", + "model.language_model.layers.22.self_attn.q_norm.weight": "model-00001-of-00004.safetensors", + "model.language_model.layers.22.self_attn.q_proj.weight": "model-00004-of-00004.safetensors", + "model.language_model.layers.22.self_attn.v_proj.weight": "model-00003-of-00004.safetensors", + "model.language_model.layers.23.input_layernorm.weight": "model-00003-of-00004.safetensors", + "model.language_model.layers.23.mlp.down_proj.weight": "model-00001-of-00004.safetensors", + "model.language_model.layers.23.mlp.gate_proj.weight": "model-00004-of-00004.safetensors", + "model.language_model.layers.23.mlp.up_proj.weight": "model-00003-of-00004.safetensors", + "model.language_model.layers.23.post_attention_layernorm.weight": "model-00004-of-00004.safetensors", + "model.language_model.layers.23.self_attn.k_norm.weight": "model-00003-of-00004.safetensors", + "model.language_model.layers.23.self_attn.k_proj.weight": "model-00002-of-00004.safetensors", + "model.language_model.layers.23.self_attn.o_proj.weight": "model-00002-of-00004.safetensors", + "model.language_model.layers.23.self_attn.q_norm.weight": "model-00001-of-00004.safetensors", + "model.language_model.layers.23.self_attn.q_proj.weight": "model-00004-of-00004.safetensors", + "model.language_model.layers.23.self_attn.v_proj.weight": "model-00004-of-00004.safetensors", + "model.language_model.layers.24.input_layernorm.weight": "model-00001-of-00004.safetensors", + "model.language_model.layers.24.mlp.down_proj.weight": "model-00003-of-00004.safetensors", + "model.language_model.layers.24.mlp.gate_proj.weight": "model-00002-of-00004.safetensors", + "model.language_model.layers.24.mlp.up_proj.weight": "model-00001-of-00004.safetensors", + "model.language_model.layers.24.post_attention_layernorm.weight": "model-00001-of-00004.safetensors", + "model.language_model.layers.24.self_attn.k_norm.weight": "model-00002-of-00004.safetensors", + "model.language_model.layers.24.self_attn.k_proj.weight": "model-00004-of-00004.safetensors", + "model.language_model.layers.24.self_attn.o_proj.weight": "model-00003-of-00004.safetensors", + "model.language_model.layers.24.self_attn.q_norm.weight": "model-00001-of-00004.safetensors", + "model.language_model.layers.24.self_attn.q_proj.weight": "model-00004-of-00004.safetensors", + "model.language_model.layers.24.self_attn.v_proj.weight": "model-00002-of-00004.safetensors", + "model.language_model.layers.25.input_layernorm.weight": "model-00001-of-00004.safetensors", + "model.language_model.layers.25.mlp.down_proj.weight": "model-00004-of-00004.safetensors", + "model.language_model.layers.25.mlp.gate_proj.weight": "model-00004-of-00004.safetensors", + "model.language_model.layers.25.mlp.up_proj.weight": "model-00004-of-00004.safetensors", + "model.language_model.layers.25.post_attention_layernorm.weight": "model-00002-of-00004.safetensors", + "model.language_model.layers.25.self_attn.k_norm.weight": "model-00002-of-00004.safetensors", + "model.language_model.layers.25.self_attn.k_proj.weight": "model-00002-of-00004.safetensors", + "model.language_model.layers.25.self_attn.o_proj.weight": "model-00004-of-00004.safetensors", + "model.language_model.layers.25.self_attn.q_norm.weight": "model-00003-of-00004.safetensors", + "model.language_model.layers.25.self_attn.q_proj.weight": "model-00001-of-00004.safetensors", + "model.language_model.layers.25.self_attn.v_proj.weight": "model-00002-of-00004.safetensors", + "model.language_model.layers.26.input_layernorm.weight": "model-00004-of-00004.safetensors", + "model.language_model.layers.26.mlp.down_proj.weight": "model-00003-of-00004.safetensors", + "model.language_model.layers.26.mlp.gate_proj.weight": "model-00004-of-00004.safetensors", + "model.language_model.layers.26.mlp.up_proj.weight": "model-00002-of-00004.safetensors", + "model.language_model.layers.26.post_attention_layernorm.weight": "model-00001-of-00004.safetensors", + "model.language_model.layers.26.self_attn.k_norm.weight": "model-00001-of-00004.safetensors", + "model.language_model.layers.26.self_attn.k_proj.weight": "model-00001-of-00004.safetensors", + "model.language_model.layers.26.self_attn.o_proj.weight": "model-00004-of-00004.safetensors", + "model.language_model.layers.26.self_attn.q_norm.weight": "model-00001-of-00004.safetensors", + "model.language_model.layers.26.self_attn.q_proj.weight": "model-00001-of-00004.safetensors", + "model.language_model.layers.26.self_attn.v_proj.weight": "model-00001-of-00004.safetensors", + "model.language_model.layers.27.input_layernorm.weight": "model-00004-of-00004.safetensors", + "model.language_model.layers.27.mlp.down_proj.weight": "model-00004-of-00004.safetensors", + "model.language_model.layers.27.mlp.gate_proj.weight": "model-00001-of-00004.safetensors", + "model.language_model.layers.27.mlp.up_proj.weight": "model-00002-of-00004.safetensors", + "model.language_model.layers.27.post_attention_layernorm.weight": "model-00002-of-00004.safetensors", + "model.language_model.layers.27.self_attn.k_norm.weight": "model-00002-of-00004.safetensors", + "model.language_model.layers.27.self_attn.k_proj.weight": "model-00002-of-00004.safetensors", + "model.language_model.layers.27.self_attn.o_proj.weight": "model-00001-of-00004.safetensors", + "model.language_model.layers.27.self_attn.q_norm.weight": "model-00002-of-00004.safetensors", + "model.language_model.layers.27.self_attn.q_proj.weight": "model-00004-of-00004.safetensors", + "model.language_model.layers.27.self_attn.v_proj.weight": "model-00003-of-00004.safetensors", + "model.language_model.layers.28.input_layernorm.weight": "model-00001-of-00004.safetensors", + "model.language_model.layers.28.mlp.down_proj.weight": "model-00001-of-00004.safetensors", + "model.language_model.layers.28.mlp.gate_proj.weight": "model-00002-of-00004.safetensors", + "model.language_model.layers.28.mlp.up_proj.weight": "model-00003-of-00004.safetensors", + "model.language_model.layers.28.post_attention_layernorm.weight": "model-00001-of-00004.safetensors", + "model.language_model.layers.28.self_attn.k_norm.weight": "model-00003-of-00004.safetensors", + "model.language_model.layers.28.self_attn.k_proj.weight": "model-00002-of-00004.safetensors", + "model.language_model.layers.28.self_attn.o_proj.weight": "model-00001-of-00004.safetensors", + "model.language_model.layers.28.self_attn.q_norm.weight": "model-00002-of-00004.safetensors", + "model.language_model.layers.28.self_attn.q_proj.weight": "model-00002-of-00004.safetensors", + "model.language_model.layers.28.self_attn.v_proj.weight": "model-00003-of-00004.safetensors", + "model.language_model.layers.29.input_layernorm.weight": "model-00001-of-00004.safetensors", + "model.language_model.layers.29.mlp.down_proj.weight": "model-00001-of-00004.safetensors", + "model.language_model.layers.29.mlp.gate_proj.weight": "model-00003-of-00004.safetensors", + "model.language_model.layers.29.mlp.up_proj.weight": "model-00001-of-00004.safetensors", + "model.language_model.layers.29.post_attention_layernorm.weight": "model-00003-of-00004.safetensors", + "model.language_model.layers.29.self_attn.k_norm.weight": "model-00002-of-00004.safetensors", + "model.language_model.layers.29.self_attn.k_proj.weight": "model-00004-of-00004.safetensors", + "model.language_model.layers.29.self_attn.o_proj.weight": "model-00003-of-00004.safetensors", + "model.language_model.layers.29.self_attn.q_norm.weight": "model-00001-of-00004.safetensors", + "model.language_model.layers.29.self_attn.q_proj.weight": "model-00004-of-00004.safetensors", + "model.language_model.layers.29.self_attn.v_proj.weight": "model-00001-of-00004.safetensors", + "model.language_model.layers.3.input_layernorm.weight": "model-00002-of-00004.safetensors", + "model.language_model.layers.3.mlp.down_proj.weight": "model-00004-of-00004.safetensors", + "model.language_model.layers.3.mlp.gate_proj.weight": "model-00003-of-00004.safetensors", + "model.language_model.layers.3.mlp.up_proj.weight": "model-00002-of-00004.safetensors", + "model.language_model.layers.3.post_attention_layernorm.weight": "model-00003-of-00004.safetensors", + "model.language_model.layers.3.self_attn.k_norm.weight": "model-00003-of-00004.safetensors", + "model.language_model.layers.3.self_attn.k_proj.weight": "model-00003-of-00004.safetensors", + "model.language_model.layers.3.self_attn.o_proj.weight": "model-00001-of-00004.safetensors", + "model.language_model.layers.3.self_attn.q_norm.weight": "model-00003-of-00004.safetensors", + "model.language_model.layers.3.self_attn.q_proj.weight": "model-00002-of-00004.safetensors", + "model.language_model.layers.3.self_attn.v_proj.weight": "model-00003-of-00004.safetensors", + "model.language_model.layers.30.input_layernorm.weight": "model-00004-of-00004.safetensors", + "model.language_model.layers.30.mlp.down_proj.weight": "model-00004-of-00004.safetensors", + "model.language_model.layers.30.mlp.gate_proj.weight": "model-00003-of-00004.safetensors", + "model.language_model.layers.30.mlp.up_proj.weight": "model-00003-of-00004.safetensors", + "model.language_model.layers.30.post_attention_layernorm.weight": "model-00003-of-00004.safetensors", + "model.language_model.layers.30.self_attn.k_norm.weight": "model-00003-of-00004.safetensors", + "model.language_model.layers.30.self_attn.k_proj.weight": "model-00001-of-00004.safetensors", + "model.language_model.layers.30.self_attn.o_proj.weight": "model-00001-of-00004.safetensors", + "model.language_model.layers.30.self_attn.q_norm.weight": "model-00004-of-00004.safetensors", + "model.language_model.layers.30.self_attn.q_proj.weight": "model-00003-of-00004.safetensors", + "model.language_model.layers.30.self_attn.v_proj.weight": "model-00001-of-00004.safetensors", + "model.language_model.layers.31.input_layernorm.weight": "model-00004-of-00004.safetensors", + "model.language_model.layers.31.mlp.down_proj.weight": "model-00003-of-00004.safetensors", + "model.language_model.layers.31.mlp.gate_proj.weight": "model-00003-of-00004.safetensors", + "model.language_model.layers.31.mlp.up_proj.weight": "model-00002-of-00004.safetensors", + "model.language_model.layers.31.post_attention_layernorm.weight": "model-00004-of-00004.safetensors", + "model.language_model.layers.31.self_attn.k_norm.weight": "model-00004-of-00004.safetensors", + "model.language_model.layers.31.self_attn.k_proj.weight": "model-00003-of-00004.safetensors", + "model.language_model.layers.31.self_attn.o_proj.weight": "model-00002-of-00004.safetensors", + "model.language_model.layers.31.self_attn.q_norm.weight": "model-00002-of-00004.safetensors", + "model.language_model.layers.31.self_attn.q_proj.weight": "model-00001-of-00004.safetensors", + "model.language_model.layers.31.self_attn.v_proj.weight": "model-00004-of-00004.safetensors", + "model.language_model.layers.32.input_layernorm.weight": "model-00001-of-00004.safetensors", + "model.language_model.layers.32.mlp.down_proj.weight": "model-00003-of-00004.safetensors", + "model.language_model.layers.32.mlp.gate_proj.weight": "model-00001-of-00004.safetensors", + "model.language_model.layers.32.mlp.up_proj.weight": "model-00002-of-00004.safetensors", + "model.language_model.layers.32.post_attention_layernorm.weight": "model-00003-of-00004.safetensors", + "model.language_model.layers.32.self_attn.k_norm.weight": "model-00001-of-00004.safetensors", + "model.language_model.layers.32.self_attn.k_proj.weight": "model-00004-of-00004.safetensors", + "model.language_model.layers.32.self_attn.o_proj.weight": "model-00002-of-00004.safetensors", + "model.language_model.layers.32.self_attn.q_norm.weight": "model-00002-of-00004.safetensors", + "model.language_model.layers.32.self_attn.q_proj.weight": "model-00002-of-00004.safetensors", + "model.language_model.layers.32.self_attn.v_proj.weight": "model-00002-of-00004.safetensors", + "model.language_model.layers.33.input_layernorm.weight": "model-00003-of-00004.safetensors", + "model.language_model.layers.33.mlp.down_proj.weight": "model-00001-of-00004.safetensors", + "model.language_model.layers.33.mlp.gate_proj.weight": "model-00003-of-00004.safetensors", + "model.language_model.layers.33.mlp.up_proj.weight": "model-00004-of-00004.safetensors", + "model.language_model.layers.33.post_attention_layernorm.weight": "model-00002-of-00004.safetensors", + "model.language_model.layers.33.self_attn.k_norm.weight": "model-00002-of-00004.safetensors", + "model.language_model.layers.33.self_attn.k_proj.weight": "model-00002-of-00004.safetensors", + "model.language_model.layers.33.self_attn.o_proj.weight": "model-00002-of-00004.safetensors", + "model.language_model.layers.33.self_attn.q_norm.weight": "model-00001-of-00004.safetensors", + "model.language_model.layers.33.self_attn.q_proj.weight": "model-00001-of-00004.safetensors", + "model.language_model.layers.33.self_attn.v_proj.weight": "model-00001-of-00004.safetensors", + "model.language_model.layers.34.input_layernorm.weight": "model-00004-of-00004.safetensors", + "model.language_model.layers.34.mlp.down_proj.weight": "model-00001-of-00004.safetensors", + "model.language_model.layers.34.mlp.gate_proj.weight": "model-00002-of-00004.safetensors", + "model.language_model.layers.34.mlp.up_proj.weight": "model-00001-of-00004.safetensors", + "model.language_model.layers.34.post_attention_layernorm.weight": "model-00004-of-00004.safetensors", + "model.language_model.layers.34.self_attn.k_norm.weight": "model-00001-of-00004.safetensors", + "model.language_model.layers.34.self_attn.k_proj.weight": "model-00003-of-00004.safetensors", + "model.language_model.layers.34.self_attn.o_proj.weight": "model-00001-of-00004.safetensors", + "model.language_model.layers.34.self_attn.q_norm.weight": "model-00001-of-00004.safetensors", + "model.language_model.layers.34.self_attn.q_proj.weight": "model-00003-of-00004.safetensors", + "model.language_model.layers.34.self_attn.v_proj.weight": "model-00001-of-00004.safetensors", + "model.language_model.layers.35.input_layernorm.weight": "model-00004-of-00004.safetensors", + "model.language_model.layers.35.mlp.down_proj.weight": "model-00003-of-00004.safetensors", + "model.language_model.layers.35.mlp.gate_proj.weight": "model-00003-of-00004.safetensors", + "model.language_model.layers.35.mlp.up_proj.weight": "model-00004-of-00004.safetensors", + "model.language_model.layers.35.post_attention_layernorm.weight": "model-00002-of-00004.safetensors", + "model.language_model.layers.35.self_attn.k_norm.weight": "model-00004-of-00004.safetensors", + "model.language_model.layers.35.self_attn.k_proj.weight": "model-00002-of-00004.safetensors", + "model.language_model.layers.35.self_attn.o_proj.weight": "model-00004-of-00004.safetensors", + "model.language_model.layers.35.self_attn.q_norm.weight": "model-00003-of-00004.safetensors", + "model.language_model.layers.35.self_attn.q_proj.weight": "model-00001-of-00004.safetensors", + "model.language_model.layers.35.self_attn.v_proj.weight": "model-00004-of-00004.safetensors", + "model.language_model.layers.4.input_layernorm.weight": "model-00003-of-00004.safetensors", + "model.language_model.layers.4.mlp.down_proj.weight": "model-00004-of-00004.safetensors", + "model.language_model.layers.4.mlp.gate_proj.weight": "model-00002-of-00004.safetensors", + "model.language_model.layers.4.mlp.up_proj.weight": "model-00003-of-00004.safetensors", + "model.language_model.layers.4.post_attention_layernorm.weight": "model-00003-of-00004.safetensors", + "model.language_model.layers.4.self_attn.k_norm.weight": "model-00002-of-00004.safetensors", + "model.language_model.layers.4.self_attn.k_proj.weight": "model-00001-of-00004.safetensors", + "model.language_model.layers.4.self_attn.o_proj.weight": "model-00004-of-00004.safetensors", + "model.language_model.layers.4.self_attn.q_norm.weight": "model-00002-of-00004.safetensors", + "model.language_model.layers.4.self_attn.q_proj.weight": "model-00002-of-00004.safetensors", + "model.language_model.layers.4.self_attn.v_proj.weight": "model-00003-of-00004.safetensors", + "model.language_model.layers.5.input_layernorm.weight": "model-00003-of-00004.safetensors", + "model.language_model.layers.5.mlp.down_proj.weight": "model-00001-of-00004.safetensors", + "model.language_model.layers.5.mlp.gate_proj.weight": "model-00002-of-00004.safetensors", + "model.language_model.layers.5.mlp.up_proj.weight": "model-00001-of-00004.safetensors", + "model.language_model.layers.5.post_attention_layernorm.weight": "model-00002-of-00004.safetensors", + "model.language_model.layers.5.self_attn.k_norm.weight": "model-00002-of-00004.safetensors", + "model.language_model.layers.5.self_attn.k_proj.weight": "model-00001-of-00004.safetensors", + "model.language_model.layers.5.self_attn.o_proj.weight": "model-00001-of-00004.safetensors", + "model.language_model.layers.5.self_attn.q_norm.weight": "model-00004-of-00004.safetensors", + "model.language_model.layers.5.self_attn.q_proj.weight": "model-00002-of-00004.safetensors", + "model.language_model.layers.5.self_attn.v_proj.weight": "model-00002-of-00004.safetensors", + "model.language_model.layers.6.input_layernorm.weight": "model-00001-of-00004.safetensors", + "model.language_model.layers.6.mlp.down_proj.weight": "model-00002-of-00004.safetensors", + "model.language_model.layers.6.mlp.gate_proj.weight": "model-00002-of-00004.safetensors", + "model.language_model.layers.6.mlp.up_proj.weight": "model-00002-of-00004.safetensors", + "model.language_model.layers.6.post_attention_layernorm.weight": "model-00003-of-00004.safetensors", + "model.language_model.layers.6.self_attn.k_norm.weight": "model-00003-of-00004.safetensors", + "model.language_model.layers.6.self_attn.k_proj.weight": "model-00002-of-00004.safetensors", + "model.language_model.layers.6.self_attn.o_proj.weight": "model-00002-of-00004.safetensors", + "model.language_model.layers.6.self_attn.q_norm.weight": "model-00002-of-00004.safetensors", + "model.language_model.layers.6.self_attn.q_proj.weight": "model-00002-of-00004.safetensors", + "model.language_model.layers.6.self_attn.v_proj.weight": "model-00002-of-00004.safetensors", + "model.language_model.layers.7.input_layernorm.weight": "model-00003-of-00004.safetensors", + "model.language_model.layers.7.mlp.down_proj.weight": "model-00004-of-00004.safetensors", + "model.language_model.layers.7.mlp.gate_proj.weight": "model-00003-of-00004.safetensors", + "model.language_model.layers.7.mlp.up_proj.weight": "model-00002-of-00004.safetensors", + "model.language_model.layers.7.post_attention_layernorm.weight": "model-00004-of-00004.safetensors", + "model.language_model.layers.7.self_attn.k_norm.weight": "model-00001-of-00004.safetensors", + "model.language_model.layers.7.self_attn.k_proj.weight": "model-00002-of-00004.safetensors", + "model.language_model.layers.7.self_attn.o_proj.weight": "model-00004-of-00004.safetensors", + "model.language_model.layers.7.self_attn.q_norm.weight": "model-00002-of-00004.safetensors", + "model.language_model.layers.7.self_attn.q_proj.weight": "model-00004-of-00004.safetensors", + "model.language_model.layers.7.self_attn.v_proj.weight": "model-00004-of-00004.safetensors", + "model.language_model.layers.8.input_layernorm.weight": "model-00001-of-00004.safetensors", + "model.language_model.layers.8.mlp.down_proj.weight": "model-00001-of-00004.safetensors", + "model.language_model.layers.8.mlp.gate_proj.weight": "model-00002-of-00004.safetensors", + "model.language_model.layers.8.mlp.up_proj.weight": "model-00002-of-00004.safetensors", + "model.language_model.layers.8.post_attention_layernorm.weight": "model-00003-of-00004.safetensors", + "model.language_model.layers.8.self_attn.k_norm.weight": "model-00002-of-00004.safetensors", + "model.language_model.layers.8.self_attn.k_proj.weight": "model-00003-of-00004.safetensors", + "model.language_model.layers.8.self_attn.o_proj.weight": "model-00003-of-00004.safetensors", + "model.language_model.layers.8.self_attn.q_norm.weight": "model-00002-of-00004.safetensors", + "model.language_model.layers.8.self_attn.q_proj.weight": "model-00002-of-00004.safetensors", + "model.language_model.layers.8.self_attn.v_proj.weight": "model-00004-of-00004.safetensors", + "model.language_model.layers.9.input_layernorm.weight": "model-00002-of-00004.safetensors", + "model.language_model.layers.9.mlp.down_proj.weight": "model-00002-of-00004.safetensors", + "model.language_model.layers.9.mlp.gate_proj.weight": "model-00002-of-00004.safetensors", + "model.language_model.layers.9.mlp.up_proj.weight": "model-00001-of-00004.safetensors", + "model.language_model.layers.9.post_attention_layernorm.weight": "model-00002-of-00004.safetensors", + "model.language_model.layers.9.self_attn.k_norm.weight": "model-00003-of-00004.safetensors", + "model.language_model.layers.9.self_attn.k_proj.weight": "model-00003-of-00004.safetensors", + "model.language_model.layers.9.self_attn.o_proj.weight": "model-00002-of-00004.safetensors", + "model.language_model.layers.9.self_attn.q_norm.weight": "model-00004-of-00004.safetensors", + "model.language_model.layers.9.self_attn.q_proj.weight": "model-00003-of-00004.safetensors", + "model.language_model.layers.9.self_attn.v_proj.weight": "model-00001-of-00004.safetensors", + "model.language_model.norm.weight": "model-00004-of-00004.safetensors", + "model.visual.blocks.0.attn.proj.bias": "model-00004-of-00004.safetensors", + "model.visual.blocks.0.attn.proj.weight": "model-00002-of-00004.safetensors", + "model.visual.blocks.0.attn.qkv.bias": "model-00001-of-00004.safetensors", + "model.visual.blocks.0.attn.qkv.weight": "model-00001-of-00004.safetensors", + "model.visual.blocks.0.mlp.linear_fc1.bias": "model-00004-of-00004.safetensors", + "model.visual.blocks.0.mlp.linear_fc1.weight": "model-00004-of-00004.safetensors", + "model.visual.blocks.0.mlp.linear_fc2.bias": "model-00004-of-00004.safetensors", + "model.visual.blocks.0.mlp.linear_fc2.weight": "model-00002-of-00004.safetensors", + "model.visual.blocks.0.norm1.bias": "model-00004-of-00004.safetensors", + "model.visual.blocks.0.norm1.weight": "model-00003-of-00004.safetensors", + "model.visual.blocks.0.norm2.bias": "model-00004-of-00004.safetensors", + "model.visual.blocks.0.norm2.weight": "model-00001-of-00004.safetensors", + "model.visual.blocks.1.attn.proj.bias": "model-00002-of-00004.safetensors", + "model.visual.blocks.1.attn.proj.weight": "model-00004-of-00004.safetensors", + "model.visual.blocks.1.attn.qkv.bias": "model-00004-of-00004.safetensors", + "model.visual.blocks.1.attn.qkv.weight": "model-00002-of-00004.safetensors", + "model.visual.blocks.1.mlp.linear_fc1.bias": "model-00002-of-00004.safetensors", + "model.visual.blocks.1.mlp.linear_fc1.weight": "model-00003-of-00004.safetensors", + "model.visual.blocks.1.mlp.linear_fc2.bias": "model-00001-of-00004.safetensors", + "model.visual.blocks.1.mlp.linear_fc2.weight": "model-00003-of-00004.safetensors", + "model.visual.blocks.1.norm1.bias": "model-00003-of-00004.safetensors", + "model.visual.blocks.1.norm1.weight": "model-00001-of-00004.safetensors", + "model.visual.blocks.1.norm2.bias": "model-00004-of-00004.safetensors", + "model.visual.blocks.1.norm2.weight": "model-00003-of-00004.safetensors", + "model.visual.blocks.10.attn.proj.bias": "model-00002-of-00004.safetensors", + "model.visual.blocks.10.attn.proj.weight": "model-00002-of-00004.safetensors", + "model.visual.blocks.10.attn.qkv.bias": "model-00004-of-00004.safetensors", + "model.visual.blocks.10.attn.qkv.weight": "model-00004-of-00004.safetensors", + "model.visual.blocks.10.mlp.linear_fc1.bias": "model-00002-of-00004.safetensors", + "model.visual.blocks.10.mlp.linear_fc1.weight": "model-00004-of-00004.safetensors", + "model.visual.blocks.10.mlp.linear_fc2.bias": "model-00002-of-00004.safetensors", + "model.visual.blocks.10.mlp.linear_fc2.weight": "model-00002-of-00004.safetensors", + "model.visual.blocks.10.norm1.bias": "model-00001-of-00004.safetensors", + "model.visual.blocks.10.norm1.weight": "model-00004-of-00004.safetensors", + "model.visual.blocks.10.norm2.bias": "model-00002-of-00004.safetensors", + "model.visual.blocks.10.norm2.weight": "model-00004-of-00004.safetensors", + "model.visual.blocks.11.attn.proj.bias": "model-00002-of-00004.safetensors", + "model.visual.blocks.11.attn.proj.weight": "model-00002-of-00004.safetensors", + "model.visual.blocks.11.attn.qkv.bias": "model-00004-of-00004.safetensors", + "model.visual.blocks.11.attn.qkv.weight": "model-00001-of-00004.safetensors", + "model.visual.blocks.11.mlp.linear_fc1.bias": "model-00001-of-00004.safetensors", + "model.visual.blocks.11.mlp.linear_fc1.weight": "model-00003-of-00004.safetensors", + "model.visual.blocks.11.mlp.linear_fc2.bias": "model-00003-of-00004.safetensors", + "model.visual.blocks.11.mlp.linear_fc2.weight": "model-00002-of-00004.safetensors", + "model.visual.blocks.11.norm1.bias": "model-00004-of-00004.safetensors", + "model.visual.blocks.11.norm1.weight": "model-00001-of-00004.safetensors", + "model.visual.blocks.11.norm2.bias": "model-00001-of-00004.safetensors", + "model.visual.blocks.11.norm2.weight": "model-00003-of-00004.safetensors", + "model.visual.blocks.12.attn.proj.bias": "model-00001-of-00004.safetensors", + "model.visual.blocks.12.attn.proj.weight": "model-00001-of-00004.safetensors", + "model.visual.blocks.12.attn.qkv.bias": "model-00001-of-00004.safetensors", + "model.visual.blocks.12.attn.qkv.weight": "model-00001-of-00004.safetensors", + "model.visual.blocks.12.mlp.linear_fc1.bias": "model-00002-of-00004.safetensors", + "model.visual.blocks.12.mlp.linear_fc1.weight": "model-00003-of-00004.safetensors", + "model.visual.blocks.12.mlp.linear_fc2.bias": "model-00002-of-00004.safetensors", + "model.visual.blocks.12.mlp.linear_fc2.weight": "model-00001-of-00004.safetensors", + "model.visual.blocks.12.norm1.bias": "model-00003-of-00004.safetensors", + "model.visual.blocks.12.norm1.weight": "model-00002-of-00004.safetensors", + "model.visual.blocks.12.norm2.bias": "model-00002-of-00004.safetensors", + "model.visual.blocks.12.norm2.weight": "model-00004-of-00004.safetensors", + "model.visual.blocks.13.attn.proj.bias": "model-00002-of-00004.safetensors", + "model.visual.blocks.13.attn.proj.weight": "model-00001-of-00004.safetensors", + "model.visual.blocks.13.attn.qkv.bias": "model-00002-of-00004.safetensors", + "model.visual.blocks.13.attn.qkv.weight": "model-00001-of-00004.safetensors", + "model.visual.blocks.13.mlp.linear_fc1.bias": "model-00002-of-00004.safetensors", + "model.visual.blocks.13.mlp.linear_fc1.weight": "model-00002-of-00004.safetensors", + "model.visual.blocks.13.mlp.linear_fc2.bias": "model-00003-of-00004.safetensors", + "model.visual.blocks.13.mlp.linear_fc2.weight": "model-00001-of-00004.safetensors", + "model.visual.blocks.13.norm1.bias": "model-00004-of-00004.safetensors", + "model.visual.blocks.13.norm1.weight": "model-00003-of-00004.safetensors", + "model.visual.blocks.13.norm2.bias": "model-00002-of-00004.safetensors", + "model.visual.blocks.13.norm2.weight": "model-00001-of-00004.safetensors", + "model.visual.blocks.14.attn.proj.bias": "model-00002-of-00004.safetensors", + "model.visual.blocks.14.attn.proj.weight": "model-00001-of-00004.safetensors", + "model.visual.blocks.14.attn.qkv.bias": "model-00004-of-00004.safetensors", + "model.visual.blocks.14.attn.qkv.weight": "model-00002-of-00004.safetensors", + "model.visual.blocks.14.mlp.linear_fc1.bias": "model-00003-of-00004.safetensors", + "model.visual.blocks.14.mlp.linear_fc1.weight": "model-00002-of-00004.safetensors", + "model.visual.blocks.14.mlp.linear_fc2.bias": "model-00002-of-00004.safetensors", + "model.visual.blocks.14.mlp.linear_fc2.weight": "model-00003-of-00004.safetensors", + "model.visual.blocks.14.norm1.bias": "model-00001-of-00004.safetensors", + "model.visual.blocks.14.norm1.weight": "model-00004-of-00004.safetensors", + "model.visual.blocks.14.norm2.bias": "model-00002-of-00004.safetensors", + "model.visual.blocks.14.norm2.weight": "model-00003-of-00004.safetensors", + "model.visual.blocks.15.attn.proj.bias": "model-00002-of-00004.safetensors", + "model.visual.blocks.15.attn.proj.weight": "model-00001-of-00004.safetensors", + "model.visual.blocks.15.attn.qkv.bias": "model-00002-of-00004.safetensors", + "model.visual.blocks.15.attn.qkv.weight": "model-00002-of-00004.safetensors", + "model.visual.blocks.15.mlp.linear_fc1.bias": "model-00003-of-00004.safetensors", + "model.visual.blocks.15.mlp.linear_fc1.weight": "model-00001-of-00004.safetensors", + "model.visual.blocks.15.mlp.linear_fc2.bias": "model-00002-of-00004.safetensors", + "model.visual.blocks.15.mlp.linear_fc2.weight": "model-00003-of-00004.safetensors", + "model.visual.blocks.15.norm1.bias": "model-00003-of-00004.safetensors", + "model.visual.blocks.15.norm1.weight": "model-00002-of-00004.safetensors", + "model.visual.blocks.15.norm2.bias": "model-00002-of-00004.safetensors", + "model.visual.blocks.15.norm2.weight": "model-00001-of-00004.safetensors", + "model.visual.blocks.16.attn.proj.bias": "model-00002-of-00004.safetensors", + "model.visual.blocks.16.attn.proj.weight": "model-00002-of-00004.safetensors", + "model.visual.blocks.16.attn.qkv.bias": "model-00004-of-00004.safetensors", + "model.visual.blocks.16.attn.qkv.weight": "model-00001-of-00004.safetensors", + "model.visual.blocks.16.mlp.linear_fc1.bias": "model-00001-of-00004.safetensors", + "model.visual.blocks.16.mlp.linear_fc1.weight": "model-00001-of-00004.safetensors", + "model.visual.blocks.16.mlp.linear_fc2.bias": "model-00003-of-00004.safetensors", + "model.visual.blocks.16.mlp.linear_fc2.weight": "model-00004-of-00004.safetensors", + "model.visual.blocks.16.norm1.bias": "model-00003-of-00004.safetensors", + "model.visual.blocks.16.norm1.weight": "model-00002-of-00004.safetensors", + "model.visual.blocks.16.norm2.bias": "model-00001-of-00004.safetensors", + "model.visual.blocks.16.norm2.weight": "model-00003-of-00004.safetensors", + "model.visual.blocks.17.attn.proj.bias": "model-00001-of-00004.safetensors", + "model.visual.blocks.17.attn.proj.weight": "model-00004-of-00004.safetensors", + "model.visual.blocks.17.attn.qkv.bias": "model-00003-of-00004.safetensors", + "model.visual.blocks.17.attn.qkv.weight": "model-00001-of-00004.safetensors", + "model.visual.blocks.17.mlp.linear_fc1.bias": "model-00003-of-00004.safetensors", + "model.visual.blocks.17.mlp.linear_fc1.weight": "model-00001-of-00004.safetensors", + "model.visual.blocks.17.mlp.linear_fc2.bias": "model-00003-of-00004.safetensors", + "model.visual.blocks.17.mlp.linear_fc2.weight": "model-00002-of-00004.safetensors", + "model.visual.blocks.17.norm1.bias": "model-00003-of-00004.safetensors", + "model.visual.blocks.17.norm1.weight": "model-00004-of-00004.safetensors", + "model.visual.blocks.17.norm2.bias": "model-00003-of-00004.safetensors", + "model.visual.blocks.17.norm2.weight": "model-00001-of-00004.safetensors", + "model.visual.blocks.18.attn.proj.bias": "model-00003-of-00004.safetensors", + "model.visual.blocks.18.attn.proj.weight": "model-00001-of-00004.safetensors", + "model.visual.blocks.18.attn.qkv.bias": "model-00004-of-00004.safetensors", + "model.visual.blocks.18.attn.qkv.weight": "model-00003-of-00004.safetensors", + "model.visual.blocks.18.mlp.linear_fc1.bias": "model-00001-of-00004.safetensors", + "model.visual.blocks.18.mlp.linear_fc1.weight": "model-00003-of-00004.safetensors", + "model.visual.blocks.18.mlp.linear_fc2.bias": "model-00004-of-00004.safetensors", + "model.visual.blocks.18.mlp.linear_fc2.weight": "model-00002-of-00004.safetensors", + "model.visual.blocks.18.norm1.bias": "model-00002-of-00004.safetensors", + "model.visual.blocks.18.norm1.weight": "model-00001-of-00004.safetensors", + "model.visual.blocks.18.norm2.bias": "model-00001-of-00004.safetensors", + "model.visual.blocks.18.norm2.weight": "model-00003-of-00004.safetensors", + "model.visual.blocks.19.attn.proj.bias": "model-00002-of-00004.safetensors", + "model.visual.blocks.19.attn.proj.weight": "model-00003-of-00004.safetensors", + "model.visual.blocks.19.attn.qkv.bias": "model-00001-of-00004.safetensors", + "model.visual.blocks.19.attn.qkv.weight": "model-00004-of-00004.safetensors", + "model.visual.blocks.19.mlp.linear_fc1.bias": "model-00003-of-00004.safetensors", + "model.visual.blocks.19.mlp.linear_fc1.weight": "model-00002-of-00004.safetensors", + "model.visual.blocks.19.mlp.linear_fc2.bias": "model-00004-of-00004.safetensors", + "model.visual.blocks.19.mlp.linear_fc2.weight": "model-00004-of-00004.safetensors", + "model.visual.blocks.19.norm1.bias": "model-00001-of-00004.safetensors", + "model.visual.blocks.19.norm1.weight": "model-00004-of-00004.safetensors", + "model.visual.blocks.19.norm2.bias": "model-00002-of-00004.safetensors", + "model.visual.blocks.19.norm2.weight": "model-00002-of-00004.safetensors", + "model.visual.blocks.2.attn.proj.bias": "model-00002-of-00004.safetensors", + "model.visual.blocks.2.attn.proj.weight": "model-00002-of-00004.safetensors", + "model.visual.blocks.2.attn.qkv.bias": "model-00002-of-00004.safetensors", + "model.visual.blocks.2.attn.qkv.weight": "model-00001-of-00004.safetensors", + "model.visual.blocks.2.mlp.linear_fc1.bias": "model-00002-of-00004.safetensors", + "model.visual.blocks.2.mlp.linear_fc1.weight": "model-00002-of-00004.safetensors", + "model.visual.blocks.2.mlp.linear_fc2.bias": "model-00001-of-00004.safetensors", + "model.visual.blocks.2.mlp.linear_fc2.weight": "model-00004-of-00004.safetensors", + "model.visual.blocks.2.norm1.bias": "model-00003-of-00004.safetensors", + "model.visual.blocks.2.norm1.weight": "model-00002-of-00004.safetensors", + "model.visual.blocks.2.norm2.bias": "model-00003-of-00004.safetensors", + "model.visual.blocks.2.norm2.weight": "model-00003-of-00004.safetensors", + "model.visual.blocks.20.attn.proj.bias": "model-00003-of-00004.safetensors", + "model.visual.blocks.20.attn.proj.weight": "model-00002-of-00004.safetensors", + "model.visual.blocks.20.attn.qkv.bias": "model-00002-of-00004.safetensors", + "model.visual.blocks.20.attn.qkv.weight": "model-00003-of-00004.safetensors", + "model.visual.blocks.20.mlp.linear_fc1.bias": "model-00003-of-00004.safetensors", + "model.visual.blocks.20.mlp.linear_fc1.weight": "model-00004-of-00004.safetensors", + "model.visual.blocks.20.mlp.linear_fc2.bias": "model-00002-of-00004.safetensors", + "model.visual.blocks.20.mlp.linear_fc2.weight": "model-00001-of-00004.safetensors", + "model.visual.blocks.20.norm1.bias": "model-00004-of-00004.safetensors", + "model.visual.blocks.20.norm1.weight": "model-00002-of-00004.safetensors", + "model.visual.blocks.20.norm2.bias": "model-00004-of-00004.safetensors", + "model.visual.blocks.20.norm2.weight": "model-00001-of-00004.safetensors", + "model.visual.blocks.21.attn.proj.bias": "model-00003-of-00004.safetensors", + "model.visual.blocks.21.attn.proj.weight": "model-00004-of-00004.safetensors", + "model.visual.blocks.21.attn.qkv.bias": "model-00003-of-00004.safetensors", + "model.visual.blocks.21.attn.qkv.weight": "model-00001-of-00004.safetensors", + "model.visual.blocks.21.mlp.linear_fc1.bias": "model-00003-of-00004.safetensors", + "model.visual.blocks.21.mlp.linear_fc1.weight": "model-00003-of-00004.safetensors", + "model.visual.blocks.21.mlp.linear_fc2.bias": "model-00004-of-00004.safetensors", + "model.visual.blocks.21.mlp.linear_fc2.weight": "model-00001-of-00004.safetensors", + "model.visual.blocks.21.norm1.bias": "model-00003-of-00004.safetensors", + "model.visual.blocks.21.norm1.weight": "model-00003-of-00004.safetensors", + "model.visual.blocks.21.norm2.bias": "model-00004-of-00004.safetensors", + "model.visual.blocks.21.norm2.weight": "model-00002-of-00004.safetensors", + "model.visual.blocks.22.attn.proj.bias": "model-00001-of-00004.safetensors", + "model.visual.blocks.22.attn.proj.weight": "model-00003-of-00004.safetensors", + "model.visual.blocks.22.attn.qkv.bias": "model-00001-of-00004.safetensors", + "model.visual.blocks.22.attn.qkv.weight": "model-00004-of-00004.safetensors", + "model.visual.blocks.22.mlp.linear_fc1.bias": "model-00003-of-00004.safetensors", + "model.visual.blocks.22.mlp.linear_fc1.weight": "model-00001-of-00004.safetensors", + "model.visual.blocks.22.mlp.linear_fc2.bias": "model-00002-of-00004.safetensors", + "model.visual.blocks.22.mlp.linear_fc2.weight": "model-00003-of-00004.safetensors", + "model.visual.blocks.22.norm1.bias": "model-00003-of-00004.safetensors", + "model.visual.blocks.22.norm1.weight": "model-00002-of-00004.safetensors", + "model.visual.blocks.22.norm2.bias": "model-00004-of-00004.safetensors", + "model.visual.blocks.22.norm2.weight": "model-00003-of-00004.safetensors", + "model.visual.blocks.23.attn.proj.bias": "model-00003-of-00004.safetensors", + "model.visual.blocks.23.attn.proj.weight": "model-00004-of-00004.safetensors", + "model.visual.blocks.23.attn.qkv.bias": "model-00002-of-00004.safetensors", + "model.visual.blocks.23.attn.qkv.weight": "model-00004-of-00004.safetensors", + "model.visual.blocks.23.mlp.linear_fc1.bias": "model-00001-of-00004.safetensors", + "model.visual.blocks.23.mlp.linear_fc1.weight": "model-00001-of-00004.safetensors", + "model.visual.blocks.23.mlp.linear_fc2.bias": "model-00003-of-00004.safetensors", + "model.visual.blocks.23.mlp.linear_fc2.weight": "model-00002-of-00004.safetensors", + "model.visual.blocks.23.norm1.bias": "model-00003-of-00004.safetensors", + "model.visual.blocks.23.norm1.weight": "model-00002-of-00004.safetensors", + "model.visual.blocks.23.norm2.bias": "model-00003-of-00004.safetensors", + "model.visual.blocks.23.norm2.weight": "model-00004-of-00004.safetensors", + "model.visual.blocks.24.attn.proj.bias": "model-00002-of-00004.safetensors", + "model.visual.blocks.24.attn.proj.weight": "model-00001-of-00004.safetensors", + "model.visual.blocks.24.attn.qkv.bias": "model-00002-of-00004.safetensors", + "model.visual.blocks.24.attn.qkv.weight": "model-00004-of-00004.safetensors", + "model.visual.blocks.24.mlp.linear_fc1.bias": "model-00002-of-00004.safetensors", + "model.visual.blocks.24.mlp.linear_fc1.weight": "model-00003-of-00004.safetensors", + "model.visual.blocks.24.mlp.linear_fc2.bias": "model-00003-of-00004.safetensors", + "model.visual.blocks.24.mlp.linear_fc2.weight": "model-00003-of-00004.safetensors", + "model.visual.blocks.24.norm1.bias": "model-00002-of-00004.safetensors", + "model.visual.blocks.24.norm1.weight": "model-00001-of-00004.safetensors", + "model.visual.blocks.24.norm2.bias": "model-00004-of-00004.safetensors", + "model.visual.blocks.24.norm2.weight": "model-00001-of-00004.safetensors", + "model.visual.blocks.25.attn.proj.bias": "model-00001-of-00004.safetensors", + "model.visual.blocks.25.attn.proj.weight": "model-00001-of-00004.safetensors", + "model.visual.blocks.25.attn.qkv.bias": "model-00004-of-00004.safetensors", + "model.visual.blocks.25.attn.qkv.weight": "model-00004-of-00004.safetensors", + "model.visual.blocks.25.mlp.linear_fc1.bias": "model-00003-of-00004.safetensors", + "model.visual.blocks.25.mlp.linear_fc1.weight": "model-00003-of-00004.safetensors", + "model.visual.blocks.25.mlp.linear_fc2.bias": "model-00003-of-00004.safetensors", + "model.visual.blocks.25.mlp.linear_fc2.weight": "model-00004-of-00004.safetensors", + "model.visual.blocks.25.norm1.bias": "model-00004-of-00004.safetensors", + "model.visual.blocks.25.norm1.weight": "model-00004-of-00004.safetensors", + "model.visual.blocks.25.norm2.bias": "model-00003-of-00004.safetensors", + "model.visual.blocks.25.norm2.weight": "model-00004-of-00004.safetensors", + "model.visual.blocks.26.attn.proj.bias": "model-00003-of-00004.safetensors", + "model.visual.blocks.26.attn.proj.weight": "model-00003-of-00004.safetensors", + "model.visual.blocks.26.attn.qkv.bias": "model-00002-of-00004.safetensors", + "model.visual.blocks.26.attn.qkv.weight": "model-00002-of-00004.safetensors", + "model.visual.blocks.26.mlp.linear_fc1.bias": "model-00002-of-00004.safetensors", + "model.visual.blocks.26.mlp.linear_fc1.weight": "model-00001-of-00004.safetensors", + "model.visual.blocks.26.mlp.linear_fc2.bias": "model-00002-of-00004.safetensors", + "model.visual.blocks.26.mlp.linear_fc2.weight": "model-00001-of-00004.safetensors", + "model.visual.blocks.26.norm1.bias": "model-00003-of-00004.safetensors", + "model.visual.blocks.26.norm1.weight": "model-00004-of-00004.safetensors", + "model.visual.blocks.26.norm2.bias": "model-00001-of-00004.safetensors", + "model.visual.blocks.26.norm2.weight": "model-00001-of-00004.safetensors", + "model.visual.blocks.3.attn.proj.bias": "model-00001-of-00004.safetensors", + "model.visual.blocks.3.attn.proj.weight": "model-00003-of-00004.safetensors", + "model.visual.blocks.3.attn.qkv.bias": "model-00001-of-00004.safetensors", + "model.visual.blocks.3.attn.qkv.weight": "model-00001-of-00004.safetensors", + "model.visual.blocks.3.mlp.linear_fc1.bias": "model-00004-of-00004.safetensors", + "model.visual.blocks.3.mlp.linear_fc1.weight": "model-00001-of-00004.safetensors", + "model.visual.blocks.3.mlp.linear_fc2.bias": "model-00002-of-00004.safetensors", + "model.visual.blocks.3.mlp.linear_fc2.weight": "model-00001-of-00004.safetensors", + "model.visual.blocks.3.norm1.bias": "model-00002-of-00004.safetensors", + "model.visual.blocks.3.norm1.weight": "model-00002-of-00004.safetensors", + "model.visual.blocks.3.norm2.bias": "model-00003-of-00004.safetensors", + "model.visual.blocks.3.norm2.weight": "model-00004-of-00004.safetensors", + "model.visual.blocks.4.attn.proj.bias": "model-00001-of-00004.safetensors", + "model.visual.blocks.4.attn.proj.weight": "model-00002-of-00004.safetensors", + "model.visual.blocks.4.attn.qkv.bias": "model-00002-of-00004.safetensors", + "model.visual.blocks.4.attn.qkv.weight": "model-00003-of-00004.safetensors", + "model.visual.blocks.4.mlp.linear_fc1.bias": "model-00002-of-00004.safetensors", + "model.visual.blocks.4.mlp.linear_fc1.weight": "model-00004-of-00004.safetensors", + "model.visual.blocks.4.mlp.linear_fc2.bias": "model-00001-of-00004.safetensors", + "model.visual.blocks.4.mlp.linear_fc2.weight": "model-00001-of-00004.safetensors", + "model.visual.blocks.4.norm1.bias": "model-00001-of-00004.safetensors", + "model.visual.blocks.4.norm1.weight": "model-00002-of-00004.safetensors", + "model.visual.blocks.4.norm2.bias": "model-00002-of-00004.safetensors", + "model.visual.blocks.4.norm2.weight": "model-00002-of-00004.safetensors", + "model.visual.blocks.5.attn.proj.bias": "model-00003-of-00004.safetensors", + "model.visual.blocks.5.attn.proj.weight": "model-00004-of-00004.safetensors", + "model.visual.blocks.5.attn.qkv.bias": "model-00001-of-00004.safetensors", + "model.visual.blocks.5.attn.qkv.weight": "model-00004-of-00004.safetensors", + "model.visual.blocks.5.mlp.linear_fc1.bias": "model-00002-of-00004.safetensors", + "model.visual.blocks.5.mlp.linear_fc1.weight": "model-00003-of-00004.safetensors", + "model.visual.blocks.5.mlp.linear_fc2.bias": "model-00003-of-00004.safetensors", + "model.visual.blocks.5.mlp.linear_fc2.weight": "model-00002-of-00004.safetensors", + "model.visual.blocks.5.norm1.bias": "model-00001-of-00004.safetensors", + "model.visual.blocks.5.norm1.weight": "model-00002-of-00004.safetensors", + "model.visual.blocks.5.norm2.bias": "model-00002-of-00004.safetensors", + "model.visual.blocks.5.norm2.weight": "model-00002-of-00004.safetensors", + "model.visual.blocks.6.attn.proj.bias": "model-00003-of-00004.safetensors", + "model.visual.blocks.6.attn.proj.weight": "model-00003-of-00004.safetensors", + "model.visual.blocks.6.attn.qkv.bias": "model-00001-of-00004.safetensors", + "model.visual.blocks.6.attn.qkv.weight": "model-00001-of-00004.safetensors", + "model.visual.blocks.6.mlp.linear_fc1.bias": "model-00003-of-00004.safetensors", + "model.visual.blocks.6.mlp.linear_fc1.weight": "model-00004-of-00004.safetensors", + "model.visual.blocks.6.mlp.linear_fc2.bias": "model-00001-of-00004.safetensors", + "model.visual.blocks.6.mlp.linear_fc2.weight": "model-00003-of-00004.safetensors", + "model.visual.blocks.6.norm1.bias": "model-00002-of-00004.safetensors", + "model.visual.blocks.6.norm1.weight": "model-00001-of-00004.safetensors", + "model.visual.blocks.6.norm2.bias": "model-00001-of-00004.safetensors", + "model.visual.blocks.6.norm2.weight": "model-00001-of-00004.safetensors", + "model.visual.blocks.7.attn.proj.bias": "model-00003-of-00004.safetensors", + "model.visual.blocks.7.attn.proj.weight": "model-00002-of-00004.safetensors", + "model.visual.blocks.7.attn.qkv.bias": "model-00002-of-00004.safetensors", + "model.visual.blocks.7.attn.qkv.weight": "model-00003-of-00004.safetensors", + "model.visual.blocks.7.mlp.linear_fc1.bias": "model-00004-of-00004.safetensors", + "model.visual.blocks.7.mlp.linear_fc1.weight": "model-00003-of-00004.safetensors", + "model.visual.blocks.7.mlp.linear_fc2.bias": "model-00001-of-00004.safetensors", + "model.visual.blocks.7.mlp.linear_fc2.weight": "model-00002-of-00004.safetensors", + "model.visual.blocks.7.norm1.bias": "model-00002-of-00004.safetensors", + "model.visual.blocks.7.norm1.weight": "model-00003-of-00004.safetensors", + "model.visual.blocks.7.norm2.bias": "model-00001-of-00004.safetensors", + "model.visual.blocks.7.norm2.weight": "model-00002-of-00004.safetensors", + "model.visual.blocks.8.attn.proj.bias": "model-00003-of-00004.safetensors", + "model.visual.blocks.8.attn.proj.weight": "model-00003-of-00004.safetensors", + "model.visual.blocks.8.attn.qkv.bias": "model-00003-of-00004.safetensors", + "model.visual.blocks.8.attn.qkv.weight": "model-00003-of-00004.safetensors", + "model.visual.blocks.8.mlp.linear_fc1.bias": "model-00001-of-00004.safetensors", + "model.visual.blocks.8.mlp.linear_fc1.weight": "model-00001-of-00004.safetensors", + "model.visual.blocks.8.mlp.linear_fc2.bias": "model-00002-of-00004.safetensors", + "model.visual.blocks.8.mlp.linear_fc2.weight": "model-00001-of-00004.safetensors", + "model.visual.blocks.8.norm1.bias": "model-00001-of-00004.safetensors", + "model.visual.blocks.8.norm1.weight": "model-00002-of-00004.safetensors", + "model.visual.blocks.8.norm2.bias": "model-00002-of-00004.safetensors", + "model.visual.blocks.8.norm2.weight": "model-00003-of-00004.safetensors", + "model.visual.blocks.9.attn.proj.bias": "model-00003-of-00004.safetensors", + "model.visual.blocks.9.attn.proj.weight": "model-00004-of-00004.safetensors", + "model.visual.blocks.9.attn.qkv.bias": "model-00004-of-00004.safetensors", + "model.visual.blocks.9.attn.qkv.weight": "model-00003-of-00004.safetensors", + "model.visual.blocks.9.mlp.linear_fc1.bias": "model-00003-of-00004.safetensors", + "model.visual.blocks.9.mlp.linear_fc1.weight": "model-00004-of-00004.safetensors", + "model.visual.blocks.9.mlp.linear_fc2.bias": "model-00001-of-00004.safetensors", + "model.visual.blocks.9.mlp.linear_fc2.weight": "model-00001-of-00004.safetensors", + "model.visual.blocks.9.norm1.bias": "model-00002-of-00004.safetensors", + "model.visual.blocks.9.norm1.weight": "model-00001-of-00004.safetensors", + "model.visual.blocks.9.norm2.bias": "model-00001-of-00004.safetensors", + "model.visual.blocks.9.norm2.weight": "model-00002-of-00004.safetensors", + "model.visual.deepstack_merger_list.0.linear_fc1.bias": "model-00003-of-00004.safetensors", + "model.visual.deepstack_merger_list.0.linear_fc1.weight": "model-00003-of-00004.safetensors", + "model.visual.deepstack_merger_list.0.linear_fc2.bias": "model-00003-of-00004.safetensors", + "model.visual.deepstack_merger_list.0.linear_fc2.weight": "model-00001-of-00004.safetensors", + "model.visual.deepstack_merger_list.0.norm.bias": "model-00002-of-00004.safetensors", + "model.visual.deepstack_merger_list.0.norm.weight": "model-00001-of-00004.safetensors", + "model.visual.deepstack_merger_list.1.linear_fc1.bias": "model-00003-of-00004.safetensors", + "model.visual.deepstack_merger_list.1.linear_fc1.weight": "model-00002-of-00004.safetensors", + "model.visual.deepstack_merger_list.1.linear_fc2.bias": "model-00004-of-00004.safetensors", + "model.visual.deepstack_merger_list.1.linear_fc2.weight": "model-00002-of-00004.safetensors", + "model.visual.deepstack_merger_list.1.norm.bias": "model-00002-of-00004.safetensors", + "model.visual.deepstack_merger_list.1.norm.weight": "model-00002-of-00004.safetensors", + "model.visual.deepstack_merger_list.2.linear_fc1.bias": "model-00004-of-00004.safetensors", + "model.visual.deepstack_merger_list.2.linear_fc1.weight": "model-00002-of-00004.safetensors", + "model.visual.deepstack_merger_list.2.linear_fc2.bias": "model-00001-of-00004.safetensors", + "model.visual.deepstack_merger_list.2.linear_fc2.weight": "model-00002-of-00004.safetensors", + "model.visual.deepstack_merger_list.2.norm.bias": "model-00003-of-00004.safetensors", + "model.visual.deepstack_merger_list.2.norm.weight": "model-00001-of-00004.safetensors", + "model.visual.merger.linear_fc1.bias": "model-00003-of-00004.safetensors", + "model.visual.merger.linear_fc1.weight": "model-00001-of-00004.safetensors", + "model.visual.merger.linear_fc2.bias": "model-00001-of-00004.safetensors", + "model.visual.merger.linear_fc2.weight": "model-00003-of-00004.safetensors", + "model.visual.merger.norm.bias": "model-00003-of-00004.safetensors", + "model.visual.merger.norm.weight": "model-00002-of-00004.safetensors", + "model.visual.patch_embed.proj.bias": "model-00002-of-00004.safetensors", + "model.visual.patch_embed.proj.weight": "model-00002-of-00004.safetensors", + "model.visual.pos_embed.weight": "model-00003-of-00004.safetensors" + } +} diff --git a/posttrain_models/8b_cold_start_sft_46k_multi_32k_timelens_1epoch_grpo_format_tiou_f1_cacc_multi_10k/global_step_280/processor_config.json b/posttrain_models/8b_cold_start_sft_46k_multi_32k_timelens_1epoch_grpo_format_tiou_f1_cacc_multi_10k/global_step_280/processor_config.json new file mode 100644 index 0000000000000000000000000000000000000000..a4fe483a685ede297970ea7a13145d5aabe63a1c --- /dev/null +++ b/posttrain_models/8b_cold_start_sft_46k_multi_32k_timelens_1epoch_grpo_format_tiou_f1_cacc_multi_10k/global_step_280/processor_config.json @@ -0,0 +1,85 @@ +{ + "image_processor": { + "crop_size": null, + "data_format": "channels_first", + "device": null, + "disable_grouping": null, + "do_center_crop": null, + "do_convert_rgb": true, + "do_normalize": true, + "do_pad": null, + "do_rescale": true, + "do_resize": true, + "image_mean": [ + 0.5, + 0.5, + 0.5 + ], + "image_processor_type": "Qwen2VLImageProcessorFast", + "image_seq_length": null, + "image_std": [ + 0.5, + 0.5, + 0.5 + ], + "input_data_format": null, + "max_pixels": null, + "merge_size": 2, + "min_pixels": null, + "pad_size": null, + "patch_size": 16, + "processor_class": "Qwen3VLProcessor", + "resample": 3, + "rescale_factor": 0.00392156862745098, + "return_tensors": null, + "size": { + "longest_edge": 16777216, + "shortest_edge": 65536 + }, + "temporal_patch_size": 2 + }, + "processor_class": "Qwen3VLProcessor", + "video_processor": { + "crop_size": null, + "data_format": "channels_first", + "default_to_square": true, + "device": null, + "do_center_crop": null, + "do_convert_rgb": true, + "do_normalize": true, + "do_pad": null, + "do_rescale": true, + "do_resize": true, + "do_sample_frames": true, + "fps": 2, + "image_mean": [ + 0.5, + 0.5, + 0.5 + ], + "image_std": [ + 0.5, + 0.5, + 0.5 + ], + "input_data_format": null, + "max_frames": 768, + "merge_size": 2, + "min_frames": 4, + "num_frames": null, + "pad_size": null, + "patch_size": 16, + "processor_class": "Qwen3VLProcessor", + "resample": 3, + "rescale_factor": 0.00392156862745098, + "return_metadata": false, + "return_tensors": null, + "size": { + "longest_edge": 25165824, + "shortest_edge": 4096 + }, + "temporal_patch_size": 2, + "video_metadata": null, + "video_processor_type": "Qwen3VLVideoProcessor" + } +} diff --git a/posttrain_models/8b_cold_start_sft_46k_multi_32k_timelens_1epoch_grpo_format_tiou_f1_cacc_multi_10k/global_step_280/special_tokens_map.json b/posttrain_models/8b_cold_start_sft_46k_multi_32k_timelens_1epoch_grpo_format_tiou_f1_cacc_multi_10k/global_step_280/special_tokens_map.json new file mode 100644 index 0000000000000000000000000000000000000000..ac23c0aaa2434523c494330aeb79c58395378103 --- /dev/null +++ b/posttrain_models/8b_cold_start_sft_46k_multi_32k_timelens_1epoch_grpo_format_tiou_f1_cacc_multi_10k/global_step_280/special_tokens_map.json @@ -0,0 +1,31 @@ +{ + "additional_special_tokens": [ + "<|im_start|>", + "<|im_end|>", + "<|object_ref_start|>", + "<|object_ref_end|>", + "<|box_start|>", + "<|box_end|>", + "<|quad_start|>", + "<|quad_end|>", + "<|vision_start|>", + "<|vision_end|>", + "<|vision_pad|>", + "<|image_pad|>", + "<|video_pad|>" + ], + "eos_token": { + "content": "<|im_end|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false + }, + "pad_token": { + "content": "<|endoftext|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false + } +} diff --git a/posttrain_models/8b_cold_start_sft_46k_multi_32k_timelens_1epoch_grpo_format_tiou_f1_cacc_multi_10k/global_step_280/tokenizer_config.json b/posttrain_models/8b_cold_start_sft_46k_multi_32k_timelens_1epoch_grpo_format_tiou_f1_cacc_multi_10k/global_step_280/tokenizer_config.json new file mode 100644 index 0000000000000000000000000000000000000000..fec7f182135c384a08ea2fc9635566ca79c45afc --- /dev/null +++ b/posttrain_models/8b_cold_start_sft_46k_multi_32k_timelens_1epoch_grpo_format_tiou_f1_cacc_multi_10k/global_step_280/tokenizer_config.json @@ -0,0 +1,240 @@ +{ + "add_bos_token": false, + "add_prefix_space": false, + "added_tokens_decoder": { + "151643": { + "content": "<|endoftext|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "151644": { + "content": "<|im_start|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "151645": { + "content": "<|im_end|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "151646": { + "content": "<|object_ref_start|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "151647": { + "content": "<|object_ref_end|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "151648": { + "content": "<|box_start|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "151649": { + "content": "<|box_end|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "151650": { + "content": "<|quad_start|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "151651": { + "content": "<|quad_end|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "151652": { + "content": "<|vision_start|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "151653": { + "content": "<|vision_end|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "151654": { + "content": "<|vision_pad|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "151655": { + "content": "<|image_pad|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "151656": { + "content": "<|video_pad|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "151657": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "151658": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "151659": { + "content": "<|fim_prefix|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "151660": { + "content": "<|fim_middle|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "151661": { + "content": "<|fim_suffix|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "151662": { + "content": "<|fim_pad|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "151663": { + "content": "<|repo_name|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "151664": { + "content": "<|file_sep|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "151665": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "151666": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "151667": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "151668": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + } + }, + "additional_special_tokens": [ + "<|im_start|>", + "<|im_end|>", + "<|object_ref_start|>", + "<|object_ref_end|>", + "<|box_start|>", + "<|box_end|>", + "<|quad_start|>", + "<|quad_end|>", + "<|vision_start|>", + "<|vision_end|>", + "<|vision_pad|>", + "<|image_pad|>", + "<|video_pad|>" + ], + "bos_token": null, + "clean_up_tokenization_spaces": false, + "eos_token": "<|im_end|>", + "errors": "replace", + "extra_special_tokens": {}, + "model_max_length": 262144, + "pad_token": "<|endoftext|>", + "processor_class": "Qwen3VLProcessor", + "split_special_tokens": false, + "tokenizer_class": "Qwen2Tokenizer", + "unk_token": null +} diff --git a/posttrain_models/8b_cold_start_sft_46k_multi_32k_timelens_1epoch_grpo_format_tiou_f1_cacc_multi_10k/global_step_300/added_tokens.json b/posttrain_models/8b_cold_start_sft_46k_multi_32k_timelens_1epoch_grpo_format_tiou_f1_cacc_multi_10k/global_step_300/added_tokens.json new file mode 100644 index 0000000000000000000000000000000000000000..b54f9135e44c1e81047e8d05cb027af8bc039eed --- /dev/null +++ b/posttrain_models/8b_cold_start_sft_46k_multi_32k_timelens_1epoch_grpo_format_tiou_f1_cacc_multi_10k/global_step_300/added_tokens.json @@ -0,0 +1,28 @@ +{ + "": 151668, + "": 151658, + "": 151666, + "": 151667, + "": 151657, + "": 151665, + "<|box_end|>": 151649, + "<|box_start|>": 151648, + "<|endoftext|>": 151643, + "<|file_sep|>": 151664, + "<|fim_middle|>": 151660, + "<|fim_pad|>": 151662, + "<|fim_prefix|>": 151659, + "<|fim_suffix|>": 151661, + "<|im_end|>": 151645, + "<|im_start|>": 151644, + "<|image_pad|>": 151655, + "<|object_ref_end|>": 151647, + "<|object_ref_start|>": 151646, + "<|quad_end|>": 151651, + "<|quad_start|>": 151650, + "<|repo_name|>": 151663, + "<|video_pad|>": 151656, + "<|vision_end|>": 151653, + "<|vision_pad|>": 151654, + "<|vision_start|>": 151652 +} diff --git a/posttrain_models/8b_cold_start_sft_46k_multi_32k_timelens_1epoch_grpo_format_tiou_f1_cacc_multi_10k/global_step_300/chat_template.jinja b/posttrain_models/8b_cold_start_sft_46k_multi_32k_timelens_1epoch_grpo_format_tiou_f1_cacc_multi_10k/global_step_300/chat_template.jinja new file mode 100644 index 0000000000000000000000000000000000000000..124386803f142761528f710e77ae483f5f8c4fc4 --- /dev/null +++ b/posttrain_models/8b_cold_start_sft_46k_multi_32k_timelens_1epoch_grpo_format_tiou_f1_cacc_multi_10k/global_step_300/chat_template.jinja @@ -0,0 +1,120 @@ +{%- if tools %} + {{- '<|im_start|>system\n' }} + {%- if messages[0].role == 'system' %} + {%- if messages[0].content is string %} + {{- messages[0].content }} + {%- else %} + {%- for content in messages[0].content %} + {%- if 'text' in content %} + {{- content.text }} + {%- endif %} + {%- endfor %} + {%- endif %} + {{- '\n\n' }} + {%- endif %} + {{- "# Tools\n\nYou may call one or more functions to assist with the user query.\n\nYou are provided with function signatures within XML tags:\n" }} + {%- for tool in tools %} + {{- "\n" }} + {{- tool | tojson }} + {%- endfor %} + {{- "\n\n\nFor each function call, return a json object with function name and arguments within XML tags:\n\n{\"name\": , \"arguments\": }\n<|im_end|>\n" }} +{%- else %} + {%- if messages[0].role == 'system' %} + {{- '<|im_start|>system\n' }} + {%- if messages[0].content is string %} + {{- messages[0].content }} + {%- else %} + {%- for content in messages[0].content %} + {%- if 'text' in content %} + {{- content.text }} + {%- endif %} + {%- endfor %} + {%- endif %} + {{- '<|im_end|>\n' }} + {%- endif %} +{%- endif %} +{%- set image_count = namespace(value=0) %} +{%- set video_count = namespace(value=0) %} +{%- for message in messages %} + {%- if message.role == "user" %} + {{- '<|im_start|>' + message.role + '\n' }} + {%- if message.content is string %} + {{- message.content }} + {%- else %} + {%- for content in message.content %} + {%- if content.type == 'image' or 'image' in content or 'image_url' in content %} + {%- set image_count.value = image_count.value + 1 %} + {%- if add_vision_id %}Picture {{ image_count.value }}: {% endif -%} + <|vision_start|><|image_pad|><|vision_end|> + {%- elif content.type == 'video' or 'video' in content %} + {%- set video_count.value = video_count.value + 1 %} + {%- if add_vision_id %}Video {{ video_count.value }}: {% endif -%} + <|vision_start|><|video_pad|><|vision_end|> + {%- elif 'text' in content %} + {{- content.text }} + {%- endif %} + {%- endfor %} + {%- endif %} + {{- '<|im_end|>\n' }} + {%- elif message.role == "assistant" %} + {{- '<|im_start|>' + message.role + '\n' }} + {%- if message.content is string %} + {{- message.content }} + {%- else %} + {%- for content_item in message.content %} + {%- if 'text' in content_item %} + {{- content_item.text }} + {%- endif %} + {%- endfor %} + {%- endif %} + {%- if message.tool_calls %} + {%- for tool_call in message.tool_calls %} + {%- if (loop.first and message.content) or (not loop.first) %} + {{- '\n' }} + {%- endif %} + {%- if tool_call.function %} + {%- set tool_call = tool_call.function %} + {%- endif %} + {{- '\n{"name": "' }} + {{- tool_call.name }} + {{- '", "arguments": ' }} + {%- if tool_call.arguments is string %} + {{- tool_call.arguments }} + {%- else %} + {{- tool_call.arguments | tojson }} + {%- endif %} + {{- '}\n' }} + {%- endfor %} + {%- endif %} + {{- '<|im_end|>\n' }} + {%- elif message.role == "tool" %} + {%- if loop.first or (messages[loop.index0 - 1].role != "tool") %} + {{- '<|im_start|>user' }} + {%- endif %} + {{- '\n\n' }} + {%- if message.content is string %} + {{- message.content }} + {%- else %} + {%- for content in message.content %} + {%- if content.type == 'image' or 'image' in content or 'image_url' in content %} + {%- set image_count.value = image_count.value + 1 %} + {%- if add_vision_id %}Picture {{ image_count.value }}: {% endif -%} + <|vision_start|><|image_pad|><|vision_end|> + {%- elif content.type == 'video' or 'video' in content %} + {%- set video_count.value = video_count.value + 1 %} + {%- if add_vision_id %}Video {{ video_count.value }}: {% endif -%} + <|vision_start|><|video_pad|><|vision_end|> + {%- elif 'text' in content %} + {{- content.text }} + {%- endif %} + {%- endfor %} + {%- endif %} + {{- '\n' }} + {%- if loop.last or (messages[loop.index0 + 1].role != "tool") %} + {{- '<|im_end|>\n' }} + {%- endif %} + {%- endif %} +{%- endfor %} +{%- if add_generation_prompt %} + {{- '<|im_start|>assistant\n' }} +{%- endif %} diff --git a/posttrain_models/8b_cold_start_sft_46k_multi_32k_timelens_1epoch_grpo_format_tiou_f1_cacc_multi_10k/global_step_300/config.json b/posttrain_models/8b_cold_start_sft_46k_multi_32k_timelens_1epoch_grpo_format_tiou_f1_cacc_multi_10k/global_step_300/config.json new file mode 100644 index 0000000000000000000000000000000000000000..db69009e04e5bed06b6959b1b3f36d034048a1df --- /dev/null +++ b/posttrain_models/8b_cold_start_sft_46k_multi_32k_timelens_1epoch_grpo_format_tiou_f1_cacc_multi_10k/global_step_300/config.json @@ -0,0 +1,70 @@ +{ + "architectures": [ + "Qwen3VLForConditionalGeneration" + ], + "dtype": "float32", + "eos_token_id": 151645, + "hidden_size": 4096, + "image_token_id": 151655, + "model_type": "qwen3_vl", + "pad_token_id": 151643, + "text_config": { + "attention_bias": false, + "attention_dropout": 0.0, + "bos_token_id": 151643, + "dtype": "float32", + "eos_token_id": 151645, + "head_dim": 128, + "hidden_act": "silu", + "hidden_size": 4096, + "initializer_range": 0.02, + "intermediate_size": 12288, + "max_position_embeddings": 262144, + "model_type": "qwen3_vl_text", + "num_attention_heads": 32, + "num_hidden_layers": 36, + "num_key_value_heads": 8, + "pad_token_id": 151643, + "rms_norm_eps": 1e-06, + "rope_theta": 5000000, + "use_cache": false, + "vocab_size": 151936, + "rope_scaling": { + "mrope_interleaved": true, + "mrope_section": [ + 24, + 20, + 20 + ], + "rope_theta": 5000000, + "rope_type": "default" + } + }, + "tie_word_embeddings": false, + "transformers_version": "5.0.0.dev0", + "video_token_id": 151656, + "vision_config": { + "deepstack_visual_indexes": [ + 8, + 16, + 24 + ], + "depth": 27, + "dtype": "float32", + "hidden_act": "gelu_pytorch_tanh", + "hidden_size": 1152, + "in_channels": 3, + "initializer_range": 0.02, + "intermediate_size": 4304, + "model_type": "qwen3_vl", + "num_heads": 16, + "num_position_embeddings": 2304, + "out_hidden_size": 4096, + "pad_token_id": 151643, + "patch_size": 16, + "spatial_merge_size": 2, + "temporal_patch_size": 2 + }, + "vision_end_token_id": 151653, + "vision_start_token_id": 151652 +} \ No newline at end of file diff --git a/posttrain_models/8b_cold_start_sft_46k_multi_32k_timelens_1epoch_grpo_format_tiou_f1_cacc_multi_10k/global_step_300/generation_config.json b/posttrain_models/8b_cold_start_sft_46k_multi_32k_timelens_1epoch_grpo_format_tiou_f1_cacc_multi_10k/global_step_300/generation_config.json new file mode 100644 index 0000000000000000000000000000000000000000..8458f29aec2209e3f63ca59bca95814310a80de1 --- /dev/null +++ b/posttrain_models/8b_cold_start_sft_46k_multi_32k_timelens_1epoch_grpo_format_tiou_f1_cacc_multi_10k/global_step_300/generation_config.json @@ -0,0 +1,13 @@ +{ + "bos_token_id": 151643, + "do_sample": true, + "eos_token_id": [ + 151645, + 151643 + ], + "pad_token_id": 151643, + "temperature": 0.7, + "top_k": 20, + "top_p": 0.8, + "transformers_version": "5.0.0.dev0" +} diff --git a/posttrain_models/8b_cold_start_sft_46k_multi_32k_timelens_1epoch_grpo_format_tiou_f1_cacc_multi_10k/global_step_300/model.safetensors.index.json b/posttrain_models/8b_cold_start_sft_46k_multi_32k_timelens_1epoch_grpo_format_tiou_f1_cacc_multi_10k/global_step_300/model.safetensors.index.json new file mode 100644 index 0000000000000000000000000000000000000000..111dba8f0a25b33412f9c633585913e60dd5f8e4 --- /dev/null +++ b/posttrain_models/8b_cold_start_sft_46k_multi_32k_timelens_1epoch_grpo_format_tiou_f1_cacc_multi_10k/global_step_300/model.safetensors.index.json @@ -0,0 +1,758 @@ +{ + "metadata": { + "total_parameters": 8767123696, + "total_size": 17534247392 + }, + "weight_map": { + "lm_head.weight": "model-00004-of-00004.safetensors", + "model.language_model.embed_tokens.weight": "model-00002-of-00004.safetensors", + "model.language_model.layers.0.input_layernorm.weight": "model-00004-of-00004.safetensors", + "model.language_model.layers.0.mlp.down_proj.weight": "model-00001-of-00004.safetensors", + "model.language_model.layers.0.mlp.gate_proj.weight": "model-00003-of-00004.safetensors", + "model.language_model.layers.0.mlp.up_proj.weight": "model-00002-of-00004.safetensors", + "model.language_model.layers.0.post_attention_layernorm.weight": "model-00003-of-00004.safetensors", + "model.language_model.layers.0.self_attn.k_norm.weight": "model-00002-of-00004.safetensors", + "model.language_model.layers.0.self_attn.k_proj.weight": "model-00001-of-00004.safetensors", + "model.language_model.layers.0.self_attn.o_proj.weight": "model-00002-of-00004.safetensors", + "model.language_model.layers.0.self_attn.q_norm.weight": "model-00001-of-00004.safetensors", + "model.language_model.layers.0.self_attn.q_proj.weight": "model-00003-of-00004.safetensors", + "model.language_model.layers.0.self_attn.v_proj.weight": "model-00003-of-00004.safetensors", + "model.language_model.layers.1.input_layernorm.weight": "model-00003-of-00004.safetensors", + "model.language_model.layers.1.mlp.down_proj.weight": "model-00003-of-00004.safetensors", + "model.language_model.layers.1.mlp.gate_proj.weight": "model-00003-of-00004.safetensors", + "model.language_model.layers.1.mlp.up_proj.weight": "model-00002-of-00004.safetensors", + "model.language_model.layers.1.post_attention_layernorm.weight": "model-00003-of-00004.safetensors", + "model.language_model.layers.1.self_attn.k_norm.weight": "model-00003-of-00004.safetensors", + "model.language_model.layers.1.self_attn.k_proj.weight": "model-00003-of-00004.safetensors", + "model.language_model.layers.1.self_attn.o_proj.weight": "model-00002-of-00004.safetensors", + "model.language_model.layers.1.self_attn.q_norm.weight": "model-00004-of-00004.safetensors", + "model.language_model.layers.1.self_attn.q_proj.weight": "model-00003-of-00004.safetensors", + "model.language_model.layers.1.self_attn.v_proj.weight": "model-00004-of-00004.safetensors", + "model.language_model.layers.10.input_layernorm.weight": "model-00004-of-00004.safetensors", + "model.language_model.layers.10.mlp.down_proj.weight": "model-00002-of-00004.safetensors", + "model.language_model.layers.10.mlp.gate_proj.weight": "model-00001-of-00004.safetensors", + "model.language_model.layers.10.mlp.up_proj.weight": "model-00002-of-00004.safetensors", + "model.language_model.layers.10.post_attention_layernorm.weight": "model-00001-of-00004.safetensors", + "model.language_model.layers.10.self_attn.k_norm.weight": "model-00003-of-00004.safetensors", + "model.language_model.layers.10.self_attn.k_proj.weight": "model-00002-of-00004.safetensors", + "model.language_model.layers.10.self_attn.o_proj.weight": "model-00001-of-00004.safetensors", + "model.language_model.layers.10.self_attn.q_norm.weight": "model-00003-of-00004.safetensors", + "model.language_model.layers.10.self_attn.q_proj.weight": "model-00001-of-00004.safetensors", + "model.language_model.layers.10.self_attn.v_proj.weight": "model-00003-of-00004.safetensors", + "model.language_model.layers.11.input_layernorm.weight": "model-00003-of-00004.safetensors", + "model.language_model.layers.11.mlp.down_proj.weight": "model-00003-of-00004.safetensors", + "model.language_model.layers.11.mlp.gate_proj.weight": "model-00002-of-00004.safetensors", + "model.language_model.layers.11.mlp.up_proj.weight": "model-00004-of-00004.safetensors", + "model.language_model.layers.11.post_attention_layernorm.weight": "model-00001-of-00004.safetensors", + "model.language_model.layers.11.self_attn.k_norm.weight": "model-00003-of-00004.safetensors", + "model.language_model.layers.11.self_attn.k_proj.weight": "model-00003-of-00004.safetensors", + "model.language_model.layers.11.self_attn.o_proj.weight": "model-00001-of-00004.safetensors", + "model.language_model.layers.11.self_attn.q_norm.weight": "model-00001-of-00004.safetensors", + "model.language_model.layers.11.self_attn.q_proj.weight": "model-00002-of-00004.safetensors", + "model.language_model.layers.11.self_attn.v_proj.weight": "model-00004-of-00004.safetensors", + "model.language_model.layers.12.input_layernorm.weight": "model-00002-of-00004.safetensors", + "model.language_model.layers.12.mlp.down_proj.weight": "model-00003-of-00004.safetensors", + "model.language_model.layers.12.mlp.gate_proj.weight": "model-00002-of-00004.safetensors", + "model.language_model.layers.12.mlp.up_proj.weight": "model-00003-of-00004.safetensors", + "model.language_model.layers.12.post_attention_layernorm.weight": "model-00002-of-00004.safetensors", + "model.language_model.layers.12.self_attn.k_norm.weight": "model-00003-of-00004.safetensors", + "model.language_model.layers.12.self_attn.k_proj.weight": "model-00004-of-00004.safetensors", + "model.language_model.layers.12.self_attn.o_proj.weight": "model-00003-of-00004.safetensors", + "model.language_model.layers.12.self_attn.q_norm.weight": "model-00004-of-00004.safetensors", + "model.language_model.layers.12.self_attn.q_proj.weight": "model-00003-of-00004.safetensors", + "model.language_model.layers.12.self_attn.v_proj.weight": "model-00001-of-00004.safetensors", + "model.language_model.layers.13.input_layernorm.weight": "model-00002-of-00004.safetensors", + "model.language_model.layers.13.mlp.down_proj.weight": "model-00001-of-00004.safetensors", + "model.language_model.layers.13.mlp.gate_proj.weight": "model-00003-of-00004.safetensors", + "model.language_model.layers.13.mlp.up_proj.weight": "model-00003-of-00004.safetensors", + "model.language_model.layers.13.post_attention_layernorm.weight": "model-00001-of-00004.safetensors", + "model.language_model.layers.13.self_attn.k_norm.weight": "model-00004-of-00004.safetensors", + "model.language_model.layers.13.self_attn.k_proj.weight": "model-00002-of-00004.safetensors", + "model.language_model.layers.13.self_attn.o_proj.weight": "model-00003-of-00004.safetensors", + "model.language_model.layers.13.self_attn.q_norm.weight": "model-00003-of-00004.safetensors", + "model.language_model.layers.13.self_attn.q_proj.weight": "model-00002-of-00004.safetensors", + "model.language_model.layers.13.self_attn.v_proj.weight": "model-00001-of-00004.safetensors", + "model.language_model.layers.14.input_layernorm.weight": "model-00002-of-00004.safetensors", + "model.language_model.layers.14.mlp.down_proj.weight": "model-00002-of-00004.safetensors", + "model.language_model.layers.14.mlp.gate_proj.weight": "model-00004-of-00004.safetensors", + "model.language_model.layers.14.mlp.up_proj.weight": "model-00003-of-00004.safetensors", + "model.language_model.layers.14.post_attention_layernorm.weight": "model-00001-of-00004.safetensors", + "model.language_model.layers.14.self_attn.k_norm.weight": "model-00004-of-00004.safetensors", + "model.language_model.layers.14.self_attn.k_proj.weight": "model-00003-of-00004.safetensors", + "model.language_model.layers.14.self_attn.o_proj.weight": "model-00003-of-00004.safetensors", + "model.language_model.layers.14.self_attn.q_norm.weight": "model-00003-of-00004.safetensors", + "model.language_model.layers.14.self_attn.q_proj.weight": "model-00004-of-00004.safetensors", + "model.language_model.layers.14.self_attn.v_proj.weight": "model-00001-of-00004.safetensors", + "model.language_model.layers.15.input_layernorm.weight": "model-00002-of-00004.safetensors", + "model.language_model.layers.15.mlp.down_proj.weight": "model-00003-of-00004.safetensors", + "model.language_model.layers.15.mlp.gate_proj.weight": "model-00001-of-00004.safetensors", + "model.language_model.layers.15.mlp.up_proj.weight": "model-00001-of-00004.safetensors", + "model.language_model.layers.15.post_attention_layernorm.weight": "model-00003-of-00004.safetensors", + "model.language_model.layers.15.self_attn.k_norm.weight": "model-00004-of-00004.safetensors", + "model.language_model.layers.15.self_attn.k_proj.weight": "model-00003-of-00004.safetensors", + "model.language_model.layers.15.self_attn.o_proj.weight": "model-00004-of-00004.safetensors", + "model.language_model.layers.15.self_attn.q_norm.weight": "model-00002-of-00004.safetensors", + "model.language_model.layers.15.self_attn.q_proj.weight": "model-00003-of-00004.safetensors", + "model.language_model.layers.15.self_attn.v_proj.weight": "model-00003-of-00004.safetensors", + "model.language_model.layers.16.input_layernorm.weight": "model-00001-of-00004.safetensors", + "model.language_model.layers.16.mlp.down_proj.weight": "model-00001-of-00004.safetensors", + "model.language_model.layers.16.mlp.gate_proj.weight": "model-00002-of-00004.safetensors", + "model.language_model.layers.16.mlp.up_proj.weight": "model-00001-of-00004.safetensors", + "model.language_model.layers.16.post_attention_layernorm.weight": "model-00003-of-00004.safetensors", + "model.language_model.layers.16.self_attn.k_norm.weight": "model-00003-of-00004.safetensors", + "model.language_model.layers.16.self_attn.k_proj.weight": "model-00003-of-00004.safetensors", + "model.language_model.layers.16.self_attn.o_proj.weight": "model-00002-of-00004.safetensors", + "model.language_model.layers.16.self_attn.q_norm.weight": "model-00001-of-00004.safetensors", + "model.language_model.layers.16.self_attn.q_proj.weight": "model-00003-of-00004.safetensors", + "model.language_model.layers.16.self_attn.v_proj.weight": "model-00001-of-00004.safetensors", + "model.language_model.layers.17.input_layernorm.weight": "model-00001-of-00004.safetensors", + "model.language_model.layers.17.mlp.down_proj.weight": "model-00002-of-00004.safetensors", + "model.language_model.layers.17.mlp.gate_proj.weight": "model-00002-of-00004.safetensors", + "model.language_model.layers.17.mlp.up_proj.weight": "model-00003-of-00004.safetensors", + "model.language_model.layers.17.post_attention_layernorm.weight": "model-00004-of-00004.safetensors", + "model.language_model.layers.17.self_attn.k_norm.weight": "model-00001-of-00004.safetensors", + "model.language_model.layers.17.self_attn.k_proj.weight": "model-00003-of-00004.safetensors", + "model.language_model.layers.17.self_attn.o_proj.weight": "model-00003-of-00004.safetensors", + "model.language_model.layers.17.self_attn.q_norm.weight": "model-00002-of-00004.safetensors", + "model.language_model.layers.17.self_attn.q_proj.weight": "model-00002-of-00004.safetensors", + "model.language_model.layers.17.self_attn.v_proj.weight": "model-00003-of-00004.safetensors", + "model.language_model.layers.18.input_layernorm.weight": "model-00002-of-00004.safetensors", + "model.language_model.layers.18.mlp.down_proj.weight": "model-00001-of-00004.safetensors", + "model.language_model.layers.18.mlp.gate_proj.weight": "model-00002-of-00004.safetensors", + "model.language_model.layers.18.mlp.up_proj.weight": "model-00002-of-00004.safetensors", + "model.language_model.layers.18.post_attention_layernorm.weight": "model-00004-of-00004.safetensors", + "model.language_model.layers.18.self_attn.k_norm.weight": "model-00002-of-00004.safetensors", + "model.language_model.layers.18.self_attn.k_proj.weight": "model-00003-of-00004.safetensors", + "model.language_model.layers.18.self_attn.o_proj.weight": "model-00002-of-00004.safetensors", + "model.language_model.layers.18.self_attn.q_norm.weight": "model-00001-of-00004.safetensors", + "model.language_model.layers.18.self_attn.q_proj.weight": "model-00001-of-00004.safetensors", + "model.language_model.layers.18.self_attn.v_proj.weight": "model-00001-of-00004.safetensors", + "model.language_model.layers.19.input_layernorm.weight": "model-00002-of-00004.safetensors", + "model.language_model.layers.19.mlp.down_proj.weight": "model-00001-of-00004.safetensors", + "model.language_model.layers.19.mlp.gate_proj.weight": "model-00004-of-00004.safetensors", + "model.language_model.layers.19.mlp.up_proj.weight": "model-00002-of-00004.safetensors", + "model.language_model.layers.19.post_attention_layernorm.weight": "model-00001-of-00004.safetensors", + "model.language_model.layers.19.self_attn.k_norm.weight": "model-00001-of-00004.safetensors", + "model.language_model.layers.19.self_attn.k_proj.weight": "model-00003-of-00004.safetensors", + "model.language_model.layers.19.self_attn.o_proj.weight": "model-00001-of-00004.safetensors", + "model.language_model.layers.19.self_attn.q_norm.weight": "model-00003-of-00004.safetensors", + "model.language_model.layers.19.self_attn.q_proj.weight": "model-00002-of-00004.safetensors", + "model.language_model.layers.19.self_attn.v_proj.weight": "model-00003-of-00004.safetensors", + "model.language_model.layers.2.input_layernorm.weight": "model-00001-of-00004.safetensors", + "model.language_model.layers.2.mlp.down_proj.weight": "model-00001-of-00004.safetensors", + "model.language_model.layers.2.mlp.gate_proj.weight": "model-00001-of-00004.safetensors", + "model.language_model.layers.2.mlp.up_proj.weight": "model-00001-of-00004.safetensors", + "model.language_model.layers.2.post_attention_layernorm.weight": "model-00001-of-00004.safetensors", + "model.language_model.layers.2.self_attn.k_norm.weight": "model-00003-of-00004.safetensors", + "model.language_model.layers.2.self_attn.k_proj.weight": "model-00003-of-00004.safetensors", + "model.language_model.layers.2.self_attn.o_proj.weight": "model-00004-of-00004.safetensors", + "model.language_model.layers.2.self_attn.q_norm.weight": "model-00001-of-00004.safetensors", + "model.language_model.layers.2.self_attn.q_proj.weight": "model-00001-of-00004.safetensors", + "model.language_model.layers.2.self_attn.v_proj.weight": "model-00001-of-00004.safetensors", + "model.language_model.layers.20.input_layernorm.weight": "model-00003-of-00004.safetensors", + "model.language_model.layers.20.mlp.down_proj.weight": "model-00002-of-00004.safetensors", + "model.language_model.layers.20.mlp.gate_proj.weight": "model-00001-of-00004.safetensors", + "model.language_model.layers.20.mlp.up_proj.weight": "model-00003-of-00004.safetensors", + "model.language_model.layers.20.post_attention_layernorm.weight": "model-00002-of-00004.safetensors", + "model.language_model.layers.20.self_attn.k_norm.weight": "model-00002-of-00004.safetensors", + "model.language_model.layers.20.self_attn.k_proj.weight": "model-00002-of-00004.safetensors", + "model.language_model.layers.20.self_attn.o_proj.weight": "model-00002-of-00004.safetensors", + "model.language_model.layers.20.self_attn.q_norm.weight": "model-00002-of-00004.safetensors", + "model.language_model.layers.20.self_attn.q_proj.weight": "model-00001-of-00004.safetensors", + "model.language_model.layers.20.self_attn.v_proj.weight": "model-00003-of-00004.safetensors", + "model.language_model.layers.21.input_layernorm.weight": "model-00001-of-00004.safetensors", + "model.language_model.layers.21.mlp.down_proj.weight": "model-00001-of-00004.safetensors", + "model.language_model.layers.21.mlp.gate_proj.weight": "model-00003-of-00004.safetensors", + "model.language_model.layers.21.mlp.up_proj.weight": "model-00001-of-00004.safetensors", + "model.language_model.layers.21.post_attention_layernorm.weight": "model-00003-of-00004.safetensors", + "model.language_model.layers.21.self_attn.k_norm.weight": "model-00004-of-00004.safetensors", + "model.language_model.layers.21.self_attn.k_proj.weight": "model-00004-of-00004.safetensors", + "model.language_model.layers.21.self_attn.o_proj.weight": "model-00001-of-00004.safetensors", + "model.language_model.layers.21.self_attn.q_norm.weight": "model-00001-of-00004.safetensors", + "model.language_model.layers.21.self_attn.q_proj.weight": "model-00003-of-00004.safetensors", + "model.language_model.layers.21.self_attn.v_proj.weight": "model-00003-of-00004.safetensors", + "model.language_model.layers.22.input_layernorm.weight": "model-00003-of-00004.safetensors", + "model.language_model.layers.22.mlp.down_proj.weight": "model-00001-of-00004.safetensors", + "model.language_model.layers.22.mlp.gate_proj.weight": "model-00003-of-00004.safetensors", + "model.language_model.layers.22.mlp.up_proj.weight": "model-00002-of-00004.safetensors", + "model.language_model.layers.22.post_attention_layernorm.weight": "model-00003-of-00004.safetensors", + "model.language_model.layers.22.self_attn.k_norm.weight": "model-00002-of-00004.safetensors", + "model.language_model.layers.22.self_attn.k_proj.weight": "model-00003-of-00004.safetensors", + "model.language_model.layers.22.self_attn.o_proj.weight": "model-00001-of-00004.safetensors", + "model.language_model.layers.22.self_attn.q_norm.weight": "model-00001-of-00004.safetensors", + "model.language_model.layers.22.self_attn.q_proj.weight": "model-00002-of-00004.safetensors", + "model.language_model.layers.22.self_attn.v_proj.weight": "model-00001-of-00004.safetensors", + "model.language_model.layers.23.input_layernorm.weight": "model-00001-of-00004.safetensors", + "model.language_model.layers.23.mlp.down_proj.weight": "model-00002-of-00004.safetensors", + "model.language_model.layers.23.mlp.gate_proj.weight": "model-00001-of-00004.safetensors", + "model.language_model.layers.23.mlp.up_proj.weight": "model-00003-of-00004.safetensors", + "model.language_model.layers.23.post_attention_layernorm.weight": "model-00001-of-00004.safetensors", + "model.language_model.layers.23.self_attn.k_norm.weight": "model-00002-of-00004.safetensors", + "model.language_model.layers.23.self_attn.k_proj.weight": "model-00001-of-00004.safetensors", + "model.language_model.layers.23.self_attn.o_proj.weight": "model-00001-of-00004.safetensors", + "model.language_model.layers.23.self_attn.q_norm.weight": "model-00001-of-00004.safetensors", + "model.language_model.layers.23.self_attn.q_proj.weight": "model-00003-of-00004.safetensors", + "model.language_model.layers.23.self_attn.v_proj.weight": "model-00003-of-00004.safetensors", + "model.language_model.layers.24.input_layernorm.weight": "model-00001-of-00004.safetensors", + "model.language_model.layers.24.mlp.down_proj.weight": "model-00001-of-00004.safetensors", + "model.language_model.layers.24.mlp.gate_proj.weight": "model-00001-of-00004.safetensors", + "model.language_model.layers.24.mlp.up_proj.weight": "model-00003-of-00004.safetensors", + "model.language_model.layers.24.post_attention_layernorm.weight": "model-00001-of-00004.safetensors", + "model.language_model.layers.24.self_attn.k_norm.weight": "model-00003-of-00004.safetensors", + "model.language_model.layers.24.self_attn.k_proj.weight": "model-00001-of-00004.safetensors", + "model.language_model.layers.24.self_attn.o_proj.weight": "model-00001-of-00004.safetensors", + "model.language_model.layers.24.self_attn.q_norm.weight": "model-00001-of-00004.safetensors", + "model.language_model.layers.24.self_attn.q_proj.weight": "model-00004-of-00004.safetensors", + "model.language_model.layers.24.self_attn.v_proj.weight": "model-00002-of-00004.safetensors", + "model.language_model.layers.25.input_layernorm.weight": "model-00003-of-00004.safetensors", + "model.language_model.layers.25.mlp.down_proj.weight": "model-00003-of-00004.safetensors", + "model.language_model.layers.25.mlp.gate_proj.weight": "model-00003-of-00004.safetensors", + "model.language_model.layers.25.mlp.up_proj.weight": "model-00001-of-00004.safetensors", + "model.language_model.layers.25.post_attention_layernorm.weight": "model-00001-of-00004.safetensors", + "model.language_model.layers.25.self_attn.k_norm.weight": "model-00001-of-00004.safetensors", + "model.language_model.layers.25.self_attn.k_proj.weight": "model-00003-of-00004.safetensors", + "model.language_model.layers.25.self_attn.o_proj.weight": "model-00001-of-00004.safetensors", + "model.language_model.layers.25.self_attn.q_norm.weight": "model-00001-of-00004.safetensors", + "model.language_model.layers.25.self_attn.q_proj.weight": "model-00001-of-00004.safetensors", + "model.language_model.layers.25.self_attn.v_proj.weight": "model-00003-of-00004.safetensors", + "model.language_model.layers.26.input_layernorm.weight": "model-00003-of-00004.safetensors", + "model.language_model.layers.26.mlp.down_proj.weight": "model-00001-of-00004.safetensors", + "model.language_model.layers.26.mlp.gate_proj.weight": "model-00001-of-00004.safetensors", + "model.language_model.layers.26.mlp.up_proj.weight": "model-00003-of-00004.safetensors", + "model.language_model.layers.26.post_attention_layernorm.weight": "model-00002-of-00004.safetensors", + "model.language_model.layers.26.self_attn.k_norm.weight": "model-00001-of-00004.safetensors", + "model.language_model.layers.26.self_attn.k_proj.weight": "model-00003-of-00004.safetensors", + "model.language_model.layers.26.self_attn.o_proj.weight": "model-00002-of-00004.safetensors", + "model.language_model.layers.26.self_attn.q_norm.weight": "model-00002-of-00004.safetensors", + "model.language_model.layers.26.self_attn.q_proj.weight": "model-00004-of-00004.safetensors", + "model.language_model.layers.26.self_attn.v_proj.weight": "model-00003-of-00004.safetensors", + "model.language_model.layers.27.input_layernorm.weight": "model-00001-of-00004.safetensors", + "model.language_model.layers.27.mlp.down_proj.weight": "model-00002-of-00004.safetensors", + "model.language_model.layers.27.mlp.gate_proj.weight": "model-00001-of-00004.safetensors", + "model.language_model.layers.27.mlp.up_proj.weight": "model-00002-of-00004.safetensors", + "model.language_model.layers.27.post_attention_layernorm.weight": "model-00003-of-00004.safetensors", + "model.language_model.layers.27.self_attn.k_norm.weight": "model-00002-of-00004.safetensors", + "model.language_model.layers.27.self_attn.k_proj.weight": "model-00003-of-00004.safetensors", + "model.language_model.layers.27.self_attn.o_proj.weight": "model-00003-of-00004.safetensors", + "model.language_model.layers.27.self_attn.q_norm.weight": "model-00003-of-00004.safetensors", + "model.language_model.layers.27.self_attn.q_proj.weight": "model-00004-of-00004.safetensors", + "model.language_model.layers.27.self_attn.v_proj.weight": "model-00001-of-00004.safetensors", + "model.language_model.layers.28.input_layernorm.weight": "model-00001-of-00004.safetensors", + "model.language_model.layers.28.mlp.down_proj.weight": "model-00001-of-00004.safetensors", + "model.language_model.layers.28.mlp.gate_proj.weight": "model-00002-of-00004.safetensors", + "model.language_model.layers.28.mlp.up_proj.weight": "model-00001-of-00004.safetensors", + "model.language_model.layers.28.post_attention_layernorm.weight": "model-00004-of-00004.safetensors", + "model.language_model.layers.28.self_attn.k_norm.weight": "model-00003-of-00004.safetensors", + "model.language_model.layers.28.self_attn.k_proj.weight": "model-00003-of-00004.safetensors", + "model.language_model.layers.28.self_attn.o_proj.weight": "model-00002-of-00004.safetensors", + "model.language_model.layers.28.self_attn.q_norm.weight": "model-00001-of-00004.safetensors", + "model.language_model.layers.28.self_attn.q_proj.weight": "model-00003-of-00004.safetensors", + "model.language_model.layers.28.self_attn.v_proj.weight": "model-00002-of-00004.safetensors", + "model.language_model.layers.29.input_layernorm.weight": "model-00004-of-00004.safetensors", + "model.language_model.layers.29.mlp.down_proj.weight": "model-00001-of-00004.safetensors", + "model.language_model.layers.29.mlp.gate_proj.weight": "model-00004-of-00004.safetensors", + "model.language_model.layers.29.mlp.up_proj.weight": "model-00003-of-00004.safetensors", + "model.language_model.layers.29.post_attention_layernorm.weight": "model-00002-of-00004.safetensors", + "model.language_model.layers.29.self_attn.k_norm.weight": "model-00003-of-00004.safetensors", + "model.language_model.layers.29.self_attn.k_proj.weight": "model-00001-of-00004.safetensors", + "model.language_model.layers.29.self_attn.o_proj.weight": "model-00002-of-00004.safetensors", + "model.language_model.layers.29.self_attn.q_norm.weight": "model-00003-of-00004.safetensors", + "model.language_model.layers.29.self_attn.q_proj.weight": "model-00003-of-00004.safetensors", + "model.language_model.layers.29.self_attn.v_proj.weight": "model-00003-of-00004.safetensors", + "model.language_model.layers.3.input_layernorm.weight": "model-00004-of-00004.safetensors", + "model.language_model.layers.3.mlp.down_proj.weight": "model-00002-of-00004.safetensors", + "model.language_model.layers.3.mlp.gate_proj.weight": "model-00003-of-00004.safetensors", + "model.language_model.layers.3.mlp.up_proj.weight": "model-00003-of-00004.safetensors", + "model.language_model.layers.3.post_attention_layernorm.weight": "model-00003-of-00004.safetensors", + "model.language_model.layers.3.self_attn.k_norm.weight": "model-00004-of-00004.safetensors", + "model.language_model.layers.3.self_attn.k_proj.weight": "model-00001-of-00004.safetensors", + "model.language_model.layers.3.self_attn.o_proj.weight": "model-00001-of-00004.safetensors", + "model.language_model.layers.3.self_attn.q_norm.weight": "model-00001-of-00004.safetensors", + "model.language_model.layers.3.self_attn.q_proj.weight": "model-00001-of-00004.safetensors", + "model.language_model.layers.3.self_attn.v_proj.weight": "model-00001-of-00004.safetensors", + "model.language_model.layers.30.input_layernorm.weight": "model-00001-of-00004.safetensors", + "model.language_model.layers.30.mlp.down_proj.weight": "model-00004-of-00004.safetensors", + "model.language_model.layers.30.mlp.gate_proj.weight": "model-00002-of-00004.safetensors", + "model.language_model.layers.30.mlp.up_proj.weight": "model-00004-of-00004.safetensors", + "model.language_model.layers.30.post_attention_layernorm.weight": "model-00004-of-00004.safetensors", + "model.language_model.layers.30.self_attn.k_norm.weight": "model-00003-of-00004.safetensors", + "model.language_model.layers.30.self_attn.k_proj.weight": "model-00001-of-00004.safetensors", + "model.language_model.layers.30.self_attn.o_proj.weight": "model-00003-of-00004.safetensors", + "model.language_model.layers.30.self_attn.q_norm.weight": "model-00003-of-00004.safetensors", + "model.language_model.layers.30.self_attn.q_proj.weight": "model-00003-of-00004.safetensors", + "model.language_model.layers.30.self_attn.v_proj.weight": "model-00003-of-00004.safetensors", + "model.language_model.layers.31.input_layernorm.weight": "model-00001-of-00004.safetensors", + "model.language_model.layers.31.mlp.down_proj.weight": "model-00001-of-00004.safetensors", + "model.language_model.layers.31.mlp.gate_proj.weight": "model-00003-of-00004.safetensors", + "model.language_model.layers.31.mlp.up_proj.weight": "model-00003-of-00004.safetensors", + "model.language_model.layers.31.post_attention_layernorm.weight": "model-00003-of-00004.safetensors", + "model.language_model.layers.31.self_attn.k_norm.weight": "model-00001-of-00004.safetensors", + "model.language_model.layers.31.self_attn.k_proj.weight": "model-00003-of-00004.safetensors", + "model.language_model.layers.31.self_attn.o_proj.weight": "model-00004-of-00004.safetensors", + "model.language_model.layers.31.self_attn.q_norm.weight": "model-00003-of-00004.safetensors", + "model.language_model.layers.31.self_attn.q_proj.weight": "model-00003-of-00004.safetensors", + "model.language_model.layers.31.self_attn.v_proj.weight": "model-00004-of-00004.safetensors", + "model.language_model.layers.32.input_layernorm.weight": "model-00003-of-00004.safetensors", + "model.language_model.layers.32.mlp.down_proj.weight": "model-00002-of-00004.safetensors", + "model.language_model.layers.32.mlp.gate_proj.weight": "model-00001-of-00004.safetensors", + "model.language_model.layers.32.mlp.up_proj.weight": "model-00002-of-00004.safetensors", + "model.language_model.layers.32.post_attention_layernorm.weight": "model-00003-of-00004.safetensors", + "model.language_model.layers.32.self_attn.k_norm.weight": "model-00002-of-00004.safetensors", + "model.language_model.layers.32.self_attn.k_proj.weight": "model-00001-of-00004.safetensors", + "model.language_model.layers.32.self_attn.o_proj.weight": "model-00003-of-00004.safetensors", + "model.language_model.layers.32.self_attn.q_norm.weight": "model-00003-of-00004.safetensors", + "model.language_model.layers.32.self_attn.q_proj.weight": "model-00003-of-00004.safetensors", + "model.language_model.layers.32.self_attn.v_proj.weight": "model-00003-of-00004.safetensors", + "model.language_model.layers.33.input_layernorm.weight": "model-00002-of-00004.safetensors", + "model.language_model.layers.33.mlp.down_proj.weight": "model-00001-of-00004.safetensors", + "model.language_model.layers.33.mlp.gate_proj.weight": "model-00002-of-00004.safetensors", + "model.language_model.layers.33.mlp.up_proj.weight": "model-00001-of-00004.safetensors", + "model.language_model.layers.33.post_attention_layernorm.weight": "model-00001-of-00004.safetensors", + "model.language_model.layers.33.self_attn.k_norm.weight": "model-00003-of-00004.safetensors", + "model.language_model.layers.33.self_attn.k_proj.weight": "model-00003-of-00004.safetensors", + "model.language_model.layers.33.self_attn.o_proj.weight": "model-00002-of-00004.safetensors", + "model.language_model.layers.33.self_attn.q_norm.weight": "model-00004-of-00004.safetensors", + "model.language_model.layers.33.self_attn.q_proj.weight": "model-00001-of-00004.safetensors", + "model.language_model.layers.33.self_attn.v_proj.weight": "model-00001-of-00004.safetensors", + "model.language_model.layers.34.input_layernorm.weight": "model-00002-of-00004.safetensors", + "model.language_model.layers.34.mlp.down_proj.weight": "model-00003-of-00004.safetensors", + "model.language_model.layers.34.mlp.gate_proj.weight": "model-00003-of-00004.safetensors", + "model.language_model.layers.34.mlp.up_proj.weight": "model-00001-of-00004.safetensors", + "model.language_model.layers.34.post_attention_layernorm.weight": "model-00001-of-00004.safetensors", + "model.language_model.layers.34.self_attn.k_norm.weight": "model-00003-of-00004.safetensors", + "model.language_model.layers.34.self_attn.k_proj.weight": "model-00003-of-00004.safetensors", + "model.language_model.layers.34.self_attn.o_proj.weight": "model-00001-of-00004.safetensors", + "model.language_model.layers.34.self_attn.q_norm.weight": "model-00003-of-00004.safetensors", + "model.language_model.layers.34.self_attn.q_proj.weight": "model-00004-of-00004.safetensors", + "model.language_model.layers.34.self_attn.v_proj.weight": "model-00003-of-00004.safetensors", + "model.language_model.layers.35.input_layernorm.weight": "model-00003-of-00004.safetensors", + "model.language_model.layers.35.mlp.down_proj.weight": "model-00003-of-00004.safetensors", + "model.language_model.layers.35.mlp.gate_proj.weight": "model-00003-of-00004.safetensors", + "model.language_model.layers.35.mlp.up_proj.weight": "model-00001-of-00004.safetensors", + "model.language_model.layers.35.post_attention_layernorm.weight": "model-00003-of-00004.safetensors", + "model.language_model.layers.35.self_attn.k_norm.weight": "model-00003-of-00004.safetensors", + "model.language_model.layers.35.self_attn.k_proj.weight": "model-00001-of-00004.safetensors", + "model.language_model.layers.35.self_attn.o_proj.weight": "model-00002-of-00004.safetensors", + "model.language_model.layers.35.self_attn.q_norm.weight": "model-00003-of-00004.safetensors", + "model.language_model.layers.35.self_attn.q_proj.weight": "model-00001-of-00004.safetensors", + "model.language_model.layers.35.self_attn.v_proj.weight": "model-00001-of-00004.safetensors", + "model.language_model.layers.4.input_layernorm.weight": "model-00004-of-00004.safetensors", + "model.language_model.layers.4.mlp.down_proj.weight": "model-00003-of-00004.safetensors", + "model.language_model.layers.4.mlp.gate_proj.weight": "model-00001-of-00004.safetensors", + "model.language_model.layers.4.mlp.up_proj.weight": "model-00003-of-00004.safetensors", + "model.language_model.layers.4.post_attention_layernorm.weight": "model-00003-of-00004.safetensors", + "model.language_model.layers.4.self_attn.k_norm.weight": "model-00002-of-00004.safetensors", + "model.language_model.layers.4.self_attn.k_proj.weight": "model-00003-of-00004.safetensors", + "model.language_model.layers.4.self_attn.o_proj.weight": "model-00002-of-00004.safetensors", + "model.language_model.layers.4.self_attn.q_norm.weight": "model-00003-of-00004.safetensors", + "model.language_model.layers.4.self_attn.q_proj.weight": "model-00002-of-00004.safetensors", + "model.language_model.layers.4.self_attn.v_proj.weight": "model-00003-of-00004.safetensors", + "model.language_model.layers.5.input_layernorm.weight": "model-00001-of-00004.safetensors", + "model.language_model.layers.5.mlp.down_proj.weight": "model-00003-of-00004.safetensors", + "model.language_model.layers.5.mlp.gate_proj.weight": "model-00003-of-00004.safetensors", + "model.language_model.layers.5.mlp.up_proj.weight": "model-00002-of-00004.safetensors", + "model.language_model.layers.5.post_attention_layernorm.weight": "model-00001-of-00004.safetensors", + "model.language_model.layers.5.self_attn.k_norm.weight": "model-00004-of-00004.safetensors", + "model.language_model.layers.5.self_attn.k_proj.weight": "model-00004-of-00004.safetensors", + "model.language_model.layers.5.self_attn.o_proj.weight": "model-00003-of-00004.safetensors", + "model.language_model.layers.5.self_attn.q_norm.weight": "model-00003-of-00004.safetensors", + "model.language_model.layers.5.self_attn.q_proj.weight": "model-00001-of-00004.safetensors", + "model.language_model.layers.5.self_attn.v_proj.weight": "model-00003-of-00004.safetensors", + "model.language_model.layers.6.input_layernorm.weight": "model-00004-of-00004.safetensors", + "model.language_model.layers.6.mlp.down_proj.weight": "model-00001-of-00004.safetensors", + "model.language_model.layers.6.mlp.gate_proj.weight": "model-00001-of-00004.safetensors", + "model.language_model.layers.6.mlp.up_proj.weight": "model-00001-of-00004.safetensors", + "model.language_model.layers.6.post_attention_layernorm.weight": "model-00003-of-00004.safetensors", + "model.language_model.layers.6.self_attn.k_norm.weight": "model-00003-of-00004.safetensors", + "model.language_model.layers.6.self_attn.k_proj.weight": "model-00004-of-00004.safetensors", + "model.language_model.layers.6.self_attn.o_proj.weight": "model-00004-of-00004.safetensors", + "model.language_model.layers.6.self_attn.q_norm.weight": "model-00001-of-00004.safetensors", + "model.language_model.layers.6.self_attn.q_proj.weight": "model-00004-of-00004.safetensors", + "model.language_model.layers.6.self_attn.v_proj.weight": "model-00003-of-00004.safetensors", + "model.language_model.layers.7.input_layernorm.weight": "model-00003-of-00004.safetensors", + "model.language_model.layers.7.mlp.down_proj.weight": "model-00003-of-00004.safetensors", + "model.language_model.layers.7.mlp.gate_proj.weight": "model-00002-of-00004.safetensors", + "model.language_model.layers.7.mlp.up_proj.weight": "model-00002-of-00004.safetensors", + "model.language_model.layers.7.post_attention_layernorm.weight": "model-00004-of-00004.safetensors", + "model.language_model.layers.7.self_attn.k_norm.weight": "model-00002-of-00004.safetensors", + "model.language_model.layers.7.self_attn.k_proj.weight": "model-00002-of-00004.safetensors", + "model.language_model.layers.7.self_attn.o_proj.weight": "model-00003-of-00004.safetensors", + "model.language_model.layers.7.self_attn.q_norm.weight": "model-00001-of-00004.safetensors", + "model.language_model.layers.7.self_attn.q_proj.weight": "model-00003-of-00004.safetensors", + "model.language_model.layers.7.self_attn.v_proj.weight": "model-00001-of-00004.safetensors", + "model.language_model.layers.8.input_layernorm.weight": "model-00001-of-00004.safetensors", + "model.language_model.layers.8.mlp.down_proj.weight": "model-00004-of-00004.safetensors", + "model.language_model.layers.8.mlp.gate_proj.weight": "model-00002-of-00004.safetensors", + "model.language_model.layers.8.mlp.up_proj.weight": "model-00001-of-00004.safetensors", + "model.language_model.layers.8.post_attention_layernorm.weight": "model-00004-of-00004.safetensors", + "model.language_model.layers.8.self_attn.k_norm.weight": "model-00003-of-00004.safetensors", + "model.language_model.layers.8.self_attn.k_proj.weight": "model-00001-of-00004.safetensors", + "model.language_model.layers.8.self_attn.o_proj.weight": "model-00001-of-00004.safetensors", + "model.language_model.layers.8.self_attn.q_norm.weight": "model-00002-of-00004.safetensors", + "model.language_model.layers.8.self_attn.q_proj.weight": "model-00002-of-00004.safetensors", + "model.language_model.layers.8.self_attn.v_proj.weight": "model-00001-of-00004.safetensors", + "model.language_model.layers.9.input_layernorm.weight": "model-00001-of-00004.safetensors", + "model.language_model.layers.9.mlp.down_proj.weight": "model-00003-of-00004.safetensors", + "model.language_model.layers.9.mlp.gate_proj.weight": "model-00003-of-00004.safetensors", + "model.language_model.layers.9.mlp.up_proj.weight": "model-00001-of-00004.safetensors", + "model.language_model.layers.9.post_attention_layernorm.weight": "model-00003-of-00004.safetensors", + "model.language_model.layers.9.self_attn.k_norm.weight": "model-00003-of-00004.safetensors", + "model.language_model.layers.9.self_attn.k_proj.weight": "model-00001-of-00004.safetensors", + "model.language_model.layers.9.self_attn.o_proj.weight": "model-00004-of-00004.safetensors", + "model.language_model.layers.9.self_attn.q_norm.weight": "model-00001-of-00004.safetensors", + "model.language_model.layers.9.self_attn.q_proj.weight": "model-00003-of-00004.safetensors", + "model.language_model.layers.9.self_attn.v_proj.weight": "model-00003-of-00004.safetensors", + "model.language_model.norm.weight": "model-00001-of-00004.safetensors", + "model.visual.blocks.0.attn.proj.bias": "model-00003-of-00004.safetensors", + "model.visual.blocks.0.attn.proj.weight": "model-00001-of-00004.safetensors", + "model.visual.blocks.0.attn.qkv.bias": "model-00002-of-00004.safetensors", + "model.visual.blocks.0.attn.qkv.weight": "model-00003-of-00004.safetensors", + "model.visual.blocks.0.mlp.linear_fc1.bias": "model-00004-of-00004.safetensors", + "model.visual.blocks.0.mlp.linear_fc1.weight": "model-00003-of-00004.safetensors", + "model.visual.blocks.0.mlp.linear_fc2.bias": "model-00003-of-00004.safetensors", + "model.visual.blocks.0.mlp.linear_fc2.weight": "model-00003-of-00004.safetensors", + "model.visual.blocks.0.norm1.bias": "model-00001-of-00004.safetensors", + "model.visual.blocks.0.norm1.weight": "model-00001-of-00004.safetensors", + "model.visual.blocks.0.norm2.bias": "model-00001-of-00004.safetensors", + "model.visual.blocks.0.norm2.weight": "model-00001-of-00004.safetensors", + "model.visual.blocks.1.attn.proj.bias": "model-00001-of-00004.safetensors", + "model.visual.blocks.1.attn.proj.weight": "model-00001-of-00004.safetensors", + "model.visual.blocks.1.attn.qkv.bias": "model-00003-of-00004.safetensors", + "model.visual.blocks.1.attn.qkv.weight": "model-00003-of-00004.safetensors", + "model.visual.blocks.1.mlp.linear_fc1.bias": "model-00003-of-00004.safetensors", + "model.visual.blocks.1.mlp.linear_fc1.weight": "model-00001-of-00004.safetensors", + "model.visual.blocks.1.mlp.linear_fc2.bias": "model-00002-of-00004.safetensors", + "model.visual.blocks.1.mlp.linear_fc2.weight": "model-00003-of-00004.safetensors", + "model.visual.blocks.1.norm1.bias": "model-00004-of-00004.safetensors", + "model.visual.blocks.1.norm1.weight": "model-00004-of-00004.safetensors", + "model.visual.blocks.1.norm2.bias": "model-00003-of-00004.safetensors", + "model.visual.blocks.1.norm2.weight": "model-00004-of-00004.safetensors", + "model.visual.blocks.10.attn.proj.bias": "model-00003-of-00004.safetensors", + "model.visual.blocks.10.attn.proj.weight": "model-00003-of-00004.safetensors", + "model.visual.blocks.10.attn.qkv.bias": "model-00003-of-00004.safetensors", + "model.visual.blocks.10.attn.qkv.weight": "model-00003-of-00004.safetensors", + "model.visual.blocks.10.mlp.linear_fc1.bias": "model-00002-of-00004.safetensors", + "model.visual.blocks.10.mlp.linear_fc1.weight": "model-00003-of-00004.safetensors", + "model.visual.blocks.10.mlp.linear_fc2.bias": "model-00003-of-00004.safetensors", + "model.visual.blocks.10.mlp.linear_fc2.weight": "model-00003-of-00004.safetensors", + "model.visual.blocks.10.norm1.bias": "model-00003-of-00004.safetensors", + "model.visual.blocks.10.norm1.weight": "model-00001-of-00004.safetensors", + "model.visual.blocks.10.norm2.bias": "model-00003-of-00004.safetensors", + "model.visual.blocks.10.norm2.weight": "model-00001-of-00004.safetensors", + "model.visual.blocks.11.attn.proj.bias": "model-00003-of-00004.safetensors", + "model.visual.blocks.11.attn.proj.weight": "model-00003-of-00004.safetensors", + "model.visual.blocks.11.attn.qkv.bias": "model-00003-of-00004.safetensors", + "model.visual.blocks.11.attn.qkv.weight": "model-00004-of-00004.safetensors", + "model.visual.blocks.11.mlp.linear_fc1.bias": "model-00004-of-00004.safetensors", + "model.visual.blocks.11.mlp.linear_fc1.weight": "model-00002-of-00004.safetensors", + "model.visual.blocks.11.mlp.linear_fc2.bias": "model-00001-of-00004.safetensors", + "model.visual.blocks.11.mlp.linear_fc2.weight": "model-00003-of-00004.safetensors", + "model.visual.blocks.11.norm1.bias": "model-00001-of-00004.safetensors", + "model.visual.blocks.11.norm1.weight": "model-00001-of-00004.safetensors", + "model.visual.blocks.11.norm2.bias": "model-00003-of-00004.safetensors", + "model.visual.blocks.11.norm2.weight": "model-00002-of-00004.safetensors", + "model.visual.blocks.12.attn.proj.bias": "model-00003-of-00004.safetensors", + "model.visual.blocks.12.attn.proj.weight": "model-00004-of-00004.safetensors", + "model.visual.blocks.12.attn.qkv.bias": "model-00003-of-00004.safetensors", + "model.visual.blocks.12.attn.qkv.weight": "model-00002-of-00004.safetensors", + "model.visual.blocks.12.mlp.linear_fc1.bias": "model-00002-of-00004.safetensors", + "model.visual.blocks.12.mlp.linear_fc1.weight": "model-00003-of-00004.safetensors", + "model.visual.blocks.12.mlp.linear_fc2.bias": "model-00003-of-00004.safetensors", + "model.visual.blocks.12.mlp.linear_fc2.weight": "model-00003-of-00004.safetensors", + "model.visual.blocks.12.norm1.bias": "model-00003-of-00004.safetensors", + "model.visual.blocks.12.norm1.weight": "model-00001-of-00004.safetensors", + "model.visual.blocks.12.norm2.bias": "model-00004-of-00004.safetensors", + "model.visual.blocks.12.norm2.weight": "model-00003-of-00004.safetensors", + "model.visual.blocks.13.attn.proj.bias": "model-00003-of-00004.safetensors", + "model.visual.blocks.13.attn.proj.weight": "model-00001-of-00004.safetensors", + "model.visual.blocks.13.attn.qkv.bias": "model-00001-of-00004.safetensors", + "model.visual.blocks.13.attn.qkv.weight": "model-00001-of-00004.safetensors", + "model.visual.blocks.13.mlp.linear_fc1.bias": "model-00002-of-00004.safetensors", + "model.visual.blocks.13.mlp.linear_fc1.weight": "model-00004-of-00004.safetensors", + "model.visual.blocks.13.mlp.linear_fc2.bias": "model-00004-of-00004.safetensors", + "model.visual.blocks.13.mlp.linear_fc2.weight": "model-00001-of-00004.safetensors", + "model.visual.blocks.13.norm1.bias": "model-00004-of-00004.safetensors", + "model.visual.blocks.13.norm1.weight": "model-00003-of-00004.safetensors", + "model.visual.blocks.13.norm2.bias": "model-00001-of-00004.safetensors", + "model.visual.blocks.13.norm2.weight": "model-00001-of-00004.safetensors", + "model.visual.blocks.14.attn.proj.bias": "model-00003-of-00004.safetensors", + "model.visual.blocks.14.attn.proj.weight": "model-00003-of-00004.safetensors", + "model.visual.blocks.14.attn.qkv.bias": "model-00002-of-00004.safetensors", + "model.visual.blocks.14.attn.qkv.weight": "model-00003-of-00004.safetensors", + "model.visual.blocks.14.mlp.linear_fc1.bias": "model-00001-of-00004.safetensors", + "model.visual.blocks.14.mlp.linear_fc1.weight": "model-00002-of-00004.safetensors", + "model.visual.blocks.14.mlp.linear_fc2.bias": "model-00001-of-00004.safetensors", + "model.visual.blocks.14.mlp.linear_fc2.weight": "model-00001-of-00004.safetensors", + "model.visual.blocks.14.norm1.bias": "model-00002-of-00004.safetensors", + "model.visual.blocks.14.norm1.weight": "model-00003-of-00004.safetensors", + "model.visual.blocks.14.norm2.bias": "model-00002-of-00004.safetensors", + "model.visual.blocks.14.norm2.weight": "model-00001-of-00004.safetensors", + "model.visual.blocks.15.attn.proj.bias": "model-00003-of-00004.safetensors", + "model.visual.blocks.15.attn.proj.weight": "model-00002-of-00004.safetensors", + "model.visual.blocks.15.attn.qkv.bias": "model-00004-of-00004.safetensors", + "model.visual.blocks.15.attn.qkv.weight": "model-00001-of-00004.safetensors", + "model.visual.blocks.15.mlp.linear_fc1.bias": "model-00003-of-00004.safetensors", + "model.visual.blocks.15.mlp.linear_fc1.weight": "model-00004-of-00004.safetensors", + "model.visual.blocks.15.mlp.linear_fc2.bias": "model-00001-of-00004.safetensors", + "model.visual.blocks.15.mlp.linear_fc2.weight": "model-00003-of-00004.safetensors", + "model.visual.blocks.15.norm1.bias": "model-00003-of-00004.safetensors", + "model.visual.blocks.15.norm1.weight": "model-00003-of-00004.safetensors", + "model.visual.blocks.15.norm2.bias": "model-00003-of-00004.safetensors", + "model.visual.blocks.15.norm2.weight": "model-00003-of-00004.safetensors", + "model.visual.blocks.16.attn.proj.bias": "model-00001-of-00004.safetensors", + "model.visual.blocks.16.attn.proj.weight": "model-00003-of-00004.safetensors", + "model.visual.blocks.16.attn.qkv.bias": "model-00003-of-00004.safetensors", + "model.visual.blocks.16.attn.qkv.weight": "model-00001-of-00004.safetensors", + "model.visual.blocks.16.mlp.linear_fc1.bias": "model-00002-of-00004.safetensors", + "model.visual.blocks.16.mlp.linear_fc1.weight": "model-00002-of-00004.safetensors", + "model.visual.blocks.16.mlp.linear_fc2.bias": "model-00001-of-00004.safetensors", + "model.visual.blocks.16.mlp.linear_fc2.weight": "model-00003-of-00004.safetensors", + "model.visual.blocks.16.norm1.bias": "model-00001-of-00004.safetensors", + "model.visual.blocks.16.norm1.weight": "model-00003-of-00004.safetensors", + "model.visual.blocks.16.norm2.bias": "model-00001-of-00004.safetensors", + "model.visual.blocks.16.norm2.weight": "model-00001-of-00004.safetensors", + "model.visual.blocks.17.attn.proj.bias": "model-00001-of-00004.safetensors", + "model.visual.blocks.17.attn.proj.weight": "model-00001-of-00004.safetensors", + "model.visual.blocks.17.attn.qkv.bias": "model-00004-of-00004.safetensors", + "model.visual.blocks.17.attn.qkv.weight": "model-00003-of-00004.safetensors", + "model.visual.blocks.17.mlp.linear_fc1.bias": "model-00003-of-00004.safetensors", + "model.visual.blocks.17.mlp.linear_fc1.weight": "model-00002-of-00004.safetensors", + "model.visual.blocks.17.mlp.linear_fc2.bias": "model-00002-of-00004.safetensors", + "model.visual.blocks.17.mlp.linear_fc2.weight": "model-00003-of-00004.safetensors", + "model.visual.blocks.17.norm1.bias": "model-00003-of-00004.safetensors", + "model.visual.blocks.17.norm1.weight": "model-00003-of-00004.safetensors", + "model.visual.blocks.17.norm2.bias": "model-00004-of-00004.safetensors", + "model.visual.blocks.17.norm2.weight": "model-00001-of-00004.safetensors", + "model.visual.blocks.18.attn.proj.bias": "model-00002-of-00004.safetensors", + "model.visual.blocks.18.attn.proj.weight": "model-00003-of-00004.safetensors", + "model.visual.blocks.18.attn.qkv.bias": "model-00001-of-00004.safetensors", + "model.visual.blocks.18.attn.qkv.weight": "model-00003-of-00004.safetensors", + "model.visual.blocks.18.mlp.linear_fc1.bias": "model-00003-of-00004.safetensors", + "model.visual.blocks.18.mlp.linear_fc1.weight": "model-00003-of-00004.safetensors", + "model.visual.blocks.18.mlp.linear_fc2.bias": "model-00003-of-00004.safetensors", + "model.visual.blocks.18.mlp.linear_fc2.weight": "model-00003-of-00004.safetensors", + "model.visual.blocks.18.norm1.bias": "model-00004-of-00004.safetensors", + "model.visual.blocks.18.norm1.weight": "model-00002-of-00004.safetensors", + "model.visual.blocks.18.norm2.bias": "model-00001-of-00004.safetensors", + "model.visual.blocks.18.norm2.weight": "model-00002-of-00004.safetensors", + "model.visual.blocks.19.attn.proj.bias": "model-00001-of-00004.safetensors", + "model.visual.blocks.19.attn.proj.weight": "model-00003-of-00004.safetensors", + "model.visual.blocks.19.attn.qkv.bias": "model-00002-of-00004.safetensors", + "model.visual.blocks.19.attn.qkv.weight": "model-00001-of-00004.safetensors", + "model.visual.blocks.19.mlp.linear_fc1.bias": "model-00001-of-00004.safetensors", + "model.visual.blocks.19.mlp.linear_fc1.weight": "model-00004-of-00004.safetensors", + "model.visual.blocks.19.mlp.linear_fc2.bias": "model-00004-of-00004.safetensors", + "model.visual.blocks.19.mlp.linear_fc2.weight": "model-00002-of-00004.safetensors", + "model.visual.blocks.19.norm1.bias": "model-00003-of-00004.safetensors", + "model.visual.blocks.19.norm1.weight": "model-00001-of-00004.safetensors", + "model.visual.blocks.19.norm2.bias": "model-00001-of-00004.safetensors", + "model.visual.blocks.19.norm2.weight": "model-00004-of-00004.safetensors", + "model.visual.blocks.2.attn.proj.bias": "model-00004-of-00004.safetensors", + "model.visual.blocks.2.attn.proj.weight": "model-00003-of-00004.safetensors", + "model.visual.blocks.2.attn.qkv.bias": "model-00003-of-00004.safetensors", + "model.visual.blocks.2.attn.qkv.weight": "model-00003-of-00004.safetensors", + "model.visual.blocks.2.mlp.linear_fc1.bias": "model-00004-of-00004.safetensors", + "model.visual.blocks.2.mlp.linear_fc1.weight": "model-00004-of-00004.safetensors", + "model.visual.blocks.2.mlp.linear_fc2.bias": "model-00001-of-00004.safetensors", + "model.visual.blocks.2.mlp.linear_fc2.weight": "model-00004-of-00004.safetensors", + "model.visual.blocks.2.norm1.bias": "model-00003-of-00004.safetensors", + "model.visual.blocks.2.norm1.weight": "model-00003-of-00004.safetensors", + "model.visual.blocks.2.norm2.bias": "model-00001-of-00004.safetensors", + "model.visual.blocks.2.norm2.weight": "model-00003-of-00004.safetensors", + "model.visual.blocks.20.attn.proj.bias": "model-00003-of-00004.safetensors", + "model.visual.blocks.20.attn.proj.weight": "model-00003-of-00004.safetensors", + "model.visual.blocks.20.attn.qkv.bias": "model-00001-of-00004.safetensors", + "model.visual.blocks.20.attn.qkv.weight": "model-00004-of-00004.safetensors", + "model.visual.blocks.20.mlp.linear_fc1.bias": "model-00001-of-00004.safetensors", + "model.visual.blocks.20.mlp.linear_fc1.weight": "model-00001-of-00004.safetensors", + "model.visual.blocks.20.mlp.linear_fc2.bias": "model-00003-of-00004.safetensors", + "model.visual.blocks.20.mlp.linear_fc2.weight": "model-00002-of-00004.safetensors", + "model.visual.blocks.20.norm1.bias": "model-00003-of-00004.safetensors", + "model.visual.blocks.20.norm1.weight": "model-00002-of-00004.safetensors", + "model.visual.blocks.20.norm2.bias": "model-00001-of-00004.safetensors", + "model.visual.blocks.20.norm2.weight": "model-00001-of-00004.safetensors", + "model.visual.blocks.21.attn.proj.bias": "model-00003-of-00004.safetensors", + "model.visual.blocks.21.attn.proj.weight": "model-00001-of-00004.safetensors", + "model.visual.blocks.21.attn.qkv.bias": "model-00003-of-00004.safetensors", + "model.visual.blocks.21.attn.qkv.weight": "model-00002-of-00004.safetensors", + "model.visual.blocks.21.mlp.linear_fc1.bias": "model-00004-of-00004.safetensors", + "model.visual.blocks.21.mlp.linear_fc1.weight": "model-00001-of-00004.safetensors", + "model.visual.blocks.21.mlp.linear_fc2.bias": "model-00002-of-00004.safetensors", + "model.visual.blocks.21.mlp.linear_fc2.weight": "model-00003-of-00004.safetensors", + "model.visual.blocks.21.norm1.bias": "model-00002-of-00004.safetensors", + "model.visual.blocks.21.norm1.weight": "model-00001-of-00004.safetensors", + "model.visual.blocks.21.norm2.bias": "model-00001-of-00004.safetensors", + "model.visual.blocks.21.norm2.weight": "model-00003-of-00004.safetensors", + "model.visual.blocks.22.attn.proj.bias": "model-00001-of-00004.safetensors", + "model.visual.blocks.22.attn.proj.weight": "model-00004-of-00004.safetensors", + "model.visual.blocks.22.attn.qkv.bias": "model-00003-of-00004.safetensors", + "model.visual.blocks.22.attn.qkv.weight": "model-00002-of-00004.safetensors", + "model.visual.blocks.22.mlp.linear_fc1.bias": "model-00002-of-00004.safetensors", + "model.visual.blocks.22.mlp.linear_fc1.weight": "model-00001-of-00004.safetensors", + "model.visual.blocks.22.mlp.linear_fc2.bias": "model-00003-of-00004.safetensors", + "model.visual.blocks.22.mlp.linear_fc2.weight": "model-00003-of-00004.safetensors", + "model.visual.blocks.22.norm1.bias": "model-00003-of-00004.safetensors", + "model.visual.blocks.22.norm1.weight": "model-00001-of-00004.safetensors", + "model.visual.blocks.22.norm2.bias": "model-00004-of-00004.safetensors", + "model.visual.blocks.22.norm2.weight": "model-00003-of-00004.safetensors", + "model.visual.blocks.23.attn.proj.bias": "model-00002-of-00004.safetensors", + "model.visual.blocks.23.attn.proj.weight": "model-00001-of-00004.safetensors", + "model.visual.blocks.23.attn.qkv.bias": "model-00001-of-00004.safetensors", + "model.visual.blocks.23.attn.qkv.weight": "model-00004-of-00004.safetensors", + "model.visual.blocks.23.mlp.linear_fc1.bias": "model-00001-of-00004.safetensors", + "model.visual.blocks.23.mlp.linear_fc1.weight": "model-00001-of-00004.safetensors", + "model.visual.blocks.23.mlp.linear_fc2.bias": "model-00004-of-00004.safetensors", + "model.visual.blocks.23.mlp.linear_fc2.weight": "model-00003-of-00004.safetensors", + "model.visual.blocks.23.norm1.bias": "model-00003-of-00004.safetensors", + "model.visual.blocks.23.norm1.weight": "model-00004-of-00004.safetensors", + "model.visual.blocks.23.norm2.bias": "model-00001-of-00004.safetensors", + "model.visual.blocks.23.norm2.weight": "model-00004-of-00004.safetensors", + "model.visual.blocks.24.attn.proj.bias": "model-00004-of-00004.safetensors", + "model.visual.blocks.24.attn.proj.weight": "model-00002-of-00004.safetensors", + "model.visual.blocks.24.attn.qkv.bias": "model-00002-of-00004.safetensors", + "model.visual.blocks.24.attn.qkv.weight": "model-00003-of-00004.safetensors", + "model.visual.blocks.24.mlp.linear_fc1.bias": "model-00003-of-00004.safetensors", + "model.visual.blocks.24.mlp.linear_fc1.weight": "model-00003-of-00004.safetensors", + "model.visual.blocks.24.mlp.linear_fc2.bias": "model-00004-of-00004.safetensors", + "model.visual.blocks.24.mlp.linear_fc2.weight": "model-00003-of-00004.safetensors", + "model.visual.blocks.24.norm1.bias": "model-00001-of-00004.safetensors", + "model.visual.blocks.24.norm1.weight": "model-00001-of-00004.safetensors", + "model.visual.blocks.24.norm2.bias": "model-00003-of-00004.safetensors", + "model.visual.blocks.24.norm2.weight": "model-00003-of-00004.safetensors", + "model.visual.blocks.25.attn.proj.bias": "model-00001-of-00004.safetensors", + "model.visual.blocks.25.attn.proj.weight": "model-00001-of-00004.safetensors", + "model.visual.blocks.25.attn.qkv.bias": "model-00003-of-00004.safetensors", + "model.visual.blocks.25.attn.qkv.weight": "model-00002-of-00004.safetensors", + "model.visual.blocks.25.mlp.linear_fc1.bias": "model-00001-of-00004.safetensors", + "model.visual.blocks.25.mlp.linear_fc1.weight": "model-00002-of-00004.safetensors", + "model.visual.blocks.25.mlp.linear_fc2.bias": "model-00003-of-00004.safetensors", + "model.visual.blocks.25.mlp.linear_fc2.weight": "model-00004-of-00004.safetensors", + "model.visual.blocks.25.norm1.bias": "model-00003-of-00004.safetensors", + "model.visual.blocks.25.norm1.weight": "model-00003-of-00004.safetensors", + "model.visual.blocks.25.norm2.bias": "model-00002-of-00004.safetensors", + "model.visual.blocks.25.norm2.weight": "model-00003-of-00004.safetensors", + "model.visual.blocks.26.attn.proj.bias": "model-00002-of-00004.safetensors", + "model.visual.blocks.26.attn.proj.weight": "model-00001-of-00004.safetensors", + "model.visual.blocks.26.attn.qkv.bias": "model-00004-of-00004.safetensors", + "model.visual.blocks.26.attn.qkv.weight": "model-00001-of-00004.safetensors", + "model.visual.blocks.26.mlp.linear_fc1.bias": "model-00004-of-00004.safetensors", + "model.visual.blocks.26.mlp.linear_fc1.weight": "model-00004-of-00004.safetensors", + "model.visual.blocks.26.mlp.linear_fc2.bias": "model-00001-of-00004.safetensors", + "model.visual.blocks.26.mlp.linear_fc2.weight": "model-00003-of-00004.safetensors", + "model.visual.blocks.26.norm1.bias": "model-00001-of-00004.safetensors", + "model.visual.blocks.26.norm1.weight": "model-00001-of-00004.safetensors", + "model.visual.blocks.26.norm2.bias": "model-00001-of-00004.safetensors", + "model.visual.blocks.26.norm2.weight": "model-00003-of-00004.safetensors", + "model.visual.blocks.3.attn.proj.bias": "model-00001-of-00004.safetensors", + "model.visual.blocks.3.attn.proj.weight": "model-00003-of-00004.safetensors", + "model.visual.blocks.3.attn.qkv.bias": "model-00001-of-00004.safetensors", + "model.visual.blocks.3.attn.qkv.weight": "model-00001-of-00004.safetensors", + "model.visual.blocks.3.mlp.linear_fc1.bias": "model-00004-of-00004.safetensors", + "model.visual.blocks.3.mlp.linear_fc1.weight": "model-00004-of-00004.safetensors", + "model.visual.blocks.3.mlp.linear_fc2.bias": "model-00001-of-00004.safetensors", + "model.visual.blocks.3.mlp.linear_fc2.weight": "model-00003-of-00004.safetensors", + "model.visual.blocks.3.norm1.bias": "model-00002-of-00004.safetensors", + "model.visual.blocks.3.norm1.weight": "model-00001-of-00004.safetensors", + "model.visual.blocks.3.norm2.bias": "model-00002-of-00004.safetensors", + "model.visual.blocks.3.norm2.weight": "model-00003-of-00004.safetensors", + "model.visual.blocks.4.attn.proj.bias": "model-00002-of-00004.safetensors", + "model.visual.blocks.4.attn.proj.weight": "model-00001-of-00004.safetensors", + "model.visual.blocks.4.attn.qkv.bias": "model-00001-of-00004.safetensors", + "model.visual.blocks.4.attn.qkv.weight": "model-00002-of-00004.safetensors", + "model.visual.blocks.4.mlp.linear_fc1.bias": "model-00002-of-00004.safetensors", + "model.visual.blocks.4.mlp.linear_fc1.weight": "model-00001-of-00004.safetensors", + "model.visual.blocks.4.mlp.linear_fc2.bias": "model-00001-of-00004.safetensors", + "model.visual.blocks.4.mlp.linear_fc2.weight": "model-00003-of-00004.safetensors", + "model.visual.blocks.4.norm1.bias": "model-00003-of-00004.safetensors", + "model.visual.blocks.4.norm1.weight": "model-00001-of-00004.safetensors", + "model.visual.blocks.4.norm2.bias": "model-00001-of-00004.safetensors", + "model.visual.blocks.4.norm2.weight": "model-00001-of-00004.safetensors", + "model.visual.blocks.5.attn.proj.bias": "model-00003-of-00004.safetensors", + "model.visual.blocks.5.attn.proj.weight": "model-00003-of-00004.safetensors", + "model.visual.blocks.5.attn.qkv.bias": "model-00003-of-00004.safetensors", + "model.visual.blocks.5.attn.qkv.weight": "model-00001-of-00004.safetensors", + "model.visual.blocks.5.mlp.linear_fc1.bias": "model-00003-of-00004.safetensors", + "model.visual.blocks.5.mlp.linear_fc1.weight": "model-00001-of-00004.safetensors", + "model.visual.blocks.5.mlp.linear_fc2.bias": "model-00003-of-00004.safetensors", + "model.visual.blocks.5.mlp.linear_fc2.weight": "model-00003-of-00004.safetensors", + "model.visual.blocks.5.norm1.bias": "model-00001-of-00004.safetensors", + "model.visual.blocks.5.norm1.weight": "model-00002-of-00004.safetensors", + "model.visual.blocks.5.norm2.bias": "model-00003-of-00004.safetensors", + "model.visual.blocks.5.norm2.weight": "model-00003-of-00004.safetensors", + "model.visual.blocks.6.attn.proj.bias": "model-00001-of-00004.safetensors", + "model.visual.blocks.6.attn.proj.weight": "model-00002-of-00004.safetensors", + "model.visual.blocks.6.attn.qkv.bias": "model-00001-of-00004.safetensors", + "model.visual.blocks.6.attn.qkv.weight": "model-00002-of-00004.safetensors", + "model.visual.blocks.6.mlp.linear_fc1.bias": "model-00004-of-00004.safetensors", + "model.visual.blocks.6.mlp.linear_fc1.weight": "model-00001-of-00004.safetensors", + "model.visual.blocks.6.mlp.linear_fc2.bias": "model-00004-of-00004.safetensors", + "model.visual.blocks.6.mlp.linear_fc2.weight": "model-00002-of-00004.safetensors", + "model.visual.blocks.6.norm1.bias": "model-00004-of-00004.safetensors", + "model.visual.blocks.6.norm1.weight": "model-00001-of-00004.safetensors", + "model.visual.blocks.6.norm2.bias": "model-00002-of-00004.safetensors", + "model.visual.blocks.6.norm2.weight": "model-00003-of-00004.safetensors", + "model.visual.blocks.7.attn.proj.bias": "model-00004-of-00004.safetensors", + "model.visual.blocks.7.attn.proj.weight": "model-00002-of-00004.safetensors", + "model.visual.blocks.7.attn.qkv.bias": "model-00002-of-00004.safetensors", + "model.visual.blocks.7.attn.qkv.weight": "model-00001-of-00004.safetensors", + "model.visual.blocks.7.mlp.linear_fc1.bias": "model-00001-of-00004.safetensors", + "model.visual.blocks.7.mlp.linear_fc1.weight": "model-00003-of-00004.safetensors", + "model.visual.blocks.7.mlp.linear_fc2.bias": "model-00002-of-00004.safetensors", + "model.visual.blocks.7.mlp.linear_fc2.weight": "model-00003-of-00004.safetensors", + "model.visual.blocks.7.norm1.bias": "model-00002-of-00004.safetensors", + "model.visual.blocks.7.norm1.weight": "model-00001-of-00004.safetensors", + "model.visual.blocks.7.norm2.bias": "model-00001-of-00004.safetensors", + "model.visual.blocks.7.norm2.weight": "model-00001-of-00004.safetensors", + "model.visual.blocks.8.attn.proj.bias": "model-00003-of-00004.safetensors", + "model.visual.blocks.8.attn.proj.weight": "model-00001-of-00004.safetensors", + "model.visual.blocks.8.attn.qkv.bias": "model-00002-of-00004.safetensors", + "model.visual.blocks.8.attn.qkv.weight": "model-00003-of-00004.safetensors", + "model.visual.blocks.8.mlp.linear_fc1.bias": "model-00004-of-00004.safetensors", + "model.visual.blocks.8.mlp.linear_fc1.weight": "model-00001-of-00004.safetensors", + "model.visual.blocks.8.mlp.linear_fc2.bias": "model-00001-of-00004.safetensors", + "model.visual.blocks.8.mlp.linear_fc2.weight": "model-00004-of-00004.safetensors", + "model.visual.blocks.8.norm1.bias": "model-00001-of-00004.safetensors", + "model.visual.blocks.8.norm1.weight": "model-00004-of-00004.safetensors", + "model.visual.blocks.8.norm2.bias": "model-00001-of-00004.safetensors", + "model.visual.blocks.8.norm2.weight": "model-00004-of-00004.safetensors", + "model.visual.blocks.9.attn.proj.bias": "model-00003-of-00004.safetensors", + "model.visual.blocks.9.attn.proj.weight": "model-00004-of-00004.safetensors", + "model.visual.blocks.9.attn.qkv.bias": "model-00004-of-00004.safetensors", + "model.visual.blocks.9.attn.qkv.weight": "model-00002-of-00004.safetensors", + "model.visual.blocks.9.mlp.linear_fc1.bias": "model-00002-of-00004.safetensors", + "model.visual.blocks.9.mlp.linear_fc1.weight": "model-00004-of-00004.safetensors", + "model.visual.blocks.9.mlp.linear_fc2.bias": "model-00002-of-00004.safetensors", + "model.visual.blocks.9.mlp.linear_fc2.weight": "model-00002-of-00004.safetensors", + "model.visual.blocks.9.norm1.bias": "model-00004-of-00004.safetensors", + "model.visual.blocks.9.norm1.weight": "model-00002-of-00004.safetensors", + "model.visual.blocks.9.norm2.bias": "model-00003-of-00004.safetensors", + "model.visual.blocks.9.norm2.weight": "model-00003-of-00004.safetensors", + "model.visual.deepstack_merger_list.0.linear_fc1.bias": "model-00003-of-00004.safetensors", + "model.visual.deepstack_merger_list.0.linear_fc1.weight": "model-00002-of-00004.safetensors", + "model.visual.deepstack_merger_list.0.linear_fc2.bias": "model-00001-of-00004.safetensors", + "model.visual.deepstack_merger_list.0.linear_fc2.weight": "model-00003-of-00004.safetensors", + "model.visual.deepstack_merger_list.0.norm.bias": "model-00001-of-00004.safetensors", + "model.visual.deepstack_merger_list.0.norm.weight": "model-00002-of-00004.safetensors", + "model.visual.deepstack_merger_list.1.linear_fc1.bias": "model-00003-of-00004.safetensors", + "model.visual.deepstack_merger_list.1.linear_fc1.weight": "model-00001-of-00004.safetensors", + "model.visual.deepstack_merger_list.1.linear_fc2.bias": "model-00003-of-00004.safetensors", + "model.visual.deepstack_merger_list.1.linear_fc2.weight": "model-00004-of-00004.safetensors", + "model.visual.deepstack_merger_list.1.norm.bias": "model-00004-of-00004.safetensors", + "model.visual.deepstack_merger_list.1.norm.weight": "model-00003-of-00004.safetensors", + "model.visual.deepstack_merger_list.2.linear_fc1.bias": "model-00002-of-00004.safetensors", + "model.visual.deepstack_merger_list.2.linear_fc1.weight": "model-00004-of-00004.safetensors", + "model.visual.deepstack_merger_list.2.linear_fc2.bias": "model-00002-of-00004.safetensors", + "model.visual.deepstack_merger_list.2.linear_fc2.weight": "model-00001-of-00004.safetensors", + "model.visual.deepstack_merger_list.2.norm.bias": "model-00003-of-00004.safetensors", + "model.visual.deepstack_merger_list.2.norm.weight": "model-00004-of-00004.safetensors", + "model.visual.merger.linear_fc1.bias": "model-00003-of-00004.safetensors", + "model.visual.merger.linear_fc1.weight": "model-00002-of-00004.safetensors", + "model.visual.merger.linear_fc2.bias": "model-00001-of-00004.safetensors", + "model.visual.merger.linear_fc2.weight": "model-00002-of-00004.safetensors", + "model.visual.merger.norm.bias": "model-00004-of-00004.safetensors", + "model.visual.merger.norm.weight": "model-00001-of-00004.safetensors", + "model.visual.patch_embed.proj.bias": "model-00004-of-00004.safetensors", + "model.visual.patch_embed.proj.weight": "model-00002-of-00004.safetensors", + "model.visual.pos_embed.weight": "model-00001-of-00004.safetensors" + } +} diff --git a/posttrain_models/8b_cold_start_sft_46k_multi_32k_timelens_1epoch_grpo_format_tiou_f1_cacc_multi_10k/global_step_300/processor_config.json b/posttrain_models/8b_cold_start_sft_46k_multi_32k_timelens_1epoch_grpo_format_tiou_f1_cacc_multi_10k/global_step_300/processor_config.json new file mode 100644 index 0000000000000000000000000000000000000000..a4fe483a685ede297970ea7a13145d5aabe63a1c --- /dev/null +++ b/posttrain_models/8b_cold_start_sft_46k_multi_32k_timelens_1epoch_grpo_format_tiou_f1_cacc_multi_10k/global_step_300/processor_config.json @@ -0,0 +1,85 @@ +{ + "image_processor": { + "crop_size": null, + "data_format": "channels_first", + "device": null, + "disable_grouping": null, + "do_center_crop": null, + "do_convert_rgb": true, + "do_normalize": true, + "do_pad": null, + "do_rescale": true, + "do_resize": true, + "image_mean": [ + 0.5, + 0.5, + 0.5 + ], + "image_processor_type": "Qwen2VLImageProcessorFast", + "image_seq_length": null, + "image_std": [ + 0.5, + 0.5, + 0.5 + ], + "input_data_format": null, + "max_pixels": null, + "merge_size": 2, + "min_pixels": null, + "pad_size": null, + "patch_size": 16, + "processor_class": "Qwen3VLProcessor", + "resample": 3, + "rescale_factor": 0.00392156862745098, + "return_tensors": null, + "size": { + "longest_edge": 16777216, + "shortest_edge": 65536 + }, + "temporal_patch_size": 2 + }, + "processor_class": "Qwen3VLProcessor", + "video_processor": { + "crop_size": null, + "data_format": "channels_first", + "default_to_square": true, + "device": null, + "do_center_crop": null, + "do_convert_rgb": true, + "do_normalize": true, + "do_pad": null, + "do_rescale": true, + "do_resize": true, + "do_sample_frames": true, + "fps": 2, + "image_mean": [ + 0.5, + 0.5, + 0.5 + ], + "image_std": [ + 0.5, + 0.5, + 0.5 + ], + "input_data_format": null, + "max_frames": 768, + "merge_size": 2, + "min_frames": 4, + "num_frames": null, + "pad_size": null, + "patch_size": 16, + "processor_class": "Qwen3VLProcessor", + "resample": 3, + "rescale_factor": 0.00392156862745098, + "return_metadata": false, + "return_tensors": null, + "size": { + "longest_edge": 25165824, + "shortest_edge": 4096 + }, + "temporal_patch_size": 2, + "video_metadata": null, + "video_processor_type": "Qwen3VLVideoProcessor" + } +} diff --git a/posttrain_models/8b_cold_start_sft_46k_multi_32k_timelens_1epoch_grpo_format_tiou_f1_cacc_multi_10k/global_step_300/special_tokens_map.json b/posttrain_models/8b_cold_start_sft_46k_multi_32k_timelens_1epoch_grpo_format_tiou_f1_cacc_multi_10k/global_step_300/special_tokens_map.json new file mode 100644 index 0000000000000000000000000000000000000000..ac23c0aaa2434523c494330aeb79c58395378103 --- /dev/null +++ b/posttrain_models/8b_cold_start_sft_46k_multi_32k_timelens_1epoch_grpo_format_tiou_f1_cacc_multi_10k/global_step_300/special_tokens_map.json @@ -0,0 +1,31 @@ +{ + "additional_special_tokens": [ + "<|im_start|>", + "<|im_end|>", + "<|object_ref_start|>", + "<|object_ref_end|>", + "<|box_start|>", + "<|box_end|>", + "<|quad_start|>", + "<|quad_end|>", + "<|vision_start|>", + "<|vision_end|>", + "<|vision_pad|>", + "<|image_pad|>", + "<|video_pad|>" + ], + "eos_token": { + "content": "<|im_end|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false + }, + "pad_token": { + "content": "<|endoftext|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false + } +} diff --git a/posttrain_models/8b_cold_start_sft_46k_multi_32k_timelens_1epoch_grpo_format_tiou_f1_cacc_multi_10k/global_step_300/tokenizer_config.json b/posttrain_models/8b_cold_start_sft_46k_multi_32k_timelens_1epoch_grpo_format_tiou_f1_cacc_multi_10k/global_step_300/tokenizer_config.json new file mode 100644 index 0000000000000000000000000000000000000000..fec7f182135c384a08ea2fc9635566ca79c45afc --- /dev/null +++ b/posttrain_models/8b_cold_start_sft_46k_multi_32k_timelens_1epoch_grpo_format_tiou_f1_cacc_multi_10k/global_step_300/tokenizer_config.json @@ -0,0 +1,240 @@ +{ + "add_bos_token": false, + "add_prefix_space": false, + "added_tokens_decoder": { + "151643": { + "content": "<|endoftext|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "151644": { + "content": "<|im_start|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "151645": { + "content": "<|im_end|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "151646": { + "content": "<|object_ref_start|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "151647": { + "content": "<|object_ref_end|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "151648": { + "content": "<|box_start|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "151649": { + "content": "<|box_end|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "151650": { + "content": "<|quad_start|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "151651": { + "content": "<|quad_end|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "151652": { + "content": "<|vision_start|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "151653": { + "content": "<|vision_end|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "151654": { + "content": "<|vision_pad|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "151655": { + "content": "<|image_pad|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "151656": { + "content": "<|video_pad|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "151657": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "151658": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "151659": { + "content": "<|fim_prefix|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "151660": { + "content": "<|fim_middle|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "151661": { + "content": "<|fim_suffix|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "151662": { + "content": "<|fim_pad|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "151663": { + "content": "<|repo_name|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "151664": { + "content": "<|file_sep|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "151665": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "151666": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "151667": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "151668": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + } + }, + "additional_special_tokens": [ + "<|im_start|>", + "<|im_end|>", + "<|object_ref_start|>", + "<|object_ref_end|>", + "<|box_start|>", + "<|box_end|>", + "<|quad_start|>", + "<|quad_end|>", + "<|vision_start|>", + "<|vision_end|>", + "<|vision_pad|>", + "<|image_pad|>", + "<|video_pad|>" + ], + "bos_token": null, + "clean_up_tokenization_spaces": false, + "eos_token": "<|im_end|>", + "errors": "replace", + "extra_special_tokens": {}, + "model_max_length": 262144, + "pad_token": "<|endoftext|>", + "processor_class": "Qwen3VLProcessor", + "split_special_tokens": false, + "tokenizer_class": "Qwen2Tokenizer", + "unk_token": null +} diff --git a/posttrain_models/Qwen3-4B-Instruct-2507/.gitattributes b/posttrain_models/Qwen3-4B-Instruct-2507/.gitattributes new file mode 100644 index 0000000000000000000000000000000000000000..52373fe24473b1aa44333d318f578ae6bf04b49b --- /dev/null +++ b/posttrain_models/Qwen3-4B-Instruct-2507/.gitattributes @@ -0,0 +1,36 @@ +*.7z filter=lfs diff=lfs merge=lfs -text +*.arrow filter=lfs diff=lfs merge=lfs -text +*.bin filter=lfs diff=lfs merge=lfs -text +*.bz2 filter=lfs diff=lfs merge=lfs -text +*.ckpt filter=lfs diff=lfs merge=lfs -text +*.ftz filter=lfs diff=lfs merge=lfs -text +*.gz filter=lfs diff=lfs merge=lfs -text +*.h5 filter=lfs diff=lfs merge=lfs -text +*.joblib filter=lfs diff=lfs merge=lfs -text +*.lfs.* filter=lfs diff=lfs merge=lfs -text +*.mlmodel filter=lfs diff=lfs merge=lfs -text +*.model filter=lfs diff=lfs merge=lfs -text +*.msgpack filter=lfs diff=lfs merge=lfs -text +*.npy filter=lfs diff=lfs merge=lfs -text +*.npz filter=lfs diff=lfs merge=lfs -text +*.onnx filter=lfs diff=lfs merge=lfs -text +*.ot filter=lfs diff=lfs merge=lfs -text +*.parquet filter=lfs diff=lfs merge=lfs -text +*.pb filter=lfs diff=lfs merge=lfs -text +*.pickle filter=lfs diff=lfs merge=lfs -text +*.pkl filter=lfs diff=lfs merge=lfs -text +*.pt filter=lfs diff=lfs merge=lfs -text +*.pth filter=lfs diff=lfs merge=lfs -text +*.rar filter=lfs diff=lfs merge=lfs -text +*.safetensors filter=lfs diff=lfs merge=lfs -text +saved_model/**/* filter=lfs diff=lfs merge=lfs -text +*.tar.* filter=lfs diff=lfs merge=lfs -text +*.tar filter=lfs diff=lfs merge=lfs -text +*.tflite filter=lfs diff=lfs merge=lfs -text +*.tgz filter=lfs diff=lfs merge=lfs -text +*.wasm filter=lfs diff=lfs merge=lfs -text +*.xz filter=lfs diff=lfs merge=lfs -text +*.zip filter=lfs diff=lfs merge=lfs -text +*.zst filter=lfs diff=lfs merge=lfs -text +*tfevents* filter=lfs diff=lfs merge=lfs -text +tokenizer.json filter=lfs diff=lfs merge=lfs -text diff --git a/posttrain_models/Qwen3-4B-Instruct-2507/LICENSE b/posttrain_models/Qwen3-4B-Instruct-2507/LICENSE new file mode 100644 index 0000000000000000000000000000000000000000..6634c8cc3133b3848ec74b9f275acaaa1ea618ab --- /dev/null +++ b/posttrain_models/Qwen3-4B-Instruct-2507/LICENSE @@ -0,0 +1,202 @@ + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright 2024 Alibaba Cloud + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. \ No newline at end of file diff --git a/posttrain_models/Qwen3-4B-Instruct-2507/README.md b/posttrain_models/Qwen3-4B-Instruct-2507/README.md new file mode 100644 index 0000000000000000000000000000000000000000..b85879f8e465b5b02bd7ea13dd7ffaf22816bf59 --- /dev/null +++ b/posttrain_models/Qwen3-4B-Instruct-2507/README.md @@ -0,0 +1,211 @@ +--- +library_name: transformers +license: apache-2.0 +license_link: https://huggingface.co/Qwen/Qwen3-4B-Instruct-2507/blob/main/LICENSE +pipeline_tag: text-generation +--- + +# Qwen3-4B-Instruct-2507 + + Chat + + +## Highlights + +We introduce the updated version of the **Qwen3-4B non-thinking mode**, named **Qwen3-4B-Instruct-2507**, featuring the following key enhancements: + +- **Significant improvements** in general capabilities, including **instruction following, logical reasoning, text comprehension, mathematics, science, coding and tool usage**. +- **Substantial gains** in long-tail knowledge coverage across **multiple languages**. +- **Markedly better alignment** with user preferences in **subjective and open-ended tasks**, enabling more helpful responses and higher-quality text generation. +- **Enhanced capabilities** in **256K long-context understanding**. + +![image/jpeg](https://qianwen-res.oss-accelerate.aliyuncs.com/Qwen3-2507/Qwen3-4B-Instruct.001.jpeg) + +## Model Overview + +**Qwen3-4B-Instruct-2507** has the following features: +- Type: Causal Language Models +- Training Stage: Pretraining & Post-training +- Number of Parameters: 4.0B +- Number of Paramaters (Non-Embedding): 3.6B +- Number of Layers: 36 +- Number of Attention Heads (GQA): 32 for Q and 8 for KV +- Context Length: **262,144 natively**. + +**NOTE: This model supports only non-thinking mode and does not generate ```` blocks in its output. Meanwhile, specifying `enable_thinking=False` is no longer required.** + +For more details, including benchmark evaluation, hardware requirements, and inference performance, please refer to our [blog](https://qwenlm.github.io/blog/qwen3/), [GitHub](https://github.com/QwenLM/Qwen3), and [Documentation](https://qwen.readthedocs.io/en/latest/). + + +## Performance + +| | GPT-4.1-nano-2025-04-14 | Qwen3-30B-A3B Non-Thinking | Qwen3-4B Non-Thinking | Qwen3-4B-Instruct-2507 | +|--- | --- | --- | --- | --- | +| **Knowledge** | | | | +| MMLU-Pro | 62.8 | 69.1 | 58.0 | **69.6** | +| MMLU-Redux | 80.2 | 84.1 | 77.3 | **84.2** | +| GPQA | 50.3 | 54.8 | 41.7 | **62.0** | +| SuperGPQA | 32.2 | 42.2 | 32.0 | **42.8** | +| **Reasoning** | | | | +| AIME25 | 22.7 | 21.6 | 19.1 | **47.4** | +| HMMT25 | 9.7 | 12.0 | 12.1 | **31.0** | +| ZebraLogic | 14.8 | 33.2 | 35.2 | **80.2** | +| LiveBench 20241125 | 41.5 | 59.4 | 48.4 | **63.0** | +| **Coding** | | | | +| LiveCodeBench v6 (25.02-25.05) | 31.5 | 29.0 | 26.4 | **35.1** | +| MultiPL-E | 76.3 | 74.6 | 66.6 | **76.8** | +| Aider-Polyglot | 9.8 | **24.4** | 13.8 | 12.9 | +| **Alignment** | | | | +| IFEval | 74.5 | **83.7** | 81.2 | 83.4 | +| Arena-Hard v2* | 15.9 | 24.8 | 9.5 | **43.4** | +| Creative Writing v3 | 72.7 | 68.1 | 53.6 | **83.5** | +| WritingBench | 66.9 | 72.2 | 68.5 | **83.4** | +| **Agent** | | | | +| BFCL-v3 | 53.0 | 58.6 | 57.6 | **61.9** | +| TAU1-Retail | 23.5 | 38.3 | 24.3 | **48.7** | +| TAU1-Airline | 14.0 | 18.0 | 16.0 | **32.0** | +| TAU2-Retail | - | 31.6 | 28.1 | **40.4** | +| TAU2-Airline | - | 18.0 | 12.0 | **24.0** | +| TAU2-Telecom | - | **18.4** | 17.5 | 13.2 | +| **Multilingualism** | | | | +| MultiIF | 60.7 | **70.8** | 61.3 | 69.0 | +| MMLU-ProX | 56.2 | **65.1** | 49.6 | 61.6 | +| INCLUDE | 58.6 | **67.8** | 53.8 | 60.1 | +| PolyMATH | 15.6 | 23.3 | 16.6 | **31.1** | + +*: For reproducibility, we report the win rates evaluated by GPT-4.1. + + +## Quickstart + +The code of Qwen3 has been in the latest Hugging Face `transformers` and we advise you to use the latest version of `transformers`. + +With `transformers<4.51.0`, you will encounter the following error: +``` +KeyError: 'qwen3' +``` + +The following contains a code snippet illustrating how to use the model generate content based on given inputs. +```python +from transformers import AutoModelForCausalLM, AutoTokenizer + +model_name = "Qwen/Qwen3-4B-Instruct-2507" + +# load the tokenizer and the model +tokenizer = AutoTokenizer.from_pretrained(model_name) +model = AutoModelForCausalLM.from_pretrained( + model_name, + torch_dtype="auto", + device_map="auto" +) + +# prepare the model input +prompt = "Give me a short introduction to large language model." +messages = [ + {"role": "user", "content": prompt} +] +text = tokenizer.apply_chat_template( + messages, + tokenize=False, + add_generation_prompt=True, +) +model_inputs = tokenizer([text], return_tensors="pt").to(model.device) + +# conduct text completion +generated_ids = model.generate( + **model_inputs, + max_new_tokens=16384 +) +output_ids = generated_ids[0][len(model_inputs.input_ids[0]):].tolist() + +content = tokenizer.decode(output_ids, skip_special_tokens=True) + +print("content:", content) +``` + +For deployment, you can use `sglang>=0.4.6.post1` or `vllm>=0.8.5` or to create an OpenAI-compatible API endpoint: +- SGLang: + ```shell + python -m sglang.launch_server --model-path Qwen/Qwen3-4B-Instruct-2507 --context-length 262144 + ``` +- vLLM: + ```shell + vllm serve Qwen/Qwen3-4B-Instruct-2507 --max-model-len 262144 + ``` + +**Note: If you encounter out-of-memory (OOM) issues, consider reducing the context length to a shorter value, such as `32,768`.** + +For local use, applications such as Ollama, LMStudio, MLX-LM, llama.cpp, and KTransformers have also supported Qwen3. + +## Agentic Use + +Qwen3 excels in tool calling capabilities. We recommend using [Qwen-Agent](https://github.com/QwenLM/Qwen-Agent) to make the best use of agentic ability of Qwen3. Qwen-Agent encapsulates tool-calling templates and tool-calling parsers internally, greatly reducing coding complexity. + +To define the available tools, you can use the MCP configuration file, use the integrated tool of Qwen-Agent, or integrate other tools by yourself. +```python +from qwen_agent.agents import Assistant + +# Define LLM +llm_cfg = { + 'model': 'Qwen3-4B-Instruct-2507', + + # Use a custom endpoint compatible with OpenAI API: + 'model_server': 'http://localhost:8000/v1', # api_base + 'api_key': 'EMPTY', +} + +# Define Tools +tools = [ + {'mcpServers': { # You can specify the MCP configuration file + 'time': { + 'command': 'uvx', + 'args': ['mcp-server-time', '--local-timezone=Asia/Shanghai'] + }, + "fetch": { + "command": "uvx", + "args": ["mcp-server-fetch"] + } + } + }, + 'code_interpreter', # Built-in tools +] + +# Define Agent +bot = Assistant(llm=llm_cfg, function_list=tools) + +# Streaming generation +messages = [{'role': 'user', 'content': 'https://qwenlm.github.io/blog/ Introduce the latest developments of Qwen'}] +for responses in bot.run(messages=messages): + pass +print(responses) +``` + +## Best Practices + +To achieve optimal performance, we recommend the following settings: + +1. **Sampling Parameters**: + - We suggest using `Temperature=0.7`, `TopP=0.8`, `TopK=20`, and `MinP=0`. + - For supported frameworks, you can adjust the `presence_penalty` parameter between 0 and 2 to reduce endless repetitions. However, using a higher value may occasionally result in language mixing and a slight decrease in model performance. + +2. **Adequate Output Length**: We recommend using an output length of 16,384 tokens for most queries, which is adequate for instruct models. + +3. **Standardize Output Format**: We recommend using prompts to standardize model outputs when benchmarking. + - **Math Problems**: Include "Please reason step by step, and put your final answer within \boxed{}." in the prompt. + - **Multiple-Choice Questions**: Add the following JSON structure to the prompt to standardize responses: "Please show your choice in the `answer` field with only the choice letter, e.g., `"answer": "C"`." + +### Citation + +If you find our work helpful, feel free to give us a cite. + +``` +@misc{qwen3technicalreport, + title={Qwen3 Technical Report}, + author={Qwen Team}, + year={2025}, + eprint={2505.09388}, + archivePrefix={arXiv}, + primaryClass={cs.CL}, + url={https://arxiv.org/abs/2505.09388}, +} +``` \ No newline at end of file diff --git a/posttrain_models/Qwen3-4B-Instruct-2507/config.json b/posttrain_models/Qwen3-4B-Instruct-2507/config.json new file mode 100644 index 0000000000000000000000000000000000000000..6988f134db143052042f2bd6e0c897bc6a605189 --- /dev/null +++ b/posttrain_models/Qwen3-4B-Instruct-2507/config.json @@ -0,0 +1,30 @@ +{ + "architectures": [ + "Qwen3ForCausalLM" + ], + "attention_bias": false, + "attention_dropout": 0.0, + "bos_token_id": 151643, + "eos_token_id": 151645, + "head_dim": 128, + "hidden_act": "silu", + "hidden_size": 2560, + "initializer_range": 0.02, + "intermediate_size": 9728, + "max_position_embeddings": 262144, + "max_window_layers": 36, + "model_type": "qwen3", + "num_attention_heads": 32, + "num_hidden_layers": 36, + "num_key_value_heads": 8, + "rms_norm_eps": 1e-06, + "rope_scaling": null, + "rope_theta": 5000000, + "sliding_window": null, + "tie_word_embeddings": true, + "torch_dtype": "bfloat16", + "transformers_version": "4.51.0", + "use_cache": true, + "use_sliding_window": false, + "vocab_size": 151936 +} \ No newline at end of file diff --git a/posttrain_models/Qwen3-4B-Instruct-2507/generation_config.json b/posttrain_models/Qwen3-4B-Instruct-2507/generation_config.json new file mode 100644 index 0000000000000000000000000000000000000000..432531a002c181a19de338313d2375e9d7494d7e --- /dev/null +++ b/posttrain_models/Qwen3-4B-Instruct-2507/generation_config.json @@ -0,0 +1,13 @@ +{ + "bos_token_id": 151643, + "do_sample": true, + "eos_token_id": [ + 151645, + 151643 + ], + "pad_token_id": 151643, + "temperature": 0.7, + "top_k": 20, + "top_p": 0.8, + "transformers_version": "4.51.0" +} \ No newline at end of file diff --git a/posttrain_models/Qwen3-4B-Instruct-2507/model.safetensors.index.json b/posttrain_models/Qwen3-4B-Instruct-2507/model.safetensors.index.json new file mode 100644 index 0000000000000000000000000000000000000000..4747b0297d3109f14db49886972e3369c9a00b2a --- /dev/null +++ b/posttrain_models/Qwen3-4B-Instruct-2507/model.safetensors.index.json @@ -0,0 +1,405 @@ +{ + "metadata": { + "total_size": 8045591552 + }, + "weight_map": { + "model.embed_tokens.weight": "model-00001-of-00003.safetensors", + "model.layers.0.input_layernorm.weight": "model-00001-of-00003.safetensors", + "model.layers.0.mlp.down_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.0.mlp.gate_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.0.mlp.up_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.0.post_attention_layernorm.weight": "model-00001-of-00003.safetensors", + "model.layers.0.self_attn.k_norm.weight": "model-00001-of-00003.safetensors", + "model.layers.0.self_attn.k_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.0.self_attn.o_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.0.self_attn.q_norm.weight": "model-00001-of-00003.safetensors", + "model.layers.0.self_attn.q_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.0.self_attn.v_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.1.input_layernorm.weight": "model-00001-of-00003.safetensors", + "model.layers.1.mlp.down_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.1.mlp.gate_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.1.mlp.up_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.1.post_attention_layernorm.weight": "model-00001-of-00003.safetensors", + "model.layers.1.self_attn.k_norm.weight": "model-00001-of-00003.safetensors", + "model.layers.1.self_attn.k_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.1.self_attn.o_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.1.self_attn.q_norm.weight": "model-00001-of-00003.safetensors", + "model.layers.1.self_attn.q_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.1.self_attn.v_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.10.input_layernorm.weight": "model-00001-of-00003.safetensors", + "model.layers.10.mlp.down_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.10.mlp.gate_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.10.mlp.up_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.10.post_attention_layernorm.weight": "model-00001-of-00003.safetensors", + "model.layers.10.self_attn.k_norm.weight": "model-00001-of-00003.safetensors", + "model.layers.10.self_attn.k_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.10.self_attn.o_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.10.self_attn.q_norm.weight": "model-00001-of-00003.safetensors", + "model.layers.10.self_attn.q_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.10.self_attn.v_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.11.input_layernorm.weight": "model-00001-of-00003.safetensors", + "model.layers.11.mlp.down_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.11.mlp.gate_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.11.mlp.up_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.11.post_attention_layernorm.weight": "model-00001-of-00003.safetensors", + "model.layers.11.self_attn.k_norm.weight": "model-00001-of-00003.safetensors", + "model.layers.11.self_attn.k_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.11.self_attn.o_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.11.self_attn.q_norm.weight": "model-00001-of-00003.safetensors", + "model.layers.11.self_attn.q_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.11.self_attn.v_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.12.input_layernorm.weight": "model-00001-of-00003.safetensors", + "model.layers.12.mlp.down_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.12.mlp.gate_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.12.mlp.up_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.12.post_attention_layernorm.weight": "model-00001-of-00003.safetensors", + "model.layers.12.self_attn.k_norm.weight": "model-00001-of-00003.safetensors", + "model.layers.12.self_attn.k_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.12.self_attn.o_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.12.self_attn.q_norm.weight": "model-00001-of-00003.safetensors", + "model.layers.12.self_attn.q_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.12.self_attn.v_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.13.input_layernorm.weight": "model-00001-of-00003.safetensors", + "model.layers.13.mlp.down_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.13.mlp.gate_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.13.mlp.up_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.13.post_attention_layernorm.weight": "model-00001-of-00003.safetensors", + "model.layers.13.self_attn.k_norm.weight": "model-00001-of-00003.safetensors", + "model.layers.13.self_attn.k_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.13.self_attn.o_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.13.self_attn.q_norm.weight": "model-00001-of-00003.safetensors", + "model.layers.13.self_attn.q_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.13.self_attn.v_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.14.input_layernorm.weight": "model-00001-of-00003.safetensors", + "model.layers.14.mlp.down_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.14.mlp.gate_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.14.mlp.up_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.14.post_attention_layernorm.weight": "model-00001-of-00003.safetensors", + "model.layers.14.self_attn.k_norm.weight": "model-00001-of-00003.safetensors", + "model.layers.14.self_attn.k_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.14.self_attn.o_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.14.self_attn.q_norm.weight": "model-00001-of-00003.safetensors", + "model.layers.14.self_attn.q_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.14.self_attn.v_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.15.input_layernorm.weight": "model-00002-of-00003.safetensors", + "model.layers.15.mlp.down_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.15.mlp.gate_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.15.mlp.up_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.15.post_attention_layernorm.weight": "model-00002-of-00003.safetensors", + "model.layers.15.self_attn.k_norm.weight": "model-00001-of-00003.safetensors", + "model.layers.15.self_attn.k_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.15.self_attn.o_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.15.self_attn.q_norm.weight": "model-00001-of-00003.safetensors", + "model.layers.15.self_attn.q_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.15.self_attn.v_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.16.input_layernorm.weight": "model-00002-of-00003.safetensors", + "model.layers.16.mlp.down_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.16.mlp.gate_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.16.mlp.up_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.16.post_attention_layernorm.weight": "model-00002-of-00003.safetensors", + "model.layers.16.self_attn.k_norm.weight": "model-00002-of-00003.safetensors", + "model.layers.16.self_attn.k_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.16.self_attn.o_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.16.self_attn.q_norm.weight": "model-00002-of-00003.safetensors", + "model.layers.16.self_attn.q_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.16.self_attn.v_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.17.input_layernorm.weight": "model-00002-of-00003.safetensors", + "model.layers.17.mlp.down_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.17.mlp.gate_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.17.mlp.up_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.17.post_attention_layernorm.weight": "model-00002-of-00003.safetensors", + "model.layers.17.self_attn.k_norm.weight": "model-00002-of-00003.safetensors", + "model.layers.17.self_attn.k_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.17.self_attn.o_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.17.self_attn.q_norm.weight": "model-00002-of-00003.safetensors", + "model.layers.17.self_attn.q_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.17.self_attn.v_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.18.input_layernorm.weight": "model-00002-of-00003.safetensors", + "model.layers.18.mlp.down_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.18.mlp.gate_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.18.mlp.up_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.18.post_attention_layernorm.weight": "model-00002-of-00003.safetensors", + "model.layers.18.self_attn.k_norm.weight": "model-00002-of-00003.safetensors", + "model.layers.18.self_attn.k_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.18.self_attn.o_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.18.self_attn.q_norm.weight": "model-00002-of-00003.safetensors", + "model.layers.18.self_attn.q_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.18.self_attn.v_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.19.input_layernorm.weight": "model-00002-of-00003.safetensors", + "model.layers.19.mlp.down_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.19.mlp.gate_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.19.mlp.up_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.19.post_attention_layernorm.weight": "model-00002-of-00003.safetensors", + "model.layers.19.self_attn.k_norm.weight": "model-00002-of-00003.safetensors", + "model.layers.19.self_attn.k_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.19.self_attn.o_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.19.self_attn.q_norm.weight": "model-00002-of-00003.safetensors", + "model.layers.19.self_attn.q_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.19.self_attn.v_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.2.input_layernorm.weight": "model-00001-of-00003.safetensors", + "model.layers.2.mlp.down_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.2.mlp.gate_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.2.mlp.up_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.2.post_attention_layernorm.weight": "model-00001-of-00003.safetensors", + "model.layers.2.self_attn.k_norm.weight": "model-00001-of-00003.safetensors", + "model.layers.2.self_attn.k_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.2.self_attn.o_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.2.self_attn.q_norm.weight": "model-00001-of-00003.safetensors", + "model.layers.2.self_attn.q_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.2.self_attn.v_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.20.input_layernorm.weight": "model-00002-of-00003.safetensors", + "model.layers.20.mlp.down_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.20.mlp.gate_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.20.mlp.up_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.20.post_attention_layernorm.weight": "model-00002-of-00003.safetensors", + "model.layers.20.self_attn.k_norm.weight": "model-00002-of-00003.safetensors", + "model.layers.20.self_attn.k_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.20.self_attn.o_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.20.self_attn.q_norm.weight": "model-00002-of-00003.safetensors", + "model.layers.20.self_attn.q_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.20.self_attn.v_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.21.input_layernorm.weight": "model-00002-of-00003.safetensors", + "model.layers.21.mlp.down_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.21.mlp.gate_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.21.mlp.up_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.21.post_attention_layernorm.weight": "model-00002-of-00003.safetensors", + "model.layers.21.self_attn.k_norm.weight": "model-00002-of-00003.safetensors", + "model.layers.21.self_attn.k_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.21.self_attn.o_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.21.self_attn.q_norm.weight": "model-00002-of-00003.safetensors", + "model.layers.21.self_attn.q_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.21.self_attn.v_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.22.input_layernorm.weight": "model-00002-of-00003.safetensors", + "model.layers.22.mlp.down_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.22.mlp.gate_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.22.mlp.up_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.22.post_attention_layernorm.weight": "model-00002-of-00003.safetensors", + "model.layers.22.self_attn.k_norm.weight": "model-00002-of-00003.safetensors", + "model.layers.22.self_attn.k_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.22.self_attn.o_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.22.self_attn.q_norm.weight": "model-00002-of-00003.safetensors", + "model.layers.22.self_attn.q_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.22.self_attn.v_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.23.input_layernorm.weight": "model-00002-of-00003.safetensors", + "model.layers.23.mlp.down_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.23.mlp.gate_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.23.mlp.up_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.23.post_attention_layernorm.weight": "model-00002-of-00003.safetensors", + "model.layers.23.self_attn.k_norm.weight": "model-00002-of-00003.safetensors", + "model.layers.23.self_attn.k_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.23.self_attn.o_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.23.self_attn.q_norm.weight": "model-00002-of-00003.safetensors", + "model.layers.23.self_attn.q_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.23.self_attn.v_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.24.input_layernorm.weight": "model-00002-of-00003.safetensors", + "model.layers.24.mlp.down_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.24.mlp.gate_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.24.mlp.up_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.24.post_attention_layernorm.weight": "model-00002-of-00003.safetensors", + "model.layers.24.self_attn.k_norm.weight": "model-00002-of-00003.safetensors", + "model.layers.24.self_attn.k_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.24.self_attn.o_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.24.self_attn.q_norm.weight": "model-00002-of-00003.safetensors", + "model.layers.24.self_attn.q_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.24.self_attn.v_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.25.input_layernorm.weight": "model-00002-of-00003.safetensors", + "model.layers.25.mlp.down_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.25.mlp.gate_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.25.mlp.up_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.25.post_attention_layernorm.weight": "model-00002-of-00003.safetensors", + "model.layers.25.self_attn.k_norm.weight": "model-00002-of-00003.safetensors", + "model.layers.25.self_attn.k_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.25.self_attn.o_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.25.self_attn.q_norm.weight": "model-00002-of-00003.safetensors", + "model.layers.25.self_attn.q_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.25.self_attn.v_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.26.input_layernorm.weight": "model-00002-of-00003.safetensors", + "model.layers.26.mlp.down_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.26.mlp.gate_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.26.mlp.up_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.26.post_attention_layernorm.weight": "model-00002-of-00003.safetensors", + "model.layers.26.self_attn.k_norm.weight": "model-00002-of-00003.safetensors", + "model.layers.26.self_attn.k_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.26.self_attn.o_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.26.self_attn.q_norm.weight": "model-00002-of-00003.safetensors", + "model.layers.26.self_attn.q_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.26.self_attn.v_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.27.input_layernorm.weight": "model-00002-of-00003.safetensors", + "model.layers.27.mlp.down_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.27.mlp.gate_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.27.mlp.up_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.27.post_attention_layernorm.weight": "model-00002-of-00003.safetensors", + "model.layers.27.self_attn.k_norm.weight": "model-00002-of-00003.safetensors", + "model.layers.27.self_attn.k_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.27.self_attn.o_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.27.self_attn.q_norm.weight": "model-00002-of-00003.safetensors", + "model.layers.27.self_attn.q_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.27.self_attn.v_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.28.input_layernorm.weight": "model-00002-of-00003.safetensors", + "model.layers.28.mlp.down_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.28.mlp.gate_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.28.mlp.up_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.28.post_attention_layernorm.weight": "model-00002-of-00003.safetensors", + "model.layers.28.self_attn.k_norm.weight": "model-00002-of-00003.safetensors", + "model.layers.28.self_attn.k_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.28.self_attn.o_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.28.self_attn.q_norm.weight": "model-00002-of-00003.safetensors", + "model.layers.28.self_attn.q_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.28.self_attn.v_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.29.input_layernorm.weight": "model-00002-of-00003.safetensors", + "model.layers.29.mlp.down_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.29.mlp.gate_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.29.mlp.up_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.29.post_attention_layernorm.weight": "model-00002-of-00003.safetensors", + "model.layers.29.self_attn.k_norm.weight": "model-00002-of-00003.safetensors", + "model.layers.29.self_attn.k_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.29.self_attn.o_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.29.self_attn.q_norm.weight": "model-00002-of-00003.safetensors", + "model.layers.29.self_attn.q_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.29.self_attn.v_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.3.input_layernorm.weight": "model-00001-of-00003.safetensors", + "model.layers.3.mlp.down_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.3.mlp.gate_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.3.mlp.up_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.3.post_attention_layernorm.weight": "model-00001-of-00003.safetensors", + "model.layers.3.self_attn.k_norm.weight": "model-00001-of-00003.safetensors", + "model.layers.3.self_attn.k_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.3.self_attn.o_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.3.self_attn.q_norm.weight": "model-00001-of-00003.safetensors", + "model.layers.3.self_attn.q_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.3.self_attn.v_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.30.input_layernorm.weight": "model-00002-of-00003.safetensors", + "model.layers.30.mlp.down_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.30.mlp.gate_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.30.mlp.up_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.30.post_attention_layernorm.weight": "model-00002-of-00003.safetensors", + "model.layers.30.self_attn.k_norm.weight": "model-00002-of-00003.safetensors", + "model.layers.30.self_attn.k_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.30.self_attn.o_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.30.self_attn.q_norm.weight": "model-00002-of-00003.safetensors", + "model.layers.30.self_attn.q_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.30.self_attn.v_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.31.input_layernorm.weight": "model-00002-of-00003.safetensors", + "model.layers.31.mlp.down_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.31.mlp.gate_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.31.mlp.up_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.31.post_attention_layernorm.weight": "model-00002-of-00003.safetensors", + "model.layers.31.self_attn.k_norm.weight": "model-00002-of-00003.safetensors", + "model.layers.31.self_attn.k_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.31.self_attn.o_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.31.self_attn.q_norm.weight": "model-00002-of-00003.safetensors", + "model.layers.31.self_attn.q_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.31.self_attn.v_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.32.input_layernorm.weight": "model-00002-of-00003.safetensors", + "model.layers.32.mlp.down_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.32.mlp.gate_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.32.mlp.up_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.32.post_attention_layernorm.weight": "model-00002-of-00003.safetensors", + "model.layers.32.self_attn.k_norm.weight": "model-00002-of-00003.safetensors", + "model.layers.32.self_attn.k_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.32.self_attn.o_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.32.self_attn.q_norm.weight": "model-00002-of-00003.safetensors", + "model.layers.32.self_attn.q_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.32.self_attn.v_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.33.input_layernorm.weight": "model-00002-of-00003.safetensors", + "model.layers.33.mlp.down_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.33.mlp.gate_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.33.mlp.up_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.33.post_attention_layernorm.weight": "model-00002-of-00003.safetensors", + "model.layers.33.self_attn.k_norm.weight": "model-00002-of-00003.safetensors", + "model.layers.33.self_attn.k_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.33.self_attn.o_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.33.self_attn.q_norm.weight": "model-00002-of-00003.safetensors", + "model.layers.33.self_attn.q_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.33.self_attn.v_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.34.input_layernorm.weight": "model-00002-of-00003.safetensors", + "model.layers.34.mlp.down_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.34.mlp.gate_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.34.mlp.up_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.34.post_attention_layernorm.weight": "model-00002-of-00003.safetensors", + "model.layers.34.self_attn.k_norm.weight": "model-00002-of-00003.safetensors", + "model.layers.34.self_attn.k_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.34.self_attn.o_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.34.self_attn.q_norm.weight": "model-00002-of-00003.safetensors", + "model.layers.34.self_attn.q_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.34.self_attn.v_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.35.input_layernorm.weight": "model-00003-of-00003.safetensors", + "model.layers.35.mlp.down_proj.weight": "model-00003-of-00003.safetensors", + "model.layers.35.mlp.gate_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.35.mlp.up_proj.weight": "model-00003-of-00003.safetensors", + "model.layers.35.post_attention_layernorm.weight": "model-00003-of-00003.safetensors", + "model.layers.35.self_attn.k_norm.weight": "model-00002-of-00003.safetensors", + "model.layers.35.self_attn.k_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.35.self_attn.o_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.35.self_attn.q_norm.weight": "model-00002-of-00003.safetensors", + "model.layers.35.self_attn.q_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.35.self_attn.v_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.4.input_layernorm.weight": "model-00001-of-00003.safetensors", + "model.layers.4.mlp.down_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.4.mlp.gate_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.4.mlp.up_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.4.post_attention_layernorm.weight": "model-00001-of-00003.safetensors", + "model.layers.4.self_attn.k_norm.weight": "model-00001-of-00003.safetensors", + "model.layers.4.self_attn.k_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.4.self_attn.o_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.4.self_attn.q_norm.weight": "model-00001-of-00003.safetensors", + "model.layers.4.self_attn.q_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.4.self_attn.v_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.5.input_layernorm.weight": "model-00001-of-00003.safetensors", + "model.layers.5.mlp.down_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.5.mlp.gate_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.5.mlp.up_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.5.post_attention_layernorm.weight": "model-00001-of-00003.safetensors", + "model.layers.5.self_attn.k_norm.weight": "model-00001-of-00003.safetensors", + "model.layers.5.self_attn.k_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.5.self_attn.o_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.5.self_attn.q_norm.weight": "model-00001-of-00003.safetensors", + "model.layers.5.self_attn.q_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.5.self_attn.v_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.6.input_layernorm.weight": "model-00001-of-00003.safetensors", + "model.layers.6.mlp.down_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.6.mlp.gate_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.6.mlp.up_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.6.post_attention_layernorm.weight": "model-00001-of-00003.safetensors", + "model.layers.6.self_attn.k_norm.weight": "model-00001-of-00003.safetensors", + "model.layers.6.self_attn.k_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.6.self_attn.o_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.6.self_attn.q_norm.weight": "model-00001-of-00003.safetensors", + "model.layers.6.self_attn.q_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.6.self_attn.v_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.7.input_layernorm.weight": "model-00001-of-00003.safetensors", + "model.layers.7.mlp.down_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.7.mlp.gate_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.7.mlp.up_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.7.post_attention_layernorm.weight": "model-00001-of-00003.safetensors", + "model.layers.7.self_attn.k_norm.weight": "model-00001-of-00003.safetensors", + "model.layers.7.self_attn.k_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.7.self_attn.o_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.7.self_attn.q_norm.weight": "model-00001-of-00003.safetensors", + "model.layers.7.self_attn.q_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.7.self_attn.v_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.8.input_layernorm.weight": "model-00001-of-00003.safetensors", + "model.layers.8.mlp.down_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.8.mlp.gate_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.8.mlp.up_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.8.post_attention_layernorm.weight": "model-00001-of-00003.safetensors", + "model.layers.8.self_attn.k_norm.weight": "model-00001-of-00003.safetensors", + "model.layers.8.self_attn.k_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.8.self_attn.o_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.8.self_attn.q_norm.weight": "model-00001-of-00003.safetensors", + "model.layers.8.self_attn.q_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.8.self_attn.v_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.9.input_layernorm.weight": "model-00001-of-00003.safetensors", + "model.layers.9.mlp.down_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.9.mlp.gate_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.9.mlp.up_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.9.post_attention_layernorm.weight": "model-00001-of-00003.safetensors", + "model.layers.9.self_attn.k_norm.weight": "model-00001-of-00003.safetensors", + "model.layers.9.self_attn.k_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.9.self_attn.o_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.9.self_attn.q_norm.weight": "model-00001-of-00003.safetensors", + "model.layers.9.self_attn.q_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.9.self_attn.v_proj.weight": "model-00001-of-00003.safetensors", + "model.norm.weight": "model-00003-of-00003.safetensors" + } +} diff --git a/posttrain_models/Qwen3-4B-Instruct-2507/tokenizer_config.json b/posttrain_models/Qwen3-4B-Instruct-2507/tokenizer_config.json new file mode 100644 index 0000000000000000000000000000000000000000..51c1be0d9192e7f6e6596de71d0f07d58fbc32ac --- /dev/null +++ b/posttrain_models/Qwen3-4B-Instruct-2507/tokenizer_config.json @@ -0,0 +1,239 @@ +{ + "add_prefix_space": false, + "added_tokens_decoder": { + "151643": { + "content": "<|endoftext|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "151644": { + "content": "<|im_start|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "151645": { + "content": "<|im_end|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "151646": { + "content": "<|object_ref_start|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "151647": { + "content": "<|object_ref_end|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "151648": { + "content": "<|box_start|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "151649": { + "content": "<|box_end|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "151650": { + "content": "<|quad_start|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "151651": { + "content": "<|quad_end|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "151652": { + "content": "<|vision_start|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "151653": { + "content": "<|vision_end|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "151654": { + "content": "<|vision_pad|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "151655": { + "content": "<|image_pad|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "151656": { + "content": "<|video_pad|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "151657": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "151658": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "151659": { + "content": "<|fim_prefix|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "151660": { + "content": "<|fim_middle|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "151661": { + "content": "<|fim_suffix|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "151662": { + "content": "<|fim_pad|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "151663": { + "content": "<|repo_name|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "151664": { + "content": "<|file_sep|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "151665": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "151666": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "151667": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "151668": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + } + }, + "additional_special_tokens": [ + "<|im_start|>", + "<|im_end|>", + "<|object_ref_start|>", + "<|object_ref_end|>", + "<|box_start|>", + "<|box_end|>", + "<|quad_start|>", + "<|quad_end|>", + "<|vision_start|>", + "<|vision_end|>", + "<|vision_pad|>", + "<|image_pad|>", + "<|video_pad|>" + ], + "bos_token": null, + "chat_template": "{%- if tools %}\n {{- '<|im_start|>system\\n' }}\n {%- if messages[0].role == 'system' %}\n {{- messages[0].content + '\\n\\n' }}\n {%- endif %}\n {{- \"# Tools\\n\\nYou may call one or more functions to assist with the user query.\\n\\nYou are provided with function signatures within XML tags:\\n\" }}\n {%- for tool in tools %}\n {{- \"\\n\" }}\n {{- tool | tojson }}\n {%- endfor %}\n {{- \"\\n\\n\\nFor each function call, return a json object with function name and arguments within XML tags:\\n\\n{\\\"name\\\": , \\\"arguments\\\": }\\n<|im_end|>\\n\" }}\n{%- else %}\n {%- if messages[0].role == 'system' %}\n {{- '<|im_start|>system\\n' + messages[0].content + '<|im_end|>\\n' }}\n {%- endif %}\n{%- endif %}\n{%- for message in messages %}\n {%- if message.content is string %}\n {%- set content = message.content %}\n {%- else %}\n {%- set content = '' %}\n {%- endif %}\n {%- if (message.role == \"user\") or (message.role == \"system\" and not loop.first) %}\n {{- '<|im_start|>' + message.role + '\\n' + content + '<|im_end|>' + '\\n' }}\n {%- elif message.role == \"assistant\" %}\n {{- '<|im_start|>' + message.role + '\\n' + content }}\n {%- if message.tool_calls %}\n {%- for tool_call in message.tool_calls %}\n {%- if (loop.first and content) or (not loop.first) %}\n {{- '\\n' }}\n {%- endif %}\n {%- if tool_call.function %}\n {%- set tool_call = tool_call.function %}\n {%- endif %}\n {{- '\\n{\"name\": \"' }}\n {{- tool_call.name }}\n {{- '\", \"arguments\": ' }}\n {%- if tool_call.arguments is string %}\n {{- tool_call.arguments }}\n {%- else %}\n {{- tool_call.arguments | tojson }}\n {%- endif %}\n {{- '}\\n' }}\n {%- endfor %}\n {%- endif %}\n {{- '<|im_end|>\\n' }}\n {%- elif message.role == \"tool\" %}\n {%- if loop.first or (messages[loop.index0 - 1].role != \"tool\") %}\n {{- '<|im_start|>user' }}\n {%- endif %}\n {{- '\\n\\n' }}\n {{- content }}\n {{- '\\n' }}\n {%- if loop.last or (messages[loop.index0 + 1].role != \"tool\") %}\n {{- '<|im_end|>\\n' }}\n {%- endif %}\n {%- endif %}\n{%- endfor %}\n{%- if add_generation_prompt %}\n {{- '<|im_start|>assistant\\n' }}\n{%- endif %}", + "clean_up_tokenization_spaces": false, + "eos_token": "<|im_end|>", + "errors": "replace", + "model_max_length": 1010000, + "pad_token": "<|endoftext|>", + "split_special_tokens": false, + "tokenizer_class": "Qwen2Tokenizer", + "unk_token": null, + "add_bos_token": false +} \ No newline at end of file diff --git a/posttrain_models/Qwen3-VL-2B-Instruct/.gitattributes b/posttrain_models/Qwen3-VL-2B-Instruct/.gitattributes new file mode 100644 index 0000000000000000000000000000000000000000..a6344aac8c09253b3b630fb776ae94478aa0275b --- /dev/null +++ b/posttrain_models/Qwen3-VL-2B-Instruct/.gitattributes @@ -0,0 +1,35 @@ +*.7z filter=lfs diff=lfs merge=lfs -text +*.arrow filter=lfs diff=lfs merge=lfs -text +*.bin filter=lfs diff=lfs merge=lfs -text +*.bz2 filter=lfs diff=lfs merge=lfs -text +*.ckpt filter=lfs diff=lfs merge=lfs -text +*.ftz filter=lfs diff=lfs merge=lfs -text +*.gz filter=lfs diff=lfs merge=lfs -text +*.h5 filter=lfs diff=lfs merge=lfs -text +*.joblib filter=lfs diff=lfs merge=lfs -text +*.lfs.* filter=lfs diff=lfs merge=lfs -text +*.mlmodel filter=lfs diff=lfs merge=lfs -text +*.model filter=lfs diff=lfs merge=lfs -text +*.msgpack filter=lfs diff=lfs merge=lfs -text +*.npy filter=lfs diff=lfs merge=lfs -text +*.npz filter=lfs diff=lfs merge=lfs -text +*.onnx filter=lfs diff=lfs merge=lfs -text +*.ot filter=lfs diff=lfs merge=lfs -text +*.parquet filter=lfs diff=lfs merge=lfs -text +*.pb filter=lfs diff=lfs merge=lfs -text +*.pickle filter=lfs diff=lfs merge=lfs -text +*.pkl filter=lfs diff=lfs merge=lfs -text +*.pt filter=lfs diff=lfs merge=lfs -text +*.pth filter=lfs diff=lfs merge=lfs -text +*.rar filter=lfs diff=lfs merge=lfs -text +*.safetensors filter=lfs diff=lfs merge=lfs -text +saved_model/**/* filter=lfs diff=lfs merge=lfs -text +*.tar.* filter=lfs diff=lfs merge=lfs -text +*.tar filter=lfs diff=lfs merge=lfs -text +*.tflite filter=lfs diff=lfs merge=lfs -text +*.tgz filter=lfs diff=lfs merge=lfs -text +*.wasm filter=lfs diff=lfs merge=lfs -text +*.xz filter=lfs diff=lfs merge=lfs -text +*.zip filter=lfs diff=lfs merge=lfs -text +*.zst filter=lfs diff=lfs merge=lfs -text +*tfevents* filter=lfs diff=lfs merge=lfs -text diff --git a/posttrain_models/Qwen3-VL-2B-Instruct/README.md b/posttrain_models/Qwen3-VL-2B-Instruct/README.md new file mode 100644 index 0000000000000000000000000000000000000000..13afcb1d2c695b57ebe4ee740db48671069566ce --- /dev/null +++ b/posttrain_models/Qwen3-VL-2B-Instruct/README.md @@ -0,0 +1,192 @@ +--- +license: apache-2.0 +pipeline_tag: image-text-to-text +library_name: transformers +--- + + Demo + + + +# Qwen3-VL-2B-Instruct + + +Meet Qwen3-VL — the most powerful vision-language model in the Qwen series to date. + +This generation delivers comprehensive upgrades across the board: superior text understanding & generation, deeper visual perception & reasoning, extended context length, enhanced spatial and video dynamics comprehension, and stronger agent interaction capabilities. + +Available in Dense and MoE architectures that scale from edge to cloud, with Instruct and reasoning‑enhanced Thinking editions for flexible, on‑demand deployment. + + +#### Key Enhancements: + +* **Visual Agent**: Operates PC/mobile GUIs—recognizes elements, understands functions, invokes tools, completes tasks. + +* **Visual Coding Boost**: Generates Draw.io/HTML/CSS/JS from images/videos. + +* **Advanced Spatial Perception**: Judges object positions, viewpoints, and occlusions; provides stronger 2D grounding and enables 3D grounding for spatial reasoning and embodied AI. + +* **Long Context & Video Understanding**: Native 256K context, expandable to 1M; handles books and hours-long video with full recall and second-level indexing. + +* **Enhanced Multimodal Reasoning**: Excels in STEM/Math—causal analysis and logical, evidence-based answers. + +* **Upgraded Visual Recognition**: Broader, higher-quality pretraining is able to “recognize everything”—celebrities, anime, products, landmarks, flora/fauna, etc. + +* **Expanded OCR**: Supports 32 languages (up from 19); robust in low light, blur, and tilt; better with rare/ancient characters and jargon; improved long-document structure parsing. + +* **Text Understanding on par with pure LLMs**: Seamless text–vision fusion for lossless, unified comprehension. + + +#### Model Architecture Updates: + +

+ +

+ + +1. **Interleaved-MRoPE**: Full‑frequency allocation over time, width, and height via robust positional embeddings, enhancing long‑horizon video reasoning. + +2. **DeepStack**: Fuses multi‑level ViT features to capture fine‑grained details and sharpen image–text alignment. + +3. **Text–Timestamp Alignment:** Moves beyond T‑RoPE to precise, timestamp‑grounded event localization for stronger video temporal modeling. + +This is the weight repository for Qwen3-VL-2B-Instruct. + + +--- + +## Model Performance + +**Multimodal performance** + +![](https://qianwen-res.oss-accelerate.aliyuncs.com/Qwen3-VL/qwen3vl_2b_32b_vl_instruct.jpg) + +**Pure text performance** +![](https://qianwen-res.oss-accelerate.aliyuncs.com/Qwen3-VL/qwen3vl_2b_32b_text_instruct.jpg) + +## Quickstart + +Below, we provide simple examples to show how to use Qwen3-VL with 🤖 ModelScope and 🤗 Transformers. + +The code of Qwen3-VL has been in the latest Hugging Face transformers and we advise you to build from source with command: +``` +pip install git+https://github.com/huggingface/transformers +# pip install transformers==4.57.0 # currently, V4.57.0 is not released +``` + +### Using 🤗 Transformers to Chat + +Here we show a code snippet to show how to use the chat model with `transformers`: + +```python +from transformers import Qwen3VLForConditionalGeneration, AutoProcessor + +# default: Load the model on the available device(s) +model = Qwen3VLForConditionalGeneration.from_pretrained( + "Qwen/Qwen3-VL-2B-Instruct", dtype="auto", device_map="auto" +) + +# We recommend enabling flash_attention_2 for better acceleration and memory saving, especially in multi-image and video scenarios. +# model = Qwen3VLForConditionalGeneration.from_pretrained( +# "Qwen/Qwen3-VL-2B-Instruct", +# dtype=torch.bfloat16, +# attn_implementation="flash_attention_2", +# device_map="auto", +# ) + +processor = AutoProcessor.from_pretrained("Qwen/Qwen3-VL-2B-Instruct") + +messages = [ + { + "role": "user", + "content": [ + { + "type": "image", + "image": "https://qianwen-res.oss-cn-beijing.aliyuncs.com/Qwen-VL/assets/demo.jpeg", + }, + {"type": "text", "text": "Describe this image."}, + ], + } +] + +# Preparation for inference +inputs = processor.apply_chat_template( + messages, + tokenize=True, + add_generation_prompt=True, + return_dict=True, + return_tensors="pt" +) +inputs = inputs.to(model.device) + +# Inference: Generation of the output +generated_ids = model.generate(**inputs, max_new_tokens=128) +generated_ids_trimmed = [ + out_ids[len(in_ids) :] for in_ids, out_ids in zip(inputs.input_ids, generated_ids) +] +output_text = processor.batch_decode( + generated_ids_trimmed, skip_special_tokens=True, clean_up_tokenization_spaces=False +) +print(output_text) +``` + +### Generation Hyperparameters +#### VL +```bash +export greedy='false' +export top_p=0.8 +export top_k=20 +export temperature=0.7 +export repetition_penalty=1.0 +export presence_penalty=1.5 +export out_seq_length=16384 +``` + +#### Text +```bash +export greedy='false' +export top_p=1.0 +export top_k=40 +export repetition_penalty=1.0 +export presence_penalty=2.0 +export temperature=1.0 +export out_seq_length=32768 +``` + + +## Citation + +If you find our work helpful, feel free to give us a cite. + +``` +@misc{qwen3technicalreport, + title={Qwen3 Technical Report}, + author={Qwen Team}, + year={2025}, + eprint={2505.09388}, + archivePrefix={arXiv}, + primaryClass={cs.CL}, + url={https://arxiv.org/abs/2505.09388}, +} + +@article{Qwen2.5-VL, + title={Qwen2.5-VL Technical Report}, + author={Bai, Shuai and Chen, Keqin and Liu, Xuejing and Wang, Jialin and Ge, Wenbin and Song, Sibo and Dang, Kai and Wang, Peng and Wang, Shijie and Tang, Jun and Zhong, Humen and Zhu, Yuanzhi and Yang, Mingkun and Li, Zhaohai and Wan, Jianqiang and Wang, Pengfei and Ding, Wei and Fu, Zheren and Xu, Yiheng and Ye, Jiabo and Zhang, Xi and Xie, Tianbao and Cheng, Zesen and Zhang, Hang and Yang, Zhibo and Xu, Haiyang and Lin, Junyang}, + journal={arXiv preprint arXiv:2502.13923}, + year={2025} +} + +@article{Qwen2VL, + title={Qwen2-VL: Enhancing Vision-Language Model's Perception of the World at Any Resolution}, + author={Wang, Peng and Bai, Shuai and Tan, Sinan and Wang, Shijie and Fan, Zhihao and Bai, Jinze and Chen, Keqin and Liu, Xuejing and Wang, Jialin and Ge, Wenbin and Fan, Yang and Dang, Kai and Du, Mengfei and Ren, Xuancheng and Men, Rui and Liu, Dayiheng and Zhou, Chang and Zhou, Jingren and Lin, Junyang}, + journal={arXiv preprint arXiv:2409.12191}, + year={2024} +} + +@article{Qwen-VL, + title={Qwen-VL: A Versatile Vision-Language Model for Understanding, Localization, Text Reading, and Beyond}, + author={Bai, Jinze and Bai, Shuai and Yang, Shusheng and Wang, Shijie and Tan, Sinan and Wang, Peng and Lin, Junyang and Zhou, Chang and Zhou, Jingren}, + journal={arXiv preprint arXiv:2308.12966}, + year={2023} +} +``` \ No newline at end of file diff --git a/posttrain_models/Qwen3-VL-2B-Instruct/chat_template.json b/posttrain_models/Qwen3-VL-2B-Instruct/chat_template.json new file mode 100644 index 0000000000000000000000000000000000000000..e49e75bdda104c8de941eb20e0b3765184fb27bc --- /dev/null +++ b/posttrain_models/Qwen3-VL-2B-Instruct/chat_template.json @@ -0,0 +1,4 @@ +{ + "chat_template": "{%- if tools %}\n {{- '<|im_start|>system\\n' }}\n {%- if messages[0].role == 'system' %}\n {%- if messages[0].content is string %}\n {{- messages[0].content }}\n {%- else %}\n {%- for content in messages[0].content %}\n {%- if 'text' in content %}\n {{- content.text }}\n {%- endif %}\n {%- endfor %}\n {%- endif %}\n {{- '\\n\\n' }}\n {%- endif %}\n {{- \"# Tools\\n\\nYou may call one or more functions to assist with the user query.\\n\\nYou are provided with function signatures within XML tags:\\n\" }}\n {%- for tool in tools %}\n {{- \"\\n\" }}\n {{- tool | tojson }}\n {%- endfor %}\n {{- \"\\n\\n\\nFor each function call, return a json object with function name and arguments within XML tags:\\n\\n{\\\"name\\\": , \\\"arguments\\\": }\\n<|im_end|>\\n\" }}\n{%- else %}\n {%- if messages[0].role == 'system' %}\n {{- '<|im_start|>system\\n' }}\n {%- if messages[0].content is string %}\n {{- messages[0].content }}\n {%- else %}\n {%- for content in messages[0].content %}\n {%- if 'text' in content %}\n {{- content.text }}\n {%- endif %}\n {%- endfor %}\n {%- endif %}\n {{- '<|im_end|>\\n' }}\n {%- endif %}\n{%- endif %}\n{%- set image_count = namespace(value=0) %}\n{%- set video_count = namespace(value=0) %}\n{%- for message in messages %}\n {%- if message.role == \"user\" %}\n {{- '<|im_start|>' + message.role + '\\n' }}\n {%- if message.content is string %}\n {{- message.content }}\n {%- else %}\n {%- for content in message.content %}\n {%- if content.type == 'image' or 'image' in content or 'image_url' in content %}\n {%- set image_count.value = image_count.value + 1 %}\n {%- if add_vision_id %}Picture {{ image_count.value }}: {% endif -%}\n <|vision_start|><|image_pad|><|vision_end|>\n {%- elif content.type == 'video' or 'video' in content %}\n {%- set video_count.value = video_count.value + 1 %}\n {%- if add_vision_id %}Video {{ video_count.value }}: {% endif -%}\n <|vision_start|><|video_pad|><|vision_end|>\n {%- elif 'text' in content %}\n {{- content.text }}\n {%- endif %}\n {%- endfor %}\n {%- endif %}\n {{- '<|im_end|>\\n' }}\n {%- elif message.role == \"assistant\" %}\n {{- '<|im_start|>' + message.role + '\\n' }}\n {%- if message.content is string %}\n {{- message.content }}\n {%- else %}\n {%- for content_item in message.content %}\n {%- if 'text' in content_item %}\n {{- content_item.text }}\n {%- endif %}\n {%- endfor %}\n {%- endif %}\n {%- if message.tool_calls %}\n {%- for tool_call in message.tool_calls %}\n {%- if (loop.first and message.content) or (not loop.first) %}\n {{- '\\n' }}\n {%- endif %}\n {%- if tool_call.function %}\n {%- set tool_call = tool_call.function %}\n {%- endif %}\n {{- '\\n{\"name\": \"' }}\n {{- tool_call.name }}\n {{- '\", \"arguments\": ' }}\n {%- if tool_call.arguments is string %}\n {{- tool_call.arguments }}\n {%- else %}\n {{- tool_call.arguments | tojson }}\n {%- endif %}\n {{- '}\\n' }}\n {%- endfor %}\n {%- endif %}\n {{- '<|im_end|>\\n' }}\n {%- elif message.role == \"tool\" %}\n {%- if loop.first or (messages[loop.index0 - 1].role != \"tool\") %}\n {{- '<|im_start|>user' }}\n {%- endif %}\n {{- '\\n\\n' }}\n {%- if message.content is string %}\n {{- message.content }}\n {%- else %}\n {%- for content in message.content %}\n {%- if content.type == 'image' or 'image' in content or 'image_url' in content %}\n {%- set image_count.value = image_count.value + 1 %}\n {%- if add_vision_id %}Picture {{ image_count.value }}: {% endif -%}\n <|vision_start|><|image_pad|><|vision_end|>\n {%- elif content.type == 'video' or 'video' in content %}\n {%- set video_count.value = video_count.value + 1 %}\n {%- if add_vision_id %}Video {{ video_count.value }}: {% endif -%}\n <|vision_start|><|video_pad|><|vision_end|>\n {%- elif 'text' in content %}\n {{- content.text }}\n {%- endif %}\n {%- endfor %}\n {%- endif %}\n {{- '\\n' }}\n {%- if loop.last or (messages[loop.index0 + 1].role != \"tool\") %}\n {{- '<|im_end|>\\n' }}\n {%- endif %}\n {%- endif %}\n{%- endfor %}\n{%- if add_generation_prompt %}\n {{- '<|im_start|>assistant\\n' }}\n{%- endif %}\n" +} + \ No newline at end of file diff --git a/posttrain_models/Qwen3-VL-2B-Instruct/config.json b/posttrain_models/Qwen3-VL-2B-Instruct/config.json new file mode 100644 index 0000000000000000000000000000000000000000..0cd8c646eb55794594b14fefcd1d3acea5cc000f --- /dev/null +++ b/posttrain_models/Qwen3-VL-2B-Instruct/config.json @@ -0,0 +1,63 @@ +{ + "architectures": [ + "Qwen3VLForConditionalGeneration" + ], + "image_token_id": 151655, + "model_type": "qwen3_vl", + "text_config": { + "attention_bias": false, + "attention_dropout": 0.0, + "bos_token_id": 151643, + "dtype": "bfloat16", + "eos_token_id": 151645, + "head_dim": 128, + "hidden_act": "silu", + "hidden_size": 2048, + "initializer_range": 0.02, + "intermediate_size": 6144, + "max_position_embeddings": 262144, + "model_type": "qwen3_vl_text", + "num_attention_heads": 16, + "num_hidden_layers": 28, + "num_key_value_heads": 8, + "rms_norm_eps": 1e-06, + "rope_scaling": { + "mrope_interleaved": true, + "mrope_section": [ + 24, + 20, + 20 + ], + "rope_type": "default" + }, + "rope_theta": 5000000, + "tie_word_embeddings": true, + "use_cache": true, + "vocab_size": 151936 + }, + "tie_word_embeddings": true, + "transformers_version": "4.57.0.dev0", + "video_token_id": 151656, + "vision_config": { + "deepstack_visual_indexes": [ + 5, + 11, + 17 + ], + "depth": 24, + "hidden_act": "gelu_pytorch_tanh", + "hidden_size": 1024, + "in_channels": 3, + "initializer_range": 0.02, + "intermediate_size": 4096, + "model_type": "qwen3_vl", + "num_heads": 16, + "num_position_embeddings": 2304, + "out_hidden_size": 2048, + "patch_size": 16, + "spatial_merge_size": 2, + "temporal_patch_size": 2 + }, + "vision_end_token_id": 151653, + "vision_start_token_id": 151652 +} diff --git a/posttrain_models/Qwen3-VL-2B-Instruct/generation_config.json b/posttrain_models/Qwen3-VL-2B-Instruct/generation_config.json new file mode 100644 index 0000000000000000000000000000000000000000..7bb37933d2d86e4ceaabcc6540be989658f238e9 --- /dev/null +++ b/posttrain_models/Qwen3-VL-2B-Instruct/generation_config.json @@ -0,0 +1,14 @@ +{ + "bos_token_id": 151643, + "pad_token_id": 151643, + "do_sample": true, + "eos_token_id": [ + 151645, + 151643 + ], + "top_p": 0.8, + "top_k": 20, + "temperature": 0.7, + "repetition_penalty": 1.0, + "transformers_version": "4.56.0" +} \ No newline at end of file diff --git a/posttrain_models/Qwen3-VL-2B-Instruct/preprocessor_config.json b/posttrain_models/Qwen3-VL-2B-Instruct/preprocessor_config.json new file mode 100644 index 0000000000000000000000000000000000000000..2ea84a437d448ff71b08df68fdd949d5cc4ebb64 --- /dev/null +++ b/posttrain_models/Qwen3-VL-2B-Instruct/preprocessor_config.json @@ -0,0 +1,21 @@ +{ + "size": { + "longest_edge": 16777216, + "shortest_edge": 65536 + }, + "patch_size": 16, + "temporal_patch_size": 2, + "merge_size": 2, + "image_mean": [ + 0.5, + 0.5, + 0.5 + ], + "image_std": [ + 0.5, + 0.5, + 0.5 + ], + "processor_class": "Qwen3VLProcessor", + "image_processor_type": "Qwen2VLImageProcessorFast" +} \ No newline at end of file diff --git a/posttrain_models/Qwen3-VL-2B-Instruct/tokenizer_config.json b/posttrain_models/Qwen3-VL-2B-Instruct/tokenizer_config.json new file mode 100644 index 0000000000000000000000000000000000000000..d3d3763207692c78780f4bf42d4dadf49a5c8012 --- /dev/null +++ b/posttrain_models/Qwen3-VL-2B-Instruct/tokenizer_config.json @@ -0,0 +1,239 @@ +{ + "add_bos_token": false, + "add_prefix_space": false, + "added_tokens_decoder": { + "151643": { + "content": "<|endoftext|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "151644": { + "content": "<|im_start|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "151645": { + "content": "<|im_end|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "151646": { + "content": "<|object_ref_start|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "151647": { + "content": "<|object_ref_end|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "151648": { + "content": "<|box_start|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "151649": { + "content": "<|box_end|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "151650": { + "content": "<|quad_start|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "151651": { + "content": "<|quad_end|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "151652": { + "content": "<|vision_start|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "151653": { + "content": "<|vision_end|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "151654": { + "content": "<|vision_pad|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "151655": { + "content": "<|image_pad|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "151656": { + "content": "<|video_pad|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "151657": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "151658": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "151659": { + "content": "<|fim_prefix|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "151660": { + "content": "<|fim_middle|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "151661": { + "content": "<|fim_suffix|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "151662": { + "content": "<|fim_pad|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "151663": { + "content": "<|repo_name|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "151664": { + "content": "<|file_sep|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "151665": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "151666": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "151667": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "151668": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + } + }, + "additional_special_tokens": [ + "<|im_start|>", + "<|im_end|>", + "<|object_ref_start|>", + "<|object_ref_end|>", + "<|box_start|>", + "<|box_end|>", + "<|quad_start|>", + "<|quad_end|>", + "<|vision_start|>", + "<|vision_end|>", + "<|vision_pad|>", + "<|image_pad|>", + "<|video_pad|>" + ], + "bos_token": null, + "chat_template": "{%- if tools %}\n {{- '<|im_start|>system\\n' }}\n {%- if messages[0].role == 'system' %}\n {%- if messages[0].content is string %}\n {{- messages[0].content }}\n {%- else %}\n {%- for content in messages[0].content %}\n {%- if 'text' in content %}\n {{- content.text }}\n {%- endif %}\n {%- endfor %}\n {%- endif %}\n {{- '\\n\\n' }}\n {%- endif %}\n {{- \"# Tools\\n\\nYou may call one or more functions to assist with the user query.\\n\\nYou are provided with function signatures within XML tags:\\n\" }}\n {%- for tool in tools %}\n {{- \"\\n\" }}\n {{- tool | tojson }}\n {%- endfor %}\n {{- \"\\n\\n\\nFor each function call, return a json object with function name and arguments within XML tags:\\n\\n{\\\"name\\\": , \\\"arguments\\\": }\\n<|im_end|>\\n\" }}\n{%- else %}\n {%- if messages[0].role == 'system' %}\n {{- '<|im_start|>system\\n' }}\n {%- if messages[0].content is string %}\n {{- messages[0].content }}\n {%- else %}\n {%- for content in messages[0].content %}\n {%- if 'text' in content %}\n {{- content.text }}\n {%- endif %}\n {%- endfor %}\n {%- endif %}\n {{- '<|im_end|>\\n' }}\n {%- endif %}\n{%- endif %}\n{%- set image_count = namespace(value=0) %}\n{%- set video_count = namespace(value=0) %}\n{%- for message in messages %}\n {%- if message.role == \"user\" %}\n {{- '<|im_start|>' + message.role + '\\n' }}\n {%- if message.content is string %}\n {{- message.content }}\n {%- else %}\n {%- for content in message.content %}\n {%- if content.type == 'image' or 'image' in content or 'image_url' in content %}\n {%- set image_count.value = image_count.value + 1 %}\n {%- if add_vision_id %}Picture {{ image_count.value }}: {% endif -%}\n <|vision_start|><|image_pad|><|vision_end|>\n {%- elif content.type == 'video' or 'video' in content %}\n {%- set video_count.value = video_count.value + 1 %}\n {%- if add_vision_id %}Video {{ video_count.value }}: {% endif -%}\n <|vision_start|><|video_pad|><|vision_end|>\n {%- elif 'text' in content %}\n {{- content.text }}\n {%- endif %}\n {%- endfor %}\n {%- endif %}\n {{- '<|im_end|>\\n' }}\n {%- elif message.role == \"assistant\" %}\n {{- '<|im_start|>' + message.role + '\\n' }}\n {%- if message.content is string %}\n {{- message.content }}\n {%- else %}\n {%- for content_item in message.content %}\n {%- if 'text' in content_item %}\n {{- content_item.text }}\n {%- endif %}\n {%- endfor %}\n {%- endif %}\n {%- if message.tool_calls %}\n {%- for tool_call in message.tool_calls %}\n {%- if (loop.first and message.content) or (not loop.first) %}\n {{- '\\n' }}\n {%- endif %}\n {%- if tool_call.function %}\n {%- set tool_call = tool_call.function %}\n {%- endif %}\n {{- '\\n{\"name\": \"' }}\n {{- tool_call.name }}\n {{- '\", \"arguments\": ' }}\n {%- if tool_call.arguments is string %}\n {{- tool_call.arguments }}\n {%- else %}\n {{- tool_call.arguments | tojson }}\n {%- endif %}\n {{- '}\\n' }}\n {%- endfor %}\n {%- endif %}\n {{- '<|im_end|>\\n' }}\n {%- elif message.role == \"tool\" %}\n {%- if loop.first or (messages[loop.index0 - 1].role != \"tool\") %}\n {{- '<|im_start|>user' }}\n {%- endif %}\n {{- '\\n\\n' }}\n {%- if message.content is string %}\n {{- message.content }}\n {%- else %}\n {%- for content in message.content %}\n {%- if content.type == 'image' or 'image' in content or 'image_url' in content %}\n {%- set image_count.value = image_count.value + 1 %}\n {%- if add_vision_id %}Picture {{ image_count.value }}: {% endif -%}\n <|vision_start|><|image_pad|><|vision_end|>\n {%- elif content.type == 'video' or 'video' in content %}\n {%- set video_count.value = video_count.value + 1 %}\n {%- if add_vision_id %}Video {{ video_count.value }}: {% endif -%}\n <|vision_start|><|video_pad|><|vision_end|>\n {%- elif 'text' in content %}\n {{- content.text }}\n {%- endif %}\n {%- endfor %}\n {%- endif %}\n {{- '\\n' }}\n {%- if loop.last or (messages[loop.index0 + 1].role != \"tool\") %}\n {{- '<|im_end|>\\n' }}\n {%- endif %}\n {%- endif %}\n{%- endfor %}\n{%- if add_generation_prompt %}\n {{- '<|im_start|>assistant\\n' }}\n{%- endif %}\n", + "clean_up_tokenization_spaces": false, + "eos_token": "<|im_end|>", + "errors": "replace", + "model_max_length": 262144, + "pad_token": "<|endoftext|>", + "split_special_tokens": false, + "tokenizer_class": "Qwen2Tokenizer", + "unk_token": null +} \ No newline at end of file diff --git a/posttrain_models/Qwen3-VL-2B-Instruct/video_preprocessor_config.json b/posttrain_models/Qwen3-VL-2B-Instruct/video_preprocessor_config.json new file mode 100644 index 0000000000000000000000000000000000000000..3ba673a5ad7d4d13f54155ecd38b2a94a6dac8fe --- /dev/null +++ b/posttrain_models/Qwen3-VL-2B-Instruct/video_preprocessor_config.json @@ -0,0 +1,21 @@ +{ + "size": { + "longest_edge": 25165824, + "shortest_edge": 4096 + }, + "patch_size": 16, + "temporal_patch_size": 2, + "merge_size": 2, + "image_mean": [ + 0.5, + 0.5, + 0.5 + ], + "image_std": [ + 0.5, + 0.5, + 0.5 + ], + "processor_class": "Qwen3VLProcessor", + "video_processor_type": "Qwen3VLVideoProcessor" +} \ No newline at end of file diff --git a/posttrain_models/Qwen3VL_2B_lora32_batch124578_epoch4/added_tokens.json b/posttrain_models/Qwen3VL_2B_lora32_batch124578_epoch4/added_tokens.json new file mode 100644 index 0000000000000000000000000000000000000000..b54f9135e44c1e81047e8d05cb027af8bc039eed --- /dev/null +++ b/posttrain_models/Qwen3VL_2B_lora32_batch124578_epoch4/added_tokens.json @@ -0,0 +1,28 @@ +{ + "": 151668, + "": 151658, + "": 151666, + "": 151667, + "": 151657, + "": 151665, + "<|box_end|>": 151649, + "<|box_start|>": 151648, + "<|endoftext|>": 151643, + "<|file_sep|>": 151664, + "<|fim_middle|>": 151660, + "<|fim_pad|>": 151662, + "<|fim_prefix|>": 151659, + "<|fim_suffix|>": 151661, + "<|im_end|>": 151645, + "<|im_start|>": 151644, + "<|image_pad|>": 151655, + "<|object_ref_end|>": 151647, + "<|object_ref_start|>": 151646, + "<|quad_end|>": 151651, + "<|quad_start|>": 151650, + "<|repo_name|>": 151663, + "<|video_pad|>": 151656, + "<|vision_end|>": 151653, + "<|vision_pad|>": 151654, + "<|vision_start|>": 151652 +} diff --git a/posttrain_models/Qwen3VL_2B_lora32_batch124578_epoch4/chat_template.jinja b/posttrain_models/Qwen3VL_2B_lora32_batch124578_epoch4/chat_template.jinja new file mode 100644 index 0000000000000000000000000000000000000000..124386803f142761528f710e77ae483f5f8c4fc4 --- /dev/null +++ b/posttrain_models/Qwen3VL_2B_lora32_batch124578_epoch4/chat_template.jinja @@ -0,0 +1,120 @@ +{%- if tools %} + {{- '<|im_start|>system\n' }} + {%- if messages[0].role == 'system' %} + {%- if messages[0].content is string %} + {{- messages[0].content }} + {%- else %} + {%- for content in messages[0].content %} + {%- if 'text' in content %} + {{- content.text }} + {%- endif %} + {%- endfor %} + {%- endif %} + {{- '\n\n' }} + {%- endif %} + {{- "# Tools\n\nYou may call one or more functions to assist with the user query.\n\nYou are provided with function signatures within XML tags:\n" }} + {%- for tool in tools %} + {{- "\n" }} + {{- tool | tojson }} + {%- endfor %} + {{- "\n\n\nFor each function call, return a json object with function name and arguments within XML tags:\n\n{\"name\": , \"arguments\": }\n<|im_end|>\n" }} +{%- else %} + {%- if messages[0].role == 'system' %} + {{- '<|im_start|>system\n' }} + {%- if messages[0].content is string %} + {{- messages[0].content }} + {%- else %} + {%- for content in messages[0].content %} + {%- if 'text' in content %} + {{- content.text }} + {%- endif %} + {%- endfor %} + {%- endif %} + {{- '<|im_end|>\n' }} + {%- endif %} +{%- endif %} +{%- set image_count = namespace(value=0) %} +{%- set video_count = namespace(value=0) %} +{%- for message in messages %} + {%- if message.role == "user" %} + {{- '<|im_start|>' + message.role + '\n' }} + {%- if message.content is string %} + {{- message.content }} + {%- else %} + {%- for content in message.content %} + {%- if content.type == 'image' or 'image' in content or 'image_url' in content %} + {%- set image_count.value = image_count.value + 1 %} + {%- if add_vision_id %}Picture {{ image_count.value }}: {% endif -%} + <|vision_start|><|image_pad|><|vision_end|> + {%- elif content.type == 'video' or 'video' in content %} + {%- set video_count.value = video_count.value + 1 %} + {%- if add_vision_id %}Video {{ video_count.value }}: {% endif -%} + <|vision_start|><|video_pad|><|vision_end|> + {%- elif 'text' in content %} + {{- content.text }} + {%- endif %} + {%- endfor %} + {%- endif %} + {{- '<|im_end|>\n' }} + {%- elif message.role == "assistant" %} + {{- '<|im_start|>' + message.role + '\n' }} + {%- if message.content is string %} + {{- message.content }} + {%- else %} + {%- for content_item in message.content %} + {%- if 'text' in content_item %} + {{- content_item.text }} + {%- endif %} + {%- endfor %} + {%- endif %} + {%- if message.tool_calls %} + {%- for tool_call in message.tool_calls %} + {%- if (loop.first and message.content) or (not loop.first) %} + {{- '\n' }} + {%- endif %} + {%- if tool_call.function %} + {%- set tool_call = tool_call.function %} + {%- endif %} + {{- '\n{"name": "' }} + {{- tool_call.name }} + {{- '", "arguments": ' }} + {%- if tool_call.arguments is string %} + {{- tool_call.arguments }} + {%- else %} + {{- tool_call.arguments | tojson }} + {%- endif %} + {{- '}\n' }} + {%- endfor %} + {%- endif %} + {{- '<|im_end|>\n' }} + {%- elif message.role == "tool" %} + {%- if loop.first or (messages[loop.index0 - 1].role != "tool") %} + {{- '<|im_start|>user' }} + {%- endif %} + {{- '\n\n' }} + {%- if message.content is string %} + {{- message.content }} + {%- else %} + {%- for content in message.content %} + {%- if content.type == 'image' or 'image' in content or 'image_url' in content %} + {%- set image_count.value = image_count.value + 1 %} + {%- if add_vision_id %}Picture {{ image_count.value }}: {% endif -%} + <|vision_start|><|image_pad|><|vision_end|> + {%- elif content.type == 'video' or 'video' in content %} + {%- set video_count.value = video_count.value + 1 %} + {%- if add_vision_id %}Video {{ video_count.value }}: {% endif -%} + <|vision_start|><|video_pad|><|vision_end|> + {%- elif 'text' in content %} + {{- content.text }} + {%- endif %} + {%- endfor %} + {%- endif %} + {{- '\n' }} + {%- if loop.last or (messages[loop.index0 + 1].role != "tool") %} + {{- '<|im_end|>\n' }} + {%- endif %} + {%- endif %} +{%- endfor %} +{%- if add_generation_prompt %} + {{- '<|im_start|>assistant\n' }} +{%- endif %} diff --git a/posttrain_models/Qwen3VL_2B_lora32_batch124578_epoch4/config.json b/posttrain_models/Qwen3VL_2B_lora32_batch124578_epoch4/config.json new file mode 100644 index 0000000000000000000000000000000000000000..712a1c7294a489dce13a882ce6ca357e9a42bf6e --- /dev/null +++ b/posttrain_models/Qwen3VL_2B_lora32_batch124578_epoch4/config.json @@ -0,0 +1,65 @@ +{ + "architectures": [ + "Qwen3VLForConditionalGeneration" + ], + "dtype": "bfloat16", + "image_token_id": 151655, + "model_type": "qwen3_vl", + "text_config": { + "attention_bias": false, + "attention_dropout": 0.0, + "bos_token_id": 151643, + "dtype": "bfloat16", + "eos_token_id": 151645, + "head_dim": 128, + "hidden_act": "silu", + "hidden_size": 2048, + "initializer_range": 0.02, + "intermediate_size": 6144, + "max_position_embeddings": 262144, + "model_type": "qwen3_vl_text", + "num_attention_heads": 16, + "num_hidden_layers": 28, + "num_key_value_heads": 8, + "rms_norm_eps": 1e-06, + "rope_scaling": { + "mrope_interleaved": true, + "mrope_section": [ + 24, + 20, + 20 + ], + "rope_type": "default" + }, + "rope_theta": 5000000, + "tie_word_embeddings": true, + "use_cache": true, + "vocab_size": 151936 + }, + "tie_word_embeddings": true, + "transformers_version": "4.57.3", + "video_token_id": 151656, + "vision_config": { + "deepstack_visual_indexes": [ + 5, + 11, + 17 + ], + "depth": 24, + "dtype": "bfloat16", + "hidden_act": "gelu_pytorch_tanh", + "hidden_size": 1024, + "in_channels": 3, + "initializer_range": 0.02, + "intermediate_size": 4096, + "model_type": "qwen3_vl", + "num_heads": 16, + "num_position_embeddings": 2304, + "out_hidden_size": 2048, + "patch_size": 16, + "spatial_merge_size": 2, + "temporal_patch_size": 2 + }, + "vision_end_token_id": 151653, + "vision_start_token_id": 151652 +} diff --git a/posttrain_models/Qwen3VL_2B_lora32_batch124578_epoch4/generation_config.json b/posttrain_models/Qwen3VL_2B_lora32_batch124578_epoch4/generation_config.json new file mode 100644 index 0000000000000000000000000000000000000000..3dd2c545b96b4d51586680d7bb27fdca5c90dfe9 --- /dev/null +++ b/posttrain_models/Qwen3VL_2B_lora32_batch124578_epoch4/generation_config.json @@ -0,0 +1,13 @@ +{ + "bos_token_id": 151643, + "do_sample": true, + "eos_token_id": [ + 151645, + 151643 + ], + "pad_token_id": 151643, + "temperature": 0.7, + "top_k": 20, + "top_p": 0.8, + "transformers_version": "4.57.3" +} diff --git a/posttrain_models/Qwen3VL_2B_lora32_batch124578_epoch4/preprocessor_config.json b/posttrain_models/Qwen3VL_2B_lora32_batch124578_epoch4/preprocessor_config.json new file mode 100644 index 0000000000000000000000000000000000000000..2fa6553554f69b4e3a6b742a75264ff9dff90ad0 --- /dev/null +++ b/posttrain_models/Qwen3VL_2B_lora32_batch124578_epoch4/preprocessor_config.json @@ -0,0 +1,39 @@ +{ + "crop_size": null, + "data_format": "channels_first", + "default_to_square": true, + "device": null, + "disable_grouping": null, + "do_center_crop": null, + "do_convert_rgb": true, + "do_normalize": true, + "do_pad": null, + "do_rescale": true, + "do_resize": true, + "image_mean": [ + 0.5, + 0.5, + 0.5 + ], + "image_processor_type": "Qwen2VLImageProcessorFast", + "image_std": [ + 0.5, + 0.5, + 0.5 + ], + "input_data_format": null, + "max_pixels": null, + "merge_size": 2, + "min_pixels": null, + "pad_size": null, + "patch_size": 16, + "processor_class": "Qwen3VLProcessor", + "resample": 3, + "rescale_factor": 0.00392156862745098, + "return_tensors": null, + "size": { + "longest_edge": 16777216, + "shortest_edge": 65536 + }, + "temporal_patch_size": 2 +} diff --git a/posttrain_models/Qwen3VL_2B_lora32_batch124578_epoch4/special_tokens_map.json b/posttrain_models/Qwen3VL_2B_lora32_batch124578_epoch4/special_tokens_map.json new file mode 100644 index 0000000000000000000000000000000000000000..ac23c0aaa2434523c494330aeb79c58395378103 --- /dev/null +++ b/posttrain_models/Qwen3VL_2B_lora32_batch124578_epoch4/special_tokens_map.json @@ -0,0 +1,31 @@ +{ + "additional_special_tokens": [ + "<|im_start|>", + "<|im_end|>", + "<|object_ref_start|>", + "<|object_ref_end|>", + "<|box_start|>", + "<|box_end|>", + "<|quad_start|>", + "<|quad_end|>", + "<|vision_start|>", + "<|vision_end|>", + "<|vision_pad|>", + "<|image_pad|>", + "<|video_pad|>" + ], + "eos_token": { + "content": "<|im_end|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false + }, + "pad_token": { + "content": "<|endoftext|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false + } +} diff --git a/posttrain_models/Qwen3VL_2B_lora32_batch124578_epoch4/tokenizer_config.json b/posttrain_models/Qwen3VL_2B_lora32_batch124578_epoch4/tokenizer_config.json new file mode 100644 index 0000000000000000000000000000000000000000..fec7f182135c384a08ea2fc9635566ca79c45afc --- /dev/null +++ b/posttrain_models/Qwen3VL_2B_lora32_batch124578_epoch4/tokenizer_config.json @@ -0,0 +1,240 @@ +{ + "add_bos_token": false, + "add_prefix_space": false, + "added_tokens_decoder": { + "151643": { + "content": "<|endoftext|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "151644": { + "content": "<|im_start|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "151645": { + "content": "<|im_end|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "151646": { + "content": "<|object_ref_start|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "151647": { + "content": "<|object_ref_end|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "151648": { + "content": "<|box_start|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "151649": { + "content": "<|box_end|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "151650": { + "content": "<|quad_start|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "151651": { + "content": "<|quad_end|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "151652": { + "content": "<|vision_start|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "151653": { + "content": "<|vision_end|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "151654": { + "content": "<|vision_pad|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "151655": { + "content": "<|image_pad|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "151656": { + "content": "<|video_pad|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "151657": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "151658": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "151659": { + "content": "<|fim_prefix|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "151660": { + "content": "<|fim_middle|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "151661": { + "content": "<|fim_suffix|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "151662": { + "content": "<|fim_pad|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "151663": { + "content": "<|repo_name|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "151664": { + "content": "<|file_sep|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "151665": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "151666": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "151667": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "151668": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + } + }, + "additional_special_tokens": [ + "<|im_start|>", + "<|im_end|>", + "<|object_ref_start|>", + "<|object_ref_end|>", + "<|box_start|>", + "<|box_end|>", + "<|quad_start|>", + "<|quad_end|>", + "<|vision_start|>", + "<|vision_end|>", + "<|vision_pad|>", + "<|image_pad|>", + "<|video_pad|>" + ], + "bos_token": null, + "clean_up_tokenization_spaces": false, + "eos_token": "<|im_end|>", + "errors": "replace", + "extra_special_tokens": {}, + "model_max_length": 262144, + "pad_token": "<|endoftext|>", + "processor_class": "Qwen3VLProcessor", + "split_special_tokens": false, + "tokenizer_class": "Qwen2Tokenizer", + "unk_token": null +} diff --git a/posttrain_models/Qwen3VL_2B_lora32_batch124578_epoch4/video_preprocessor_config.json b/posttrain_models/Qwen3VL_2B_lora32_batch124578_epoch4/video_preprocessor_config.json new file mode 100644 index 0000000000000000000000000000000000000000..e32b1d90356f74457edbfca27aa1e121dab7cdc7 --- /dev/null +++ b/posttrain_models/Qwen3VL_2B_lora32_batch124578_epoch4/video_preprocessor_config.json @@ -0,0 +1,41 @@ +{ + "crop_size": null, + "data_format": "channels_first", + "default_to_square": true, + "device": null, + "do_center_crop": null, + "do_convert_rgb": true, + "do_normalize": true, + "do_rescale": true, + "do_resize": true, + "do_sample_frames": true, + "fps": 2, + "image_mean": [ + 0.5, + 0.5, + 0.5 + ], + "image_std": [ + 0.5, + 0.5, + 0.5 + ], + "input_data_format": null, + "max_frames": 768, + "merge_size": 2, + "min_frames": 4, + "num_frames": null, + "pad_size": null, + "patch_size": 16, + "processor_class": "Qwen3VLProcessor", + "resample": 3, + "rescale_factor": 0.00392156862745098, + "return_metadata": false, + "size": { + "longest_edge": 25165824, + "shortest_edge": 4096 + }, + "temporal_patch_size": 2, + "video_metadata": null, + "video_processor_type": "Qwen3VLVideoProcessor" +} diff --git a/posttrain_models/Qwen3VL_4B_lora32_batch1245789_epoch4/added_tokens.json b/posttrain_models/Qwen3VL_4B_lora32_batch1245789_epoch4/added_tokens.json new file mode 100644 index 0000000000000000000000000000000000000000..b54f9135e44c1e81047e8d05cb027af8bc039eed --- /dev/null +++ b/posttrain_models/Qwen3VL_4B_lora32_batch1245789_epoch4/added_tokens.json @@ -0,0 +1,28 @@ +{ + "": 151668, + "": 151658, + "": 151666, + "": 151667, + "": 151657, + "": 151665, + "<|box_end|>": 151649, + "<|box_start|>": 151648, + "<|endoftext|>": 151643, + "<|file_sep|>": 151664, + "<|fim_middle|>": 151660, + "<|fim_pad|>": 151662, + "<|fim_prefix|>": 151659, + "<|fim_suffix|>": 151661, + "<|im_end|>": 151645, + "<|im_start|>": 151644, + "<|image_pad|>": 151655, + "<|object_ref_end|>": 151647, + "<|object_ref_start|>": 151646, + "<|quad_end|>": 151651, + "<|quad_start|>": 151650, + "<|repo_name|>": 151663, + "<|video_pad|>": 151656, + "<|vision_end|>": 151653, + "<|vision_pad|>": 151654, + "<|vision_start|>": 151652 +} diff --git a/posttrain_models/Qwen3VL_4B_lora32_batch1245789_epoch4/chat_template.jinja b/posttrain_models/Qwen3VL_4B_lora32_batch1245789_epoch4/chat_template.jinja new file mode 100644 index 0000000000000000000000000000000000000000..124386803f142761528f710e77ae483f5f8c4fc4 --- /dev/null +++ b/posttrain_models/Qwen3VL_4B_lora32_batch1245789_epoch4/chat_template.jinja @@ -0,0 +1,120 @@ +{%- if tools %} + {{- '<|im_start|>system\n' }} + {%- if messages[0].role == 'system' %} + {%- if messages[0].content is string %} + {{- messages[0].content }} + {%- else %} + {%- for content in messages[0].content %} + {%- if 'text' in content %} + {{- content.text }} + {%- endif %} + {%- endfor %} + {%- endif %} + {{- '\n\n' }} + {%- endif %} + {{- "# Tools\n\nYou may call one or more functions to assist with the user query.\n\nYou are provided with function signatures within XML tags:\n" }} + {%- for tool in tools %} + {{- "\n" }} + {{- tool | tojson }} + {%- endfor %} + {{- "\n\n\nFor each function call, return a json object with function name and arguments within XML tags:\n\n{\"name\": , \"arguments\": }\n<|im_end|>\n" }} +{%- else %} + {%- if messages[0].role == 'system' %} + {{- '<|im_start|>system\n' }} + {%- if messages[0].content is string %} + {{- messages[0].content }} + {%- else %} + {%- for content in messages[0].content %} + {%- if 'text' in content %} + {{- content.text }} + {%- endif %} + {%- endfor %} + {%- endif %} + {{- '<|im_end|>\n' }} + {%- endif %} +{%- endif %} +{%- set image_count = namespace(value=0) %} +{%- set video_count = namespace(value=0) %} +{%- for message in messages %} + {%- if message.role == "user" %} + {{- '<|im_start|>' + message.role + '\n' }} + {%- if message.content is string %} + {{- message.content }} + {%- else %} + {%- for content in message.content %} + {%- if content.type == 'image' or 'image' in content or 'image_url' in content %} + {%- set image_count.value = image_count.value + 1 %} + {%- if add_vision_id %}Picture {{ image_count.value }}: {% endif -%} + <|vision_start|><|image_pad|><|vision_end|> + {%- elif content.type == 'video' or 'video' in content %} + {%- set video_count.value = video_count.value + 1 %} + {%- if add_vision_id %}Video {{ video_count.value }}: {% endif -%} + <|vision_start|><|video_pad|><|vision_end|> + {%- elif 'text' in content %} + {{- content.text }} + {%- endif %} + {%- endfor %} + {%- endif %} + {{- '<|im_end|>\n' }} + {%- elif message.role == "assistant" %} + {{- '<|im_start|>' + message.role + '\n' }} + {%- if message.content is string %} + {{- message.content }} + {%- else %} + {%- for content_item in message.content %} + {%- if 'text' in content_item %} + {{- content_item.text }} + {%- endif %} + {%- endfor %} + {%- endif %} + {%- if message.tool_calls %} + {%- for tool_call in message.tool_calls %} + {%- if (loop.first and message.content) or (not loop.first) %} + {{- '\n' }} + {%- endif %} + {%- if tool_call.function %} + {%- set tool_call = tool_call.function %} + {%- endif %} + {{- '\n{"name": "' }} + {{- tool_call.name }} + {{- '", "arguments": ' }} + {%- if tool_call.arguments is string %} + {{- tool_call.arguments }} + {%- else %} + {{- tool_call.arguments | tojson }} + {%- endif %} + {{- '}\n' }} + {%- endfor %} + {%- endif %} + {{- '<|im_end|>\n' }} + {%- elif message.role == "tool" %} + {%- if loop.first or (messages[loop.index0 - 1].role != "tool") %} + {{- '<|im_start|>user' }} + {%- endif %} + {{- '\n\n' }} + {%- if message.content is string %} + {{- message.content }} + {%- else %} + {%- for content in message.content %} + {%- if content.type == 'image' or 'image' in content or 'image_url' in content %} + {%- set image_count.value = image_count.value + 1 %} + {%- if add_vision_id %}Picture {{ image_count.value }}: {% endif -%} + <|vision_start|><|image_pad|><|vision_end|> + {%- elif content.type == 'video' or 'video' in content %} + {%- set video_count.value = video_count.value + 1 %} + {%- if add_vision_id %}Video {{ video_count.value }}: {% endif -%} + <|vision_start|><|video_pad|><|vision_end|> + {%- elif 'text' in content %} + {{- content.text }} + {%- endif %} + {%- endfor %} + {%- endif %} + {{- '\n' }} + {%- if loop.last or (messages[loop.index0 + 1].role != "tool") %} + {{- '<|im_end|>\n' }} + {%- endif %} + {%- endif %} +{%- endfor %} +{%- if add_generation_prompt %} + {{- '<|im_start|>assistant\n' }} +{%- endif %} diff --git a/posttrain_models/Qwen3VL_4B_lora32_batch1245789_epoch4/config.json b/posttrain_models/Qwen3VL_4B_lora32_batch1245789_epoch4/config.json new file mode 100644 index 0000000000000000000000000000000000000000..85b02b30a3b2935dc814fc358440e2a482dfaa6a --- /dev/null +++ b/posttrain_models/Qwen3VL_4B_lora32_batch1245789_epoch4/config.json @@ -0,0 +1,65 @@ +{ + "architectures": [ + "Qwen3VLForConditionalGeneration" + ], + "dtype": "bfloat16", + "image_token_id": 151655, + "model_type": "qwen3_vl", + "text_config": { + "attention_bias": false, + "attention_dropout": 0.0, + "bos_token_id": 151643, + "dtype": "bfloat16", + "eos_token_id": 151645, + "head_dim": 128, + "hidden_act": "silu", + "hidden_size": 2560, + "initializer_range": 0.02, + "intermediate_size": 9728, + "max_position_embeddings": 262144, + "model_type": "qwen3_vl_text", + "num_attention_heads": 32, + "num_hidden_layers": 36, + "num_key_value_heads": 8, + "rms_norm_eps": 1e-06, + "rope_scaling": { + "mrope_interleaved": true, + "mrope_section": [ + 24, + 20, + 20 + ], + "rope_type": "default" + }, + "rope_theta": 5000000, + "tie_word_embeddings": true, + "use_cache": true, + "vocab_size": 151936 + }, + "tie_word_embeddings": true, + "transformers_version": "4.57.1", + "video_token_id": 151656, + "vision_config": { + "deepstack_visual_indexes": [ + 5, + 11, + 17 + ], + "depth": 24, + "dtype": "bfloat16", + "hidden_act": "gelu_pytorch_tanh", + "hidden_size": 1024, + "in_channels": 3, + "initializer_range": 0.02, + "intermediate_size": 4096, + "model_type": "qwen3_vl", + "num_heads": 16, + "num_position_embeddings": 2304, + "out_hidden_size": 2560, + "patch_size": 16, + "spatial_merge_size": 2, + "temporal_patch_size": 2 + }, + "vision_end_token_id": 151653, + "vision_start_token_id": 151652 +} diff --git a/posttrain_models/Qwen3VL_4B_lora32_batch1245789_epoch4/generation_config.json b/posttrain_models/Qwen3VL_4B_lora32_batch1245789_epoch4/generation_config.json new file mode 100644 index 0000000000000000000000000000000000000000..bdb4e0372b04220c80df250f2b3354d1c81c97cb --- /dev/null +++ b/posttrain_models/Qwen3VL_4B_lora32_batch1245789_epoch4/generation_config.json @@ -0,0 +1,13 @@ +{ + "bos_token_id": 151643, + "do_sample": true, + "eos_token_id": [ + 151645, + 151643 + ], + "pad_token_id": 151643, + "temperature": 0.7, + "top_k": 20, + "top_p": 0.8, + "transformers_version": "4.57.1" +} diff --git a/posttrain_models/Qwen3VL_4B_lora32_batch1245789_epoch4/model.safetensors.index.json b/posttrain_models/Qwen3VL_4B_lora32_batch1245789_epoch4/model.safetensors.index.json new file mode 100644 index 0000000000000000000000000000000000000000..58d118e8ce51a42ed3b147a55f9c92f39d273c60 --- /dev/null +++ b/posttrain_models/Qwen3VL_4B_lora32_batch1245789_epoch4/model.safetensors.index.json @@ -0,0 +1,721 @@ +{ + "metadata": { + "total_parameters": 4437815808, + "total_size": 8875631616 + }, + "weight_map": { + "model.language_model.embed_tokens.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.0.input_layernorm.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.0.mlp.down_proj.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.0.mlp.gate_proj.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.0.mlp.up_proj.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.0.post_attention_layernorm.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.0.self_attn.k_norm.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.0.self_attn.k_proj.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.0.self_attn.o_proj.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.0.self_attn.q_norm.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.0.self_attn.q_proj.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.0.self_attn.v_proj.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.1.input_layernorm.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.1.mlp.down_proj.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.1.mlp.gate_proj.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.1.mlp.up_proj.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.1.post_attention_layernorm.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.1.self_attn.k_norm.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.1.self_attn.k_proj.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.1.self_attn.o_proj.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.1.self_attn.q_norm.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.1.self_attn.q_proj.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.1.self_attn.v_proj.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.10.input_layernorm.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.10.mlp.down_proj.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.10.mlp.gate_proj.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.10.mlp.up_proj.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.10.post_attention_layernorm.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.10.self_attn.k_norm.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.10.self_attn.k_proj.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.10.self_attn.o_proj.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.10.self_attn.q_norm.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.10.self_attn.q_proj.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.10.self_attn.v_proj.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.11.input_layernorm.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.11.mlp.down_proj.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.11.mlp.gate_proj.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.11.mlp.up_proj.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.11.post_attention_layernorm.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.11.self_attn.k_norm.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.11.self_attn.k_proj.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.11.self_attn.o_proj.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.11.self_attn.q_norm.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.11.self_attn.q_proj.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.11.self_attn.v_proj.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.12.input_layernorm.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.12.mlp.down_proj.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.12.mlp.gate_proj.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.12.mlp.up_proj.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.12.post_attention_layernorm.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.12.self_attn.k_norm.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.12.self_attn.k_proj.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.12.self_attn.o_proj.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.12.self_attn.q_norm.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.12.self_attn.q_proj.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.12.self_attn.v_proj.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.13.input_layernorm.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.13.mlp.down_proj.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.13.mlp.gate_proj.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.13.mlp.up_proj.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.13.post_attention_layernorm.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.13.self_attn.k_norm.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.13.self_attn.k_proj.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.13.self_attn.o_proj.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.13.self_attn.q_norm.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.13.self_attn.q_proj.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.13.self_attn.v_proj.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.14.input_layernorm.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.14.mlp.down_proj.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.14.mlp.gate_proj.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.14.mlp.up_proj.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.14.post_attention_layernorm.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.14.self_attn.k_norm.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.14.self_attn.k_proj.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.14.self_attn.o_proj.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.14.self_attn.q_norm.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.14.self_attn.q_proj.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.14.self_attn.v_proj.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.15.input_layernorm.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.15.mlp.down_proj.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.15.mlp.gate_proj.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.15.mlp.up_proj.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.15.post_attention_layernorm.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.15.self_attn.k_norm.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.15.self_attn.k_proj.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.15.self_attn.o_proj.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.15.self_attn.q_norm.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.15.self_attn.q_proj.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.15.self_attn.v_proj.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.16.input_layernorm.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.16.mlp.down_proj.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.16.mlp.gate_proj.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.16.mlp.up_proj.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.16.post_attention_layernorm.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.16.self_attn.k_norm.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.16.self_attn.k_proj.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.16.self_attn.o_proj.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.16.self_attn.q_norm.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.16.self_attn.q_proj.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.16.self_attn.v_proj.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.17.input_layernorm.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.17.mlp.down_proj.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.17.mlp.gate_proj.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.17.mlp.up_proj.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.17.post_attention_layernorm.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.17.self_attn.k_norm.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.17.self_attn.k_proj.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.17.self_attn.o_proj.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.17.self_attn.q_norm.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.17.self_attn.q_proj.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.17.self_attn.v_proj.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.18.input_layernorm.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.18.mlp.down_proj.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.18.mlp.gate_proj.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.18.mlp.up_proj.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.18.post_attention_layernorm.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.18.self_attn.k_norm.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.18.self_attn.k_proj.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.18.self_attn.o_proj.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.18.self_attn.q_norm.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.18.self_attn.q_proj.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.18.self_attn.v_proj.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.19.input_layernorm.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.19.mlp.down_proj.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.19.mlp.gate_proj.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.19.mlp.up_proj.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.19.post_attention_layernorm.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.19.self_attn.k_norm.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.19.self_attn.k_proj.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.19.self_attn.o_proj.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.19.self_attn.q_norm.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.19.self_attn.q_proj.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.19.self_attn.v_proj.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.2.input_layernorm.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.2.mlp.down_proj.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.2.mlp.gate_proj.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.2.mlp.up_proj.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.2.post_attention_layernorm.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.2.self_attn.k_norm.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.2.self_attn.k_proj.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.2.self_attn.o_proj.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.2.self_attn.q_norm.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.2.self_attn.q_proj.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.2.self_attn.v_proj.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.20.input_layernorm.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.20.mlp.down_proj.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.20.mlp.gate_proj.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.20.mlp.up_proj.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.20.post_attention_layernorm.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.20.self_attn.k_norm.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.20.self_attn.k_proj.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.20.self_attn.o_proj.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.20.self_attn.q_norm.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.20.self_attn.q_proj.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.20.self_attn.v_proj.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.21.input_layernorm.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.21.mlp.down_proj.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.21.mlp.gate_proj.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.21.mlp.up_proj.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.21.post_attention_layernorm.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.21.self_attn.k_norm.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.21.self_attn.k_proj.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.21.self_attn.o_proj.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.21.self_attn.q_norm.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.21.self_attn.q_proj.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.21.self_attn.v_proj.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.22.input_layernorm.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.22.mlp.down_proj.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.22.mlp.gate_proj.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.22.mlp.up_proj.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.22.post_attention_layernorm.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.22.self_attn.k_norm.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.22.self_attn.k_proj.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.22.self_attn.o_proj.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.22.self_attn.q_norm.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.22.self_attn.q_proj.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.22.self_attn.v_proj.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.23.input_layernorm.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.23.mlp.down_proj.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.23.mlp.gate_proj.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.23.mlp.up_proj.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.23.post_attention_layernorm.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.23.self_attn.k_norm.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.23.self_attn.k_proj.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.23.self_attn.o_proj.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.23.self_attn.q_norm.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.23.self_attn.q_proj.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.23.self_attn.v_proj.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.24.input_layernorm.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.24.mlp.down_proj.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.24.mlp.gate_proj.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.24.mlp.up_proj.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.24.post_attention_layernorm.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.24.self_attn.k_norm.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.24.self_attn.k_proj.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.24.self_attn.o_proj.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.24.self_attn.q_norm.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.24.self_attn.q_proj.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.24.self_attn.v_proj.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.25.input_layernorm.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.25.mlp.down_proj.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.25.mlp.gate_proj.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.25.mlp.up_proj.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.25.post_attention_layernorm.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.25.self_attn.k_norm.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.25.self_attn.k_proj.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.25.self_attn.o_proj.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.25.self_attn.q_norm.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.25.self_attn.q_proj.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.25.self_attn.v_proj.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.26.input_layernorm.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.26.mlp.down_proj.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.26.mlp.gate_proj.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.26.mlp.up_proj.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.26.post_attention_layernorm.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.26.self_attn.k_norm.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.26.self_attn.k_proj.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.26.self_attn.o_proj.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.26.self_attn.q_norm.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.26.self_attn.q_proj.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.26.self_attn.v_proj.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.27.input_layernorm.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.27.mlp.down_proj.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.27.mlp.gate_proj.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.27.mlp.up_proj.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.27.post_attention_layernorm.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.27.self_attn.k_norm.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.27.self_attn.k_proj.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.27.self_attn.o_proj.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.27.self_attn.q_norm.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.27.self_attn.q_proj.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.27.self_attn.v_proj.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.28.input_layernorm.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.28.mlp.down_proj.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.28.mlp.gate_proj.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.28.mlp.up_proj.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.28.post_attention_layernorm.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.28.self_attn.k_norm.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.28.self_attn.k_proj.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.28.self_attn.o_proj.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.28.self_attn.q_norm.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.28.self_attn.q_proj.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.28.self_attn.v_proj.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.29.input_layernorm.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.29.mlp.down_proj.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.29.mlp.gate_proj.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.29.mlp.up_proj.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.29.post_attention_layernorm.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.29.self_attn.k_norm.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.29.self_attn.k_proj.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.29.self_attn.o_proj.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.29.self_attn.q_norm.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.29.self_attn.q_proj.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.29.self_attn.v_proj.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.3.input_layernorm.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.3.mlp.down_proj.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.3.mlp.gate_proj.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.3.mlp.up_proj.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.3.post_attention_layernorm.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.3.self_attn.k_norm.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.3.self_attn.k_proj.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.3.self_attn.o_proj.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.3.self_attn.q_norm.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.3.self_attn.q_proj.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.3.self_attn.v_proj.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.30.input_layernorm.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.30.mlp.down_proj.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.30.mlp.gate_proj.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.30.mlp.up_proj.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.30.post_attention_layernorm.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.30.self_attn.k_norm.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.30.self_attn.k_proj.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.30.self_attn.o_proj.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.30.self_attn.q_norm.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.30.self_attn.q_proj.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.30.self_attn.v_proj.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.31.input_layernorm.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.31.mlp.down_proj.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.31.mlp.gate_proj.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.31.mlp.up_proj.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.31.post_attention_layernorm.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.31.self_attn.k_norm.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.31.self_attn.k_proj.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.31.self_attn.o_proj.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.31.self_attn.q_norm.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.31.self_attn.q_proj.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.31.self_attn.v_proj.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.32.input_layernorm.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.32.mlp.down_proj.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.32.mlp.gate_proj.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.32.mlp.up_proj.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.32.post_attention_layernorm.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.32.self_attn.k_norm.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.32.self_attn.k_proj.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.32.self_attn.o_proj.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.32.self_attn.q_norm.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.32.self_attn.q_proj.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.32.self_attn.v_proj.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.33.input_layernorm.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.33.mlp.down_proj.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.33.mlp.gate_proj.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.33.mlp.up_proj.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.33.post_attention_layernorm.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.33.self_attn.k_norm.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.33.self_attn.k_proj.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.33.self_attn.o_proj.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.33.self_attn.q_norm.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.33.self_attn.q_proj.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.33.self_attn.v_proj.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.34.input_layernorm.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.34.mlp.down_proj.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.34.mlp.gate_proj.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.34.mlp.up_proj.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.34.post_attention_layernorm.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.34.self_attn.k_norm.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.34.self_attn.k_proj.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.34.self_attn.o_proj.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.34.self_attn.q_norm.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.34.self_attn.q_proj.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.34.self_attn.v_proj.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.35.input_layernorm.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.35.mlp.down_proj.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.35.mlp.gate_proj.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.35.mlp.up_proj.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.35.post_attention_layernorm.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.35.self_attn.k_norm.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.35.self_attn.k_proj.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.35.self_attn.o_proj.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.35.self_attn.q_norm.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.35.self_attn.q_proj.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.35.self_attn.v_proj.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.4.input_layernorm.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.4.mlp.down_proj.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.4.mlp.gate_proj.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.4.mlp.up_proj.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.4.post_attention_layernorm.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.4.self_attn.k_norm.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.4.self_attn.k_proj.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.4.self_attn.o_proj.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.4.self_attn.q_norm.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.4.self_attn.q_proj.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.4.self_attn.v_proj.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.5.input_layernorm.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.5.mlp.down_proj.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.5.mlp.gate_proj.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.5.mlp.up_proj.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.5.post_attention_layernorm.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.5.self_attn.k_norm.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.5.self_attn.k_proj.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.5.self_attn.o_proj.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.5.self_attn.q_norm.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.5.self_attn.q_proj.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.5.self_attn.v_proj.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.6.input_layernorm.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.6.mlp.down_proj.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.6.mlp.gate_proj.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.6.mlp.up_proj.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.6.post_attention_layernorm.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.6.self_attn.k_norm.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.6.self_attn.k_proj.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.6.self_attn.o_proj.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.6.self_attn.q_norm.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.6.self_attn.q_proj.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.6.self_attn.v_proj.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.7.input_layernorm.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.7.mlp.down_proj.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.7.mlp.gate_proj.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.7.mlp.up_proj.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.7.post_attention_layernorm.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.7.self_attn.k_norm.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.7.self_attn.k_proj.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.7.self_attn.o_proj.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.7.self_attn.q_norm.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.7.self_attn.q_proj.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.7.self_attn.v_proj.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.8.input_layernorm.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.8.mlp.down_proj.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.8.mlp.gate_proj.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.8.mlp.up_proj.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.8.post_attention_layernorm.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.8.self_attn.k_norm.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.8.self_attn.k_proj.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.8.self_attn.o_proj.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.8.self_attn.q_norm.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.8.self_attn.q_proj.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.8.self_attn.v_proj.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.9.input_layernorm.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.9.mlp.down_proj.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.9.mlp.gate_proj.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.9.mlp.up_proj.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.9.post_attention_layernorm.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.9.self_attn.k_norm.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.9.self_attn.k_proj.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.9.self_attn.o_proj.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.9.self_attn.q_norm.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.9.self_attn.q_proj.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.9.self_attn.v_proj.weight": "model-00001-of-00002.safetensors", + "model.language_model.norm.weight": "model-00002-of-00002.safetensors", + "model.visual.blocks.0.attn.proj.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.0.attn.proj.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.0.attn.qkv.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.0.attn.qkv.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.0.mlp.linear_fc1.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.0.mlp.linear_fc1.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.0.mlp.linear_fc2.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.0.mlp.linear_fc2.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.0.norm1.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.0.norm1.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.0.norm2.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.0.norm2.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.1.attn.proj.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.1.attn.proj.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.1.attn.qkv.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.1.attn.qkv.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.1.mlp.linear_fc1.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.1.mlp.linear_fc1.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.1.mlp.linear_fc2.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.1.mlp.linear_fc2.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.1.norm1.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.1.norm1.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.1.norm2.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.1.norm2.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.10.attn.proj.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.10.attn.proj.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.10.attn.qkv.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.10.attn.qkv.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.10.mlp.linear_fc1.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.10.mlp.linear_fc1.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.10.mlp.linear_fc2.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.10.mlp.linear_fc2.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.10.norm1.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.10.norm1.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.10.norm2.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.10.norm2.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.11.attn.proj.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.11.attn.proj.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.11.attn.qkv.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.11.attn.qkv.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.11.mlp.linear_fc1.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.11.mlp.linear_fc1.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.11.mlp.linear_fc2.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.11.mlp.linear_fc2.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.11.norm1.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.11.norm1.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.11.norm2.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.11.norm2.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.12.attn.proj.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.12.attn.proj.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.12.attn.qkv.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.12.attn.qkv.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.12.mlp.linear_fc1.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.12.mlp.linear_fc1.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.12.mlp.linear_fc2.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.12.mlp.linear_fc2.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.12.norm1.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.12.norm1.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.12.norm2.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.12.norm2.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.13.attn.proj.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.13.attn.proj.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.13.attn.qkv.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.13.attn.qkv.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.13.mlp.linear_fc1.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.13.mlp.linear_fc1.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.13.mlp.linear_fc2.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.13.mlp.linear_fc2.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.13.norm1.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.13.norm1.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.13.norm2.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.13.norm2.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.14.attn.proj.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.14.attn.proj.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.14.attn.qkv.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.14.attn.qkv.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.14.mlp.linear_fc1.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.14.mlp.linear_fc1.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.14.mlp.linear_fc2.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.14.mlp.linear_fc2.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.14.norm1.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.14.norm1.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.14.norm2.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.14.norm2.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.15.attn.proj.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.15.attn.proj.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.15.attn.qkv.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.15.attn.qkv.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.15.mlp.linear_fc1.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.15.mlp.linear_fc1.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.15.mlp.linear_fc2.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.15.mlp.linear_fc2.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.15.norm1.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.15.norm1.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.15.norm2.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.15.norm2.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.16.attn.proj.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.16.attn.proj.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.16.attn.qkv.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.16.attn.qkv.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.16.mlp.linear_fc1.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.16.mlp.linear_fc1.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.16.mlp.linear_fc2.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.16.mlp.linear_fc2.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.16.norm1.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.16.norm1.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.16.norm2.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.16.norm2.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.17.attn.proj.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.17.attn.proj.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.17.attn.qkv.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.17.attn.qkv.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.17.mlp.linear_fc1.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.17.mlp.linear_fc1.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.17.mlp.linear_fc2.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.17.mlp.linear_fc2.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.17.norm1.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.17.norm1.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.17.norm2.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.17.norm2.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.18.attn.proj.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.18.attn.proj.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.18.attn.qkv.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.18.attn.qkv.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.18.mlp.linear_fc1.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.18.mlp.linear_fc1.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.18.mlp.linear_fc2.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.18.mlp.linear_fc2.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.18.norm1.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.18.norm1.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.18.norm2.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.18.norm2.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.19.attn.proj.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.19.attn.proj.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.19.attn.qkv.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.19.attn.qkv.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.19.mlp.linear_fc1.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.19.mlp.linear_fc1.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.19.mlp.linear_fc2.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.19.mlp.linear_fc2.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.19.norm1.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.19.norm1.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.19.norm2.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.19.norm2.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.2.attn.proj.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.2.attn.proj.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.2.attn.qkv.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.2.attn.qkv.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.2.mlp.linear_fc1.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.2.mlp.linear_fc1.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.2.mlp.linear_fc2.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.2.mlp.linear_fc2.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.2.norm1.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.2.norm1.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.2.norm2.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.2.norm2.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.20.attn.proj.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.20.attn.proj.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.20.attn.qkv.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.20.attn.qkv.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.20.mlp.linear_fc1.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.20.mlp.linear_fc1.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.20.mlp.linear_fc2.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.20.mlp.linear_fc2.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.20.norm1.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.20.norm1.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.20.norm2.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.20.norm2.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.21.attn.proj.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.21.attn.proj.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.21.attn.qkv.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.21.attn.qkv.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.21.mlp.linear_fc1.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.21.mlp.linear_fc1.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.21.mlp.linear_fc2.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.21.mlp.linear_fc2.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.21.norm1.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.21.norm1.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.21.norm2.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.21.norm2.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.22.attn.proj.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.22.attn.proj.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.22.attn.qkv.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.22.attn.qkv.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.22.mlp.linear_fc1.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.22.mlp.linear_fc1.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.22.mlp.linear_fc2.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.22.mlp.linear_fc2.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.22.norm1.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.22.norm1.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.22.norm2.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.22.norm2.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.23.attn.proj.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.23.attn.proj.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.23.attn.qkv.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.23.attn.qkv.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.23.mlp.linear_fc1.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.23.mlp.linear_fc1.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.23.mlp.linear_fc2.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.23.mlp.linear_fc2.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.23.norm1.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.23.norm1.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.23.norm2.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.23.norm2.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.3.attn.proj.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.3.attn.proj.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.3.attn.qkv.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.3.attn.qkv.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.3.mlp.linear_fc1.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.3.mlp.linear_fc1.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.3.mlp.linear_fc2.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.3.mlp.linear_fc2.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.3.norm1.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.3.norm1.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.3.norm2.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.3.norm2.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.4.attn.proj.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.4.attn.proj.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.4.attn.qkv.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.4.attn.qkv.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.4.mlp.linear_fc1.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.4.mlp.linear_fc1.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.4.mlp.linear_fc2.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.4.mlp.linear_fc2.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.4.norm1.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.4.norm1.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.4.norm2.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.4.norm2.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.5.attn.proj.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.5.attn.proj.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.5.attn.qkv.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.5.attn.qkv.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.5.mlp.linear_fc1.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.5.mlp.linear_fc1.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.5.mlp.linear_fc2.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.5.mlp.linear_fc2.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.5.norm1.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.5.norm1.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.5.norm2.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.5.norm2.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.6.attn.proj.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.6.attn.proj.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.6.attn.qkv.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.6.attn.qkv.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.6.mlp.linear_fc1.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.6.mlp.linear_fc1.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.6.mlp.linear_fc2.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.6.mlp.linear_fc2.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.6.norm1.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.6.norm1.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.6.norm2.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.6.norm2.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.7.attn.proj.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.7.attn.proj.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.7.attn.qkv.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.7.attn.qkv.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.7.mlp.linear_fc1.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.7.mlp.linear_fc1.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.7.mlp.linear_fc2.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.7.mlp.linear_fc2.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.7.norm1.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.7.norm1.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.7.norm2.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.7.norm2.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.8.attn.proj.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.8.attn.proj.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.8.attn.qkv.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.8.attn.qkv.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.8.mlp.linear_fc1.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.8.mlp.linear_fc1.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.8.mlp.linear_fc2.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.8.mlp.linear_fc2.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.8.norm1.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.8.norm1.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.8.norm2.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.8.norm2.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.9.attn.proj.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.9.attn.proj.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.9.attn.qkv.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.9.attn.qkv.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.9.mlp.linear_fc1.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.9.mlp.linear_fc1.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.9.mlp.linear_fc2.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.9.mlp.linear_fc2.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.9.norm1.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.9.norm1.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.9.norm2.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.9.norm2.weight": "model-00001-of-00002.safetensors", + "model.visual.deepstack_merger_list.0.linear_fc1.bias": "model-00001-of-00002.safetensors", + "model.visual.deepstack_merger_list.0.linear_fc1.weight": "model-00001-of-00002.safetensors", + "model.visual.deepstack_merger_list.0.linear_fc2.bias": "model-00001-of-00002.safetensors", + "model.visual.deepstack_merger_list.0.linear_fc2.weight": "model-00001-of-00002.safetensors", + "model.visual.deepstack_merger_list.0.norm.bias": "model-00001-of-00002.safetensors", + "model.visual.deepstack_merger_list.0.norm.weight": "model-00001-of-00002.safetensors", + "model.visual.deepstack_merger_list.1.linear_fc1.bias": "model-00001-of-00002.safetensors", + "model.visual.deepstack_merger_list.1.linear_fc1.weight": "model-00001-of-00002.safetensors", + "model.visual.deepstack_merger_list.1.linear_fc2.bias": "model-00001-of-00002.safetensors", + "model.visual.deepstack_merger_list.1.linear_fc2.weight": "model-00001-of-00002.safetensors", + "model.visual.deepstack_merger_list.1.norm.bias": "model-00001-of-00002.safetensors", + "model.visual.deepstack_merger_list.1.norm.weight": "model-00001-of-00002.safetensors", + "model.visual.deepstack_merger_list.2.linear_fc1.bias": "model-00001-of-00002.safetensors", + "model.visual.deepstack_merger_list.2.linear_fc1.weight": "model-00001-of-00002.safetensors", + "model.visual.deepstack_merger_list.2.linear_fc2.bias": "model-00001-of-00002.safetensors", + "model.visual.deepstack_merger_list.2.linear_fc2.weight": "model-00001-of-00002.safetensors", + "model.visual.deepstack_merger_list.2.norm.bias": "model-00001-of-00002.safetensors", + "model.visual.deepstack_merger_list.2.norm.weight": "model-00001-of-00002.safetensors", + "model.visual.merger.linear_fc1.bias": "model-00001-of-00002.safetensors", + "model.visual.merger.linear_fc1.weight": "model-00001-of-00002.safetensors", + "model.visual.merger.linear_fc2.bias": "model-00001-of-00002.safetensors", + "model.visual.merger.linear_fc2.weight": "model-00001-of-00002.safetensors", + "model.visual.merger.norm.bias": "model-00001-of-00002.safetensors", + "model.visual.merger.norm.weight": "model-00001-of-00002.safetensors", + "model.visual.patch_embed.proj.bias": "model-00001-of-00002.safetensors", + "model.visual.patch_embed.proj.weight": "model-00001-of-00002.safetensors", + "model.visual.pos_embed.weight": "model-00001-of-00002.safetensors" + } +} diff --git a/posttrain_models/Qwen3VL_4B_lora32_batch1245789_epoch4/preprocessor_config.json b/posttrain_models/Qwen3VL_4B_lora32_batch1245789_epoch4/preprocessor_config.json new file mode 100644 index 0000000000000000000000000000000000000000..2fa6553554f69b4e3a6b742a75264ff9dff90ad0 --- /dev/null +++ b/posttrain_models/Qwen3VL_4B_lora32_batch1245789_epoch4/preprocessor_config.json @@ -0,0 +1,39 @@ +{ + "crop_size": null, + "data_format": "channels_first", + "default_to_square": true, + "device": null, + "disable_grouping": null, + "do_center_crop": null, + "do_convert_rgb": true, + "do_normalize": true, + "do_pad": null, + "do_rescale": true, + "do_resize": true, + "image_mean": [ + 0.5, + 0.5, + 0.5 + ], + "image_processor_type": "Qwen2VLImageProcessorFast", + "image_std": [ + 0.5, + 0.5, + 0.5 + ], + "input_data_format": null, + "max_pixels": null, + "merge_size": 2, + "min_pixels": null, + "pad_size": null, + "patch_size": 16, + "processor_class": "Qwen3VLProcessor", + "resample": 3, + "rescale_factor": 0.00392156862745098, + "return_tensors": null, + "size": { + "longest_edge": 16777216, + "shortest_edge": 65536 + }, + "temporal_patch_size": 2 +} diff --git a/posttrain_models/Qwen3VL_4B_lora32_batch1245789_epoch4/special_tokens_map.json b/posttrain_models/Qwen3VL_4B_lora32_batch1245789_epoch4/special_tokens_map.json new file mode 100644 index 0000000000000000000000000000000000000000..ac23c0aaa2434523c494330aeb79c58395378103 --- /dev/null +++ b/posttrain_models/Qwen3VL_4B_lora32_batch1245789_epoch4/special_tokens_map.json @@ -0,0 +1,31 @@ +{ + "additional_special_tokens": [ + "<|im_start|>", + "<|im_end|>", + "<|object_ref_start|>", + "<|object_ref_end|>", + "<|box_start|>", + "<|box_end|>", + "<|quad_start|>", + "<|quad_end|>", + "<|vision_start|>", + "<|vision_end|>", + "<|vision_pad|>", + "<|image_pad|>", + "<|video_pad|>" + ], + "eos_token": { + "content": "<|im_end|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false + }, + "pad_token": { + "content": "<|endoftext|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false + } +} diff --git a/posttrain_models/Qwen3VL_4B_lora32_batch1245789_epoch4/tokenizer_config.json b/posttrain_models/Qwen3VL_4B_lora32_batch1245789_epoch4/tokenizer_config.json new file mode 100644 index 0000000000000000000000000000000000000000..fec7f182135c384a08ea2fc9635566ca79c45afc --- /dev/null +++ b/posttrain_models/Qwen3VL_4B_lora32_batch1245789_epoch4/tokenizer_config.json @@ -0,0 +1,240 @@ +{ + "add_bos_token": false, + "add_prefix_space": false, + "added_tokens_decoder": { + "151643": { + "content": "<|endoftext|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "151644": { + "content": "<|im_start|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "151645": { + "content": "<|im_end|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "151646": { + "content": "<|object_ref_start|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "151647": { + "content": "<|object_ref_end|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "151648": { + "content": "<|box_start|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "151649": { + "content": "<|box_end|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "151650": { + "content": "<|quad_start|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "151651": { + "content": "<|quad_end|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "151652": { + "content": "<|vision_start|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "151653": { + "content": "<|vision_end|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "151654": { + "content": "<|vision_pad|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "151655": { + "content": "<|image_pad|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "151656": { + "content": "<|video_pad|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "151657": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "151658": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "151659": { + "content": "<|fim_prefix|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "151660": { + "content": "<|fim_middle|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "151661": { + "content": "<|fim_suffix|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "151662": { + "content": "<|fim_pad|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "151663": { + "content": "<|repo_name|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "151664": { + "content": "<|file_sep|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "151665": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "151666": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "151667": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "151668": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + } + }, + "additional_special_tokens": [ + "<|im_start|>", + "<|im_end|>", + "<|object_ref_start|>", + "<|object_ref_end|>", + "<|box_start|>", + "<|box_end|>", + "<|quad_start|>", + "<|quad_end|>", + "<|vision_start|>", + "<|vision_end|>", + "<|vision_pad|>", + "<|image_pad|>", + "<|video_pad|>" + ], + "bos_token": null, + "clean_up_tokenization_spaces": false, + "eos_token": "<|im_end|>", + "errors": "replace", + "extra_special_tokens": {}, + "model_max_length": 262144, + "pad_token": "<|endoftext|>", + "processor_class": "Qwen3VLProcessor", + "split_special_tokens": false, + "tokenizer_class": "Qwen2Tokenizer", + "unk_token": null +} diff --git a/posttrain_models/Qwen3VL_4B_lora32_batch1245789_epoch4/video_preprocessor_config.json b/posttrain_models/Qwen3VL_4B_lora32_batch1245789_epoch4/video_preprocessor_config.json new file mode 100644 index 0000000000000000000000000000000000000000..e32b1d90356f74457edbfca27aa1e121dab7cdc7 --- /dev/null +++ b/posttrain_models/Qwen3VL_4B_lora32_batch1245789_epoch4/video_preprocessor_config.json @@ -0,0 +1,41 @@ +{ + "crop_size": null, + "data_format": "channels_first", + "default_to_square": true, + "device": null, + "do_center_crop": null, + "do_convert_rgb": true, + "do_normalize": true, + "do_rescale": true, + "do_resize": true, + "do_sample_frames": true, + "fps": 2, + "image_mean": [ + 0.5, + 0.5, + 0.5 + ], + "image_std": [ + 0.5, + 0.5, + 0.5 + ], + "input_data_format": null, + "max_frames": 768, + "merge_size": 2, + "min_frames": 4, + "num_frames": null, + "pad_size": null, + "patch_size": 16, + "processor_class": "Qwen3VLProcessor", + "resample": 3, + "rescale_factor": 0.00392156862745098, + "return_metadata": false, + "size": { + "longest_edge": 25165824, + "shortest_edge": 4096 + }, + "temporal_patch_size": 2, + "video_metadata": null, + "video_processor_type": "Qwen3VLVideoProcessor" +} diff --git a/posttrain_models/Qwen3VL_4B_lora32_batch124578_epoch4/added_tokens.json b/posttrain_models/Qwen3VL_4B_lora32_batch124578_epoch4/added_tokens.json new file mode 100644 index 0000000000000000000000000000000000000000..b54f9135e44c1e81047e8d05cb027af8bc039eed --- /dev/null +++ b/posttrain_models/Qwen3VL_4B_lora32_batch124578_epoch4/added_tokens.json @@ -0,0 +1,28 @@ +{ + "": 151668, + "": 151658, + "": 151666, + "": 151667, + "": 151657, + "": 151665, + "<|box_end|>": 151649, + "<|box_start|>": 151648, + "<|endoftext|>": 151643, + "<|file_sep|>": 151664, + "<|fim_middle|>": 151660, + "<|fim_pad|>": 151662, + "<|fim_prefix|>": 151659, + "<|fim_suffix|>": 151661, + "<|im_end|>": 151645, + "<|im_start|>": 151644, + "<|image_pad|>": 151655, + "<|object_ref_end|>": 151647, + "<|object_ref_start|>": 151646, + "<|quad_end|>": 151651, + "<|quad_start|>": 151650, + "<|repo_name|>": 151663, + "<|video_pad|>": 151656, + "<|vision_end|>": 151653, + "<|vision_pad|>": 151654, + "<|vision_start|>": 151652 +} diff --git a/posttrain_models/Qwen3VL_4B_lora32_batch124578_epoch4/chat_template.jinja b/posttrain_models/Qwen3VL_4B_lora32_batch124578_epoch4/chat_template.jinja new file mode 100644 index 0000000000000000000000000000000000000000..124386803f142761528f710e77ae483f5f8c4fc4 --- /dev/null +++ b/posttrain_models/Qwen3VL_4B_lora32_batch124578_epoch4/chat_template.jinja @@ -0,0 +1,120 @@ +{%- if tools %} + {{- '<|im_start|>system\n' }} + {%- if messages[0].role == 'system' %} + {%- if messages[0].content is string %} + {{- messages[0].content }} + {%- else %} + {%- for content in messages[0].content %} + {%- if 'text' in content %} + {{- content.text }} + {%- endif %} + {%- endfor %} + {%- endif %} + {{- '\n\n' }} + {%- endif %} + {{- "# Tools\n\nYou may call one or more functions to assist with the user query.\n\nYou are provided with function signatures within XML tags:\n" }} + {%- for tool in tools %} + {{- "\n" }} + {{- tool | tojson }} + {%- endfor %} + {{- "\n\n\nFor each function call, return a json object with function name and arguments within XML tags:\n\n{\"name\": , \"arguments\": }\n<|im_end|>\n" }} +{%- else %} + {%- if messages[0].role == 'system' %} + {{- '<|im_start|>system\n' }} + {%- if messages[0].content is string %} + {{- messages[0].content }} + {%- else %} + {%- for content in messages[0].content %} + {%- if 'text' in content %} + {{- content.text }} + {%- endif %} + {%- endfor %} + {%- endif %} + {{- '<|im_end|>\n' }} + {%- endif %} +{%- endif %} +{%- set image_count = namespace(value=0) %} +{%- set video_count = namespace(value=0) %} +{%- for message in messages %} + {%- if message.role == "user" %} + {{- '<|im_start|>' + message.role + '\n' }} + {%- if message.content is string %} + {{- message.content }} + {%- else %} + {%- for content in message.content %} + {%- if content.type == 'image' or 'image' in content or 'image_url' in content %} + {%- set image_count.value = image_count.value + 1 %} + {%- if add_vision_id %}Picture {{ image_count.value }}: {% endif -%} + <|vision_start|><|image_pad|><|vision_end|> + {%- elif content.type == 'video' or 'video' in content %} + {%- set video_count.value = video_count.value + 1 %} + {%- if add_vision_id %}Video {{ video_count.value }}: {% endif -%} + <|vision_start|><|video_pad|><|vision_end|> + {%- elif 'text' in content %} + {{- content.text }} + {%- endif %} + {%- endfor %} + {%- endif %} + {{- '<|im_end|>\n' }} + {%- elif message.role == "assistant" %} + {{- '<|im_start|>' + message.role + '\n' }} + {%- if message.content is string %} + {{- message.content }} + {%- else %} + {%- for content_item in message.content %} + {%- if 'text' in content_item %} + {{- content_item.text }} + {%- endif %} + {%- endfor %} + {%- endif %} + {%- if message.tool_calls %} + {%- for tool_call in message.tool_calls %} + {%- if (loop.first and message.content) or (not loop.first) %} + {{- '\n' }} + {%- endif %} + {%- if tool_call.function %} + {%- set tool_call = tool_call.function %} + {%- endif %} + {{- '\n{"name": "' }} + {{- tool_call.name }} + {{- '", "arguments": ' }} + {%- if tool_call.arguments is string %} + {{- tool_call.arguments }} + {%- else %} + {{- tool_call.arguments | tojson }} + {%- endif %} + {{- '}\n' }} + {%- endfor %} + {%- endif %} + {{- '<|im_end|>\n' }} + {%- elif message.role == "tool" %} + {%- if loop.first or (messages[loop.index0 - 1].role != "tool") %} + {{- '<|im_start|>user' }} + {%- endif %} + {{- '\n\n' }} + {%- if message.content is string %} + {{- message.content }} + {%- else %} + {%- for content in message.content %} + {%- if content.type == 'image' or 'image' in content or 'image_url' in content %} + {%- set image_count.value = image_count.value + 1 %} + {%- if add_vision_id %}Picture {{ image_count.value }}: {% endif -%} + <|vision_start|><|image_pad|><|vision_end|> + {%- elif content.type == 'video' or 'video' in content %} + {%- set video_count.value = video_count.value + 1 %} + {%- if add_vision_id %}Video {{ video_count.value }}: {% endif -%} + <|vision_start|><|video_pad|><|vision_end|> + {%- elif 'text' in content %} + {{- content.text }} + {%- endif %} + {%- endfor %} + {%- endif %} + {{- '\n' }} + {%- if loop.last or (messages[loop.index0 + 1].role != "tool") %} + {{- '<|im_end|>\n' }} + {%- endif %} + {%- endif %} +{%- endfor %} +{%- if add_generation_prompt %} + {{- '<|im_start|>assistant\n' }} +{%- endif %} diff --git a/posttrain_models/Qwen3VL_4B_lora32_batch124578_epoch4/config.json b/posttrain_models/Qwen3VL_4B_lora32_batch124578_epoch4/config.json new file mode 100644 index 0000000000000000000000000000000000000000..adb449916091c26098a94af0fa4add48558a9494 --- /dev/null +++ b/posttrain_models/Qwen3VL_4B_lora32_batch124578_epoch4/config.json @@ -0,0 +1,65 @@ +{ + "architectures": [ + "Qwen3VLForConditionalGeneration" + ], + "dtype": "bfloat16", + "image_token_id": 151655, + "model_type": "qwen3_vl", + "text_config": { + "attention_bias": false, + "attention_dropout": 0.0, + "bos_token_id": 151643, + "dtype": "bfloat16", + "eos_token_id": 151645, + "head_dim": 128, + "hidden_act": "silu", + "hidden_size": 2560, + "initializer_range": 0.02, + "intermediate_size": 9728, + "max_position_embeddings": 262144, + "model_type": "qwen3_vl_text", + "num_attention_heads": 32, + "num_hidden_layers": 36, + "num_key_value_heads": 8, + "rms_norm_eps": 1e-06, + "rope_scaling": { + "mrope_interleaved": true, + "mrope_section": [ + 24, + 20, + 20 + ], + "rope_type": "default" + }, + "rope_theta": 5000000, + "tie_word_embeddings": true, + "use_cache": true, + "vocab_size": 151936 + }, + "tie_word_embeddings": true, + "transformers_version": "4.57.3", + "video_token_id": 151656, + "vision_config": { + "deepstack_visual_indexes": [ + 5, + 11, + 17 + ], + "depth": 24, + "dtype": "bfloat16", + "hidden_act": "gelu_pytorch_tanh", + "hidden_size": 1024, + "in_channels": 3, + "initializer_range": 0.02, + "intermediate_size": 4096, + "model_type": "qwen3_vl", + "num_heads": 16, + "num_position_embeddings": 2304, + "out_hidden_size": 2560, + "patch_size": 16, + "spatial_merge_size": 2, + "temporal_patch_size": 2 + }, + "vision_end_token_id": 151653, + "vision_start_token_id": 151652 +} diff --git a/posttrain_models/Qwen3VL_4B_lora32_batch124578_epoch4/generation_config.json b/posttrain_models/Qwen3VL_4B_lora32_batch124578_epoch4/generation_config.json new file mode 100644 index 0000000000000000000000000000000000000000..3dd2c545b96b4d51586680d7bb27fdca5c90dfe9 --- /dev/null +++ b/posttrain_models/Qwen3VL_4B_lora32_batch124578_epoch4/generation_config.json @@ -0,0 +1,13 @@ +{ + "bos_token_id": 151643, + "do_sample": true, + "eos_token_id": [ + 151645, + 151643 + ], + "pad_token_id": 151643, + "temperature": 0.7, + "top_k": 20, + "top_p": 0.8, + "transformers_version": "4.57.3" +} diff --git a/posttrain_models/Qwen3VL_4B_lora32_batch124578_epoch4/model.safetensors.index.json b/posttrain_models/Qwen3VL_4B_lora32_batch124578_epoch4/model.safetensors.index.json new file mode 100644 index 0000000000000000000000000000000000000000..58d118e8ce51a42ed3b147a55f9c92f39d273c60 --- /dev/null +++ b/posttrain_models/Qwen3VL_4B_lora32_batch124578_epoch4/model.safetensors.index.json @@ -0,0 +1,721 @@ +{ + "metadata": { + "total_parameters": 4437815808, + "total_size": 8875631616 + }, + "weight_map": { + "model.language_model.embed_tokens.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.0.input_layernorm.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.0.mlp.down_proj.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.0.mlp.gate_proj.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.0.mlp.up_proj.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.0.post_attention_layernorm.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.0.self_attn.k_norm.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.0.self_attn.k_proj.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.0.self_attn.o_proj.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.0.self_attn.q_norm.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.0.self_attn.q_proj.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.0.self_attn.v_proj.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.1.input_layernorm.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.1.mlp.down_proj.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.1.mlp.gate_proj.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.1.mlp.up_proj.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.1.post_attention_layernorm.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.1.self_attn.k_norm.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.1.self_attn.k_proj.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.1.self_attn.o_proj.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.1.self_attn.q_norm.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.1.self_attn.q_proj.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.1.self_attn.v_proj.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.10.input_layernorm.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.10.mlp.down_proj.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.10.mlp.gate_proj.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.10.mlp.up_proj.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.10.post_attention_layernorm.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.10.self_attn.k_norm.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.10.self_attn.k_proj.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.10.self_attn.o_proj.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.10.self_attn.q_norm.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.10.self_attn.q_proj.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.10.self_attn.v_proj.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.11.input_layernorm.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.11.mlp.down_proj.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.11.mlp.gate_proj.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.11.mlp.up_proj.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.11.post_attention_layernorm.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.11.self_attn.k_norm.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.11.self_attn.k_proj.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.11.self_attn.o_proj.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.11.self_attn.q_norm.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.11.self_attn.q_proj.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.11.self_attn.v_proj.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.12.input_layernorm.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.12.mlp.down_proj.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.12.mlp.gate_proj.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.12.mlp.up_proj.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.12.post_attention_layernorm.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.12.self_attn.k_norm.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.12.self_attn.k_proj.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.12.self_attn.o_proj.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.12.self_attn.q_norm.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.12.self_attn.q_proj.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.12.self_attn.v_proj.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.13.input_layernorm.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.13.mlp.down_proj.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.13.mlp.gate_proj.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.13.mlp.up_proj.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.13.post_attention_layernorm.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.13.self_attn.k_norm.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.13.self_attn.k_proj.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.13.self_attn.o_proj.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.13.self_attn.q_norm.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.13.self_attn.q_proj.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.13.self_attn.v_proj.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.14.input_layernorm.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.14.mlp.down_proj.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.14.mlp.gate_proj.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.14.mlp.up_proj.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.14.post_attention_layernorm.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.14.self_attn.k_norm.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.14.self_attn.k_proj.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.14.self_attn.o_proj.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.14.self_attn.q_norm.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.14.self_attn.q_proj.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.14.self_attn.v_proj.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.15.input_layernorm.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.15.mlp.down_proj.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.15.mlp.gate_proj.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.15.mlp.up_proj.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.15.post_attention_layernorm.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.15.self_attn.k_norm.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.15.self_attn.k_proj.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.15.self_attn.o_proj.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.15.self_attn.q_norm.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.15.self_attn.q_proj.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.15.self_attn.v_proj.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.16.input_layernorm.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.16.mlp.down_proj.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.16.mlp.gate_proj.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.16.mlp.up_proj.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.16.post_attention_layernorm.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.16.self_attn.k_norm.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.16.self_attn.k_proj.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.16.self_attn.o_proj.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.16.self_attn.q_norm.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.16.self_attn.q_proj.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.16.self_attn.v_proj.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.17.input_layernorm.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.17.mlp.down_proj.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.17.mlp.gate_proj.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.17.mlp.up_proj.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.17.post_attention_layernorm.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.17.self_attn.k_norm.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.17.self_attn.k_proj.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.17.self_attn.o_proj.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.17.self_attn.q_norm.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.17.self_attn.q_proj.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.17.self_attn.v_proj.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.18.input_layernorm.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.18.mlp.down_proj.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.18.mlp.gate_proj.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.18.mlp.up_proj.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.18.post_attention_layernorm.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.18.self_attn.k_norm.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.18.self_attn.k_proj.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.18.self_attn.o_proj.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.18.self_attn.q_norm.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.18.self_attn.q_proj.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.18.self_attn.v_proj.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.19.input_layernorm.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.19.mlp.down_proj.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.19.mlp.gate_proj.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.19.mlp.up_proj.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.19.post_attention_layernorm.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.19.self_attn.k_norm.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.19.self_attn.k_proj.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.19.self_attn.o_proj.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.19.self_attn.q_norm.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.19.self_attn.q_proj.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.19.self_attn.v_proj.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.2.input_layernorm.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.2.mlp.down_proj.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.2.mlp.gate_proj.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.2.mlp.up_proj.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.2.post_attention_layernorm.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.2.self_attn.k_norm.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.2.self_attn.k_proj.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.2.self_attn.o_proj.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.2.self_attn.q_norm.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.2.self_attn.q_proj.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.2.self_attn.v_proj.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.20.input_layernorm.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.20.mlp.down_proj.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.20.mlp.gate_proj.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.20.mlp.up_proj.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.20.post_attention_layernorm.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.20.self_attn.k_norm.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.20.self_attn.k_proj.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.20.self_attn.o_proj.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.20.self_attn.q_norm.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.20.self_attn.q_proj.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.20.self_attn.v_proj.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.21.input_layernorm.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.21.mlp.down_proj.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.21.mlp.gate_proj.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.21.mlp.up_proj.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.21.post_attention_layernorm.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.21.self_attn.k_norm.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.21.self_attn.k_proj.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.21.self_attn.o_proj.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.21.self_attn.q_norm.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.21.self_attn.q_proj.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.21.self_attn.v_proj.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.22.input_layernorm.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.22.mlp.down_proj.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.22.mlp.gate_proj.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.22.mlp.up_proj.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.22.post_attention_layernorm.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.22.self_attn.k_norm.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.22.self_attn.k_proj.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.22.self_attn.o_proj.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.22.self_attn.q_norm.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.22.self_attn.q_proj.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.22.self_attn.v_proj.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.23.input_layernorm.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.23.mlp.down_proj.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.23.mlp.gate_proj.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.23.mlp.up_proj.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.23.post_attention_layernorm.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.23.self_attn.k_norm.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.23.self_attn.k_proj.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.23.self_attn.o_proj.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.23.self_attn.q_norm.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.23.self_attn.q_proj.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.23.self_attn.v_proj.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.24.input_layernorm.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.24.mlp.down_proj.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.24.mlp.gate_proj.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.24.mlp.up_proj.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.24.post_attention_layernorm.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.24.self_attn.k_norm.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.24.self_attn.k_proj.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.24.self_attn.o_proj.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.24.self_attn.q_norm.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.24.self_attn.q_proj.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.24.self_attn.v_proj.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.25.input_layernorm.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.25.mlp.down_proj.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.25.mlp.gate_proj.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.25.mlp.up_proj.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.25.post_attention_layernorm.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.25.self_attn.k_norm.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.25.self_attn.k_proj.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.25.self_attn.o_proj.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.25.self_attn.q_norm.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.25.self_attn.q_proj.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.25.self_attn.v_proj.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.26.input_layernorm.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.26.mlp.down_proj.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.26.mlp.gate_proj.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.26.mlp.up_proj.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.26.post_attention_layernorm.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.26.self_attn.k_norm.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.26.self_attn.k_proj.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.26.self_attn.o_proj.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.26.self_attn.q_norm.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.26.self_attn.q_proj.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.26.self_attn.v_proj.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.27.input_layernorm.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.27.mlp.down_proj.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.27.mlp.gate_proj.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.27.mlp.up_proj.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.27.post_attention_layernorm.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.27.self_attn.k_norm.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.27.self_attn.k_proj.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.27.self_attn.o_proj.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.27.self_attn.q_norm.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.27.self_attn.q_proj.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.27.self_attn.v_proj.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.28.input_layernorm.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.28.mlp.down_proj.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.28.mlp.gate_proj.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.28.mlp.up_proj.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.28.post_attention_layernorm.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.28.self_attn.k_norm.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.28.self_attn.k_proj.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.28.self_attn.o_proj.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.28.self_attn.q_norm.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.28.self_attn.q_proj.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.28.self_attn.v_proj.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.29.input_layernorm.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.29.mlp.down_proj.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.29.mlp.gate_proj.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.29.mlp.up_proj.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.29.post_attention_layernorm.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.29.self_attn.k_norm.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.29.self_attn.k_proj.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.29.self_attn.o_proj.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.29.self_attn.q_norm.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.29.self_attn.q_proj.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.29.self_attn.v_proj.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.3.input_layernorm.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.3.mlp.down_proj.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.3.mlp.gate_proj.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.3.mlp.up_proj.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.3.post_attention_layernorm.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.3.self_attn.k_norm.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.3.self_attn.k_proj.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.3.self_attn.o_proj.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.3.self_attn.q_norm.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.3.self_attn.q_proj.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.3.self_attn.v_proj.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.30.input_layernorm.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.30.mlp.down_proj.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.30.mlp.gate_proj.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.30.mlp.up_proj.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.30.post_attention_layernorm.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.30.self_attn.k_norm.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.30.self_attn.k_proj.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.30.self_attn.o_proj.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.30.self_attn.q_norm.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.30.self_attn.q_proj.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.30.self_attn.v_proj.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.31.input_layernorm.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.31.mlp.down_proj.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.31.mlp.gate_proj.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.31.mlp.up_proj.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.31.post_attention_layernorm.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.31.self_attn.k_norm.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.31.self_attn.k_proj.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.31.self_attn.o_proj.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.31.self_attn.q_norm.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.31.self_attn.q_proj.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.31.self_attn.v_proj.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.32.input_layernorm.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.32.mlp.down_proj.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.32.mlp.gate_proj.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.32.mlp.up_proj.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.32.post_attention_layernorm.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.32.self_attn.k_norm.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.32.self_attn.k_proj.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.32.self_attn.o_proj.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.32.self_attn.q_norm.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.32.self_attn.q_proj.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.32.self_attn.v_proj.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.33.input_layernorm.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.33.mlp.down_proj.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.33.mlp.gate_proj.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.33.mlp.up_proj.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.33.post_attention_layernorm.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.33.self_attn.k_norm.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.33.self_attn.k_proj.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.33.self_attn.o_proj.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.33.self_attn.q_norm.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.33.self_attn.q_proj.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.33.self_attn.v_proj.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.34.input_layernorm.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.34.mlp.down_proj.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.34.mlp.gate_proj.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.34.mlp.up_proj.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.34.post_attention_layernorm.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.34.self_attn.k_norm.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.34.self_attn.k_proj.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.34.self_attn.o_proj.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.34.self_attn.q_norm.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.34.self_attn.q_proj.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.34.self_attn.v_proj.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.35.input_layernorm.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.35.mlp.down_proj.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.35.mlp.gate_proj.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.35.mlp.up_proj.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.35.post_attention_layernorm.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.35.self_attn.k_norm.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.35.self_attn.k_proj.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.35.self_attn.o_proj.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.35.self_attn.q_norm.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.35.self_attn.q_proj.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.35.self_attn.v_proj.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.4.input_layernorm.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.4.mlp.down_proj.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.4.mlp.gate_proj.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.4.mlp.up_proj.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.4.post_attention_layernorm.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.4.self_attn.k_norm.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.4.self_attn.k_proj.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.4.self_attn.o_proj.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.4.self_attn.q_norm.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.4.self_attn.q_proj.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.4.self_attn.v_proj.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.5.input_layernorm.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.5.mlp.down_proj.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.5.mlp.gate_proj.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.5.mlp.up_proj.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.5.post_attention_layernorm.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.5.self_attn.k_norm.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.5.self_attn.k_proj.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.5.self_attn.o_proj.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.5.self_attn.q_norm.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.5.self_attn.q_proj.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.5.self_attn.v_proj.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.6.input_layernorm.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.6.mlp.down_proj.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.6.mlp.gate_proj.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.6.mlp.up_proj.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.6.post_attention_layernorm.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.6.self_attn.k_norm.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.6.self_attn.k_proj.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.6.self_attn.o_proj.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.6.self_attn.q_norm.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.6.self_attn.q_proj.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.6.self_attn.v_proj.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.7.input_layernorm.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.7.mlp.down_proj.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.7.mlp.gate_proj.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.7.mlp.up_proj.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.7.post_attention_layernorm.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.7.self_attn.k_norm.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.7.self_attn.k_proj.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.7.self_attn.o_proj.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.7.self_attn.q_norm.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.7.self_attn.q_proj.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.7.self_attn.v_proj.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.8.input_layernorm.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.8.mlp.down_proj.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.8.mlp.gate_proj.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.8.mlp.up_proj.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.8.post_attention_layernorm.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.8.self_attn.k_norm.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.8.self_attn.k_proj.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.8.self_attn.o_proj.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.8.self_attn.q_norm.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.8.self_attn.q_proj.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.8.self_attn.v_proj.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.9.input_layernorm.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.9.mlp.down_proj.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.9.mlp.gate_proj.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.9.mlp.up_proj.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.9.post_attention_layernorm.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.9.self_attn.k_norm.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.9.self_attn.k_proj.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.9.self_attn.o_proj.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.9.self_attn.q_norm.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.9.self_attn.q_proj.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.9.self_attn.v_proj.weight": "model-00001-of-00002.safetensors", + "model.language_model.norm.weight": "model-00002-of-00002.safetensors", + "model.visual.blocks.0.attn.proj.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.0.attn.proj.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.0.attn.qkv.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.0.attn.qkv.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.0.mlp.linear_fc1.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.0.mlp.linear_fc1.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.0.mlp.linear_fc2.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.0.mlp.linear_fc2.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.0.norm1.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.0.norm1.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.0.norm2.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.0.norm2.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.1.attn.proj.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.1.attn.proj.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.1.attn.qkv.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.1.attn.qkv.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.1.mlp.linear_fc1.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.1.mlp.linear_fc1.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.1.mlp.linear_fc2.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.1.mlp.linear_fc2.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.1.norm1.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.1.norm1.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.1.norm2.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.1.norm2.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.10.attn.proj.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.10.attn.proj.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.10.attn.qkv.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.10.attn.qkv.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.10.mlp.linear_fc1.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.10.mlp.linear_fc1.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.10.mlp.linear_fc2.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.10.mlp.linear_fc2.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.10.norm1.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.10.norm1.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.10.norm2.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.10.norm2.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.11.attn.proj.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.11.attn.proj.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.11.attn.qkv.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.11.attn.qkv.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.11.mlp.linear_fc1.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.11.mlp.linear_fc1.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.11.mlp.linear_fc2.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.11.mlp.linear_fc2.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.11.norm1.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.11.norm1.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.11.norm2.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.11.norm2.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.12.attn.proj.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.12.attn.proj.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.12.attn.qkv.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.12.attn.qkv.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.12.mlp.linear_fc1.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.12.mlp.linear_fc1.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.12.mlp.linear_fc2.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.12.mlp.linear_fc2.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.12.norm1.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.12.norm1.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.12.norm2.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.12.norm2.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.13.attn.proj.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.13.attn.proj.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.13.attn.qkv.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.13.attn.qkv.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.13.mlp.linear_fc1.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.13.mlp.linear_fc1.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.13.mlp.linear_fc2.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.13.mlp.linear_fc2.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.13.norm1.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.13.norm1.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.13.norm2.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.13.norm2.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.14.attn.proj.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.14.attn.proj.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.14.attn.qkv.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.14.attn.qkv.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.14.mlp.linear_fc1.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.14.mlp.linear_fc1.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.14.mlp.linear_fc2.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.14.mlp.linear_fc2.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.14.norm1.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.14.norm1.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.14.norm2.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.14.norm2.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.15.attn.proj.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.15.attn.proj.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.15.attn.qkv.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.15.attn.qkv.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.15.mlp.linear_fc1.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.15.mlp.linear_fc1.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.15.mlp.linear_fc2.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.15.mlp.linear_fc2.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.15.norm1.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.15.norm1.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.15.norm2.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.15.norm2.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.16.attn.proj.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.16.attn.proj.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.16.attn.qkv.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.16.attn.qkv.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.16.mlp.linear_fc1.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.16.mlp.linear_fc1.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.16.mlp.linear_fc2.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.16.mlp.linear_fc2.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.16.norm1.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.16.norm1.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.16.norm2.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.16.norm2.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.17.attn.proj.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.17.attn.proj.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.17.attn.qkv.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.17.attn.qkv.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.17.mlp.linear_fc1.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.17.mlp.linear_fc1.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.17.mlp.linear_fc2.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.17.mlp.linear_fc2.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.17.norm1.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.17.norm1.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.17.norm2.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.17.norm2.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.18.attn.proj.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.18.attn.proj.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.18.attn.qkv.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.18.attn.qkv.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.18.mlp.linear_fc1.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.18.mlp.linear_fc1.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.18.mlp.linear_fc2.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.18.mlp.linear_fc2.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.18.norm1.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.18.norm1.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.18.norm2.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.18.norm2.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.19.attn.proj.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.19.attn.proj.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.19.attn.qkv.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.19.attn.qkv.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.19.mlp.linear_fc1.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.19.mlp.linear_fc1.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.19.mlp.linear_fc2.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.19.mlp.linear_fc2.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.19.norm1.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.19.norm1.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.19.norm2.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.19.norm2.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.2.attn.proj.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.2.attn.proj.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.2.attn.qkv.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.2.attn.qkv.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.2.mlp.linear_fc1.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.2.mlp.linear_fc1.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.2.mlp.linear_fc2.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.2.mlp.linear_fc2.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.2.norm1.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.2.norm1.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.2.norm2.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.2.norm2.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.20.attn.proj.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.20.attn.proj.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.20.attn.qkv.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.20.attn.qkv.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.20.mlp.linear_fc1.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.20.mlp.linear_fc1.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.20.mlp.linear_fc2.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.20.mlp.linear_fc2.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.20.norm1.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.20.norm1.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.20.norm2.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.20.norm2.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.21.attn.proj.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.21.attn.proj.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.21.attn.qkv.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.21.attn.qkv.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.21.mlp.linear_fc1.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.21.mlp.linear_fc1.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.21.mlp.linear_fc2.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.21.mlp.linear_fc2.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.21.norm1.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.21.norm1.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.21.norm2.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.21.norm2.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.22.attn.proj.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.22.attn.proj.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.22.attn.qkv.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.22.attn.qkv.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.22.mlp.linear_fc1.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.22.mlp.linear_fc1.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.22.mlp.linear_fc2.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.22.mlp.linear_fc2.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.22.norm1.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.22.norm1.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.22.norm2.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.22.norm2.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.23.attn.proj.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.23.attn.proj.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.23.attn.qkv.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.23.attn.qkv.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.23.mlp.linear_fc1.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.23.mlp.linear_fc1.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.23.mlp.linear_fc2.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.23.mlp.linear_fc2.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.23.norm1.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.23.norm1.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.23.norm2.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.23.norm2.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.3.attn.proj.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.3.attn.proj.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.3.attn.qkv.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.3.attn.qkv.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.3.mlp.linear_fc1.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.3.mlp.linear_fc1.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.3.mlp.linear_fc2.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.3.mlp.linear_fc2.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.3.norm1.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.3.norm1.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.3.norm2.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.3.norm2.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.4.attn.proj.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.4.attn.proj.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.4.attn.qkv.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.4.attn.qkv.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.4.mlp.linear_fc1.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.4.mlp.linear_fc1.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.4.mlp.linear_fc2.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.4.mlp.linear_fc2.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.4.norm1.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.4.norm1.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.4.norm2.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.4.norm2.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.5.attn.proj.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.5.attn.proj.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.5.attn.qkv.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.5.attn.qkv.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.5.mlp.linear_fc1.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.5.mlp.linear_fc1.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.5.mlp.linear_fc2.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.5.mlp.linear_fc2.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.5.norm1.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.5.norm1.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.5.norm2.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.5.norm2.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.6.attn.proj.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.6.attn.proj.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.6.attn.qkv.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.6.attn.qkv.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.6.mlp.linear_fc1.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.6.mlp.linear_fc1.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.6.mlp.linear_fc2.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.6.mlp.linear_fc2.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.6.norm1.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.6.norm1.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.6.norm2.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.6.norm2.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.7.attn.proj.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.7.attn.proj.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.7.attn.qkv.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.7.attn.qkv.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.7.mlp.linear_fc1.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.7.mlp.linear_fc1.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.7.mlp.linear_fc2.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.7.mlp.linear_fc2.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.7.norm1.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.7.norm1.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.7.norm2.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.7.norm2.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.8.attn.proj.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.8.attn.proj.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.8.attn.qkv.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.8.attn.qkv.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.8.mlp.linear_fc1.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.8.mlp.linear_fc1.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.8.mlp.linear_fc2.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.8.mlp.linear_fc2.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.8.norm1.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.8.norm1.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.8.norm2.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.8.norm2.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.9.attn.proj.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.9.attn.proj.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.9.attn.qkv.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.9.attn.qkv.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.9.mlp.linear_fc1.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.9.mlp.linear_fc1.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.9.mlp.linear_fc2.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.9.mlp.linear_fc2.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.9.norm1.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.9.norm1.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.9.norm2.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.9.norm2.weight": "model-00001-of-00002.safetensors", + "model.visual.deepstack_merger_list.0.linear_fc1.bias": "model-00001-of-00002.safetensors", + "model.visual.deepstack_merger_list.0.linear_fc1.weight": "model-00001-of-00002.safetensors", + "model.visual.deepstack_merger_list.0.linear_fc2.bias": "model-00001-of-00002.safetensors", + "model.visual.deepstack_merger_list.0.linear_fc2.weight": "model-00001-of-00002.safetensors", + "model.visual.deepstack_merger_list.0.norm.bias": "model-00001-of-00002.safetensors", + "model.visual.deepstack_merger_list.0.norm.weight": "model-00001-of-00002.safetensors", + "model.visual.deepstack_merger_list.1.linear_fc1.bias": "model-00001-of-00002.safetensors", + "model.visual.deepstack_merger_list.1.linear_fc1.weight": "model-00001-of-00002.safetensors", + "model.visual.deepstack_merger_list.1.linear_fc2.bias": "model-00001-of-00002.safetensors", + "model.visual.deepstack_merger_list.1.linear_fc2.weight": "model-00001-of-00002.safetensors", + "model.visual.deepstack_merger_list.1.norm.bias": "model-00001-of-00002.safetensors", + "model.visual.deepstack_merger_list.1.norm.weight": "model-00001-of-00002.safetensors", + "model.visual.deepstack_merger_list.2.linear_fc1.bias": "model-00001-of-00002.safetensors", + "model.visual.deepstack_merger_list.2.linear_fc1.weight": "model-00001-of-00002.safetensors", + "model.visual.deepstack_merger_list.2.linear_fc2.bias": "model-00001-of-00002.safetensors", + "model.visual.deepstack_merger_list.2.linear_fc2.weight": "model-00001-of-00002.safetensors", + "model.visual.deepstack_merger_list.2.norm.bias": "model-00001-of-00002.safetensors", + "model.visual.deepstack_merger_list.2.norm.weight": "model-00001-of-00002.safetensors", + "model.visual.merger.linear_fc1.bias": "model-00001-of-00002.safetensors", + "model.visual.merger.linear_fc1.weight": "model-00001-of-00002.safetensors", + "model.visual.merger.linear_fc2.bias": "model-00001-of-00002.safetensors", + "model.visual.merger.linear_fc2.weight": "model-00001-of-00002.safetensors", + "model.visual.merger.norm.bias": "model-00001-of-00002.safetensors", + "model.visual.merger.norm.weight": "model-00001-of-00002.safetensors", + "model.visual.patch_embed.proj.bias": "model-00001-of-00002.safetensors", + "model.visual.patch_embed.proj.weight": "model-00001-of-00002.safetensors", + "model.visual.pos_embed.weight": "model-00001-of-00002.safetensors" + } +} diff --git a/posttrain_models/Qwen3VL_4B_lora32_batch124578_epoch4/preprocessor_config.json b/posttrain_models/Qwen3VL_4B_lora32_batch124578_epoch4/preprocessor_config.json new file mode 100644 index 0000000000000000000000000000000000000000..2fa6553554f69b4e3a6b742a75264ff9dff90ad0 --- /dev/null +++ b/posttrain_models/Qwen3VL_4B_lora32_batch124578_epoch4/preprocessor_config.json @@ -0,0 +1,39 @@ +{ + "crop_size": null, + "data_format": "channels_first", + "default_to_square": true, + "device": null, + "disable_grouping": null, + "do_center_crop": null, + "do_convert_rgb": true, + "do_normalize": true, + "do_pad": null, + "do_rescale": true, + "do_resize": true, + "image_mean": [ + 0.5, + 0.5, + 0.5 + ], + "image_processor_type": "Qwen2VLImageProcessorFast", + "image_std": [ + 0.5, + 0.5, + 0.5 + ], + "input_data_format": null, + "max_pixels": null, + "merge_size": 2, + "min_pixels": null, + "pad_size": null, + "patch_size": 16, + "processor_class": "Qwen3VLProcessor", + "resample": 3, + "rescale_factor": 0.00392156862745098, + "return_tensors": null, + "size": { + "longest_edge": 16777216, + "shortest_edge": 65536 + }, + "temporal_patch_size": 2 +} diff --git a/posttrain_models/Qwen3VL_4B_lora32_batch124578_epoch4/special_tokens_map.json b/posttrain_models/Qwen3VL_4B_lora32_batch124578_epoch4/special_tokens_map.json new file mode 100644 index 0000000000000000000000000000000000000000..ac23c0aaa2434523c494330aeb79c58395378103 --- /dev/null +++ b/posttrain_models/Qwen3VL_4B_lora32_batch124578_epoch4/special_tokens_map.json @@ -0,0 +1,31 @@ +{ + "additional_special_tokens": [ + "<|im_start|>", + "<|im_end|>", + "<|object_ref_start|>", + "<|object_ref_end|>", + "<|box_start|>", + "<|box_end|>", + "<|quad_start|>", + "<|quad_end|>", + "<|vision_start|>", + "<|vision_end|>", + "<|vision_pad|>", + "<|image_pad|>", + "<|video_pad|>" + ], + "eos_token": { + "content": "<|im_end|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false + }, + "pad_token": { + "content": "<|endoftext|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false + } +} diff --git a/posttrain_models/Qwen3VL_4B_lora32_batch124578_epoch4/tokenizer_config.json b/posttrain_models/Qwen3VL_4B_lora32_batch124578_epoch4/tokenizer_config.json new file mode 100644 index 0000000000000000000000000000000000000000..fec7f182135c384a08ea2fc9635566ca79c45afc --- /dev/null +++ b/posttrain_models/Qwen3VL_4B_lora32_batch124578_epoch4/tokenizer_config.json @@ -0,0 +1,240 @@ +{ + "add_bos_token": false, + "add_prefix_space": false, + "added_tokens_decoder": { + "151643": { + "content": "<|endoftext|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "151644": { + "content": "<|im_start|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "151645": { + "content": "<|im_end|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "151646": { + "content": "<|object_ref_start|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "151647": { + "content": "<|object_ref_end|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "151648": { + "content": "<|box_start|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "151649": { + "content": "<|box_end|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "151650": { + "content": "<|quad_start|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "151651": { + "content": "<|quad_end|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "151652": { + "content": "<|vision_start|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "151653": { + "content": "<|vision_end|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "151654": { + "content": "<|vision_pad|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "151655": { + "content": "<|image_pad|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "151656": { + "content": "<|video_pad|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "151657": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "151658": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "151659": { + "content": "<|fim_prefix|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "151660": { + "content": "<|fim_middle|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "151661": { + "content": "<|fim_suffix|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "151662": { + "content": "<|fim_pad|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "151663": { + "content": "<|repo_name|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "151664": { + "content": "<|file_sep|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "151665": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "151666": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "151667": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "151668": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + } + }, + "additional_special_tokens": [ + "<|im_start|>", + "<|im_end|>", + "<|object_ref_start|>", + "<|object_ref_end|>", + "<|box_start|>", + "<|box_end|>", + "<|quad_start|>", + "<|quad_end|>", + "<|vision_start|>", + "<|vision_end|>", + "<|vision_pad|>", + "<|image_pad|>", + "<|video_pad|>" + ], + "bos_token": null, + "clean_up_tokenization_spaces": false, + "eos_token": "<|im_end|>", + "errors": "replace", + "extra_special_tokens": {}, + "model_max_length": 262144, + "pad_token": "<|endoftext|>", + "processor_class": "Qwen3VLProcessor", + "split_special_tokens": false, + "tokenizer_class": "Qwen2Tokenizer", + "unk_token": null +} diff --git a/posttrain_models/Qwen3VL_4B_lora32_batch124578_epoch4/video_preprocessor_config.json b/posttrain_models/Qwen3VL_4B_lora32_batch124578_epoch4/video_preprocessor_config.json new file mode 100644 index 0000000000000000000000000000000000000000..e32b1d90356f74457edbfca27aa1e121dab7cdc7 --- /dev/null +++ b/posttrain_models/Qwen3VL_4B_lora32_batch124578_epoch4/video_preprocessor_config.json @@ -0,0 +1,41 @@ +{ + "crop_size": null, + "data_format": "channels_first", + "default_to_square": true, + "device": null, + "do_center_crop": null, + "do_convert_rgb": true, + "do_normalize": true, + "do_rescale": true, + "do_resize": true, + "do_sample_frames": true, + "fps": 2, + "image_mean": [ + 0.5, + 0.5, + 0.5 + ], + "image_std": [ + 0.5, + 0.5, + 0.5 + ], + "input_data_format": null, + "max_frames": 768, + "merge_size": 2, + "min_frames": 4, + "num_frames": null, + "pad_size": null, + "patch_size": 16, + "processor_class": "Qwen3VLProcessor", + "resample": 3, + "rescale_factor": 0.00392156862745098, + "return_metadata": false, + "size": { + "longest_edge": 25165824, + "shortest_edge": 4096 + }, + "temporal_patch_size": 2, + "video_metadata": null, + "video_processor_type": "Qwen3VLVideoProcessor" +} diff --git a/posttrain_models/Qwen3VL_4B_lora32_batch1245910_epoch4/added_tokens.json b/posttrain_models/Qwen3VL_4B_lora32_batch1245910_epoch4/added_tokens.json new file mode 100644 index 0000000000000000000000000000000000000000..b54f9135e44c1e81047e8d05cb027af8bc039eed --- /dev/null +++ b/posttrain_models/Qwen3VL_4B_lora32_batch1245910_epoch4/added_tokens.json @@ -0,0 +1,28 @@ +{ + "": 151668, + "": 151658, + "": 151666, + "": 151667, + "": 151657, + "": 151665, + "<|box_end|>": 151649, + "<|box_start|>": 151648, + "<|endoftext|>": 151643, + "<|file_sep|>": 151664, + "<|fim_middle|>": 151660, + "<|fim_pad|>": 151662, + "<|fim_prefix|>": 151659, + "<|fim_suffix|>": 151661, + "<|im_end|>": 151645, + "<|im_start|>": 151644, + "<|image_pad|>": 151655, + "<|object_ref_end|>": 151647, + "<|object_ref_start|>": 151646, + "<|quad_end|>": 151651, + "<|quad_start|>": 151650, + "<|repo_name|>": 151663, + "<|video_pad|>": 151656, + "<|vision_end|>": 151653, + "<|vision_pad|>": 151654, + "<|vision_start|>": 151652 +} diff --git a/posttrain_models/Qwen3VL_4B_lora32_batch1245910_epoch4/chat_template.jinja b/posttrain_models/Qwen3VL_4B_lora32_batch1245910_epoch4/chat_template.jinja new file mode 100644 index 0000000000000000000000000000000000000000..124386803f142761528f710e77ae483f5f8c4fc4 --- /dev/null +++ b/posttrain_models/Qwen3VL_4B_lora32_batch1245910_epoch4/chat_template.jinja @@ -0,0 +1,120 @@ +{%- if tools %} + {{- '<|im_start|>system\n' }} + {%- if messages[0].role == 'system' %} + {%- if messages[0].content is string %} + {{- messages[0].content }} + {%- else %} + {%- for content in messages[0].content %} + {%- if 'text' in content %} + {{- content.text }} + {%- endif %} + {%- endfor %} + {%- endif %} + {{- '\n\n' }} + {%- endif %} + {{- "# Tools\n\nYou may call one or more functions to assist with the user query.\n\nYou are provided with function signatures within XML tags:\n" }} + {%- for tool in tools %} + {{- "\n" }} + {{- tool | tojson }} + {%- endfor %} + {{- "\n\n\nFor each function call, return a json object with function name and arguments within XML tags:\n\n{\"name\": , \"arguments\": }\n<|im_end|>\n" }} +{%- else %} + {%- if messages[0].role == 'system' %} + {{- '<|im_start|>system\n' }} + {%- if messages[0].content is string %} + {{- messages[0].content }} + {%- else %} + {%- for content in messages[0].content %} + {%- if 'text' in content %} + {{- content.text }} + {%- endif %} + {%- endfor %} + {%- endif %} + {{- '<|im_end|>\n' }} + {%- endif %} +{%- endif %} +{%- set image_count = namespace(value=0) %} +{%- set video_count = namespace(value=0) %} +{%- for message in messages %} + {%- if message.role == "user" %} + {{- '<|im_start|>' + message.role + '\n' }} + {%- if message.content is string %} + {{- message.content }} + {%- else %} + {%- for content in message.content %} + {%- if content.type == 'image' or 'image' in content or 'image_url' in content %} + {%- set image_count.value = image_count.value + 1 %} + {%- if add_vision_id %}Picture {{ image_count.value }}: {% endif -%} + <|vision_start|><|image_pad|><|vision_end|> + {%- elif content.type == 'video' or 'video' in content %} + {%- set video_count.value = video_count.value + 1 %} + {%- if add_vision_id %}Video {{ video_count.value }}: {% endif -%} + <|vision_start|><|video_pad|><|vision_end|> + {%- elif 'text' in content %} + {{- content.text }} + {%- endif %} + {%- endfor %} + {%- endif %} + {{- '<|im_end|>\n' }} + {%- elif message.role == "assistant" %} + {{- '<|im_start|>' + message.role + '\n' }} + {%- if message.content is string %} + {{- message.content }} + {%- else %} + {%- for content_item in message.content %} + {%- if 'text' in content_item %} + {{- content_item.text }} + {%- endif %} + {%- endfor %} + {%- endif %} + {%- if message.tool_calls %} + {%- for tool_call in message.tool_calls %} + {%- if (loop.first and message.content) or (not loop.first) %} + {{- '\n' }} + {%- endif %} + {%- if tool_call.function %} + {%- set tool_call = tool_call.function %} + {%- endif %} + {{- '\n{"name": "' }} + {{- tool_call.name }} + {{- '", "arguments": ' }} + {%- if tool_call.arguments is string %} + {{- tool_call.arguments }} + {%- else %} + {{- tool_call.arguments | tojson }} + {%- endif %} + {{- '}\n' }} + {%- endfor %} + {%- endif %} + {{- '<|im_end|>\n' }} + {%- elif message.role == "tool" %} + {%- if loop.first or (messages[loop.index0 - 1].role != "tool") %} + {{- '<|im_start|>user' }} + {%- endif %} + {{- '\n\n' }} + {%- if message.content is string %} + {{- message.content }} + {%- else %} + {%- for content in message.content %} + {%- if content.type == 'image' or 'image' in content or 'image_url' in content %} + {%- set image_count.value = image_count.value + 1 %} + {%- if add_vision_id %}Picture {{ image_count.value }}: {% endif -%} + <|vision_start|><|image_pad|><|vision_end|> + {%- elif content.type == 'video' or 'video' in content %} + {%- set video_count.value = video_count.value + 1 %} + {%- if add_vision_id %}Video {{ video_count.value }}: {% endif -%} + <|vision_start|><|video_pad|><|vision_end|> + {%- elif 'text' in content %} + {{- content.text }} + {%- endif %} + {%- endfor %} + {%- endif %} + {{- '\n' }} + {%- if loop.last or (messages[loop.index0 + 1].role != "tool") %} + {{- '<|im_end|>\n' }} + {%- endif %} + {%- endif %} +{%- endfor %} +{%- if add_generation_prompt %} + {{- '<|im_start|>assistant\n' }} +{%- endif %} diff --git a/posttrain_models/Qwen3VL_4B_lora32_batch1245910_epoch4/config.json b/posttrain_models/Qwen3VL_4B_lora32_batch1245910_epoch4/config.json new file mode 100644 index 0000000000000000000000000000000000000000..adb449916091c26098a94af0fa4add48558a9494 --- /dev/null +++ b/posttrain_models/Qwen3VL_4B_lora32_batch1245910_epoch4/config.json @@ -0,0 +1,65 @@ +{ + "architectures": [ + "Qwen3VLForConditionalGeneration" + ], + "dtype": "bfloat16", + "image_token_id": 151655, + "model_type": "qwen3_vl", + "text_config": { + "attention_bias": false, + "attention_dropout": 0.0, + "bos_token_id": 151643, + "dtype": "bfloat16", + "eos_token_id": 151645, + "head_dim": 128, + "hidden_act": "silu", + "hidden_size": 2560, + "initializer_range": 0.02, + "intermediate_size": 9728, + "max_position_embeddings": 262144, + "model_type": "qwen3_vl_text", + "num_attention_heads": 32, + "num_hidden_layers": 36, + "num_key_value_heads": 8, + "rms_norm_eps": 1e-06, + "rope_scaling": { + "mrope_interleaved": true, + "mrope_section": [ + 24, + 20, + 20 + ], + "rope_type": "default" + }, + "rope_theta": 5000000, + "tie_word_embeddings": true, + "use_cache": true, + "vocab_size": 151936 + }, + "tie_word_embeddings": true, + "transformers_version": "4.57.3", + "video_token_id": 151656, + "vision_config": { + "deepstack_visual_indexes": [ + 5, + 11, + 17 + ], + "depth": 24, + "dtype": "bfloat16", + "hidden_act": "gelu_pytorch_tanh", + "hidden_size": 1024, + "in_channels": 3, + "initializer_range": 0.02, + "intermediate_size": 4096, + "model_type": "qwen3_vl", + "num_heads": 16, + "num_position_embeddings": 2304, + "out_hidden_size": 2560, + "patch_size": 16, + "spatial_merge_size": 2, + "temporal_patch_size": 2 + }, + "vision_end_token_id": 151653, + "vision_start_token_id": 151652 +} diff --git a/posttrain_models/Qwen3VL_4B_lora32_batch1245910_epoch4/generation_config.json b/posttrain_models/Qwen3VL_4B_lora32_batch1245910_epoch4/generation_config.json new file mode 100644 index 0000000000000000000000000000000000000000..3dd2c545b96b4d51586680d7bb27fdca5c90dfe9 --- /dev/null +++ b/posttrain_models/Qwen3VL_4B_lora32_batch1245910_epoch4/generation_config.json @@ -0,0 +1,13 @@ +{ + "bos_token_id": 151643, + "do_sample": true, + "eos_token_id": [ + 151645, + 151643 + ], + "pad_token_id": 151643, + "temperature": 0.7, + "top_k": 20, + "top_p": 0.8, + "transformers_version": "4.57.3" +} diff --git a/posttrain_models/Qwen3VL_4B_lora32_batch1245910_epoch4/model.safetensors.index.json b/posttrain_models/Qwen3VL_4B_lora32_batch1245910_epoch4/model.safetensors.index.json new file mode 100644 index 0000000000000000000000000000000000000000..58d118e8ce51a42ed3b147a55f9c92f39d273c60 --- /dev/null +++ b/posttrain_models/Qwen3VL_4B_lora32_batch1245910_epoch4/model.safetensors.index.json @@ -0,0 +1,721 @@ +{ + "metadata": { + "total_parameters": 4437815808, + "total_size": 8875631616 + }, + "weight_map": { + "model.language_model.embed_tokens.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.0.input_layernorm.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.0.mlp.down_proj.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.0.mlp.gate_proj.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.0.mlp.up_proj.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.0.post_attention_layernorm.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.0.self_attn.k_norm.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.0.self_attn.k_proj.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.0.self_attn.o_proj.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.0.self_attn.q_norm.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.0.self_attn.q_proj.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.0.self_attn.v_proj.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.1.input_layernorm.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.1.mlp.down_proj.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.1.mlp.gate_proj.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.1.mlp.up_proj.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.1.post_attention_layernorm.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.1.self_attn.k_norm.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.1.self_attn.k_proj.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.1.self_attn.o_proj.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.1.self_attn.q_norm.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.1.self_attn.q_proj.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.1.self_attn.v_proj.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.10.input_layernorm.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.10.mlp.down_proj.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.10.mlp.gate_proj.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.10.mlp.up_proj.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.10.post_attention_layernorm.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.10.self_attn.k_norm.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.10.self_attn.k_proj.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.10.self_attn.o_proj.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.10.self_attn.q_norm.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.10.self_attn.q_proj.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.10.self_attn.v_proj.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.11.input_layernorm.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.11.mlp.down_proj.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.11.mlp.gate_proj.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.11.mlp.up_proj.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.11.post_attention_layernorm.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.11.self_attn.k_norm.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.11.self_attn.k_proj.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.11.self_attn.o_proj.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.11.self_attn.q_norm.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.11.self_attn.q_proj.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.11.self_attn.v_proj.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.12.input_layernorm.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.12.mlp.down_proj.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.12.mlp.gate_proj.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.12.mlp.up_proj.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.12.post_attention_layernorm.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.12.self_attn.k_norm.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.12.self_attn.k_proj.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.12.self_attn.o_proj.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.12.self_attn.q_norm.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.12.self_attn.q_proj.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.12.self_attn.v_proj.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.13.input_layernorm.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.13.mlp.down_proj.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.13.mlp.gate_proj.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.13.mlp.up_proj.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.13.post_attention_layernorm.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.13.self_attn.k_norm.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.13.self_attn.k_proj.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.13.self_attn.o_proj.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.13.self_attn.q_norm.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.13.self_attn.q_proj.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.13.self_attn.v_proj.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.14.input_layernorm.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.14.mlp.down_proj.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.14.mlp.gate_proj.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.14.mlp.up_proj.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.14.post_attention_layernorm.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.14.self_attn.k_norm.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.14.self_attn.k_proj.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.14.self_attn.o_proj.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.14.self_attn.q_norm.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.14.self_attn.q_proj.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.14.self_attn.v_proj.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.15.input_layernorm.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.15.mlp.down_proj.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.15.mlp.gate_proj.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.15.mlp.up_proj.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.15.post_attention_layernorm.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.15.self_attn.k_norm.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.15.self_attn.k_proj.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.15.self_attn.o_proj.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.15.self_attn.q_norm.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.15.self_attn.q_proj.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.15.self_attn.v_proj.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.16.input_layernorm.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.16.mlp.down_proj.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.16.mlp.gate_proj.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.16.mlp.up_proj.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.16.post_attention_layernorm.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.16.self_attn.k_norm.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.16.self_attn.k_proj.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.16.self_attn.o_proj.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.16.self_attn.q_norm.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.16.self_attn.q_proj.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.16.self_attn.v_proj.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.17.input_layernorm.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.17.mlp.down_proj.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.17.mlp.gate_proj.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.17.mlp.up_proj.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.17.post_attention_layernorm.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.17.self_attn.k_norm.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.17.self_attn.k_proj.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.17.self_attn.o_proj.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.17.self_attn.q_norm.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.17.self_attn.q_proj.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.17.self_attn.v_proj.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.18.input_layernorm.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.18.mlp.down_proj.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.18.mlp.gate_proj.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.18.mlp.up_proj.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.18.post_attention_layernorm.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.18.self_attn.k_norm.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.18.self_attn.k_proj.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.18.self_attn.o_proj.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.18.self_attn.q_norm.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.18.self_attn.q_proj.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.18.self_attn.v_proj.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.19.input_layernorm.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.19.mlp.down_proj.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.19.mlp.gate_proj.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.19.mlp.up_proj.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.19.post_attention_layernorm.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.19.self_attn.k_norm.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.19.self_attn.k_proj.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.19.self_attn.o_proj.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.19.self_attn.q_norm.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.19.self_attn.q_proj.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.19.self_attn.v_proj.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.2.input_layernorm.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.2.mlp.down_proj.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.2.mlp.gate_proj.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.2.mlp.up_proj.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.2.post_attention_layernorm.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.2.self_attn.k_norm.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.2.self_attn.k_proj.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.2.self_attn.o_proj.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.2.self_attn.q_norm.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.2.self_attn.q_proj.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.2.self_attn.v_proj.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.20.input_layernorm.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.20.mlp.down_proj.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.20.mlp.gate_proj.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.20.mlp.up_proj.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.20.post_attention_layernorm.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.20.self_attn.k_norm.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.20.self_attn.k_proj.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.20.self_attn.o_proj.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.20.self_attn.q_norm.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.20.self_attn.q_proj.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.20.self_attn.v_proj.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.21.input_layernorm.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.21.mlp.down_proj.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.21.mlp.gate_proj.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.21.mlp.up_proj.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.21.post_attention_layernorm.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.21.self_attn.k_norm.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.21.self_attn.k_proj.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.21.self_attn.o_proj.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.21.self_attn.q_norm.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.21.self_attn.q_proj.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.21.self_attn.v_proj.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.22.input_layernorm.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.22.mlp.down_proj.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.22.mlp.gate_proj.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.22.mlp.up_proj.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.22.post_attention_layernorm.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.22.self_attn.k_norm.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.22.self_attn.k_proj.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.22.self_attn.o_proj.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.22.self_attn.q_norm.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.22.self_attn.q_proj.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.22.self_attn.v_proj.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.23.input_layernorm.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.23.mlp.down_proj.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.23.mlp.gate_proj.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.23.mlp.up_proj.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.23.post_attention_layernorm.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.23.self_attn.k_norm.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.23.self_attn.k_proj.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.23.self_attn.o_proj.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.23.self_attn.q_norm.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.23.self_attn.q_proj.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.23.self_attn.v_proj.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.24.input_layernorm.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.24.mlp.down_proj.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.24.mlp.gate_proj.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.24.mlp.up_proj.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.24.post_attention_layernorm.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.24.self_attn.k_norm.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.24.self_attn.k_proj.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.24.self_attn.o_proj.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.24.self_attn.q_norm.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.24.self_attn.q_proj.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.24.self_attn.v_proj.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.25.input_layernorm.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.25.mlp.down_proj.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.25.mlp.gate_proj.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.25.mlp.up_proj.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.25.post_attention_layernorm.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.25.self_attn.k_norm.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.25.self_attn.k_proj.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.25.self_attn.o_proj.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.25.self_attn.q_norm.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.25.self_attn.q_proj.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.25.self_attn.v_proj.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.26.input_layernorm.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.26.mlp.down_proj.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.26.mlp.gate_proj.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.26.mlp.up_proj.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.26.post_attention_layernorm.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.26.self_attn.k_norm.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.26.self_attn.k_proj.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.26.self_attn.o_proj.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.26.self_attn.q_norm.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.26.self_attn.q_proj.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.26.self_attn.v_proj.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.27.input_layernorm.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.27.mlp.down_proj.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.27.mlp.gate_proj.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.27.mlp.up_proj.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.27.post_attention_layernorm.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.27.self_attn.k_norm.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.27.self_attn.k_proj.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.27.self_attn.o_proj.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.27.self_attn.q_norm.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.27.self_attn.q_proj.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.27.self_attn.v_proj.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.28.input_layernorm.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.28.mlp.down_proj.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.28.mlp.gate_proj.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.28.mlp.up_proj.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.28.post_attention_layernorm.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.28.self_attn.k_norm.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.28.self_attn.k_proj.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.28.self_attn.o_proj.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.28.self_attn.q_norm.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.28.self_attn.q_proj.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.28.self_attn.v_proj.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.29.input_layernorm.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.29.mlp.down_proj.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.29.mlp.gate_proj.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.29.mlp.up_proj.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.29.post_attention_layernorm.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.29.self_attn.k_norm.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.29.self_attn.k_proj.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.29.self_attn.o_proj.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.29.self_attn.q_norm.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.29.self_attn.q_proj.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.29.self_attn.v_proj.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.3.input_layernorm.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.3.mlp.down_proj.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.3.mlp.gate_proj.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.3.mlp.up_proj.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.3.post_attention_layernorm.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.3.self_attn.k_norm.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.3.self_attn.k_proj.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.3.self_attn.o_proj.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.3.self_attn.q_norm.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.3.self_attn.q_proj.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.3.self_attn.v_proj.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.30.input_layernorm.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.30.mlp.down_proj.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.30.mlp.gate_proj.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.30.mlp.up_proj.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.30.post_attention_layernorm.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.30.self_attn.k_norm.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.30.self_attn.k_proj.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.30.self_attn.o_proj.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.30.self_attn.q_norm.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.30.self_attn.q_proj.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.30.self_attn.v_proj.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.31.input_layernorm.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.31.mlp.down_proj.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.31.mlp.gate_proj.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.31.mlp.up_proj.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.31.post_attention_layernorm.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.31.self_attn.k_norm.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.31.self_attn.k_proj.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.31.self_attn.o_proj.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.31.self_attn.q_norm.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.31.self_attn.q_proj.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.31.self_attn.v_proj.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.32.input_layernorm.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.32.mlp.down_proj.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.32.mlp.gate_proj.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.32.mlp.up_proj.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.32.post_attention_layernorm.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.32.self_attn.k_norm.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.32.self_attn.k_proj.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.32.self_attn.o_proj.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.32.self_attn.q_norm.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.32.self_attn.q_proj.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.32.self_attn.v_proj.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.33.input_layernorm.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.33.mlp.down_proj.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.33.mlp.gate_proj.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.33.mlp.up_proj.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.33.post_attention_layernorm.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.33.self_attn.k_norm.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.33.self_attn.k_proj.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.33.self_attn.o_proj.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.33.self_attn.q_norm.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.33.self_attn.q_proj.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.33.self_attn.v_proj.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.34.input_layernorm.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.34.mlp.down_proj.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.34.mlp.gate_proj.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.34.mlp.up_proj.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.34.post_attention_layernorm.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.34.self_attn.k_norm.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.34.self_attn.k_proj.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.34.self_attn.o_proj.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.34.self_attn.q_norm.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.34.self_attn.q_proj.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.34.self_attn.v_proj.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.35.input_layernorm.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.35.mlp.down_proj.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.35.mlp.gate_proj.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.35.mlp.up_proj.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.35.post_attention_layernorm.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.35.self_attn.k_norm.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.35.self_attn.k_proj.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.35.self_attn.o_proj.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.35.self_attn.q_norm.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.35.self_attn.q_proj.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.35.self_attn.v_proj.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.4.input_layernorm.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.4.mlp.down_proj.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.4.mlp.gate_proj.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.4.mlp.up_proj.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.4.post_attention_layernorm.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.4.self_attn.k_norm.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.4.self_attn.k_proj.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.4.self_attn.o_proj.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.4.self_attn.q_norm.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.4.self_attn.q_proj.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.4.self_attn.v_proj.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.5.input_layernorm.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.5.mlp.down_proj.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.5.mlp.gate_proj.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.5.mlp.up_proj.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.5.post_attention_layernorm.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.5.self_attn.k_norm.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.5.self_attn.k_proj.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.5.self_attn.o_proj.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.5.self_attn.q_norm.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.5.self_attn.q_proj.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.5.self_attn.v_proj.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.6.input_layernorm.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.6.mlp.down_proj.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.6.mlp.gate_proj.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.6.mlp.up_proj.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.6.post_attention_layernorm.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.6.self_attn.k_norm.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.6.self_attn.k_proj.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.6.self_attn.o_proj.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.6.self_attn.q_norm.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.6.self_attn.q_proj.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.6.self_attn.v_proj.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.7.input_layernorm.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.7.mlp.down_proj.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.7.mlp.gate_proj.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.7.mlp.up_proj.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.7.post_attention_layernorm.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.7.self_attn.k_norm.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.7.self_attn.k_proj.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.7.self_attn.o_proj.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.7.self_attn.q_norm.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.7.self_attn.q_proj.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.7.self_attn.v_proj.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.8.input_layernorm.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.8.mlp.down_proj.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.8.mlp.gate_proj.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.8.mlp.up_proj.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.8.post_attention_layernorm.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.8.self_attn.k_norm.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.8.self_attn.k_proj.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.8.self_attn.o_proj.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.8.self_attn.q_norm.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.8.self_attn.q_proj.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.8.self_attn.v_proj.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.9.input_layernorm.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.9.mlp.down_proj.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.9.mlp.gate_proj.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.9.mlp.up_proj.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.9.post_attention_layernorm.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.9.self_attn.k_norm.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.9.self_attn.k_proj.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.9.self_attn.o_proj.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.9.self_attn.q_norm.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.9.self_attn.q_proj.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.9.self_attn.v_proj.weight": "model-00001-of-00002.safetensors", + "model.language_model.norm.weight": "model-00002-of-00002.safetensors", + "model.visual.blocks.0.attn.proj.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.0.attn.proj.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.0.attn.qkv.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.0.attn.qkv.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.0.mlp.linear_fc1.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.0.mlp.linear_fc1.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.0.mlp.linear_fc2.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.0.mlp.linear_fc2.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.0.norm1.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.0.norm1.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.0.norm2.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.0.norm2.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.1.attn.proj.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.1.attn.proj.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.1.attn.qkv.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.1.attn.qkv.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.1.mlp.linear_fc1.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.1.mlp.linear_fc1.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.1.mlp.linear_fc2.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.1.mlp.linear_fc2.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.1.norm1.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.1.norm1.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.1.norm2.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.1.norm2.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.10.attn.proj.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.10.attn.proj.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.10.attn.qkv.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.10.attn.qkv.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.10.mlp.linear_fc1.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.10.mlp.linear_fc1.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.10.mlp.linear_fc2.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.10.mlp.linear_fc2.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.10.norm1.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.10.norm1.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.10.norm2.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.10.norm2.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.11.attn.proj.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.11.attn.proj.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.11.attn.qkv.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.11.attn.qkv.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.11.mlp.linear_fc1.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.11.mlp.linear_fc1.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.11.mlp.linear_fc2.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.11.mlp.linear_fc2.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.11.norm1.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.11.norm1.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.11.norm2.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.11.norm2.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.12.attn.proj.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.12.attn.proj.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.12.attn.qkv.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.12.attn.qkv.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.12.mlp.linear_fc1.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.12.mlp.linear_fc1.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.12.mlp.linear_fc2.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.12.mlp.linear_fc2.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.12.norm1.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.12.norm1.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.12.norm2.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.12.norm2.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.13.attn.proj.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.13.attn.proj.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.13.attn.qkv.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.13.attn.qkv.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.13.mlp.linear_fc1.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.13.mlp.linear_fc1.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.13.mlp.linear_fc2.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.13.mlp.linear_fc2.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.13.norm1.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.13.norm1.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.13.norm2.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.13.norm2.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.14.attn.proj.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.14.attn.proj.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.14.attn.qkv.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.14.attn.qkv.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.14.mlp.linear_fc1.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.14.mlp.linear_fc1.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.14.mlp.linear_fc2.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.14.mlp.linear_fc2.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.14.norm1.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.14.norm1.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.14.norm2.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.14.norm2.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.15.attn.proj.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.15.attn.proj.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.15.attn.qkv.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.15.attn.qkv.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.15.mlp.linear_fc1.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.15.mlp.linear_fc1.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.15.mlp.linear_fc2.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.15.mlp.linear_fc2.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.15.norm1.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.15.norm1.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.15.norm2.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.15.norm2.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.16.attn.proj.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.16.attn.proj.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.16.attn.qkv.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.16.attn.qkv.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.16.mlp.linear_fc1.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.16.mlp.linear_fc1.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.16.mlp.linear_fc2.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.16.mlp.linear_fc2.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.16.norm1.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.16.norm1.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.16.norm2.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.16.norm2.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.17.attn.proj.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.17.attn.proj.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.17.attn.qkv.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.17.attn.qkv.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.17.mlp.linear_fc1.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.17.mlp.linear_fc1.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.17.mlp.linear_fc2.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.17.mlp.linear_fc2.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.17.norm1.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.17.norm1.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.17.norm2.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.17.norm2.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.18.attn.proj.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.18.attn.proj.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.18.attn.qkv.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.18.attn.qkv.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.18.mlp.linear_fc1.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.18.mlp.linear_fc1.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.18.mlp.linear_fc2.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.18.mlp.linear_fc2.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.18.norm1.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.18.norm1.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.18.norm2.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.18.norm2.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.19.attn.proj.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.19.attn.proj.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.19.attn.qkv.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.19.attn.qkv.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.19.mlp.linear_fc1.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.19.mlp.linear_fc1.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.19.mlp.linear_fc2.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.19.mlp.linear_fc2.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.19.norm1.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.19.norm1.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.19.norm2.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.19.norm2.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.2.attn.proj.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.2.attn.proj.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.2.attn.qkv.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.2.attn.qkv.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.2.mlp.linear_fc1.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.2.mlp.linear_fc1.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.2.mlp.linear_fc2.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.2.mlp.linear_fc2.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.2.norm1.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.2.norm1.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.2.norm2.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.2.norm2.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.20.attn.proj.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.20.attn.proj.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.20.attn.qkv.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.20.attn.qkv.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.20.mlp.linear_fc1.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.20.mlp.linear_fc1.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.20.mlp.linear_fc2.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.20.mlp.linear_fc2.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.20.norm1.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.20.norm1.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.20.norm2.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.20.norm2.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.21.attn.proj.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.21.attn.proj.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.21.attn.qkv.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.21.attn.qkv.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.21.mlp.linear_fc1.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.21.mlp.linear_fc1.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.21.mlp.linear_fc2.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.21.mlp.linear_fc2.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.21.norm1.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.21.norm1.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.21.norm2.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.21.norm2.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.22.attn.proj.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.22.attn.proj.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.22.attn.qkv.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.22.attn.qkv.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.22.mlp.linear_fc1.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.22.mlp.linear_fc1.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.22.mlp.linear_fc2.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.22.mlp.linear_fc2.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.22.norm1.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.22.norm1.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.22.norm2.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.22.norm2.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.23.attn.proj.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.23.attn.proj.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.23.attn.qkv.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.23.attn.qkv.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.23.mlp.linear_fc1.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.23.mlp.linear_fc1.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.23.mlp.linear_fc2.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.23.mlp.linear_fc2.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.23.norm1.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.23.norm1.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.23.norm2.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.23.norm2.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.3.attn.proj.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.3.attn.proj.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.3.attn.qkv.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.3.attn.qkv.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.3.mlp.linear_fc1.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.3.mlp.linear_fc1.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.3.mlp.linear_fc2.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.3.mlp.linear_fc2.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.3.norm1.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.3.norm1.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.3.norm2.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.3.norm2.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.4.attn.proj.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.4.attn.proj.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.4.attn.qkv.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.4.attn.qkv.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.4.mlp.linear_fc1.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.4.mlp.linear_fc1.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.4.mlp.linear_fc2.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.4.mlp.linear_fc2.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.4.norm1.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.4.norm1.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.4.norm2.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.4.norm2.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.5.attn.proj.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.5.attn.proj.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.5.attn.qkv.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.5.attn.qkv.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.5.mlp.linear_fc1.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.5.mlp.linear_fc1.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.5.mlp.linear_fc2.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.5.mlp.linear_fc2.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.5.norm1.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.5.norm1.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.5.norm2.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.5.norm2.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.6.attn.proj.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.6.attn.proj.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.6.attn.qkv.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.6.attn.qkv.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.6.mlp.linear_fc1.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.6.mlp.linear_fc1.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.6.mlp.linear_fc2.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.6.mlp.linear_fc2.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.6.norm1.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.6.norm1.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.6.norm2.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.6.norm2.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.7.attn.proj.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.7.attn.proj.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.7.attn.qkv.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.7.attn.qkv.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.7.mlp.linear_fc1.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.7.mlp.linear_fc1.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.7.mlp.linear_fc2.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.7.mlp.linear_fc2.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.7.norm1.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.7.norm1.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.7.norm2.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.7.norm2.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.8.attn.proj.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.8.attn.proj.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.8.attn.qkv.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.8.attn.qkv.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.8.mlp.linear_fc1.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.8.mlp.linear_fc1.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.8.mlp.linear_fc2.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.8.mlp.linear_fc2.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.8.norm1.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.8.norm1.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.8.norm2.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.8.norm2.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.9.attn.proj.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.9.attn.proj.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.9.attn.qkv.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.9.attn.qkv.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.9.mlp.linear_fc1.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.9.mlp.linear_fc1.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.9.mlp.linear_fc2.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.9.mlp.linear_fc2.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.9.norm1.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.9.norm1.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.9.norm2.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.9.norm2.weight": "model-00001-of-00002.safetensors", + "model.visual.deepstack_merger_list.0.linear_fc1.bias": "model-00001-of-00002.safetensors", + "model.visual.deepstack_merger_list.0.linear_fc1.weight": "model-00001-of-00002.safetensors", + "model.visual.deepstack_merger_list.0.linear_fc2.bias": "model-00001-of-00002.safetensors", + "model.visual.deepstack_merger_list.0.linear_fc2.weight": "model-00001-of-00002.safetensors", + "model.visual.deepstack_merger_list.0.norm.bias": "model-00001-of-00002.safetensors", + "model.visual.deepstack_merger_list.0.norm.weight": "model-00001-of-00002.safetensors", + "model.visual.deepstack_merger_list.1.linear_fc1.bias": "model-00001-of-00002.safetensors", + "model.visual.deepstack_merger_list.1.linear_fc1.weight": "model-00001-of-00002.safetensors", + "model.visual.deepstack_merger_list.1.linear_fc2.bias": "model-00001-of-00002.safetensors", + "model.visual.deepstack_merger_list.1.linear_fc2.weight": "model-00001-of-00002.safetensors", + "model.visual.deepstack_merger_list.1.norm.bias": "model-00001-of-00002.safetensors", + "model.visual.deepstack_merger_list.1.norm.weight": "model-00001-of-00002.safetensors", + "model.visual.deepstack_merger_list.2.linear_fc1.bias": "model-00001-of-00002.safetensors", + "model.visual.deepstack_merger_list.2.linear_fc1.weight": "model-00001-of-00002.safetensors", + "model.visual.deepstack_merger_list.2.linear_fc2.bias": "model-00001-of-00002.safetensors", + "model.visual.deepstack_merger_list.2.linear_fc2.weight": "model-00001-of-00002.safetensors", + "model.visual.deepstack_merger_list.2.norm.bias": "model-00001-of-00002.safetensors", + "model.visual.deepstack_merger_list.2.norm.weight": "model-00001-of-00002.safetensors", + "model.visual.merger.linear_fc1.bias": "model-00001-of-00002.safetensors", + "model.visual.merger.linear_fc1.weight": "model-00001-of-00002.safetensors", + "model.visual.merger.linear_fc2.bias": "model-00001-of-00002.safetensors", + "model.visual.merger.linear_fc2.weight": "model-00001-of-00002.safetensors", + "model.visual.merger.norm.bias": "model-00001-of-00002.safetensors", + "model.visual.merger.norm.weight": "model-00001-of-00002.safetensors", + "model.visual.patch_embed.proj.bias": "model-00001-of-00002.safetensors", + "model.visual.patch_embed.proj.weight": "model-00001-of-00002.safetensors", + "model.visual.pos_embed.weight": "model-00001-of-00002.safetensors" + } +} diff --git a/posttrain_models/Qwen3VL_4B_lora32_batch1245910_epoch4/preprocessor_config.json b/posttrain_models/Qwen3VL_4B_lora32_batch1245910_epoch4/preprocessor_config.json new file mode 100644 index 0000000000000000000000000000000000000000..2fa6553554f69b4e3a6b742a75264ff9dff90ad0 --- /dev/null +++ b/posttrain_models/Qwen3VL_4B_lora32_batch1245910_epoch4/preprocessor_config.json @@ -0,0 +1,39 @@ +{ + "crop_size": null, + "data_format": "channels_first", + "default_to_square": true, + "device": null, + "disable_grouping": null, + "do_center_crop": null, + "do_convert_rgb": true, + "do_normalize": true, + "do_pad": null, + "do_rescale": true, + "do_resize": true, + "image_mean": [ + 0.5, + 0.5, + 0.5 + ], + "image_processor_type": "Qwen2VLImageProcessorFast", + "image_std": [ + 0.5, + 0.5, + 0.5 + ], + "input_data_format": null, + "max_pixels": null, + "merge_size": 2, + "min_pixels": null, + "pad_size": null, + "patch_size": 16, + "processor_class": "Qwen3VLProcessor", + "resample": 3, + "rescale_factor": 0.00392156862745098, + "return_tensors": null, + "size": { + "longest_edge": 16777216, + "shortest_edge": 65536 + }, + "temporal_patch_size": 2 +} diff --git a/posttrain_models/Qwen3VL_4B_lora32_batch1245910_epoch4/special_tokens_map.json b/posttrain_models/Qwen3VL_4B_lora32_batch1245910_epoch4/special_tokens_map.json new file mode 100644 index 0000000000000000000000000000000000000000..ac23c0aaa2434523c494330aeb79c58395378103 --- /dev/null +++ b/posttrain_models/Qwen3VL_4B_lora32_batch1245910_epoch4/special_tokens_map.json @@ -0,0 +1,31 @@ +{ + "additional_special_tokens": [ + "<|im_start|>", + "<|im_end|>", + "<|object_ref_start|>", + "<|object_ref_end|>", + "<|box_start|>", + "<|box_end|>", + "<|quad_start|>", + "<|quad_end|>", + "<|vision_start|>", + "<|vision_end|>", + "<|vision_pad|>", + "<|image_pad|>", + "<|video_pad|>" + ], + "eos_token": { + "content": "<|im_end|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false + }, + "pad_token": { + "content": "<|endoftext|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false + } +} diff --git a/posttrain_models/Qwen3VL_4B_lora32_batch1245910_epoch4/tokenizer_config.json b/posttrain_models/Qwen3VL_4B_lora32_batch1245910_epoch4/tokenizer_config.json new file mode 100644 index 0000000000000000000000000000000000000000..fec7f182135c384a08ea2fc9635566ca79c45afc --- /dev/null +++ b/posttrain_models/Qwen3VL_4B_lora32_batch1245910_epoch4/tokenizer_config.json @@ -0,0 +1,240 @@ +{ + "add_bos_token": false, + "add_prefix_space": false, + "added_tokens_decoder": { + "151643": { + "content": "<|endoftext|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "151644": { + "content": "<|im_start|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "151645": { + "content": "<|im_end|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "151646": { + "content": "<|object_ref_start|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "151647": { + "content": "<|object_ref_end|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "151648": { + "content": "<|box_start|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "151649": { + "content": "<|box_end|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "151650": { + "content": "<|quad_start|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "151651": { + "content": "<|quad_end|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "151652": { + "content": "<|vision_start|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "151653": { + "content": "<|vision_end|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "151654": { + "content": "<|vision_pad|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "151655": { + "content": "<|image_pad|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "151656": { + "content": "<|video_pad|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "151657": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "151658": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "151659": { + "content": "<|fim_prefix|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "151660": { + "content": "<|fim_middle|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "151661": { + "content": "<|fim_suffix|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "151662": { + "content": "<|fim_pad|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "151663": { + "content": "<|repo_name|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "151664": { + "content": "<|file_sep|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "151665": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "151666": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "151667": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "151668": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + } + }, + "additional_special_tokens": [ + "<|im_start|>", + "<|im_end|>", + "<|object_ref_start|>", + "<|object_ref_end|>", + "<|box_start|>", + "<|box_end|>", + "<|quad_start|>", + "<|quad_end|>", + "<|vision_start|>", + "<|vision_end|>", + "<|vision_pad|>", + "<|image_pad|>", + "<|video_pad|>" + ], + "bos_token": null, + "clean_up_tokenization_spaces": false, + "eos_token": "<|im_end|>", + "errors": "replace", + "extra_special_tokens": {}, + "model_max_length": 262144, + "pad_token": "<|endoftext|>", + "processor_class": "Qwen3VLProcessor", + "split_special_tokens": false, + "tokenizer_class": "Qwen2Tokenizer", + "unk_token": null +} diff --git a/posttrain_models/Qwen3VL_4B_lora32_batch1245910_epoch4/video_preprocessor_config.json b/posttrain_models/Qwen3VL_4B_lora32_batch1245910_epoch4/video_preprocessor_config.json new file mode 100644 index 0000000000000000000000000000000000000000..e32b1d90356f74457edbfca27aa1e121dab7cdc7 --- /dev/null +++ b/posttrain_models/Qwen3VL_4B_lora32_batch1245910_epoch4/video_preprocessor_config.json @@ -0,0 +1,41 @@ +{ + "crop_size": null, + "data_format": "channels_first", + "default_to_square": true, + "device": null, + "do_center_crop": null, + "do_convert_rgb": true, + "do_normalize": true, + "do_rescale": true, + "do_resize": true, + "do_sample_frames": true, + "fps": 2, + "image_mean": [ + 0.5, + 0.5, + 0.5 + ], + "image_std": [ + 0.5, + 0.5, + 0.5 + ], + "input_data_format": null, + "max_frames": 768, + "merge_size": 2, + "min_frames": 4, + "num_frames": null, + "pad_size": null, + "patch_size": 16, + "processor_class": "Qwen3VLProcessor", + "resample": 3, + "rescale_factor": 0.00392156862745098, + "return_metadata": false, + "size": { + "longest_edge": 25165824, + "shortest_edge": 4096 + }, + "temporal_patch_size": 2, + "video_metadata": null, + "video_processor_type": "Qwen3VLVideoProcessor" +} diff --git a/posttrain_models/Qwen3VL_4B_lora32_batch12459_epoch4/added_tokens.json b/posttrain_models/Qwen3VL_4B_lora32_batch12459_epoch4/added_tokens.json new file mode 100644 index 0000000000000000000000000000000000000000..b54f9135e44c1e81047e8d05cb027af8bc039eed --- /dev/null +++ b/posttrain_models/Qwen3VL_4B_lora32_batch12459_epoch4/added_tokens.json @@ -0,0 +1,28 @@ +{ + "": 151668, + "": 151658, + "": 151666, + "": 151667, + "": 151657, + "": 151665, + "<|box_end|>": 151649, + "<|box_start|>": 151648, + "<|endoftext|>": 151643, + "<|file_sep|>": 151664, + "<|fim_middle|>": 151660, + "<|fim_pad|>": 151662, + "<|fim_prefix|>": 151659, + "<|fim_suffix|>": 151661, + "<|im_end|>": 151645, + "<|im_start|>": 151644, + "<|image_pad|>": 151655, + "<|object_ref_end|>": 151647, + "<|object_ref_start|>": 151646, + "<|quad_end|>": 151651, + "<|quad_start|>": 151650, + "<|repo_name|>": 151663, + "<|video_pad|>": 151656, + "<|vision_end|>": 151653, + "<|vision_pad|>": 151654, + "<|vision_start|>": 151652 +} diff --git a/posttrain_models/Qwen3VL_4B_lora32_batch12459_epoch4/chat_template.jinja b/posttrain_models/Qwen3VL_4B_lora32_batch12459_epoch4/chat_template.jinja new file mode 100644 index 0000000000000000000000000000000000000000..124386803f142761528f710e77ae483f5f8c4fc4 --- /dev/null +++ b/posttrain_models/Qwen3VL_4B_lora32_batch12459_epoch4/chat_template.jinja @@ -0,0 +1,120 @@ +{%- if tools %} + {{- '<|im_start|>system\n' }} + {%- if messages[0].role == 'system' %} + {%- if messages[0].content is string %} + {{- messages[0].content }} + {%- else %} + {%- for content in messages[0].content %} + {%- if 'text' in content %} + {{- content.text }} + {%- endif %} + {%- endfor %} + {%- endif %} + {{- '\n\n' }} + {%- endif %} + {{- "# Tools\n\nYou may call one or more functions to assist with the user query.\n\nYou are provided with function signatures within XML tags:\n" }} + {%- for tool in tools %} + {{- "\n" }} + {{- tool | tojson }} + {%- endfor %} + {{- "\n\n\nFor each function call, return a json object with function name and arguments within XML tags:\n\n{\"name\": , \"arguments\": }\n<|im_end|>\n" }} +{%- else %} + {%- if messages[0].role == 'system' %} + {{- '<|im_start|>system\n' }} + {%- if messages[0].content is string %} + {{- messages[0].content }} + {%- else %} + {%- for content in messages[0].content %} + {%- if 'text' in content %} + {{- content.text }} + {%- endif %} + {%- endfor %} + {%- endif %} + {{- '<|im_end|>\n' }} + {%- endif %} +{%- endif %} +{%- set image_count = namespace(value=0) %} +{%- set video_count = namespace(value=0) %} +{%- for message in messages %} + {%- if message.role == "user" %} + {{- '<|im_start|>' + message.role + '\n' }} + {%- if message.content is string %} + {{- message.content }} + {%- else %} + {%- for content in message.content %} + {%- if content.type == 'image' or 'image' in content or 'image_url' in content %} + {%- set image_count.value = image_count.value + 1 %} + {%- if add_vision_id %}Picture {{ image_count.value }}: {% endif -%} + <|vision_start|><|image_pad|><|vision_end|> + {%- elif content.type == 'video' or 'video' in content %} + {%- set video_count.value = video_count.value + 1 %} + {%- if add_vision_id %}Video {{ video_count.value }}: {% endif -%} + <|vision_start|><|video_pad|><|vision_end|> + {%- elif 'text' in content %} + {{- content.text }} + {%- endif %} + {%- endfor %} + {%- endif %} + {{- '<|im_end|>\n' }} + {%- elif message.role == "assistant" %} + {{- '<|im_start|>' + message.role + '\n' }} + {%- if message.content is string %} + {{- message.content }} + {%- else %} + {%- for content_item in message.content %} + {%- if 'text' in content_item %} + {{- content_item.text }} + {%- endif %} + {%- endfor %} + {%- endif %} + {%- if message.tool_calls %} + {%- for tool_call in message.tool_calls %} + {%- if (loop.first and message.content) or (not loop.first) %} + {{- '\n' }} + {%- endif %} + {%- if tool_call.function %} + {%- set tool_call = tool_call.function %} + {%- endif %} + {{- '\n{"name": "' }} + {{- tool_call.name }} + {{- '", "arguments": ' }} + {%- if tool_call.arguments is string %} + {{- tool_call.arguments }} + {%- else %} + {{- tool_call.arguments | tojson }} + {%- endif %} + {{- '}\n' }} + {%- endfor %} + {%- endif %} + {{- '<|im_end|>\n' }} + {%- elif message.role == "tool" %} + {%- if loop.first or (messages[loop.index0 - 1].role != "tool") %} + {{- '<|im_start|>user' }} + {%- endif %} + {{- '\n\n' }} + {%- if message.content is string %} + {{- message.content }} + {%- else %} + {%- for content in message.content %} + {%- if content.type == 'image' or 'image' in content or 'image_url' in content %} + {%- set image_count.value = image_count.value + 1 %} + {%- if add_vision_id %}Picture {{ image_count.value }}: {% endif -%} + <|vision_start|><|image_pad|><|vision_end|> + {%- elif content.type == 'video' or 'video' in content %} + {%- set video_count.value = video_count.value + 1 %} + {%- if add_vision_id %}Video {{ video_count.value }}: {% endif -%} + <|vision_start|><|video_pad|><|vision_end|> + {%- elif 'text' in content %} + {{- content.text }} + {%- endif %} + {%- endfor %} + {%- endif %} + {{- '\n' }} + {%- if loop.last or (messages[loop.index0 + 1].role != "tool") %} + {{- '<|im_end|>\n' }} + {%- endif %} + {%- endif %} +{%- endfor %} +{%- if add_generation_prompt %} + {{- '<|im_start|>assistant\n' }} +{%- endif %} diff --git a/posttrain_models/Qwen3VL_4B_lora32_batch12459_epoch4/config.json b/posttrain_models/Qwen3VL_4B_lora32_batch12459_epoch4/config.json new file mode 100644 index 0000000000000000000000000000000000000000..adb449916091c26098a94af0fa4add48558a9494 --- /dev/null +++ b/posttrain_models/Qwen3VL_4B_lora32_batch12459_epoch4/config.json @@ -0,0 +1,65 @@ +{ + "architectures": [ + "Qwen3VLForConditionalGeneration" + ], + "dtype": "bfloat16", + "image_token_id": 151655, + "model_type": "qwen3_vl", + "text_config": { + "attention_bias": false, + "attention_dropout": 0.0, + "bos_token_id": 151643, + "dtype": "bfloat16", + "eos_token_id": 151645, + "head_dim": 128, + "hidden_act": "silu", + "hidden_size": 2560, + "initializer_range": 0.02, + "intermediate_size": 9728, + "max_position_embeddings": 262144, + "model_type": "qwen3_vl_text", + "num_attention_heads": 32, + "num_hidden_layers": 36, + "num_key_value_heads": 8, + "rms_norm_eps": 1e-06, + "rope_scaling": { + "mrope_interleaved": true, + "mrope_section": [ + 24, + 20, + 20 + ], + "rope_type": "default" + }, + "rope_theta": 5000000, + "tie_word_embeddings": true, + "use_cache": true, + "vocab_size": 151936 + }, + "tie_word_embeddings": true, + "transformers_version": "4.57.3", + "video_token_id": 151656, + "vision_config": { + "deepstack_visual_indexes": [ + 5, + 11, + 17 + ], + "depth": 24, + "dtype": "bfloat16", + "hidden_act": "gelu_pytorch_tanh", + "hidden_size": 1024, + "in_channels": 3, + "initializer_range": 0.02, + "intermediate_size": 4096, + "model_type": "qwen3_vl", + "num_heads": 16, + "num_position_embeddings": 2304, + "out_hidden_size": 2560, + "patch_size": 16, + "spatial_merge_size": 2, + "temporal_patch_size": 2 + }, + "vision_end_token_id": 151653, + "vision_start_token_id": 151652 +} diff --git a/posttrain_models/Qwen3VL_4B_lora32_batch12459_epoch4/generation_config.json b/posttrain_models/Qwen3VL_4B_lora32_batch12459_epoch4/generation_config.json new file mode 100644 index 0000000000000000000000000000000000000000..3dd2c545b96b4d51586680d7bb27fdca5c90dfe9 --- /dev/null +++ b/posttrain_models/Qwen3VL_4B_lora32_batch12459_epoch4/generation_config.json @@ -0,0 +1,13 @@ +{ + "bos_token_id": 151643, + "do_sample": true, + "eos_token_id": [ + 151645, + 151643 + ], + "pad_token_id": 151643, + "temperature": 0.7, + "top_k": 20, + "top_p": 0.8, + "transformers_version": "4.57.3" +} diff --git a/posttrain_models/Qwen3VL_4B_lora32_batch12459_epoch4/model.safetensors.index.json b/posttrain_models/Qwen3VL_4B_lora32_batch12459_epoch4/model.safetensors.index.json new file mode 100644 index 0000000000000000000000000000000000000000..58d118e8ce51a42ed3b147a55f9c92f39d273c60 --- /dev/null +++ b/posttrain_models/Qwen3VL_4B_lora32_batch12459_epoch4/model.safetensors.index.json @@ -0,0 +1,721 @@ +{ + "metadata": { + "total_parameters": 4437815808, + "total_size": 8875631616 + }, + "weight_map": { + "model.language_model.embed_tokens.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.0.input_layernorm.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.0.mlp.down_proj.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.0.mlp.gate_proj.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.0.mlp.up_proj.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.0.post_attention_layernorm.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.0.self_attn.k_norm.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.0.self_attn.k_proj.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.0.self_attn.o_proj.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.0.self_attn.q_norm.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.0.self_attn.q_proj.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.0.self_attn.v_proj.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.1.input_layernorm.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.1.mlp.down_proj.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.1.mlp.gate_proj.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.1.mlp.up_proj.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.1.post_attention_layernorm.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.1.self_attn.k_norm.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.1.self_attn.k_proj.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.1.self_attn.o_proj.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.1.self_attn.q_norm.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.1.self_attn.q_proj.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.1.self_attn.v_proj.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.10.input_layernorm.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.10.mlp.down_proj.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.10.mlp.gate_proj.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.10.mlp.up_proj.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.10.post_attention_layernorm.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.10.self_attn.k_norm.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.10.self_attn.k_proj.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.10.self_attn.o_proj.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.10.self_attn.q_norm.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.10.self_attn.q_proj.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.10.self_attn.v_proj.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.11.input_layernorm.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.11.mlp.down_proj.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.11.mlp.gate_proj.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.11.mlp.up_proj.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.11.post_attention_layernorm.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.11.self_attn.k_norm.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.11.self_attn.k_proj.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.11.self_attn.o_proj.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.11.self_attn.q_norm.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.11.self_attn.q_proj.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.11.self_attn.v_proj.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.12.input_layernorm.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.12.mlp.down_proj.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.12.mlp.gate_proj.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.12.mlp.up_proj.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.12.post_attention_layernorm.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.12.self_attn.k_norm.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.12.self_attn.k_proj.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.12.self_attn.o_proj.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.12.self_attn.q_norm.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.12.self_attn.q_proj.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.12.self_attn.v_proj.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.13.input_layernorm.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.13.mlp.down_proj.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.13.mlp.gate_proj.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.13.mlp.up_proj.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.13.post_attention_layernorm.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.13.self_attn.k_norm.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.13.self_attn.k_proj.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.13.self_attn.o_proj.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.13.self_attn.q_norm.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.13.self_attn.q_proj.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.13.self_attn.v_proj.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.14.input_layernorm.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.14.mlp.down_proj.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.14.mlp.gate_proj.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.14.mlp.up_proj.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.14.post_attention_layernorm.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.14.self_attn.k_norm.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.14.self_attn.k_proj.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.14.self_attn.o_proj.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.14.self_attn.q_norm.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.14.self_attn.q_proj.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.14.self_attn.v_proj.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.15.input_layernorm.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.15.mlp.down_proj.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.15.mlp.gate_proj.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.15.mlp.up_proj.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.15.post_attention_layernorm.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.15.self_attn.k_norm.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.15.self_attn.k_proj.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.15.self_attn.o_proj.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.15.self_attn.q_norm.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.15.self_attn.q_proj.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.15.self_attn.v_proj.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.16.input_layernorm.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.16.mlp.down_proj.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.16.mlp.gate_proj.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.16.mlp.up_proj.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.16.post_attention_layernorm.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.16.self_attn.k_norm.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.16.self_attn.k_proj.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.16.self_attn.o_proj.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.16.self_attn.q_norm.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.16.self_attn.q_proj.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.16.self_attn.v_proj.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.17.input_layernorm.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.17.mlp.down_proj.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.17.mlp.gate_proj.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.17.mlp.up_proj.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.17.post_attention_layernorm.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.17.self_attn.k_norm.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.17.self_attn.k_proj.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.17.self_attn.o_proj.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.17.self_attn.q_norm.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.17.self_attn.q_proj.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.17.self_attn.v_proj.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.18.input_layernorm.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.18.mlp.down_proj.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.18.mlp.gate_proj.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.18.mlp.up_proj.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.18.post_attention_layernorm.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.18.self_attn.k_norm.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.18.self_attn.k_proj.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.18.self_attn.o_proj.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.18.self_attn.q_norm.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.18.self_attn.q_proj.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.18.self_attn.v_proj.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.19.input_layernorm.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.19.mlp.down_proj.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.19.mlp.gate_proj.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.19.mlp.up_proj.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.19.post_attention_layernorm.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.19.self_attn.k_norm.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.19.self_attn.k_proj.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.19.self_attn.o_proj.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.19.self_attn.q_norm.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.19.self_attn.q_proj.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.19.self_attn.v_proj.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.2.input_layernorm.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.2.mlp.down_proj.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.2.mlp.gate_proj.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.2.mlp.up_proj.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.2.post_attention_layernorm.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.2.self_attn.k_norm.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.2.self_attn.k_proj.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.2.self_attn.o_proj.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.2.self_attn.q_norm.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.2.self_attn.q_proj.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.2.self_attn.v_proj.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.20.input_layernorm.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.20.mlp.down_proj.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.20.mlp.gate_proj.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.20.mlp.up_proj.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.20.post_attention_layernorm.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.20.self_attn.k_norm.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.20.self_attn.k_proj.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.20.self_attn.o_proj.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.20.self_attn.q_norm.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.20.self_attn.q_proj.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.20.self_attn.v_proj.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.21.input_layernorm.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.21.mlp.down_proj.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.21.mlp.gate_proj.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.21.mlp.up_proj.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.21.post_attention_layernorm.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.21.self_attn.k_norm.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.21.self_attn.k_proj.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.21.self_attn.o_proj.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.21.self_attn.q_norm.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.21.self_attn.q_proj.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.21.self_attn.v_proj.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.22.input_layernorm.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.22.mlp.down_proj.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.22.mlp.gate_proj.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.22.mlp.up_proj.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.22.post_attention_layernorm.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.22.self_attn.k_norm.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.22.self_attn.k_proj.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.22.self_attn.o_proj.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.22.self_attn.q_norm.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.22.self_attn.q_proj.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.22.self_attn.v_proj.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.23.input_layernorm.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.23.mlp.down_proj.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.23.mlp.gate_proj.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.23.mlp.up_proj.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.23.post_attention_layernorm.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.23.self_attn.k_norm.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.23.self_attn.k_proj.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.23.self_attn.o_proj.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.23.self_attn.q_norm.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.23.self_attn.q_proj.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.23.self_attn.v_proj.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.24.input_layernorm.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.24.mlp.down_proj.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.24.mlp.gate_proj.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.24.mlp.up_proj.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.24.post_attention_layernorm.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.24.self_attn.k_norm.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.24.self_attn.k_proj.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.24.self_attn.o_proj.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.24.self_attn.q_norm.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.24.self_attn.q_proj.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.24.self_attn.v_proj.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.25.input_layernorm.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.25.mlp.down_proj.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.25.mlp.gate_proj.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.25.mlp.up_proj.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.25.post_attention_layernorm.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.25.self_attn.k_norm.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.25.self_attn.k_proj.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.25.self_attn.o_proj.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.25.self_attn.q_norm.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.25.self_attn.q_proj.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.25.self_attn.v_proj.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.26.input_layernorm.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.26.mlp.down_proj.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.26.mlp.gate_proj.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.26.mlp.up_proj.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.26.post_attention_layernorm.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.26.self_attn.k_norm.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.26.self_attn.k_proj.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.26.self_attn.o_proj.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.26.self_attn.q_norm.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.26.self_attn.q_proj.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.26.self_attn.v_proj.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.27.input_layernorm.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.27.mlp.down_proj.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.27.mlp.gate_proj.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.27.mlp.up_proj.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.27.post_attention_layernorm.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.27.self_attn.k_norm.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.27.self_attn.k_proj.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.27.self_attn.o_proj.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.27.self_attn.q_norm.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.27.self_attn.q_proj.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.27.self_attn.v_proj.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.28.input_layernorm.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.28.mlp.down_proj.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.28.mlp.gate_proj.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.28.mlp.up_proj.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.28.post_attention_layernorm.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.28.self_attn.k_norm.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.28.self_attn.k_proj.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.28.self_attn.o_proj.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.28.self_attn.q_norm.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.28.self_attn.q_proj.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.28.self_attn.v_proj.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.29.input_layernorm.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.29.mlp.down_proj.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.29.mlp.gate_proj.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.29.mlp.up_proj.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.29.post_attention_layernorm.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.29.self_attn.k_norm.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.29.self_attn.k_proj.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.29.self_attn.o_proj.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.29.self_attn.q_norm.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.29.self_attn.q_proj.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.29.self_attn.v_proj.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.3.input_layernorm.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.3.mlp.down_proj.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.3.mlp.gate_proj.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.3.mlp.up_proj.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.3.post_attention_layernorm.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.3.self_attn.k_norm.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.3.self_attn.k_proj.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.3.self_attn.o_proj.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.3.self_attn.q_norm.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.3.self_attn.q_proj.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.3.self_attn.v_proj.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.30.input_layernorm.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.30.mlp.down_proj.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.30.mlp.gate_proj.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.30.mlp.up_proj.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.30.post_attention_layernorm.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.30.self_attn.k_norm.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.30.self_attn.k_proj.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.30.self_attn.o_proj.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.30.self_attn.q_norm.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.30.self_attn.q_proj.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.30.self_attn.v_proj.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.31.input_layernorm.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.31.mlp.down_proj.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.31.mlp.gate_proj.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.31.mlp.up_proj.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.31.post_attention_layernorm.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.31.self_attn.k_norm.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.31.self_attn.k_proj.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.31.self_attn.o_proj.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.31.self_attn.q_norm.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.31.self_attn.q_proj.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.31.self_attn.v_proj.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.32.input_layernorm.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.32.mlp.down_proj.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.32.mlp.gate_proj.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.32.mlp.up_proj.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.32.post_attention_layernorm.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.32.self_attn.k_norm.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.32.self_attn.k_proj.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.32.self_attn.o_proj.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.32.self_attn.q_norm.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.32.self_attn.q_proj.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.32.self_attn.v_proj.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.33.input_layernorm.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.33.mlp.down_proj.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.33.mlp.gate_proj.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.33.mlp.up_proj.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.33.post_attention_layernorm.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.33.self_attn.k_norm.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.33.self_attn.k_proj.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.33.self_attn.o_proj.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.33.self_attn.q_norm.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.33.self_attn.q_proj.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.33.self_attn.v_proj.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.34.input_layernorm.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.34.mlp.down_proj.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.34.mlp.gate_proj.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.34.mlp.up_proj.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.34.post_attention_layernorm.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.34.self_attn.k_norm.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.34.self_attn.k_proj.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.34.self_attn.o_proj.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.34.self_attn.q_norm.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.34.self_attn.q_proj.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.34.self_attn.v_proj.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.35.input_layernorm.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.35.mlp.down_proj.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.35.mlp.gate_proj.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.35.mlp.up_proj.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.35.post_attention_layernorm.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.35.self_attn.k_norm.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.35.self_attn.k_proj.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.35.self_attn.o_proj.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.35.self_attn.q_norm.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.35.self_attn.q_proj.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.35.self_attn.v_proj.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.4.input_layernorm.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.4.mlp.down_proj.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.4.mlp.gate_proj.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.4.mlp.up_proj.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.4.post_attention_layernorm.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.4.self_attn.k_norm.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.4.self_attn.k_proj.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.4.self_attn.o_proj.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.4.self_attn.q_norm.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.4.self_attn.q_proj.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.4.self_attn.v_proj.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.5.input_layernorm.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.5.mlp.down_proj.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.5.mlp.gate_proj.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.5.mlp.up_proj.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.5.post_attention_layernorm.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.5.self_attn.k_norm.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.5.self_attn.k_proj.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.5.self_attn.o_proj.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.5.self_attn.q_norm.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.5.self_attn.q_proj.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.5.self_attn.v_proj.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.6.input_layernorm.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.6.mlp.down_proj.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.6.mlp.gate_proj.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.6.mlp.up_proj.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.6.post_attention_layernorm.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.6.self_attn.k_norm.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.6.self_attn.k_proj.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.6.self_attn.o_proj.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.6.self_attn.q_norm.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.6.self_attn.q_proj.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.6.self_attn.v_proj.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.7.input_layernorm.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.7.mlp.down_proj.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.7.mlp.gate_proj.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.7.mlp.up_proj.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.7.post_attention_layernorm.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.7.self_attn.k_norm.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.7.self_attn.k_proj.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.7.self_attn.o_proj.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.7.self_attn.q_norm.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.7.self_attn.q_proj.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.7.self_attn.v_proj.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.8.input_layernorm.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.8.mlp.down_proj.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.8.mlp.gate_proj.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.8.mlp.up_proj.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.8.post_attention_layernorm.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.8.self_attn.k_norm.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.8.self_attn.k_proj.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.8.self_attn.o_proj.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.8.self_attn.q_norm.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.8.self_attn.q_proj.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.8.self_attn.v_proj.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.9.input_layernorm.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.9.mlp.down_proj.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.9.mlp.gate_proj.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.9.mlp.up_proj.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.9.post_attention_layernorm.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.9.self_attn.k_norm.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.9.self_attn.k_proj.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.9.self_attn.o_proj.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.9.self_attn.q_norm.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.9.self_attn.q_proj.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.9.self_attn.v_proj.weight": "model-00001-of-00002.safetensors", + "model.language_model.norm.weight": "model-00002-of-00002.safetensors", + "model.visual.blocks.0.attn.proj.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.0.attn.proj.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.0.attn.qkv.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.0.attn.qkv.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.0.mlp.linear_fc1.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.0.mlp.linear_fc1.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.0.mlp.linear_fc2.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.0.mlp.linear_fc2.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.0.norm1.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.0.norm1.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.0.norm2.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.0.norm2.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.1.attn.proj.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.1.attn.proj.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.1.attn.qkv.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.1.attn.qkv.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.1.mlp.linear_fc1.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.1.mlp.linear_fc1.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.1.mlp.linear_fc2.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.1.mlp.linear_fc2.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.1.norm1.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.1.norm1.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.1.norm2.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.1.norm2.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.10.attn.proj.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.10.attn.proj.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.10.attn.qkv.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.10.attn.qkv.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.10.mlp.linear_fc1.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.10.mlp.linear_fc1.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.10.mlp.linear_fc2.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.10.mlp.linear_fc2.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.10.norm1.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.10.norm1.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.10.norm2.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.10.norm2.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.11.attn.proj.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.11.attn.proj.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.11.attn.qkv.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.11.attn.qkv.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.11.mlp.linear_fc1.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.11.mlp.linear_fc1.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.11.mlp.linear_fc2.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.11.mlp.linear_fc2.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.11.norm1.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.11.norm1.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.11.norm2.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.11.norm2.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.12.attn.proj.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.12.attn.proj.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.12.attn.qkv.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.12.attn.qkv.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.12.mlp.linear_fc1.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.12.mlp.linear_fc1.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.12.mlp.linear_fc2.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.12.mlp.linear_fc2.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.12.norm1.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.12.norm1.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.12.norm2.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.12.norm2.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.13.attn.proj.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.13.attn.proj.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.13.attn.qkv.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.13.attn.qkv.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.13.mlp.linear_fc1.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.13.mlp.linear_fc1.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.13.mlp.linear_fc2.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.13.mlp.linear_fc2.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.13.norm1.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.13.norm1.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.13.norm2.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.13.norm2.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.14.attn.proj.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.14.attn.proj.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.14.attn.qkv.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.14.attn.qkv.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.14.mlp.linear_fc1.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.14.mlp.linear_fc1.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.14.mlp.linear_fc2.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.14.mlp.linear_fc2.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.14.norm1.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.14.norm1.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.14.norm2.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.14.norm2.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.15.attn.proj.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.15.attn.proj.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.15.attn.qkv.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.15.attn.qkv.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.15.mlp.linear_fc1.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.15.mlp.linear_fc1.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.15.mlp.linear_fc2.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.15.mlp.linear_fc2.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.15.norm1.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.15.norm1.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.15.norm2.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.15.norm2.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.16.attn.proj.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.16.attn.proj.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.16.attn.qkv.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.16.attn.qkv.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.16.mlp.linear_fc1.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.16.mlp.linear_fc1.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.16.mlp.linear_fc2.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.16.mlp.linear_fc2.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.16.norm1.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.16.norm1.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.16.norm2.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.16.norm2.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.17.attn.proj.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.17.attn.proj.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.17.attn.qkv.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.17.attn.qkv.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.17.mlp.linear_fc1.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.17.mlp.linear_fc1.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.17.mlp.linear_fc2.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.17.mlp.linear_fc2.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.17.norm1.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.17.norm1.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.17.norm2.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.17.norm2.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.18.attn.proj.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.18.attn.proj.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.18.attn.qkv.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.18.attn.qkv.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.18.mlp.linear_fc1.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.18.mlp.linear_fc1.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.18.mlp.linear_fc2.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.18.mlp.linear_fc2.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.18.norm1.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.18.norm1.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.18.norm2.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.18.norm2.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.19.attn.proj.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.19.attn.proj.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.19.attn.qkv.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.19.attn.qkv.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.19.mlp.linear_fc1.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.19.mlp.linear_fc1.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.19.mlp.linear_fc2.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.19.mlp.linear_fc2.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.19.norm1.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.19.norm1.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.19.norm2.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.19.norm2.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.2.attn.proj.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.2.attn.proj.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.2.attn.qkv.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.2.attn.qkv.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.2.mlp.linear_fc1.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.2.mlp.linear_fc1.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.2.mlp.linear_fc2.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.2.mlp.linear_fc2.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.2.norm1.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.2.norm1.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.2.norm2.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.2.norm2.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.20.attn.proj.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.20.attn.proj.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.20.attn.qkv.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.20.attn.qkv.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.20.mlp.linear_fc1.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.20.mlp.linear_fc1.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.20.mlp.linear_fc2.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.20.mlp.linear_fc2.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.20.norm1.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.20.norm1.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.20.norm2.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.20.norm2.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.21.attn.proj.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.21.attn.proj.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.21.attn.qkv.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.21.attn.qkv.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.21.mlp.linear_fc1.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.21.mlp.linear_fc1.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.21.mlp.linear_fc2.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.21.mlp.linear_fc2.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.21.norm1.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.21.norm1.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.21.norm2.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.21.norm2.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.22.attn.proj.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.22.attn.proj.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.22.attn.qkv.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.22.attn.qkv.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.22.mlp.linear_fc1.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.22.mlp.linear_fc1.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.22.mlp.linear_fc2.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.22.mlp.linear_fc2.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.22.norm1.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.22.norm1.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.22.norm2.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.22.norm2.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.23.attn.proj.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.23.attn.proj.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.23.attn.qkv.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.23.attn.qkv.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.23.mlp.linear_fc1.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.23.mlp.linear_fc1.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.23.mlp.linear_fc2.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.23.mlp.linear_fc2.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.23.norm1.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.23.norm1.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.23.norm2.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.23.norm2.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.3.attn.proj.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.3.attn.proj.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.3.attn.qkv.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.3.attn.qkv.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.3.mlp.linear_fc1.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.3.mlp.linear_fc1.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.3.mlp.linear_fc2.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.3.mlp.linear_fc2.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.3.norm1.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.3.norm1.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.3.norm2.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.3.norm2.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.4.attn.proj.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.4.attn.proj.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.4.attn.qkv.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.4.attn.qkv.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.4.mlp.linear_fc1.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.4.mlp.linear_fc1.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.4.mlp.linear_fc2.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.4.mlp.linear_fc2.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.4.norm1.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.4.norm1.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.4.norm2.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.4.norm2.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.5.attn.proj.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.5.attn.proj.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.5.attn.qkv.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.5.attn.qkv.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.5.mlp.linear_fc1.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.5.mlp.linear_fc1.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.5.mlp.linear_fc2.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.5.mlp.linear_fc2.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.5.norm1.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.5.norm1.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.5.norm2.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.5.norm2.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.6.attn.proj.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.6.attn.proj.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.6.attn.qkv.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.6.attn.qkv.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.6.mlp.linear_fc1.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.6.mlp.linear_fc1.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.6.mlp.linear_fc2.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.6.mlp.linear_fc2.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.6.norm1.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.6.norm1.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.6.norm2.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.6.norm2.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.7.attn.proj.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.7.attn.proj.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.7.attn.qkv.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.7.attn.qkv.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.7.mlp.linear_fc1.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.7.mlp.linear_fc1.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.7.mlp.linear_fc2.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.7.mlp.linear_fc2.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.7.norm1.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.7.norm1.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.7.norm2.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.7.norm2.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.8.attn.proj.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.8.attn.proj.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.8.attn.qkv.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.8.attn.qkv.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.8.mlp.linear_fc1.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.8.mlp.linear_fc1.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.8.mlp.linear_fc2.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.8.mlp.linear_fc2.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.8.norm1.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.8.norm1.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.8.norm2.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.8.norm2.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.9.attn.proj.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.9.attn.proj.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.9.attn.qkv.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.9.attn.qkv.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.9.mlp.linear_fc1.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.9.mlp.linear_fc1.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.9.mlp.linear_fc2.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.9.mlp.linear_fc2.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.9.norm1.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.9.norm1.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.9.norm2.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.9.norm2.weight": "model-00001-of-00002.safetensors", + "model.visual.deepstack_merger_list.0.linear_fc1.bias": "model-00001-of-00002.safetensors", + "model.visual.deepstack_merger_list.0.linear_fc1.weight": "model-00001-of-00002.safetensors", + "model.visual.deepstack_merger_list.0.linear_fc2.bias": "model-00001-of-00002.safetensors", + "model.visual.deepstack_merger_list.0.linear_fc2.weight": "model-00001-of-00002.safetensors", + "model.visual.deepstack_merger_list.0.norm.bias": "model-00001-of-00002.safetensors", + "model.visual.deepstack_merger_list.0.norm.weight": "model-00001-of-00002.safetensors", + "model.visual.deepstack_merger_list.1.linear_fc1.bias": "model-00001-of-00002.safetensors", + "model.visual.deepstack_merger_list.1.linear_fc1.weight": "model-00001-of-00002.safetensors", + "model.visual.deepstack_merger_list.1.linear_fc2.bias": "model-00001-of-00002.safetensors", + "model.visual.deepstack_merger_list.1.linear_fc2.weight": "model-00001-of-00002.safetensors", + "model.visual.deepstack_merger_list.1.norm.bias": "model-00001-of-00002.safetensors", + "model.visual.deepstack_merger_list.1.norm.weight": "model-00001-of-00002.safetensors", + "model.visual.deepstack_merger_list.2.linear_fc1.bias": "model-00001-of-00002.safetensors", + "model.visual.deepstack_merger_list.2.linear_fc1.weight": "model-00001-of-00002.safetensors", + "model.visual.deepstack_merger_list.2.linear_fc2.bias": "model-00001-of-00002.safetensors", + "model.visual.deepstack_merger_list.2.linear_fc2.weight": "model-00001-of-00002.safetensors", + "model.visual.deepstack_merger_list.2.norm.bias": "model-00001-of-00002.safetensors", + "model.visual.deepstack_merger_list.2.norm.weight": "model-00001-of-00002.safetensors", + "model.visual.merger.linear_fc1.bias": "model-00001-of-00002.safetensors", + "model.visual.merger.linear_fc1.weight": "model-00001-of-00002.safetensors", + "model.visual.merger.linear_fc2.bias": "model-00001-of-00002.safetensors", + "model.visual.merger.linear_fc2.weight": "model-00001-of-00002.safetensors", + "model.visual.merger.norm.bias": "model-00001-of-00002.safetensors", + "model.visual.merger.norm.weight": "model-00001-of-00002.safetensors", + "model.visual.patch_embed.proj.bias": "model-00001-of-00002.safetensors", + "model.visual.patch_embed.proj.weight": "model-00001-of-00002.safetensors", + "model.visual.pos_embed.weight": "model-00001-of-00002.safetensors" + } +} diff --git a/posttrain_models/Qwen3VL_4B_lora32_batch12459_epoch4/preprocessor_config.json b/posttrain_models/Qwen3VL_4B_lora32_batch12459_epoch4/preprocessor_config.json new file mode 100644 index 0000000000000000000000000000000000000000..2fa6553554f69b4e3a6b742a75264ff9dff90ad0 --- /dev/null +++ b/posttrain_models/Qwen3VL_4B_lora32_batch12459_epoch4/preprocessor_config.json @@ -0,0 +1,39 @@ +{ + "crop_size": null, + "data_format": "channels_first", + "default_to_square": true, + "device": null, + "disable_grouping": null, + "do_center_crop": null, + "do_convert_rgb": true, + "do_normalize": true, + "do_pad": null, + "do_rescale": true, + "do_resize": true, + "image_mean": [ + 0.5, + 0.5, + 0.5 + ], + "image_processor_type": "Qwen2VLImageProcessorFast", + "image_std": [ + 0.5, + 0.5, + 0.5 + ], + "input_data_format": null, + "max_pixels": null, + "merge_size": 2, + "min_pixels": null, + "pad_size": null, + "patch_size": 16, + "processor_class": "Qwen3VLProcessor", + "resample": 3, + "rescale_factor": 0.00392156862745098, + "return_tensors": null, + "size": { + "longest_edge": 16777216, + "shortest_edge": 65536 + }, + "temporal_patch_size": 2 +} diff --git a/posttrain_models/Qwen3VL_4B_lora32_batch12459_epoch4/special_tokens_map.json b/posttrain_models/Qwen3VL_4B_lora32_batch12459_epoch4/special_tokens_map.json new file mode 100644 index 0000000000000000000000000000000000000000..ac23c0aaa2434523c494330aeb79c58395378103 --- /dev/null +++ b/posttrain_models/Qwen3VL_4B_lora32_batch12459_epoch4/special_tokens_map.json @@ -0,0 +1,31 @@ +{ + "additional_special_tokens": [ + "<|im_start|>", + "<|im_end|>", + "<|object_ref_start|>", + "<|object_ref_end|>", + "<|box_start|>", + "<|box_end|>", + "<|quad_start|>", + "<|quad_end|>", + "<|vision_start|>", + "<|vision_end|>", + "<|vision_pad|>", + "<|image_pad|>", + "<|video_pad|>" + ], + "eos_token": { + "content": "<|im_end|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false + }, + "pad_token": { + "content": "<|endoftext|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false + } +} diff --git a/posttrain_models/Qwen3VL_4B_lora32_batch12459_epoch4/tokenizer_config.json b/posttrain_models/Qwen3VL_4B_lora32_batch12459_epoch4/tokenizer_config.json new file mode 100644 index 0000000000000000000000000000000000000000..fec7f182135c384a08ea2fc9635566ca79c45afc --- /dev/null +++ b/posttrain_models/Qwen3VL_4B_lora32_batch12459_epoch4/tokenizer_config.json @@ -0,0 +1,240 @@ +{ + "add_bos_token": false, + "add_prefix_space": false, + "added_tokens_decoder": { + "151643": { + "content": "<|endoftext|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "151644": { + "content": "<|im_start|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "151645": { + "content": "<|im_end|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "151646": { + "content": "<|object_ref_start|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "151647": { + "content": "<|object_ref_end|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "151648": { + "content": "<|box_start|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "151649": { + "content": "<|box_end|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "151650": { + "content": "<|quad_start|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "151651": { + "content": "<|quad_end|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "151652": { + "content": "<|vision_start|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "151653": { + "content": "<|vision_end|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "151654": { + "content": "<|vision_pad|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "151655": { + "content": "<|image_pad|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "151656": { + "content": "<|video_pad|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "151657": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "151658": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "151659": { + "content": "<|fim_prefix|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "151660": { + "content": "<|fim_middle|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "151661": { + "content": "<|fim_suffix|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "151662": { + "content": "<|fim_pad|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "151663": { + "content": "<|repo_name|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "151664": { + "content": "<|file_sep|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "151665": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "151666": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "151667": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "151668": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + } + }, + "additional_special_tokens": [ + "<|im_start|>", + "<|im_end|>", + "<|object_ref_start|>", + "<|object_ref_end|>", + "<|box_start|>", + "<|box_end|>", + "<|quad_start|>", + "<|quad_end|>", + "<|vision_start|>", + "<|vision_end|>", + "<|vision_pad|>", + "<|image_pad|>", + "<|video_pad|>" + ], + "bos_token": null, + "clean_up_tokenization_spaces": false, + "eos_token": "<|im_end|>", + "errors": "replace", + "extra_special_tokens": {}, + "model_max_length": 262144, + "pad_token": "<|endoftext|>", + "processor_class": "Qwen3VLProcessor", + "split_special_tokens": false, + "tokenizer_class": "Qwen2Tokenizer", + "unk_token": null +} diff --git a/posttrain_models/Qwen3VL_4B_lora32_batch12459_epoch4/video_preprocessor_config.json b/posttrain_models/Qwen3VL_4B_lora32_batch12459_epoch4/video_preprocessor_config.json new file mode 100644 index 0000000000000000000000000000000000000000..e32b1d90356f74457edbfca27aa1e121dab7cdc7 --- /dev/null +++ b/posttrain_models/Qwen3VL_4B_lora32_batch12459_epoch4/video_preprocessor_config.json @@ -0,0 +1,41 @@ +{ + "crop_size": null, + "data_format": "channels_first", + "default_to_square": true, + "device": null, + "do_center_crop": null, + "do_convert_rgb": true, + "do_normalize": true, + "do_rescale": true, + "do_resize": true, + "do_sample_frames": true, + "fps": 2, + "image_mean": [ + 0.5, + 0.5, + 0.5 + ], + "image_std": [ + 0.5, + 0.5, + 0.5 + ], + "input_data_format": null, + "max_frames": 768, + "merge_size": 2, + "min_frames": 4, + "num_frames": null, + "pad_size": null, + "patch_size": 16, + "processor_class": "Qwen3VLProcessor", + "resample": 3, + "rescale_factor": 0.00392156862745098, + "return_metadata": false, + "size": { + "longest_edge": 25165824, + "shortest_edge": 4096 + }, + "temporal_patch_size": 2, + "video_metadata": null, + "video_processor_type": "Qwen3VLVideoProcessor" +} diff --git a/posttrain_models/Qwen3VL_4B_lora32_batch1245_epoch4/added_tokens.json b/posttrain_models/Qwen3VL_4B_lora32_batch1245_epoch4/added_tokens.json new file mode 100644 index 0000000000000000000000000000000000000000..b54f9135e44c1e81047e8d05cb027af8bc039eed --- /dev/null +++ b/posttrain_models/Qwen3VL_4B_lora32_batch1245_epoch4/added_tokens.json @@ -0,0 +1,28 @@ +{ + "": 151668, + "": 151658, + "": 151666, + "": 151667, + "": 151657, + "": 151665, + "<|box_end|>": 151649, + "<|box_start|>": 151648, + "<|endoftext|>": 151643, + "<|file_sep|>": 151664, + "<|fim_middle|>": 151660, + "<|fim_pad|>": 151662, + "<|fim_prefix|>": 151659, + "<|fim_suffix|>": 151661, + "<|im_end|>": 151645, + "<|im_start|>": 151644, + "<|image_pad|>": 151655, + "<|object_ref_end|>": 151647, + "<|object_ref_start|>": 151646, + "<|quad_end|>": 151651, + "<|quad_start|>": 151650, + "<|repo_name|>": 151663, + "<|video_pad|>": 151656, + "<|vision_end|>": 151653, + "<|vision_pad|>": 151654, + "<|vision_start|>": 151652 +} diff --git a/posttrain_models/Qwen3VL_4B_lora32_batch1245_epoch4/chat_template.jinja b/posttrain_models/Qwen3VL_4B_lora32_batch1245_epoch4/chat_template.jinja new file mode 100644 index 0000000000000000000000000000000000000000..124386803f142761528f710e77ae483f5f8c4fc4 --- /dev/null +++ b/posttrain_models/Qwen3VL_4B_lora32_batch1245_epoch4/chat_template.jinja @@ -0,0 +1,120 @@ +{%- if tools %} + {{- '<|im_start|>system\n' }} + {%- if messages[0].role == 'system' %} + {%- if messages[0].content is string %} + {{- messages[0].content }} + {%- else %} + {%- for content in messages[0].content %} + {%- if 'text' in content %} + {{- content.text }} + {%- endif %} + {%- endfor %} + {%- endif %} + {{- '\n\n' }} + {%- endif %} + {{- "# Tools\n\nYou may call one or more functions to assist with the user query.\n\nYou are provided with function signatures within XML tags:\n" }} + {%- for tool in tools %} + {{- "\n" }} + {{- tool | tojson }} + {%- endfor %} + {{- "\n\n\nFor each function call, return a json object with function name and arguments within XML tags:\n\n{\"name\": , \"arguments\": }\n<|im_end|>\n" }} +{%- else %} + {%- if messages[0].role == 'system' %} + {{- '<|im_start|>system\n' }} + {%- if messages[0].content is string %} + {{- messages[0].content }} + {%- else %} + {%- for content in messages[0].content %} + {%- if 'text' in content %} + {{- content.text }} + {%- endif %} + {%- endfor %} + {%- endif %} + {{- '<|im_end|>\n' }} + {%- endif %} +{%- endif %} +{%- set image_count = namespace(value=0) %} +{%- set video_count = namespace(value=0) %} +{%- for message in messages %} + {%- if message.role == "user" %} + {{- '<|im_start|>' + message.role + '\n' }} + {%- if message.content is string %} + {{- message.content }} + {%- else %} + {%- for content in message.content %} + {%- if content.type == 'image' or 'image' in content or 'image_url' in content %} + {%- set image_count.value = image_count.value + 1 %} + {%- if add_vision_id %}Picture {{ image_count.value }}: {% endif -%} + <|vision_start|><|image_pad|><|vision_end|> + {%- elif content.type == 'video' or 'video' in content %} + {%- set video_count.value = video_count.value + 1 %} + {%- if add_vision_id %}Video {{ video_count.value }}: {% endif -%} + <|vision_start|><|video_pad|><|vision_end|> + {%- elif 'text' in content %} + {{- content.text }} + {%- endif %} + {%- endfor %} + {%- endif %} + {{- '<|im_end|>\n' }} + {%- elif message.role == "assistant" %} + {{- '<|im_start|>' + message.role + '\n' }} + {%- if message.content is string %} + {{- message.content }} + {%- else %} + {%- for content_item in message.content %} + {%- if 'text' in content_item %} + {{- content_item.text }} + {%- endif %} + {%- endfor %} + {%- endif %} + {%- if message.tool_calls %} + {%- for tool_call in message.tool_calls %} + {%- if (loop.first and message.content) or (not loop.first) %} + {{- '\n' }} + {%- endif %} + {%- if tool_call.function %} + {%- set tool_call = tool_call.function %} + {%- endif %} + {{- '\n{"name": "' }} + {{- tool_call.name }} + {{- '", "arguments": ' }} + {%- if tool_call.arguments is string %} + {{- tool_call.arguments }} + {%- else %} + {{- tool_call.arguments | tojson }} + {%- endif %} + {{- '}\n' }} + {%- endfor %} + {%- endif %} + {{- '<|im_end|>\n' }} + {%- elif message.role == "tool" %} + {%- if loop.first or (messages[loop.index0 - 1].role != "tool") %} + {{- '<|im_start|>user' }} + {%- endif %} + {{- '\n\n' }} + {%- if message.content is string %} + {{- message.content }} + {%- else %} + {%- for content in message.content %} + {%- if content.type == 'image' or 'image' in content or 'image_url' in content %} + {%- set image_count.value = image_count.value + 1 %} + {%- if add_vision_id %}Picture {{ image_count.value }}: {% endif -%} + <|vision_start|><|image_pad|><|vision_end|> + {%- elif content.type == 'video' or 'video' in content %} + {%- set video_count.value = video_count.value + 1 %} + {%- if add_vision_id %}Video {{ video_count.value }}: {% endif -%} + <|vision_start|><|video_pad|><|vision_end|> + {%- elif 'text' in content %} + {{- content.text }} + {%- endif %} + {%- endfor %} + {%- endif %} + {{- '\n' }} + {%- if loop.last or (messages[loop.index0 + 1].role != "tool") %} + {{- '<|im_end|>\n' }} + {%- endif %} + {%- endif %} +{%- endfor %} +{%- if add_generation_prompt %} + {{- '<|im_start|>assistant\n' }} +{%- endif %} diff --git a/posttrain_models/Qwen3VL_4B_lora32_batch1245_epoch4/config.json b/posttrain_models/Qwen3VL_4B_lora32_batch1245_epoch4/config.json new file mode 100644 index 0000000000000000000000000000000000000000..adb449916091c26098a94af0fa4add48558a9494 --- /dev/null +++ b/posttrain_models/Qwen3VL_4B_lora32_batch1245_epoch4/config.json @@ -0,0 +1,65 @@ +{ + "architectures": [ + "Qwen3VLForConditionalGeneration" + ], + "dtype": "bfloat16", + "image_token_id": 151655, + "model_type": "qwen3_vl", + "text_config": { + "attention_bias": false, + "attention_dropout": 0.0, + "bos_token_id": 151643, + "dtype": "bfloat16", + "eos_token_id": 151645, + "head_dim": 128, + "hidden_act": "silu", + "hidden_size": 2560, + "initializer_range": 0.02, + "intermediate_size": 9728, + "max_position_embeddings": 262144, + "model_type": "qwen3_vl_text", + "num_attention_heads": 32, + "num_hidden_layers": 36, + "num_key_value_heads": 8, + "rms_norm_eps": 1e-06, + "rope_scaling": { + "mrope_interleaved": true, + "mrope_section": [ + 24, + 20, + 20 + ], + "rope_type": "default" + }, + "rope_theta": 5000000, + "tie_word_embeddings": true, + "use_cache": true, + "vocab_size": 151936 + }, + "tie_word_embeddings": true, + "transformers_version": "4.57.3", + "video_token_id": 151656, + "vision_config": { + "deepstack_visual_indexes": [ + 5, + 11, + 17 + ], + "depth": 24, + "dtype": "bfloat16", + "hidden_act": "gelu_pytorch_tanh", + "hidden_size": 1024, + "in_channels": 3, + "initializer_range": 0.02, + "intermediate_size": 4096, + "model_type": "qwen3_vl", + "num_heads": 16, + "num_position_embeddings": 2304, + "out_hidden_size": 2560, + "patch_size": 16, + "spatial_merge_size": 2, + "temporal_patch_size": 2 + }, + "vision_end_token_id": 151653, + "vision_start_token_id": 151652 +} diff --git a/posttrain_models/Qwen3VL_4B_lora32_batch1245_epoch4/generation_config.json b/posttrain_models/Qwen3VL_4B_lora32_batch1245_epoch4/generation_config.json new file mode 100644 index 0000000000000000000000000000000000000000..3dd2c545b96b4d51586680d7bb27fdca5c90dfe9 --- /dev/null +++ b/posttrain_models/Qwen3VL_4B_lora32_batch1245_epoch4/generation_config.json @@ -0,0 +1,13 @@ +{ + "bos_token_id": 151643, + "do_sample": true, + "eos_token_id": [ + 151645, + 151643 + ], + "pad_token_id": 151643, + "temperature": 0.7, + "top_k": 20, + "top_p": 0.8, + "transformers_version": "4.57.3" +} diff --git a/posttrain_models/Qwen3VL_4B_lora32_batch1245_epoch4/model.safetensors.index.json b/posttrain_models/Qwen3VL_4B_lora32_batch1245_epoch4/model.safetensors.index.json new file mode 100644 index 0000000000000000000000000000000000000000..58d118e8ce51a42ed3b147a55f9c92f39d273c60 --- /dev/null +++ b/posttrain_models/Qwen3VL_4B_lora32_batch1245_epoch4/model.safetensors.index.json @@ -0,0 +1,721 @@ +{ + "metadata": { + "total_parameters": 4437815808, + "total_size": 8875631616 + }, + "weight_map": { + "model.language_model.embed_tokens.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.0.input_layernorm.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.0.mlp.down_proj.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.0.mlp.gate_proj.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.0.mlp.up_proj.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.0.post_attention_layernorm.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.0.self_attn.k_norm.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.0.self_attn.k_proj.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.0.self_attn.o_proj.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.0.self_attn.q_norm.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.0.self_attn.q_proj.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.0.self_attn.v_proj.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.1.input_layernorm.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.1.mlp.down_proj.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.1.mlp.gate_proj.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.1.mlp.up_proj.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.1.post_attention_layernorm.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.1.self_attn.k_norm.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.1.self_attn.k_proj.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.1.self_attn.o_proj.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.1.self_attn.q_norm.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.1.self_attn.q_proj.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.1.self_attn.v_proj.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.10.input_layernorm.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.10.mlp.down_proj.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.10.mlp.gate_proj.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.10.mlp.up_proj.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.10.post_attention_layernorm.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.10.self_attn.k_norm.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.10.self_attn.k_proj.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.10.self_attn.o_proj.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.10.self_attn.q_norm.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.10.self_attn.q_proj.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.10.self_attn.v_proj.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.11.input_layernorm.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.11.mlp.down_proj.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.11.mlp.gate_proj.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.11.mlp.up_proj.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.11.post_attention_layernorm.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.11.self_attn.k_norm.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.11.self_attn.k_proj.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.11.self_attn.o_proj.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.11.self_attn.q_norm.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.11.self_attn.q_proj.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.11.self_attn.v_proj.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.12.input_layernorm.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.12.mlp.down_proj.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.12.mlp.gate_proj.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.12.mlp.up_proj.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.12.post_attention_layernorm.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.12.self_attn.k_norm.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.12.self_attn.k_proj.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.12.self_attn.o_proj.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.12.self_attn.q_norm.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.12.self_attn.q_proj.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.12.self_attn.v_proj.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.13.input_layernorm.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.13.mlp.down_proj.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.13.mlp.gate_proj.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.13.mlp.up_proj.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.13.post_attention_layernorm.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.13.self_attn.k_norm.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.13.self_attn.k_proj.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.13.self_attn.o_proj.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.13.self_attn.q_norm.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.13.self_attn.q_proj.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.13.self_attn.v_proj.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.14.input_layernorm.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.14.mlp.down_proj.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.14.mlp.gate_proj.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.14.mlp.up_proj.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.14.post_attention_layernorm.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.14.self_attn.k_norm.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.14.self_attn.k_proj.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.14.self_attn.o_proj.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.14.self_attn.q_norm.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.14.self_attn.q_proj.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.14.self_attn.v_proj.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.15.input_layernorm.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.15.mlp.down_proj.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.15.mlp.gate_proj.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.15.mlp.up_proj.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.15.post_attention_layernorm.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.15.self_attn.k_norm.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.15.self_attn.k_proj.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.15.self_attn.o_proj.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.15.self_attn.q_norm.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.15.self_attn.q_proj.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.15.self_attn.v_proj.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.16.input_layernorm.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.16.mlp.down_proj.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.16.mlp.gate_proj.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.16.mlp.up_proj.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.16.post_attention_layernorm.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.16.self_attn.k_norm.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.16.self_attn.k_proj.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.16.self_attn.o_proj.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.16.self_attn.q_norm.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.16.self_attn.q_proj.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.16.self_attn.v_proj.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.17.input_layernorm.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.17.mlp.down_proj.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.17.mlp.gate_proj.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.17.mlp.up_proj.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.17.post_attention_layernorm.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.17.self_attn.k_norm.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.17.self_attn.k_proj.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.17.self_attn.o_proj.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.17.self_attn.q_norm.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.17.self_attn.q_proj.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.17.self_attn.v_proj.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.18.input_layernorm.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.18.mlp.down_proj.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.18.mlp.gate_proj.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.18.mlp.up_proj.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.18.post_attention_layernorm.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.18.self_attn.k_norm.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.18.self_attn.k_proj.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.18.self_attn.o_proj.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.18.self_attn.q_norm.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.18.self_attn.q_proj.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.18.self_attn.v_proj.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.19.input_layernorm.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.19.mlp.down_proj.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.19.mlp.gate_proj.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.19.mlp.up_proj.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.19.post_attention_layernorm.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.19.self_attn.k_norm.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.19.self_attn.k_proj.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.19.self_attn.o_proj.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.19.self_attn.q_norm.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.19.self_attn.q_proj.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.19.self_attn.v_proj.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.2.input_layernorm.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.2.mlp.down_proj.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.2.mlp.gate_proj.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.2.mlp.up_proj.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.2.post_attention_layernorm.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.2.self_attn.k_norm.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.2.self_attn.k_proj.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.2.self_attn.o_proj.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.2.self_attn.q_norm.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.2.self_attn.q_proj.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.2.self_attn.v_proj.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.20.input_layernorm.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.20.mlp.down_proj.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.20.mlp.gate_proj.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.20.mlp.up_proj.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.20.post_attention_layernorm.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.20.self_attn.k_norm.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.20.self_attn.k_proj.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.20.self_attn.o_proj.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.20.self_attn.q_norm.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.20.self_attn.q_proj.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.20.self_attn.v_proj.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.21.input_layernorm.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.21.mlp.down_proj.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.21.mlp.gate_proj.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.21.mlp.up_proj.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.21.post_attention_layernorm.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.21.self_attn.k_norm.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.21.self_attn.k_proj.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.21.self_attn.o_proj.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.21.self_attn.q_norm.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.21.self_attn.q_proj.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.21.self_attn.v_proj.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.22.input_layernorm.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.22.mlp.down_proj.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.22.mlp.gate_proj.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.22.mlp.up_proj.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.22.post_attention_layernorm.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.22.self_attn.k_norm.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.22.self_attn.k_proj.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.22.self_attn.o_proj.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.22.self_attn.q_norm.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.22.self_attn.q_proj.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.22.self_attn.v_proj.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.23.input_layernorm.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.23.mlp.down_proj.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.23.mlp.gate_proj.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.23.mlp.up_proj.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.23.post_attention_layernorm.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.23.self_attn.k_norm.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.23.self_attn.k_proj.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.23.self_attn.o_proj.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.23.self_attn.q_norm.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.23.self_attn.q_proj.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.23.self_attn.v_proj.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.24.input_layernorm.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.24.mlp.down_proj.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.24.mlp.gate_proj.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.24.mlp.up_proj.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.24.post_attention_layernorm.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.24.self_attn.k_norm.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.24.self_attn.k_proj.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.24.self_attn.o_proj.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.24.self_attn.q_norm.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.24.self_attn.q_proj.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.24.self_attn.v_proj.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.25.input_layernorm.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.25.mlp.down_proj.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.25.mlp.gate_proj.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.25.mlp.up_proj.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.25.post_attention_layernorm.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.25.self_attn.k_norm.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.25.self_attn.k_proj.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.25.self_attn.o_proj.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.25.self_attn.q_norm.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.25.self_attn.q_proj.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.25.self_attn.v_proj.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.26.input_layernorm.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.26.mlp.down_proj.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.26.mlp.gate_proj.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.26.mlp.up_proj.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.26.post_attention_layernorm.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.26.self_attn.k_norm.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.26.self_attn.k_proj.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.26.self_attn.o_proj.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.26.self_attn.q_norm.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.26.self_attn.q_proj.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.26.self_attn.v_proj.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.27.input_layernorm.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.27.mlp.down_proj.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.27.mlp.gate_proj.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.27.mlp.up_proj.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.27.post_attention_layernorm.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.27.self_attn.k_norm.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.27.self_attn.k_proj.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.27.self_attn.o_proj.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.27.self_attn.q_norm.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.27.self_attn.q_proj.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.27.self_attn.v_proj.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.28.input_layernorm.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.28.mlp.down_proj.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.28.mlp.gate_proj.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.28.mlp.up_proj.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.28.post_attention_layernorm.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.28.self_attn.k_norm.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.28.self_attn.k_proj.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.28.self_attn.o_proj.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.28.self_attn.q_norm.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.28.self_attn.q_proj.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.28.self_attn.v_proj.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.29.input_layernorm.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.29.mlp.down_proj.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.29.mlp.gate_proj.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.29.mlp.up_proj.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.29.post_attention_layernorm.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.29.self_attn.k_norm.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.29.self_attn.k_proj.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.29.self_attn.o_proj.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.29.self_attn.q_norm.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.29.self_attn.q_proj.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.29.self_attn.v_proj.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.3.input_layernorm.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.3.mlp.down_proj.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.3.mlp.gate_proj.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.3.mlp.up_proj.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.3.post_attention_layernorm.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.3.self_attn.k_norm.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.3.self_attn.k_proj.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.3.self_attn.o_proj.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.3.self_attn.q_norm.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.3.self_attn.q_proj.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.3.self_attn.v_proj.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.30.input_layernorm.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.30.mlp.down_proj.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.30.mlp.gate_proj.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.30.mlp.up_proj.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.30.post_attention_layernorm.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.30.self_attn.k_norm.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.30.self_attn.k_proj.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.30.self_attn.o_proj.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.30.self_attn.q_norm.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.30.self_attn.q_proj.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.30.self_attn.v_proj.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.31.input_layernorm.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.31.mlp.down_proj.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.31.mlp.gate_proj.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.31.mlp.up_proj.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.31.post_attention_layernorm.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.31.self_attn.k_norm.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.31.self_attn.k_proj.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.31.self_attn.o_proj.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.31.self_attn.q_norm.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.31.self_attn.q_proj.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.31.self_attn.v_proj.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.32.input_layernorm.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.32.mlp.down_proj.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.32.mlp.gate_proj.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.32.mlp.up_proj.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.32.post_attention_layernorm.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.32.self_attn.k_norm.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.32.self_attn.k_proj.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.32.self_attn.o_proj.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.32.self_attn.q_norm.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.32.self_attn.q_proj.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.32.self_attn.v_proj.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.33.input_layernorm.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.33.mlp.down_proj.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.33.mlp.gate_proj.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.33.mlp.up_proj.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.33.post_attention_layernorm.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.33.self_attn.k_norm.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.33.self_attn.k_proj.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.33.self_attn.o_proj.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.33.self_attn.q_norm.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.33.self_attn.q_proj.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.33.self_attn.v_proj.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.34.input_layernorm.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.34.mlp.down_proj.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.34.mlp.gate_proj.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.34.mlp.up_proj.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.34.post_attention_layernorm.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.34.self_attn.k_norm.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.34.self_attn.k_proj.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.34.self_attn.o_proj.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.34.self_attn.q_norm.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.34.self_attn.q_proj.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.34.self_attn.v_proj.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.35.input_layernorm.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.35.mlp.down_proj.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.35.mlp.gate_proj.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.35.mlp.up_proj.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.35.post_attention_layernorm.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.35.self_attn.k_norm.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.35.self_attn.k_proj.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.35.self_attn.o_proj.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.35.self_attn.q_norm.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.35.self_attn.q_proj.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.35.self_attn.v_proj.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.4.input_layernorm.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.4.mlp.down_proj.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.4.mlp.gate_proj.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.4.mlp.up_proj.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.4.post_attention_layernorm.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.4.self_attn.k_norm.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.4.self_attn.k_proj.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.4.self_attn.o_proj.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.4.self_attn.q_norm.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.4.self_attn.q_proj.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.4.self_attn.v_proj.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.5.input_layernorm.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.5.mlp.down_proj.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.5.mlp.gate_proj.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.5.mlp.up_proj.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.5.post_attention_layernorm.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.5.self_attn.k_norm.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.5.self_attn.k_proj.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.5.self_attn.o_proj.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.5.self_attn.q_norm.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.5.self_attn.q_proj.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.5.self_attn.v_proj.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.6.input_layernorm.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.6.mlp.down_proj.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.6.mlp.gate_proj.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.6.mlp.up_proj.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.6.post_attention_layernorm.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.6.self_attn.k_norm.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.6.self_attn.k_proj.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.6.self_attn.o_proj.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.6.self_attn.q_norm.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.6.self_attn.q_proj.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.6.self_attn.v_proj.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.7.input_layernorm.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.7.mlp.down_proj.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.7.mlp.gate_proj.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.7.mlp.up_proj.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.7.post_attention_layernorm.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.7.self_attn.k_norm.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.7.self_attn.k_proj.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.7.self_attn.o_proj.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.7.self_attn.q_norm.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.7.self_attn.q_proj.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.7.self_attn.v_proj.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.8.input_layernorm.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.8.mlp.down_proj.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.8.mlp.gate_proj.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.8.mlp.up_proj.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.8.post_attention_layernorm.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.8.self_attn.k_norm.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.8.self_attn.k_proj.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.8.self_attn.o_proj.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.8.self_attn.q_norm.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.8.self_attn.q_proj.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.8.self_attn.v_proj.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.9.input_layernorm.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.9.mlp.down_proj.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.9.mlp.gate_proj.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.9.mlp.up_proj.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.9.post_attention_layernorm.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.9.self_attn.k_norm.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.9.self_attn.k_proj.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.9.self_attn.o_proj.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.9.self_attn.q_norm.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.9.self_attn.q_proj.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.9.self_attn.v_proj.weight": "model-00001-of-00002.safetensors", + "model.language_model.norm.weight": "model-00002-of-00002.safetensors", + "model.visual.blocks.0.attn.proj.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.0.attn.proj.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.0.attn.qkv.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.0.attn.qkv.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.0.mlp.linear_fc1.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.0.mlp.linear_fc1.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.0.mlp.linear_fc2.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.0.mlp.linear_fc2.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.0.norm1.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.0.norm1.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.0.norm2.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.0.norm2.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.1.attn.proj.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.1.attn.proj.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.1.attn.qkv.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.1.attn.qkv.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.1.mlp.linear_fc1.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.1.mlp.linear_fc1.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.1.mlp.linear_fc2.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.1.mlp.linear_fc2.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.1.norm1.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.1.norm1.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.1.norm2.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.1.norm2.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.10.attn.proj.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.10.attn.proj.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.10.attn.qkv.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.10.attn.qkv.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.10.mlp.linear_fc1.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.10.mlp.linear_fc1.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.10.mlp.linear_fc2.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.10.mlp.linear_fc2.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.10.norm1.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.10.norm1.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.10.norm2.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.10.norm2.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.11.attn.proj.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.11.attn.proj.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.11.attn.qkv.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.11.attn.qkv.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.11.mlp.linear_fc1.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.11.mlp.linear_fc1.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.11.mlp.linear_fc2.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.11.mlp.linear_fc2.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.11.norm1.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.11.norm1.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.11.norm2.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.11.norm2.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.12.attn.proj.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.12.attn.proj.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.12.attn.qkv.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.12.attn.qkv.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.12.mlp.linear_fc1.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.12.mlp.linear_fc1.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.12.mlp.linear_fc2.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.12.mlp.linear_fc2.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.12.norm1.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.12.norm1.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.12.norm2.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.12.norm2.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.13.attn.proj.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.13.attn.proj.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.13.attn.qkv.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.13.attn.qkv.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.13.mlp.linear_fc1.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.13.mlp.linear_fc1.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.13.mlp.linear_fc2.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.13.mlp.linear_fc2.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.13.norm1.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.13.norm1.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.13.norm2.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.13.norm2.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.14.attn.proj.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.14.attn.proj.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.14.attn.qkv.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.14.attn.qkv.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.14.mlp.linear_fc1.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.14.mlp.linear_fc1.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.14.mlp.linear_fc2.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.14.mlp.linear_fc2.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.14.norm1.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.14.norm1.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.14.norm2.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.14.norm2.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.15.attn.proj.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.15.attn.proj.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.15.attn.qkv.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.15.attn.qkv.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.15.mlp.linear_fc1.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.15.mlp.linear_fc1.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.15.mlp.linear_fc2.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.15.mlp.linear_fc2.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.15.norm1.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.15.norm1.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.15.norm2.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.15.norm2.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.16.attn.proj.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.16.attn.proj.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.16.attn.qkv.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.16.attn.qkv.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.16.mlp.linear_fc1.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.16.mlp.linear_fc1.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.16.mlp.linear_fc2.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.16.mlp.linear_fc2.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.16.norm1.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.16.norm1.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.16.norm2.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.16.norm2.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.17.attn.proj.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.17.attn.proj.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.17.attn.qkv.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.17.attn.qkv.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.17.mlp.linear_fc1.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.17.mlp.linear_fc1.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.17.mlp.linear_fc2.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.17.mlp.linear_fc2.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.17.norm1.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.17.norm1.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.17.norm2.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.17.norm2.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.18.attn.proj.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.18.attn.proj.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.18.attn.qkv.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.18.attn.qkv.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.18.mlp.linear_fc1.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.18.mlp.linear_fc1.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.18.mlp.linear_fc2.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.18.mlp.linear_fc2.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.18.norm1.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.18.norm1.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.18.norm2.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.18.norm2.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.19.attn.proj.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.19.attn.proj.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.19.attn.qkv.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.19.attn.qkv.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.19.mlp.linear_fc1.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.19.mlp.linear_fc1.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.19.mlp.linear_fc2.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.19.mlp.linear_fc2.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.19.norm1.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.19.norm1.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.19.norm2.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.19.norm2.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.2.attn.proj.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.2.attn.proj.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.2.attn.qkv.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.2.attn.qkv.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.2.mlp.linear_fc1.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.2.mlp.linear_fc1.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.2.mlp.linear_fc2.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.2.mlp.linear_fc2.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.2.norm1.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.2.norm1.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.2.norm2.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.2.norm2.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.20.attn.proj.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.20.attn.proj.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.20.attn.qkv.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.20.attn.qkv.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.20.mlp.linear_fc1.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.20.mlp.linear_fc1.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.20.mlp.linear_fc2.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.20.mlp.linear_fc2.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.20.norm1.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.20.norm1.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.20.norm2.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.20.norm2.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.21.attn.proj.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.21.attn.proj.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.21.attn.qkv.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.21.attn.qkv.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.21.mlp.linear_fc1.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.21.mlp.linear_fc1.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.21.mlp.linear_fc2.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.21.mlp.linear_fc2.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.21.norm1.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.21.norm1.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.21.norm2.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.21.norm2.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.22.attn.proj.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.22.attn.proj.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.22.attn.qkv.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.22.attn.qkv.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.22.mlp.linear_fc1.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.22.mlp.linear_fc1.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.22.mlp.linear_fc2.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.22.mlp.linear_fc2.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.22.norm1.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.22.norm1.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.22.norm2.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.22.norm2.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.23.attn.proj.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.23.attn.proj.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.23.attn.qkv.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.23.attn.qkv.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.23.mlp.linear_fc1.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.23.mlp.linear_fc1.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.23.mlp.linear_fc2.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.23.mlp.linear_fc2.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.23.norm1.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.23.norm1.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.23.norm2.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.23.norm2.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.3.attn.proj.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.3.attn.proj.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.3.attn.qkv.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.3.attn.qkv.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.3.mlp.linear_fc1.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.3.mlp.linear_fc1.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.3.mlp.linear_fc2.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.3.mlp.linear_fc2.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.3.norm1.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.3.norm1.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.3.norm2.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.3.norm2.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.4.attn.proj.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.4.attn.proj.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.4.attn.qkv.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.4.attn.qkv.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.4.mlp.linear_fc1.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.4.mlp.linear_fc1.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.4.mlp.linear_fc2.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.4.mlp.linear_fc2.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.4.norm1.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.4.norm1.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.4.norm2.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.4.norm2.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.5.attn.proj.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.5.attn.proj.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.5.attn.qkv.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.5.attn.qkv.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.5.mlp.linear_fc1.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.5.mlp.linear_fc1.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.5.mlp.linear_fc2.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.5.mlp.linear_fc2.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.5.norm1.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.5.norm1.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.5.norm2.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.5.norm2.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.6.attn.proj.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.6.attn.proj.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.6.attn.qkv.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.6.attn.qkv.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.6.mlp.linear_fc1.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.6.mlp.linear_fc1.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.6.mlp.linear_fc2.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.6.mlp.linear_fc2.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.6.norm1.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.6.norm1.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.6.norm2.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.6.norm2.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.7.attn.proj.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.7.attn.proj.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.7.attn.qkv.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.7.attn.qkv.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.7.mlp.linear_fc1.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.7.mlp.linear_fc1.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.7.mlp.linear_fc2.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.7.mlp.linear_fc2.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.7.norm1.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.7.norm1.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.7.norm2.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.7.norm2.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.8.attn.proj.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.8.attn.proj.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.8.attn.qkv.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.8.attn.qkv.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.8.mlp.linear_fc1.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.8.mlp.linear_fc1.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.8.mlp.linear_fc2.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.8.mlp.linear_fc2.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.8.norm1.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.8.norm1.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.8.norm2.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.8.norm2.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.9.attn.proj.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.9.attn.proj.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.9.attn.qkv.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.9.attn.qkv.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.9.mlp.linear_fc1.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.9.mlp.linear_fc1.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.9.mlp.linear_fc2.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.9.mlp.linear_fc2.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.9.norm1.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.9.norm1.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.9.norm2.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.9.norm2.weight": "model-00001-of-00002.safetensors", + "model.visual.deepstack_merger_list.0.linear_fc1.bias": "model-00001-of-00002.safetensors", + "model.visual.deepstack_merger_list.0.linear_fc1.weight": "model-00001-of-00002.safetensors", + "model.visual.deepstack_merger_list.0.linear_fc2.bias": "model-00001-of-00002.safetensors", + "model.visual.deepstack_merger_list.0.linear_fc2.weight": "model-00001-of-00002.safetensors", + "model.visual.deepstack_merger_list.0.norm.bias": "model-00001-of-00002.safetensors", + "model.visual.deepstack_merger_list.0.norm.weight": "model-00001-of-00002.safetensors", + "model.visual.deepstack_merger_list.1.linear_fc1.bias": "model-00001-of-00002.safetensors", + "model.visual.deepstack_merger_list.1.linear_fc1.weight": "model-00001-of-00002.safetensors", + "model.visual.deepstack_merger_list.1.linear_fc2.bias": "model-00001-of-00002.safetensors", + "model.visual.deepstack_merger_list.1.linear_fc2.weight": "model-00001-of-00002.safetensors", + "model.visual.deepstack_merger_list.1.norm.bias": "model-00001-of-00002.safetensors", + "model.visual.deepstack_merger_list.1.norm.weight": "model-00001-of-00002.safetensors", + "model.visual.deepstack_merger_list.2.linear_fc1.bias": "model-00001-of-00002.safetensors", + "model.visual.deepstack_merger_list.2.linear_fc1.weight": "model-00001-of-00002.safetensors", + "model.visual.deepstack_merger_list.2.linear_fc2.bias": "model-00001-of-00002.safetensors", + "model.visual.deepstack_merger_list.2.linear_fc2.weight": "model-00001-of-00002.safetensors", + "model.visual.deepstack_merger_list.2.norm.bias": "model-00001-of-00002.safetensors", + "model.visual.deepstack_merger_list.2.norm.weight": "model-00001-of-00002.safetensors", + "model.visual.merger.linear_fc1.bias": "model-00001-of-00002.safetensors", + "model.visual.merger.linear_fc1.weight": "model-00001-of-00002.safetensors", + "model.visual.merger.linear_fc2.bias": "model-00001-of-00002.safetensors", + "model.visual.merger.linear_fc2.weight": "model-00001-of-00002.safetensors", + "model.visual.merger.norm.bias": "model-00001-of-00002.safetensors", + "model.visual.merger.norm.weight": "model-00001-of-00002.safetensors", + "model.visual.patch_embed.proj.bias": "model-00001-of-00002.safetensors", + "model.visual.patch_embed.proj.weight": "model-00001-of-00002.safetensors", + "model.visual.pos_embed.weight": "model-00001-of-00002.safetensors" + } +} diff --git a/posttrain_models/Qwen3VL_4B_lora32_batch1245_epoch4/preprocessor_config.json b/posttrain_models/Qwen3VL_4B_lora32_batch1245_epoch4/preprocessor_config.json new file mode 100644 index 0000000000000000000000000000000000000000..2fa6553554f69b4e3a6b742a75264ff9dff90ad0 --- /dev/null +++ b/posttrain_models/Qwen3VL_4B_lora32_batch1245_epoch4/preprocessor_config.json @@ -0,0 +1,39 @@ +{ + "crop_size": null, + "data_format": "channels_first", + "default_to_square": true, + "device": null, + "disable_grouping": null, + "do_center_crop": null, + "do_convert_rgb": true, + "do_normalize": true, + "do_pad": null, + "do_rescale": true, + "do_resize": true, + "image_mean": [ + 0.5, + 0.5, + 0.5 + ], + "image_processor_type": "Qwen2VLImageProcessorFast", + "image_std": [ + 0.5, + 0.5, + 0.5 + ], + "input_data_format": null, + "max_pixels": null, + "merge_size": 2, + "min_pixels": null, + "pad_size": null, + "patch_size": 16, + "processor_class": "Qwen3VLProcessor", + "resample": 3, + "rescale_factor": 0.00392156862745098, + "return_tensors": null, + "size": { + "longest_edge": 16777216, + "shortest_edge": 65536 + }, + "temporal_patch_size": 2 +} diff --git a/posttrain_models/Qwen3VL_4B_lora32_batch1245_epoch4/special_tokens_map.json b/posttrain_models/Qwen3VL_4B_lora32_batch1245_epoch4/special_tokens_map.json new file mode 100644 index 0000000000000000000000000000000000000000..ac23c0aaa2434523c494330aeb79c58395378103 --- /dev/null +++ b/posttrain_models/Qwen3VL_4B_lora32_batch1245_epoch4/special_tokens_map.json @@ -0,0 +1,31 @@ +{ + "additional_special_tokens": [ + "<|im_start|>", + "<|im_end|>", + "<|object_ref_start|>", + "<|object_ref_end|>", + "<|box_start|>", + "<|box_end|>", + "<|quad_start|>", + "<|quad_end|>", + "<|vision_start|>", + "<|vision_end|>", + "<|vision_pad|>", + "<|image_pad|>", + "<|video_pad|>" + ], + "eos_token": { + "content": "<|im_end|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false + }, + "pad_token": { + "content": "<|endoftext|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false + } +} diff --git a/posttrain_models/Qwen3VL_4B_lora32_batch1245_epoch4/tokenizer_config.json b/posttrain_models/Qwen3VL_4B_lora32_batch1245_epoch4/tokenizer_config.json new file mode 100644 index 0000000000000000000000000000000000000000..fec7f182135c384a08ea2fc9635566ca79c45afc --- /dev/null +++ b/posttrain_models/Qwen3VL_4B_lora32_batch1245_epoch4/tokenizer_config.json @@ -0,0 +1,240 @@ +{ + "add_bos_token": false, + "add_prefix_space": false, + "added_tokens_decoder": { + "151643": { + "content": "<|endoftext|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "151644": { + "content": "<|im_start|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "151645": { + "content": "<|im_end|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "151646": { + "content": "<|object_ref_start|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "151647": { + "content": "<|object_ref_end|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "151648": { + "content": "<|box_start|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "151649": { + "content": "<|box_end|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "151650": { + "content": "<|quad_start|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "151651": { + "content": "<|quad_end|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "151652": { + "content": "<|vision_start|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "151653": { + "content": "<|vision_end|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "151654": { + "content": "<|vision_pad|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "151655": { + "content": "<|image_pad|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "151656": { + "content": "<|video_pad|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "151657": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "151658": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "151659": { + "content": "<|fim_prefix|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "151660": { + "content": "<|fim_middle|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "151661": { + "content": "<|fim_suffix|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "151662": { + "content": "<|fim_pad|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "151663": { + "content": "<|repo_name|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "151664": { + "content": "<|file_sep|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "151665": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "151666": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "151667": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "151668": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + } + }, + "additional_special_tokens": [ + "<|im_start|>", + "<|im_end|>", + "<|object_ref_start|>", + "<|object_ref_end|>", + "<|box_start|>", + "<|box_end|>", + "<|quad_start|>", + "<|quad_end|>", + "<|vision_start|>", + "<|vision_end|>", + "<|vision_pad|>", + "<|image_pad|>", + "<|video_pad|>" + ], + "bos_token": null, + "clean_up_tokenization_spaces": false, + "eos_token": "<|im_end|>", + "errors": "replace", + "extra_special_tokens": {}, + "model_max_length": 262144, + "pad_token": "<|endoftext|>", + "processor_class": "Qwen3VLProcessor", + "split_special_tokens": false, + "tokenizer_class": "Qwen2Tokenizer", + "unk_token": null +} diff --git a/posttrain_models/Qwen3VL_4B_lora32_batch1245_epoch4/video_preprocessor_config.json b/posttrain_models/Qwen3VL_4B_lora32_batch1245_epoch4/video_preprocessor_config.json new file mode 100644 index 0000000000000000000000000000000000000000..e32b1d90356f74457edbfca27aa1e121dab7cdc7 --- /dev/null +++ b/posttrain_models/Qwen3VL_4B_lora32_batch1245_epoch4/video_preprocessor_config.json @@ -0,0 +1,41 @@ +{ + "crop_size": null, + "data_format": "channels_first", + "default_to_square": true, + "device": null, + "do_center_crop": null, + "do_convert_rgb": true, + "do_normalize": true, + "do_rescale": true, + "do_resize": true, + "do_sample_frames": true, + "fps": 2, + "image_mean": [ + 0.5, + 0.5, + 0.5 + ], + "image_std": [ + 0.5, + 0.5, + 0.5 + ], + "input_data_format": null, + "max_frames": 768, + "merge_size": 2, + "min_frames": 4, + "num_frames": null, + "pad_size": null, + "patch_size": 16, + "processor_class": "Qwen3VLProcessor", + "resample": 3, + "rescale_factor": 0.00392156862745098, + "return_metadata": false, + "size": { + "longest_edge": 25165824, + "shortest_edge": 4096 + }, + "temporal_patch_size": 2, + "video_metadata": null, + "video_processor_type": "Qwen3VLVideoProcessor" +} diff --git a/posttrain_models/Video-o3_SFT_RL/.gitattributes b/posttrain_models/Video-o3_SFT_RL/.gitattributes new file mode 100644 index 0000000000000000000000000000000000000000..52373fe24473b1aa44333d318f578ae6bf04b49b --- /dev/null +++ b/posttrain_models/Video-o3_SFT_RL/.gitattributes @@ -0,0 +1,36 @@ +*.7z filter=lfs diff=lfs merge=lfs -text +*.arrow filter=lfs diff=lfs merge=lfs -text +*.bin filter=lfs diff=lfs merge=lfs -text +*.bz2 filter=lfs diff=lfs merge=lfs -text +*.ckpt filter=lfs diff=lfs merge=lfs -text +*.ftz filter=lfs diff=lfs merge=lfs -text +*.gz filter=lfs diff=lfs merge=lfs -text +*.h5 filter=lfs diff=lfs merge=lfs -text +*.joblib filter=lfs diff=lfs merge=lfs -text +*.lfs.* filter=lfs diff=lfs merge=lfs -text +*.mlmodel filter=lfs diff=lfs merge=lfs -text +*.model filter=lfs diff=lfs merge=lfs -text +*.msgpack filter=lfs diff=lfs merge=lfs -text +*.npy filter=lfs diff=lfs merge=lfs -text +*.npz filter=lfs diff=lfs merge=lfs -text +*.onnx filter=lfs diff=lfs merge=lfs -text +*.ot filter=lfs diff=lfs merge=lfs -text +*.parquet filter=lfs diff=lfs merge=lfs -text +*.pb filter=lfs diff=lfs merge=lfs -text +*.pickle filter=lfs diff=lfs merge=lfs -text +*.pkl filter=lfs diff=lfs merge=lfs -text +*.pt filter=lfs diff=lfs merge=lfs -text +*.pth filter=lfs diff=lfs merge=lfs -text +*.rar filter=lfs diff=lfs merge=lfs -text +*.safetensors filter=lfs diff=lfs merge=lfs -text +saved_model/**/* filter=lfs diff=lfs merge=lfs -text +*.tar.* filter=lfs diff=lfs merge=lfs -text +*.tar filter=lfs diff=lfs merge=lfs -text +*.tflite filter=lfs diff=lfs merge=lfs -text +*.tgz filter=lfs diff=lfs merge=lfs -text +*.wasm filter=lfs diff=lfs merge=lfs -text +*.xz filter=lfs diff=lfs merge=lfs -text +*.zip filter=lfs diff=lfs merge=lfs -text +*.zst filter=lfs diff=lfs merge=lfs -text +*tfevents* filter=lfs diff=lfs merge=lfs -text +tokenizer.json filter=lfs diff=lfs merge=lfs -text diff --git a/posttrain_models/Video-o3_SFT_RL/README.md b/posttrain_models/Video-o3_SFT_RL/README.md new file mode 100644 index 0000000000000000000000000000000000000000..7be5fc7f47d5db027d120b8024982df93db95b74 --- /dev/null +++ b/posttrain_models/Video-o3_SFT_RL/README.md @@ -0,0 +1,3 @@ +--- +license: mit +--- diff --git a/posttrain_models/Video-o3_SFT_RL/added_tokens.json b/posttrain_models/Video-o3_SFT_RL/added_tokens.json new file mode 100644 index 0000000000000000000000000000000000000000..482ced4679301bf287ebb310bdd1790eb4514232 --- /dev/null +++ b/posttrain_models/Video-o3_SFT_RL/added_tokens.json @@ -0,0 +1,24 @@ +{ + "": 151658, + "": 151657, + "<|box_end|>": 151649, + "<|box_start|>": 151648, + "<|endoftext|>": 151643, + "<|file_sep|>": 151664, + "<|fim_middle|>": 151660, + "<|fim_pad|>": 151662, + "<|fim_prefix|>": 151659, + "<|fim_suffix|>": 151661, + "<|im_end|>": 151645, + "<|im_start|>": 151644, + "<|image_pad|>": 151655, + "<|object_ref_end|>": 151647, + "<|object_ref_start|>": 151646, + "<|quad_end|>": 151651, + "<|quad_start|>": 151650, + "<|repo_name|>": 151663, + "<|video_pad|>": 151656, + "<|vision_end|>": 151653, + "<|vision_pad|>": 151654, + "<|vision_start|>": 151652 +} diff --git a/posttrain_models/Video-o3_SFT_RL/chat_template.jinja b/posttrain_models/Video-o3_SFT_RL/chat_template.jinja new file mode 100644 index 0000000000000000000000000000000000000000..6c226632394ae7474b0d4b13e15793eac2e21ee9 --- /dev/null +++ b/posttrain_models/Video-o3_SFT_RL/chat_template.jinja @@ -0,0 +1,7 @@ +{% set image_count = namespace(value=0) %}{% set video_count = namespace(value=0) %}{% for message in messages %}{% if loop.first and message['role'] != 'system' %}<|im_start|>system +You are a helpful assistant.<|im_end|> +{% endif %}<|im_start|>{{ message['role'] }} +{% if message['content'] is string %}{{ message['content'] }}<|im_end|> +{% else %}{% for content in message['content'] %}{% if content['type'] == 'image' or 'image' in content or 'image_url' in content %}{% set image_count.value = image_count.value + 1 %}{% if add_vision_id %}Picture {{ image_count.value }}: {% endif %}<|vision_start|><|image_pad|><|vision_end|>{% elif content['type'] == 'video' or 'video' in content %}{% set video_count.value = video_count.value + 1 %}{% if add_vision_id %}Video {{ video_count.value }}: {% endif %}<|vision_start|><|video_pad|><|vision_end|>{% elif 'text' in content %}{{ content['text'] }}{% endif %}{% endfor %}<|im_end|> +{% endif %}{% endfor %}{% if add_generation_prompt %}<|im_start|>assistant +{% endif %} \ No newline at end of file diff --git a/posttrain_models/Video-o3_SFT_RL/config.json b/posttrain_models/Video-o3_SFT_RL/config.json new file mode 100644 index 0000000000000000000000000000000000000000..f1aea2804905d67e92aca54eb24f73e470562838 --- /dev/null +++ b/posttrain_models/Video-o3_SFT_RL/config.json @@ -0,0 +1,131 @@ +{ + "architectures": [ + "Qwen2_5_VLForConditionalGeneration" + ], + "attention_dropout": 0.0, + "dtype": "bfloat16", + "eos_token_id": 151645, + "hidden_act": "silu", + "hidden_size": 3584, + "initializer_range": 0.02, + "intermediate_size": 18944, + "max_position_embeddings": 128000, + "max_window_layers": 28, + "model_type": "qwen2_5_vl", + "num_attention_heads": 28, + "num_hidden_layers": 28, + "num_key_value_heads": 4, + "pad_token_id": 151643, + "rms_norm_eps": 1e-06, + "rope_scaling": { + "mrope_section": [ + 16, + 24, + 24 + ], + "rope_type": "default", + "type": "default" + }, + "rope_theta": 1000000.0, + "sliding_window": 32768, + "text_config": { + "architectures": [ + "Qwen2_5_VLForConditionalGeneration" + ], + "attention_dropout": 0.0, + "dtype": "bfloat16", + "eos_token_id": 151645, + "hidden_act": "silu", + "hidden_size": 3584, + "image_token_id": 151655, + "initializer_range": 0.02, + "intermediate_size": 18944, + "layer_types": [ + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention" + ], + "max_position_embeddings": 128000, + "max_window_layers": 28, + "model_type": "qwen2_5_vl_text", + "num_attention_heads": 28, + "num_hidden_layers": 28, + "num_key_value_heads": 4, + "pad_token_id": 151643, + "rms_norm_eps": 1e-06, + "rope_scaling": { + "mrope_section": [ + 16, + 24, + 24 + ], + "rope_type": "default", + "type": "default" + }, + "rope_theta": 1000000.0, + "sliding_window": null, + "use_cache": false, + "use_sliding_window": false, + "video_token_id": 151656, + "vision_token_id": 151654, + "vocab_size": 152064 + }, + "tie_word_embeddings": false, + "transformers_version": "4.57.6", + "use_cache": true, + "use_sliding_window": false, + "vision_config": { + "depth": 32, + "dtype": "bfloat16", + "fullatt_block_indexes": [ + 7, + 15, + 23, + 31 + ], + "hidden_act": "silu", + "hidden_size": 1280, + "in_channels": 3, + "in_chans": 3, + "initializer_range": 0.02, + "intermediate_size": 3420, + "model_type": "qwen2_5_vl", + "num_heads": 16, + "out_hidden_size": 3584, + "patch_size": 14, + "spatial_merge_size": 2, + "spatial_patch_size": 14, + "temporal_patch_size": 2, + "tokens_per_second": 2, + "window_size": 112 + }, + "vision_end_token_id": 151653, + "vision_start_token_id": 151652, + "vision_token_id": 151654, + "vocab_size": 152064 +} diff --git a/posttrain_models/Video-o3_SFT_RL/generation_config.json b/posttrain_models/Video-o3_SFT_RL/generation_config.json new file mode 100644 index 0000000000000000000000000000000000000000..41a103f4f84efd0f4dd67718f78b67cf73abc1ea --- /dev/null +++ b/posttrain_models/Video-o3_SFT_RL/generation_config.json @@ -0,0 +1,7 @@ +{ + "_from_model_config": true, + "eos_token_id": 151645, + "pad_token_id": 151643, + "transformers_version": "4.57.6", + "use_cache": false +} diff --git a/posttrain_models/Video-o3_SFT_RL/model.safetensors.index.json b/posttrain_models/Video-o3_SFT_RL/model.safetensors.index.json new file mode 100644 index 0000000000000000000000000000000000000000..c51c75b4d042bc1aaf5851be1de38b587868cfd2 --- /dev/null +++ b/posttrain_models/Video-o3_SFT_RL/model.safetensors.index.json @@ -0,0 +1,737 @@ +{ + "metadata": { + "total_parameters": 8292166656, + "total_size": 16584333312 + }, + "weight_map": { + "lm_head.weight": "model-00003-of-00004.safetensors", + "model.embed_tokens.weight": "model-00001-of-00004.safetensors", + "model.layers.0.input_layernorm.weight": "model-00004-of-00004.safetensors", + "model.layers.0.mlp.down_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.0.mlp.gate_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.0.mlp.up_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.0.post_attention_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.0.self_attn.k_proj.bias": "model-00001-of-00004.safetensors", + "model.layers.0.self_attn.k_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.0.self_attn.o_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.0.self_attn.q_proj.bias": "model-00001-of-00004.safetensors", + "model.layers.0.self_attn.q_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.0.self_attn.v_proj.bias": "model-00002-of-00004.safetensors", + "model.layers.0.self_attn.v_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.1.input_layernorm.weight": "model-00001-of-00004.safetensors", + "model.layers.1.mlp.down_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.1.mlp.gate_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.1.mlp.up_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.1.post_attention_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.1.self_attn.k_proj.bias": "model-00004-of-00004.safetensors", + "model.layers.1.self_attn.k_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.1.self_attn.o_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.1.self_attn.q_proj.bias": "model-00001-of-00004.safetensors", + "model.layers.1.self_attn.q_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.1.self_attn.v_proj.bias": "model-00003-of-00004.safetensors", + "model.layers.1.self_attn.v_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.10.input_layernorm.weight": "model-00001-of-00004.safetensors", + "model.layers.10.mlp.down_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.10.mlp.gate_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.10.mlp.up_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.10.post_attention_layernorm.weight": "model-00003-of-00004.safetensors", + "model.layers.10.self_attn.k_proj.bias": "model-00001-of-00004.safetensors", + "model.layers.10.self_attn.k_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.10.self_attn.o_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.10.self_attn.q_proj.bias": "model-00003-of-00004.safetensors", + "model.layers.10.self_attn.q_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.10.self_attn.v_proj.bias": "model-00002-of-00004.safetensors", + "model.layers.10.self_attn.v_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.11.input_layernorm.weight": "model-00001-of-00004.safetensors", + "model.layers.11.mlp.down_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.11.mlp.gate_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.11.mlp.up_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.11.post_attention_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.11.self_attn.k_proj.bias": "model-00001-of-00004.safetensors", + "model.layers.11.self_attn.k_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.11.self_attn.o_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.11.self_attn.q_proj.bias": "model-00002-of-00004.safetensors", + "model.layers.11.self_attn.q_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.11.self_attn.v_proj.bias": "model-00001-of-00004.safetensors", + "model.layers.11.self_attn.v_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.12.input_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.12.mlp.down_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.12.mlp.gate_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.12.mlp.up_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.12.post_attention_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.12.self_attn.k_proj.bias": "model-00003-of-00004.safetensors", + "model.layers.12.self_attn.k_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.12.self_attn.o_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.12.self_attn.q_proj.bias": "model-00003-of-00004.safetensors", + "model.layers.12.self_attn.q_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.12.self_attn.v_proj.bias": "model-00001-of-00004.safetensors", + "model.layers.12.self_attn.v_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.13.input_layernorm.weight": "model-00004-of-00004.safetensors", + "model.layers.13.mlp.down_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.13.mlp.gate_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.13.mlp.up_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.13.post_attention_layernorm.weight": "model-00003-of-00004.safetensors", + "model.layers.13.self_attn.k_proj.bias": "model-00003-of-00004.safetensors", + "model.layers.13.self_attn.k_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.13.self_attn.o_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.13.self_attn.q_proj.bias": "model-00002-of-00004.safetensors", + "model.layers.13.self_attn.q_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.13.self_attn.v_proj.bias": "model-00001-of-00004.safetensors", + "model.layers.13.self_attn.v_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.14.input_layernorm.weight": "model-00001-of-00004.safetensors", + "model.layers.14.mlp.down_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.14.mlp.gate_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.14.mlp.up_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.14.post_attention_layernorm.weight": "model-00003-of-00004.safetensors", + "model.layers.14.self_attn.k_proj.bias": "model-00001-of-00004.safetensors", + "model.layers.14.self_attn.k_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.14.self_attn.o_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.14.self_attn.q_proj.bias": "model-00002-of-00004.safetensors", + "model.layers.14.self_attn.q_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.14.self_attn.v_proj.bias": "model-00001-of-00004.safetensors", + "model.layers.14.self_attn.v_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.15.input_layernorm.weight": "model-00003-of-00004.safetensors", + "model.layers.15.mlp.down_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.15.mlp.gate_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.15.mlp.up_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.15.post_attention_layernorm.weight": "model-00003-of-00004.safetensors", + "model.layers.15.self_attn.k_proj.bias": "model-00004-of-00004.safetensors", + "model.layers.15.self_attn.k_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.15.self_attn.o_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.15.self_attn.q_proj.bias": "model-00002-of-00004.safetensors", + "model.layers.15.self_attn.q_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.15.self_attn.v_proj.bias": "model-00003-of-00004.safetensors", + "model.layers.15.self_attn.v_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.16.input_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.16.mlp.down_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.16.mlp.gate_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.16.mlp.up_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.16.post_attention_layernorm.weight": "model-00001-of-00004.safetensors", + "model.layers.16.self_attn.k_proj.bias": "model-00001-of-00004.safetensors", + "model.layers.16.self_attn.k_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.16.self_attn.o_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.16.self_attn.q_proj.bias": "model-00002-of-00004.safetensors", + "model.layers.16.self_attn.q_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.16.self_attn.v_proj.bias": "model-00003-of-00004.safetensors", + "model.layers.16.self_attn.v_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.17.input_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.17.mlp.down_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.17.mlp.gate_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.17.mlp.up_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.17.post_attention_layernorm.weight": "model-00001-of-00004.safetensors", + "model.layers.17.self_attn.k_proj.bias": "model-00003-of-00004.safetensors", + "model.layers.17.self_attn.k_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.17.self_attn.o_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.17.self_attn.q_proj.bias": "model-00002-of-00004.safetensors", + "model.layers.17.self_attn.q_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.17.self_attn.v_proj.bias": "model-00002-of-00004.safetensors", + "model.layers.17.self_attn.v_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.18.input_layernorm.weight": "model-00001-of-00004.safetensors", + "model.layers.18.mlp.down_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.18.mlp.gate_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.18.mlp.up_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.18.post_attention_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.18.self_attn.k_proj.bias": "model-00003-of-00004.safetensors", + "model.layers.18.self_attn.k_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.18.self_attn.o_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.18.self_attn.q_proj.bias": "model-00001-of-00004.safetensors", + "model.layers.18.self_attn.q_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.18.self_attn.v_proj.bias": "model-00002-of-00004.safetensors", + "model.layers.18.self_attn.v_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.19.input_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.19.mlp.down_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.19.mlp.gate_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.19.mlp.up_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.19.post_attention_layernorm.weight": "model-00003-of-00004.safetensors", + "model.layers.19.self_attn.k_proj.bias": "model-00003-of-00004.safetensors", + "model.layers.19.self_attn.k_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.19.self_attn.o_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.19.self_attn.q_proj.bias": "model-00002-of-00004.safetensors", + "model.layers.19.self_attn.q_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.19.self_attn.v_proj.bias": "model-00002-of-00004.safetensors", + "model.layers.19.self_attn.v_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.2.input_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.2.mlp.down_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.2.mlp.gate_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.2.mlp.up_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.2.post_attention_layernorm.weight": "model-00003-of-00004.safetensors", + "model.layers.2.self_attn.k_proj.bias": "model-00002-of-00004.safetensors", + "model.layers.2.self_attn.k_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.2.self_attn.o_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.2.self_attn.q_proj.bias": "model-00004-of-00004.safetensors", + "model.layers.2.self_attn.q_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.2.self_attn.v_proj.bias": "model-00001-of-00004.safetensors", + "model.layers.2.self_attn.v_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.20.input_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.20.mlp.down_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.20.mlp.gate_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.20.mlp.up_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.20.post_attention_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.20.self_attn.k_proj.bias": "model-00002-of-00004.safetensors", + "model.layers.20.self_attn.k_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.20.self_attn.o_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.20.self_attn.q_proj.bias": "model-00002-of-00004.safetensors", + "model.layers.20.self_attn.q_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.20.self_attn.v_proj.bias": "model-00004-of-00004.safetensors", + "model.layers.20.self_attn.v_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.21.input_layernorm.weight": "model-00001-of-00004.safetensors", + "model.layers.21.mlp.down_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.21.mlp.gate_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.21.mlp.up_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.21.post_attention_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.21.self_attn.k_proj.bias": "model-00004-of-00004.safetensors", + "model.layers.21.self_attn.k_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.21.self_attn.o_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.21.self_attn.q_proj.bias": "model-00002-of-00004.safetensors", + "model.layers.21.self_attn.q_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.21.self_attn.v_proj.bias": "model-00002-of-00004.safetensors", + "model.layers.21.self_attn.v_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.22.input_layernorm.weight": "model-00001-of-00004.safetensors", + "model.layers.22.mlp.down_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.22.mlp.gate_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.22.mlp.up_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.22.post_attention_layernorm.weight": "model-00003-of-00004.safetensors", + "model.layers.22.self_attn.k_proj.bias": "model-00001-of-00004.safetensors", + "model.layers.22.self_attn.k_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.22.self_attn.o_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.22.self_attn.q_proj.bias": "model-00002-of-00004.safetensors", + "model.layers.22.self_attn.q_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.22.self_attn.v_proj.bias": "model-00002-of-00004.safetensors", + "model.layers.22.self_attn.v_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.23.input_layernorm.weight": "model-00003-of-00004.safetensors", + "model.layers.23.mlp.down_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.23.mlp.gate_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.23.mlp.up_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.23.post_attention_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.23.self_attn.k_proj.bias": "model-00002-of-00004.safetensors", + "model.layers.23.self_attn.k_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.23.self_attn.o_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.23.self_attn.q_proj.bias": "model-00001-of-00004.safetensors", + "model.layers.23.self_attn.q_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.23.self_attn.v_proj.bias": "model-00003-of-00004.safetensors", + "model.layers.23.self_attn.v_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.24.input_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.24.mlp.down_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.24.mlp.gate_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.24.mlp.up_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.24.post_attention_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.24.self_attn.k_proj.bias": "model-00004-of-00004.safetensors", + "model.layers.24.self_attn.k_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.24.self_attn.o_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.24.self_attn.q_proj.bias": "model-00001-of-00004.safetensors", + "model.layers.24.self_attn.q_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.24.self_attn.v_proj.bias": "model-00003-of-00004.safetensors", + "model.layers.24.self_attn.v_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.25.input_layernorm.weight": "model-00004-of-00004.safetensors", + "model.layers.25.mlp.down_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.25.mlp.gate_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.25.mlp.up_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.25.post_attention_layernorm.weight": "model-00001-of-00004.safetensors", + "model.layers.25.self_attn.k_proj.bias": "model-00003-of-00004.safetensors", + "model.layers.25.self_attn.k_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.25.self_attn.o_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.25.self_attn.q_proj.bias": "model-00003-of-00004.safetensors", + "model.layers.25.self_attn.q_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.25.self_attn.v_proj.bias": "model-00002-of-00004.safetensors", + "model.layers.25.self_attn.v_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.26.input_layernorm.weight": "model-00003-of-00004.safetensors", + "model.layers.26.mlp.down_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.26.mlp.gate_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.26.mlp.up_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.26.post_attention_layernorm.weight": "model-00001-of-00004.safetensors", + "model.layers.26.self_attn.k_proj.bias": "model-00003-of-00004.safetensors", + "model.layers.26.self_attn.k_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.26.self_attn.o_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.26.self_attn.q_proj.bias": "model-00003-of-00004.safetensors", + "model.layers.26.self_attn.q_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.26.self_attn.v_proj.bias": "model-00002-of-00004.safetensors", + "model.layers.26.self_attn.v_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.27.input_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.27.mlp.down_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.27.mlp.gate_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.27.mlp.up_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.27.post_attention_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.27.self_attn.k_proj.bias": "model-00004-of-00004.safetensors", + "model.layers.27.self_attn.k_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.27.self_attn.o_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.27.self_attn.q_proj.bias": "model-00002-of-00004.safetensors", + "model.layers.27.self_attn.q_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.27.self_attn.v_proj.bias": "model-00004-of-00004.safetensors", + "model.layers.27.self_attn.v_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.3.input_layernorm.weight": "model-00003-of-00004.safetensors", + "model.layers.3.mlp.down_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.3.mlp.gate_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.3.mlp.up_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.3.post_attention_layernorm.weight": "model-00001-of-00004.safetensors", + "model.layers.3.self_attn.k_proj.bias": "model-00004-of-00004.safetensors", + "model.layers.3.self_attn.k_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.3.self_attn.o_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.3.self_attn.q_proj.bias": "model-00003-of-00004.safetensors", + "model.layers.3.self_attn.q_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.3.self_attn.v_proj.bias": "model-00002-of-00004.safetensors", + "model.layers.3.self_attn.v_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.4.input_layernorm.weight": "model-00001-of-00004.safetensors", + "model.layers.4.mlp.down_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.4.mlp.gate_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.4.mlp.up_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.4.post_attention_layernorm.weight": "model-00001-of-00004.safetensors", + "model.layers.4.self_attn.k_proj.bias": "model-00002-of-00004.safetensors", + "model.layers.4.self_attn.k_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.4.self_attn.o_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.4.self_attn.q_proj.bias": "model-00001-of-00004.safetensors", + "model.layers.4.self_attn.q_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.4.self_attn.v_proj.bias": "model-00003-of-00004.safetensors", + "model.layers.4.self_attn.v_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.5.input_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.5.mlp.down_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.5.mlp.gate_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.5.mlp.up_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.5.post_attention_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.5.self_attn.k_proj.bias": "model-00002-of-00004.safetensors", + "model.layers.5.self_attn.k_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.5.self_attn.o_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.5.self_attn.q_proj.bias": "model-00001-of-00004.safetensors", + "model.layers.5.self_attn.q_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.5.self_attn.v_proj.bias": "model-00002-of-00004.safetensors", + "model.layers.5.self_attn.v_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.6.input_layernorm.weight": "model-00004-of-00004.safetensors", + "model.layers.6.mlp.down_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.6.mlp.gate_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.6.mlp.up_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.6.post_attention_layernorm.weight": "model-00003-of-00004.safetensors", + "model.layers.6.self_attn.k_proj.bias": "model-00001-of-00004.safetensors", + "model.layers.6.self_attn.k_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.6.self_attn.o_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.6.self_attn.q_proj.bias": "model-00002-of-00004.safetensors", + "model.layers.6.self_attn.q_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.6.self_attn.v_proj.bias": "model-00002-of-00004.safetensors", + "model.layers.6.self_attn.v_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.7.input_layernorm.weight": "model-00001-of-00004.safetensors", + "model.layers.7.mlp.down_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.7.mlp.gate_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.7.mlp.up_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.7.post_attention_layernorm.weight": "model-00001-of-00004.safetensors", + "model.layers.7.self_attn.k_proj.bias": "model-00002-of-00004.safetensors", + "model.layers.7.self_attn.k_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.7.self_attn.o_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.7.self_attn.q_proj.bias": "model-00004-of-00004.safetensors", + "model.layers.7.self_attn.q_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.7.self_attn.v_proj.bias": "model-00004-of-00004.safetensors", + "model.layers.7.self_attn.v_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.8.input_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.8.mlp.down_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.8.mlp.gate_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.8.mlp.up_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.8.post_attention_layernorm.weight": "model-00003-of-00004.safetensors", + "model.layers.8.self_attn.k_proj.bias": "model-00003-of-00004.safetensors", + "model.layers.8.self_attn.k_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.8.self_attn.o_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.8.self_attn.q_proj.bias": "model-00001-of-00004.safetensors", + "model.layers.8.self_attn.q_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.8.self_attn.v_proj.bias": "model-00004-of-00004.safetensors", + "model.layers.8.self_attn.v_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.9.input_layernorm.weight": "model-00003-of-00004.safetensors", + "model.layers.9.mlp.down_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.9.mlp.gate_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.9.mlp.up_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.9.post_attention_layernorm.weight": "model-00001-of-00004.safetensors", + "model.layers.9.self_attn.k_proj.bias": "model-00001-of-00004.safetensors", + "model.layers.9.self_attn.k_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.9.self_attn.o_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.9.self_attn.q_proj.bias": "model-00002-of-00004.safetensors", + "model.layers.9.self_attn.q_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.9.self_attn.v_proj.bias": "model-00002-of-00004.safetensors", + "model.layers.9.self_attn.v_proj.weight": "model-00001-of-00004.safetensors", + "model.norm.weight": "model-00003-of-00004.safetensors", + "visual.blocks.0.attn.proj.bias": "model-00003-of-00004.safetensors", + "visual.blocks.0.attn.proj.weight": "model-00001-of-00004.safetensors", + "visual.blocks.0.attn.qkv.bias": "model-00002-of-00004.safetensors", + "visual.blocks.0.attn.qkv.weight": "model-00002-of-00004.safetensors", + "visual.blocks.0.mlp.down_proj.bias": "model-00003-of-00004.safetensors", + "visual.blocks.0.mlp.down_proj.weight": "model-00001-of-00004.safetensors", + "visual.blocks.0.mlp.gate_proj.bias": "model-00001-of-00004.safetensors", + "visual.blocks.0.mlp.gate_proj.weight": "model-00001-of-00004.safetensors", + "visual.blocks.0.mlp.up_proj.bias": "model-00003-of-00004.safetensors", + "visual.blocks.0.mlp.up_proj.weight": "model-00004-of-00004.safetensors", + "visual.blocks.0.norm1.weight": "model-00004-of-00004.safetensors", + "visual.blocks.0.norm2.weight": "model-00001-of-00004.safetensors", + "visual.blocks.1.attn.proj.bias": "model-00002-of-00004.safetensors", + "visual.blocks.1.attn.proj.weight": "model-00001-of-00004.safetensors", + "visual.blocks.1.attn.qkv.bias": "model-00002-of-00004.safetensors", + "visual.blocks.1.attn.qkv.weight": "model-00001-of-00004.safetensors", + "visual.blocks.1.mlp.down_proj.bias": "model-00002-of-00004.safetensors", + "visual.blocks.1.mlp.down_proj.weight": "model-00002-of-00004.safetensors", + "visual.blocks.1.mlp.gate_proj.bias": "model-00002-of-00004.safetensors", + "visual.blocks.1.mlp.gate_proj.weight": "model-00004-of-00004.safetensors", + "visual.blocks.1.mlp.up_proj.bias": "model-00004-of-00004.safetensors", + "visual.blocks.1.mlp.up_proj.weight": "model-00002-of-00004.safetensors", + "visual.blocks.1.norm1.weight": "model-00003-of-00004.safetensors", + "visual.blocks.1.norm2.weight": "model-00003-of-00004.safetensors", + "visual.blocks.10.attn.proj.bias": "model-00002-of-00004.safetensors", + "visual.blocks.10.attn.proj.weight": "model-00002-of-00004.safetensors", + "visual.blocks.10.attn.qkv.bias": "model-00002-of-00004.safetensors", + "visual.blocks.10.attn.qkv.weight": "model-00001-of-00004.safetensors", + "visual.blocks.10.mlp.down_proj.bias": "model-00004-of-00004.safetensors", + "visual.blocks.10.mlp.down_proj.weight": "model-00003-of-00004.safetensors", + "visual.blocks.10.mlp.gate_proj.bias": "model-00001-of-00004.safetensors", + "visual.blocks.10.mlp.gate_proj.weight": "model-00001-of-00004.safetensors", + "visual.blocks.10.mlp.up_proj.bias": "model-00003-of-00004.safetensors", + "visual.blocks.10.mlp.up_proj.weight": "model-00004-of-00004.safetensors", + "visual.blocks.10.norm1.weight": "model-00003-of-00004.safetensors", + "visual.blocks.10.norm2.weight": "model-00001-of-00004.safetensors", + "visual.blocks.11.attn.proj.bias": "model-00004-of-00004.safetensors", + "visual.blocks.11.attn.proj.weight": "model-00003-of-00004.safetensors", + "visual.blocks.11.attn.qkv.bias": "model-00001-of-00004.safetensors", + "visual.blocks.11.attn.qkv.weight": "model-00002-of-00004.safetensors", + "visual.blocks.11.mlp.down_proj.bias": "model-00003-of-00004.safetensors", + "visual.blocks.11.mlp.down_proj.weight": "model-00002-of-00004.safetensors", + "visual.blocks.11.mlp.gate_proj.bias": "model-00002-of-00004.safetensors", + "visual.blocks.11.mlp.gate_proj.weight": "model-00001-of-00004.safetensors", + "visual.blocks.11.mlp.up_proj.bias": "model-00002-of-00004.safetensors", + "visual.blocks.11.mlp.up_proj.weight": "model-00003-of-00004.safetensors", + "visual.blocks.11.norm1.weight": "model-00001-of-00004.safetensors", + "visual.blocks.11.norm2.weight": "model-00003-of-00004.safetensors", + "visual.blocks.12.attn.proj.bias": "model-00001-of-00004.safetensors", + "visual.blocks.12.attn.proj.weight": "model-00004-of-00004.safetensors", + "visual.blocks.12.attn.qkv.bias": "model-00003-of-00004.safetensors", + "visual.blocks.12.attn.qkv.weight": "model-00002-of-00004.safetensors", + "visual.blocks.12.mlp.down_proj.bias": "model-00001-of-00004.safetensors", + "visual.blocks.12.mlp.down_proj.weight": "model-00002-of-00004.safetensors", + "visual.blocks.12.mlp.gate_proj.bias": "model-00003-of-00004.safetensors", + "visual.blocks.12.mlp.gate_proj.weight": "model-00003-of-00004.safetensors", + "visual.blocks.12.mlp.up_proj.bias": "model-00001-of-00004.safetensors", + "visual.blocks.12.mlp.up_proj.weight": "model-00002-of-00004.safetensors", + "visual.blocks.12.norm1.weight": "model-00001-of-00004.safetensors", + "visual.blocks.12.norm2.weight": "model-00003-of-00004.safetensors", + "visual.blocks.13.attn.proj.bias": "model-00001-of-00004.safetensors", + "visual.blocks.13.attn.proj.weight": "model-00002-of-00004.safetensors", + "visual.blocks.13.attn.qkv.bias": "model-00001-of-00004.safetensors", + "visual.blocks.13.attn.qkv.weight": "model-00003-of-00004.safetensors", + "visual.blocks.13.mlp.down_proj.bias": "model-00001-of-00004.safetensors", + "visual.blocks.13.mlp.down_proj.weight": "model-00004-of-00004.safetensors", + "visual.blocks.13.mlp.gate_proj.bias": "model-00003-of-00004.safetensors", + "visual.blocks.13.mlp.gate_proj.weight": "model-00002-of-00004.safetensors", + "visual.blocks.13.mlp.up_proj.bias": "model-00002-of-00004.safetensors", + "visual.blocks.13.mlp.up_proj.weight": "model-00003-of-00004.safetensors", + "visual.blocks.13.norm1.weight": "model-00001-of-00004.safetensors", + "visual.blocks.13.norm2.weight": "model-00001-of-00004.safetensors", + "visual.blocks.14.attn.proj.bias": "model-00003-of-00004.safetensors", + "visual.blocks.14.attn.proj.weight": "model-00002-of-00004.safetensors", + "visual.blocks.14.attn.qkv.bias": "model-00001-of-00004.safetensors", + "visual.blocks.14.attn.qkv.weight": "model-00003-of-00004.safetensors", + "visual.blocks.14.mlp.down_proj.bias": "model-00004-of-00004.safetensors", + "visual.blocks.14.mlp.down_proj.weight": "model-00004-of-00004.safetensors", + "visual.blocks.14.mlp.gate_proj.bias": "model-00002-of-00004.safetensors", + "visual.blocks.14.mlp.gate_proj.weight": "model-00003-of-00004.safetensors", + "visual.blocks.14.mlp.up_proj.bias": "model-00001-of-00004.safetensors", + "visual.blocks.14.mlp.up_proj.weight": "model-00003-of-00004.safetensors", + "visual.blocks.14.norm1.weight": "model-00004-of-00004.safetensors", + "visual.blocks.14.norm2.weight": "model-00002-of-00004.safetensors", + "visual.blocks.15.attn.proj.bias": "model-00002-of-00004.safetensors", + "visual.blocks.15.attn.proj.weight": "model-00002-of-00004.safetensors", + "visual.blocks.15.attn.qkv.bias": "model-00002-of-00004.safetensors", + "visual.blocks.15.attn.qkv.weight": "model-00003-of-00004.safetensors", + "visual.blocks.15.mlp.down_proj.bias": "model-00002-of-00004.safetensors", + "visual.blocks.15.mlp.down_proj.weight": "model-00003-of-00004.safetensors", + "visual.blocks.15.mlp.gate_proj.bias": "model-00002-of-00004.safetensors", + "visual.blocks.15.mlp.gate_proj.weight": "model-00004-of-00004.safetensors", + "visual.blocks.15.mlp.up_proj.bias": "model-00001-of-00004.safetensors", + "visual.blocks.15.mlp.up_proj.weight": "model-00002-of-00004.safetensors", + "visual.blocks.15.norm1.weight": "model-00002-of-00004.safetensors", + "visual.blocks.15.norm2.weight": "model-00003-of-00004.safetensors", + "visual.blocks.16.attn.proj.bias": "model-00004-of-00004.safetensors", + "visual.blocks.16.attn.proj.weight": "model-00004-of-00004.safetensors", + "visual.blocks.16.attn.qkv.bias": "model-00004-of-00004.safetensors", + "visual.blocks.16.attn.qkv.weight": "model-00002-of-00004.safetensors", + "visual.blocks.16.mlp.down_proj.bias": "model-00004-of-00004.safetensors", + "visual.blocks.16.mlp.down_proj.weight": "model-00003-of-00004.safetensors", + "visual.blocks.16.mlp.gate_proj.bias": "model-00001-of-00004.safetensors", + "visual.blocks.16.mlp.gate_proj.weight": "model-00001-of-00004.safetensors", + "visual.blocks.16.mlp.up_proj.bias": "model-00002-of-00004.safetensors", + "visual.blocks.16.mlp.up_proj.weight": "model-00003-of-00004.safetensors", + "visual.blocks.16.norm1.weight": "model-00003-of-00004.safetensors", + "visual.blocks.16.norm2.weight": "model-00002-of-00004.safetensors", + "visual.blocks.17.attn.proj.bias": "model-00002-of-00004.safetensors", + "visual.blocks.17.attn.proj.weight": "model-00001-of-00004.safetensors", + "visual.blocks.17.attn.qkv.bias": "model-00002-of-00004.safetensors", + "visual.blocks.17.attn.qkv.weight": "model-00003-of-00004.safetensors", + "visual.blocks.17.mlp.down_proj.bias": "model-00002-of-00004.safetensors", + "visual.blocks.17.mlp.down_proj.weight": "model-00002-of-00004.safetensors", + "visual.blocks.17.mlp.gate_proj.bias": "model-00002-of-00004.safetensors", + "visual.blocks.17.mlp.gate_proj.weight": "model-00002-of-00004.safetensors", + "visual.blocks.17.mlp.up_proj.bias": "model-00002-of-00004.safetensors", + "visual.blocks.17.mlp.up_proj.weight": "model-00003-of-00004.safetensors", + "visual.blocks.17.norm1.weight": "model-00004-of-00004.safetensors", + "visual.blocks.17.norm2.weight": "model-00001-of-00004.safetensors", + "visual.blocks.18.attn.proj.bias": "model-00001-of-00004.safetensors", + "visual.blocks.18.attn.proj.weight": "model-00003-of-00004.safetensors", + "visual.blocks.18.attn.qkv.bias": "model-00003-of-00004.safetensors", + "visual.blocks.18.attn.qkv.weight": "model-00003-of-00004.safetensors", + "visual.blocks.18.mlp.down_proj.bias": "model-00002-of-00004.safetensors", + "visual.blocks.18.mlp.down_proj.weight": "model-00002-of-00004.safetensors", + "visual.blocks.18.mlp.gate_proj.bias": "model-00001-of-00004.safetensors", + "visual.blocks.18.mlp.gate_proj.weight": "model-00004-of-00004.safetensors", + "visual.blocks.18.mlp.up_proj.bias": "model-00003-of-00004.safetensors", + "visual.blocks.18.mlp.up_proj.weight": "model-00001-of-00004.safetensors", + "visual.blocks.18.norm1.weight": "model-00001-of-00004.safetensors", + "visual.blocks.18.norm2.weight": "model-00002-of-00004.safetensors", + "visual.blocks.19.attn.proj.bias": "model-00004-of-00004.safetensors", + "visual.blocks.19.attn.proj.weight": "model-00001-of-00004.safetensors", + "visual.blocks.19.attn.qkv.bias": "model-00002-of-00004.safetensors", + "visual.blocks.19.attn.qkv.weight": "model-00004-of-00004.safetensors", + "visual.blocks.19.mlp.down_proj.bias": "model-00003-of-00004.safetensors", + "visual.blocks.19.mlp.down_proj.weight": "model-00002-of-00004.safetensors", + "visual.blocks.19.mlp.gate_proj.bias": "model-00004-of-00004.safetensors", + "visual.blocks.19.mlp.gate_proj.weight": "model-00003-of-00004.safetensors", + "visual.blocks.19.mlp.up_proj.bias": "model-00001-of-00004.safetensors", + "visual.blocks.19.mlp.up_proj.weight": "model-00004-of-00004.safetensors", + "visual.blocks.19.norm1.weight": "model-00002-of-00004.safetensors", + "visual.blocks.19.norm2.weight": "model-00003-of-00004.safetensors", + "visual.blocks.2.attn.proj.bias": "model-00001-of-00004.safetensors", + "visual.blocks.2.attn.proj.weight": "model-00003-of-00004.safetensors", + "visual.blocks.2.attn.qkv.bias": "model-00004-of-00004.safetensors", + "visual.blocks.2.attn.qkv.weight": "model-00001-of-00004.safetensors", + "visual.blocks.2.mlp.down_proj.bias": "model-00001-of-00004.safetensors", + "visual.blocks.2.mlp.down_proj.weight": "model-00002-of-00004.safetensors", + "visual.blocks.2.mlp.gate_proj.bias": "model-00003-of-00004.safetensors", + "visual.blocks.2.mlp.gate_proj.weight": "model-00001-of-00004.safetensors", + "visual.blocks.2.mlp.up_proj.bias": "model-00004-of-00004.safetensors", + "visual.blocks.2.mlp.up_proj.weight": "model-00002-of-00004.safetensors", + "visual.blocks.2.norm1.weight": "model-00003-of-00004.safetensors", + "visual.blocks.2.norm2.weight": "model-00001-of-00004.safetensors", + "visual.blocks.20.attn.proj.bias": "model-00001-of-00004.safetensors", + "visual.blocks.20.attn.proj.weight": "model-00002-of-00004.safetensors", + "visual.blocks.20.attn.qkv.bias": "model-00004-of-00004.safetensors", + "visual.blocks.20.attn.qkv.weight": "model-00001-of-00004.safetensors", + "visual.blocks.20.mlp.down_proj.bias": "model-00001-of-00004.safetensors", + "visual.blocks.20.mlp.down_proj.weight": "model-00004-of-00004.safetensors", + "visual.blocks.20.mlp.gate_proj.bias": "model-00003-of-00004.safetensors", + "visual.blocks.20.mlp.gate_proj.weight": "model-00001-of-00004.safetensors", + "visual.blocks.20.mlp.up_proj.bias": "model-00003-of-00004.safetensors", + "visual.blocks.20.mlp.up_proj.weight": "model-00001-of-00004.safetensors", + "visual.blocks.20.norm1.weight": "model-00002-of-00004.safetensors", + "visual.blocks.20.norm2.weight": "model-00003-of-00004.safetensors", + "visual.blocks.21.attn.proj.bias": "model-00001-of-00004.safetensors", + "visual.blocks.21.attn.proj.weight": "model-00003-of-00004.safetensors", + "visual.blocks.21.attn.qkv.bias": "model-00003-of-00004.safetensors", + "visual.blocks.21.attn.qkv.weight": "model-00001-of-00004.safetensors", + "visual.blocks.21.mlp.down_proj.bias": "model-00004-of-00004.safetensors", + "visual.blocks.21.mlp.down_proj.weight": "model-00001-of-00004.safetensors", + "visual.blocks.21.mlp.gate_proj.bias": "model-00001-of-00004.safetensors", + "visual.blocks.21.mlp.gate_proj.weight": "model-00003-of-00004.safetensors", + "visual.blocks.21.mlp.up_proj.bias": "model-00002-of-00004.safetensors", + "visual.blocks.21.mlp.up_proj.weight": "model-00003-of-00004.safetensors", + "visual.blocks.21.norm1.weight": "model-00004-of-00004.safetensors", + "visual.blocks.21.norm2.weight": "model-00002-of-00004.safetensors", + "visual.blocks.22.attn.proj.bias": "model-00004-of-00004.safetensors", + "visual.blocks.22.attn.proj.weight": "model-00001-of-00004.safetensors", + "visual.blocks.22.attn.qkv.bias": "model-00002-of-00004.safetensors", + "visual.blocks.22.attn.qkv.weight": "model-00003-of-00004.safetensors", + "visual.blocks.22.mlp.down_proj.bias": "model-00001-of-00004.safetensors", + "visual.blocks.22.mlp.down_proj.weight": "model-00002-of-00004.safetensors", + "visual.blocks.22.mlp.gate_proj.bias": "model-00001-of-00004.safetensors", + "visual.blocks.22.mlp.gate_proj.weight": "model-00001-of-00004.safetensors", + "visual.blocks.22.mlp.up_proj.bias": "model-00003-of-00004.safetensors", + "visual.blocks.22.mlp.up_proj.weight": "model-00003-of-00004.safetensors", + "visual.blocks.22.norm1.weight": "model-00002-of-00004.safetensors", + "visual.blocks.22.norm2.weight": "model-00002-of-00004.safetensors", + "visual.blocks.23.attn.proj.bias": "model-00001-of-00004.safetensors", + "visual.blocks.23.attn.proj.weight": "model-00004-of-00004.safetensors", + "visual.blocks.23.attn.qkv.bias": "model-00004-of-00004.safetensors", + "visual.blocks.23.attn.qkv.weight": "model-00003-of-00004.safetensors", + "visual.blocks.23.mlp.down_proj.bias": "model-00002-of-00004.safetensors", + "visual.blocks.23.mlp.down_proj.weight": "model-00001-of-00004.safetensors", + "visual.blocks.23.mlp.gate_proj.bias": "model-00001-of-00004.safetensors", + "visual.blocks.23.mlp.gate_proj.weight": "model-00002-of-00004.safetensors", + "visual.blocks.23.mlp.up_proj.bias": "model-00003-of-00004.safetensors", + "visual.blocks.23.mlp.up_proj.weight": "model-00002-of-00004.safetensors", + "visual.blocks.23.norm1.weight": "model-00001-of-00004.safetensors", + "visual.blocks.23.norm2.weight": "model-00001-of-00004.safetensors", + "visual.blocks.24.attn.proj.bias": "model-00001-of-00004.safetensors", + "visual.blocks.24.attn.proj.weight": "model-00003-of-00004.safetensors", + "visual.blocks.24.attn.qkv.bias": "model-00002-of-00004.safetensors", + "visual.blocks.24.attn.qkv.weight": "model-00002-of-00004.safetensors", + "visual.blocks.24.mlp.down_proj.bias": "model-00002-of-00004.safetensors", + "visual.blocks.24.mlp.down_proj.weight": "model-00001-of-00004.safetensors", + "visual.blocks.24.mlp.gate_proj.bias": "model-00001-of-00004.safetensors", + "visual.blocks.24.mlp.gate_proj.weight": "model-00003-of-00004.safetensors", + "visual.blocks.24.mlp.up_proj.bias": "model-00003-of-00004.safetensors", + "visual.blocks.24.mlp.up_proj.weight": "model-00002-of-00004.safetensors", + "visual.blocks.24.norm1.weight": "model-00003-of-00004.safetensors", + "visual.blocks.24.norm2.weight": "model-00004-of-00004.safetensors", + "visual.blocks.25.attn.proj.bias": "model-00004-of-00004.safetensors", + "visual.blocks.25.attn.proj.weight": "model-00002-of-00004.safetensors", + "visual.blocks.25.attn.qkv.bias": "model-00001-of-00004.safetensors", + "visual.blocks.25.attn.qkv.weight": "model-00003-of-00004.safetensors", + "visual.blocks.25.mlp.down_proj.bias": "model-00002-of-00004.safetensors", + "visual.blocks.25.mlp.down_proj.weight": "model-00001-of-00004.safetensors", + "visual.blocks.25.mlp.gate_proj.bias": "model-00002-of-00004.safetensors", + "visual.blocks.25.mlp.gate_proj.weight": "model-00002-of-00004.safetensors", + "visual.blocks.25.mlp.up_proj.bias": "model-00004-of-00004.safetensors", + "visual.blocks.25.mlp.up_proj.weight": "model-00001-of-00004.safetensors", + "visual.blocks.25.norm1.weight": "model-00001-of-00004.safetensors", + "visual.blocks.25.norm2.weight": "model-00001-of-00004.safetensors", + "visual.blocks.26.attn.proj.bias": "model-00004-of-00004.safetensors", + "visual.blocks.26.attn.proj.weight": "model-00003-of-00004.safetensors", + "visual.blocks.26.attn.qkv.bias": "model-00004-of-00004.safetensors", + "visual.blocks.26.attn.qkv.weight": "model-00001-of-00004.safetensors", + "visual.blocks.26.mlp.down_proj.bias": "model-00001-of-00004.safetensors", + "visual.blocks.26.mlp.down_proj.weight": "model-00001-of-00004.safetensors", + "visual.blocks.26.mlp.gate_proj.bias": "model-00004-of-00004.safetensors", + "visual.blocks.26.mlp.gate_proj.weight": "model-00004-of-00004.safetensors", + "visual.blocks.26.mlp.up_proj.bias": "model-00001-of-00004.safetensors", + "visual.blocks.26.mlp.up_proj.weight": "model-00003-of-00004.safetensors", + "visual.blocks.26.norm1.weight": "model-00002-of-00004.safetensors", + "visual.blocks.26.norm2.weight": "model-00001-of-00004.safetensors", + "visual.blocks.27.attn.proj.bias": "model-00002-of-00004.safetensors", + "visual.blocks.27.attn.proj.weight": "model-00003-of-00004.safetensors", + "visual.blocks.27.attn.qkv.bias": "model-00002-of-00004.safetensors", + "visual.blocks.27.attn.qkv.weight": "model-00001-of-00004.safetensors", + "visual.blocks.27.mlp.down_proj.bias": "model-00002-of-00004.safetensors", + "visual.blocks.27.mlp.down_proj.weight": "model-00001-of-00004.safetensors", + "visual.blocks.27.mlp.gate_proj.bias": "model-00004-of-00004.safetensors", + "visual.blocks.27.mlp.gate_proj.weight": "model-00002-of-00004.safetensors", + "visual.blocks.27.mlp.up_proj.bias": "model-00003-of-00004.safetensors", + "visual.blocks.27.mlp.up_proj.weight": "model-00001-of-00004.safetensors", + "visual.blocks.27.norm1.weight": "model-00004-of-00004.safetensors", + "visual.blocks.27.norm2.weight": "model-00002-of-00004.safetensors", + "visual.blocks.28.attn.proj.bias": "model-00003-of-00004.safetensors", + "visual.blocks.28.attn.proj.weight": "model-00003-of-00004.safetensors", + "visual.blocks.28.attn.qkv.bias": "model-00003-of-00004.safetensors", + "visual.blocks.28.attn.qkv.weight": "model-00003-of-00004.safetensors", + "visual.blocks.28.mlp.down_proj.bias": "model-00004-of-00004.safetensors", + "visual.blocks.28.mlp.down_proj.weight": "model-00004-of-00004.safetensors", + "visual.blocks.28.mlp.gate_proj.bias": "model-00004-of-00004.safetensors", + "visual.blocks.28.mlp.gate_proj.weight": "model-00002-of-00004.safetensors", + "visual.blocks.28.mlp.up_proj.bias": "model-00002-of-00004.safetensors", + "visual.blocks.28.mlp.up_proj.weight": "model-00003-of-00004.safetensors", + "visual.blocks.28.norm1.weight": "model-00003-of-00004.safetensors", + "visual.blocks.28.norm2.weight": "model-00001-of-00004.safetensors", + "visual.blocks.29.attn.proj.bias": "model-00003-of-00004.safetensors", + "visual.blocks.29.attn.proj.weight": "model-00001-of-00004.safetensors", + "visual.blocks.29.attn.qkv.bias": "model-00001-of-00004.safetensors", + "visual.blocks.29.attn.qkv.weight": "model-00003-of-00004.safetensors", + "visual.blocks.29.mlp.down_proj.bias": "model-00001-of-00004.safetensors", + "visual.blocks.29.mlp.down_proj.weight": "model-00002-of-00004.safetensors", + "visual.blocks.29.mlp.gate_proj.bias": "model-00003-of-00004.safetensors", + "visual.blocks.29.mlp.gate_proj.weight": "model-00002-of-00004.safetensors", + "visual.blocks.29.mlp.up_proj.bias": "model-00003-of-00004.safetensors", + "visual.blocks.29.mlp.up_proj.weight": "model-00003-of-00004.safetensors", + "visual.blocks.29.norm1.weight": "model-00003-of-00004.safetensors", + "visual.blocks.29.norm2.weight": "model-00003-of-00004.safetensors", + "visual.blocks.3.attn.proj.bias": "model-00003-of-00004.safetensors", + "visual.blocks.3.attn.proj.weight": "model-00003-of-00004.safetensors", + "visual.blocks.3.attn.qkv.bias": "model-00003-of-00004.safetensors", + "visual.blocks.3.attn.qkv.weight": "model-00001-of-00004.safetensors", + "visual.blocks.3.mlp.down_proj.bias": "model-00001-of-00004.safetensors", + "visual.blocks.3.mlp.down_proj.weight": "model-00001-of-00004.safetensors", + "visual.blocks.3.mlp.gate_proj.bias": "model-00001-of-00004.safetensors", + "visual.blocks.3.mlp.gate_proj.weight": "model-00001-of-00004.safetensors", + "visual.blocks.3.mlp.up_proj.bias": "model-00003-of-00004.safetensors", + "visual.blocks.3.mlp.up_proj.weight": "model-00004-of-00004.safetensors", + "visual.blocks.3.norm1.weight": "model-00001-of-00004.safetensors", + "visual.blocks.3.norm2.weight": "model-00001-of-00004.safetensors", + "visual.blocks.30.attn.proj.bias": "model-00003-of-00004.safetensors", + "visual.blocks.30.attn.proj.weight": "model-00004-of-00004.safetensors", + "visual.blocks.30.attn.qkv.bias": "model-00001-of-00004.safetensors", + "visual.blocks.30.attn.qkv.weight": "model-00001-of-00004.safetensors", + "visual.blocks.30.mlp.down_proj.bias": "model-00003-of-00004.safetensors", + "visual.blocks.30.mlp.down_proj.weight": "model-00001-of-00004.safetensors", + "visual.blocks.30.mlp.gate_proj.bias": "model-00004-of-00004.safetensors", + "visual.blocks.30.mlp.gate_proj.weight": "model-00003-of-00004.safetensors", + "visual.blocks.30.mlp.up_proj.bias": "model-00002-of-00004.safetensors", + "visual.blocks.30.mlp.up_proj.weight": "model-00001-of-00004.safetensors", + "visual.blocks.30.norm1.weight": "model-00002-of-00004.safetensors", + "visual.blocks.30.norm2.weight": "model-00001-of-00004.safetensors", + "visual.blocks.31.attn.proj.bias": "model-00001-of-00004.safetensors", + "visual.blocks.31.attn.proj.weight": "model-00003-of-00004.safetensors", + "visual.blocks.31.attn.qkv.bias": "model-00002-of-00004.safetensors", + "visual.blocks.31.attn.qkv.weight": "model-00003-of-00004.safetensors", + "visual.blocks.31.mlp.down_proj.bias": "model-00002-of-00004.safetensors", + "visual.blocks.31.mlp.down_proj.weight": "model-00004-of-00004.safetensors", + "visual.blocks.31.mlp.gate_proj.bias": "model-00003-of-00004.safetensors", + "visual.blocks.31.mlp.gate_proj.weight": "model-00002-of-00004.safetensors", + "visual.blocks.31.mlp.up_proj.bias": "model-00004-of-00004.safetensors", + "visual.blocks.31.mlp.up_proj.weight": "model-00002-of-00004.safetensors", + "visual.blocks.31.norm1.weight": "model-00002-of-00004.safetensors", + "visual.blocks.31.norm2.weight": "model-00001-of-00004.safetensors", + "visual.blocks.4.attn.proj.bias": "model-00002-of-00004.safetensors", + "visual.blocks.4.attn.proj.weight": "model-00001-of-00004.safetensors", + "visual.blocks.4.attn.qkv.bias": "model-00003-of-00004.safetensors", + "visual.blocks.4.attn.qkv.weight": "model-00002-of-00004.safetensors", + "visual.blocks.4.mlp.down_proj.bias": "model-00001-of-00004.safetensors", + "visual.blocks.4.mlp.down_proj.weight": "model-00001-of-00004.safetensors", + "visual.blocks.4.mlp.gate_proj.bias": "model-00002-of-00004.safetensors", + "visual.blocks.4.mlp.gate_proj.weight": "model-00004-of-00004.safetensors", + "visual.blocks.4.mlp.up_proj.bias": "model-00001-of-00004.safetensors", + "visual.blocks.4.mlp.up_proj.weight": "model-00003-of-00004.safetensors", + "visual.blocks.4.norm1.weight": "model-00002-of-00004.safetensors", + "visual.blocks.4.norm2.weight": "model-00001-of-00004.safetensors", + "visual.blocks.5.attn.proj.bias": "model-00002-of-00004.safetensors", + "visual.blocks.5.attn.proj.weight": "model-00003-of-00004.safetensors", + "visual.blocks.5.attn.qkv.bias": "model-00002-of-00004.safetensors", + "visual.blocks.5.attn.qkv.weight": "model-00002-of-00004.safetensors", + "visual.blocks.5.mlp.down_proj.bias": "model-00002-of-00004.safetensors", + "visual.blocks.5.mlp.down_proj.weight": "model-00002-of-00004.safetensors", + "visual.blocks.5.mlp.gate_proj.bias": "model-00001-of-00004.safetensors", + "visual.blocks.5.mlp.gate_proj.weight": "model-00003-of-00004.safetensors", + "visual.blocks.5.mlp.up_proj.bias": "model-00003-of-00004.safetensors", + "visual.blocks.5.mlp.up_proj.weight": "model-00001-of-00004.safetensors", + "visual.blocks.5.norm1.weight": "model-00003-of-00004.safetensors", + "visual.blocks.5.norm2.weight": "model-00003-of-00004.safetensors", + "visual.blocks.6.attn.proj.bias": "model-00001-of-00004.safetensors", + "visual.blocks.6.attn.proj.weight": "model-00002-of-00004.safetensors", + "visual.blocks.6.attn.qkv.bias": "model-00004-of-00004.safetensors", + "visual.blocks.6.attn.qkv.weight": "model-00002-of-00004.safetensors", + "visual.blocks.6.mlp.down_proj.bias": "model-00004-of-00004.safetensors", + "visual.blocks.6.mlp.down_proj.weight": "model-00003-of-00004.safetensors", + "visual.blocks.6.mlp.gate_proj.bias": "model-00003-of-00004.safetensors", + "visual.blocks.6.mlp.gate_proj.weight": "model-00002-of-00004.safetensors", + "visual.blocks.6.mlp.up_proj.bias": "model-00001-of-00004.safetensors", + "visual.blocks.6.mlp.up_proj.weight": "model-00003-of-00004.safetensors", + "visual.blocks.6.norm1.weight": "model-00003-of-00004.safetensors", + "visual.blocks.6.norm2.weight": "model-00002-of-00004.safetensors", + "visual.blocks.7.attn.proj.bias": "model-00003-of-00004.safetensors", + "visual.blocks.7.attn.proj.weight": "model-00001-of-00004.safetensors", + "visual.blocks.7.attn.qkv.bias": "model-00001-of-00004.safetensors", + "visual.blocks.7.attn.qkv.weight": "model-00001-of-00004.safetensors", + "visual.blocks.7.mlp.down_proj.bias": "model-00001-of-00004.safetensors", + "visual.blocks.7.mlp.down_proj.weight": "model-00001-of-00004.safetensors", + "visual.blocks.7.mlp.gate_proj.bias": "model-00002-of-00004.safetensors", + "visual.blocks.7.mlp.gate_proj.weight": "model-00004-of-00004.safetensors", + "visual.blocks.7.mlp.up_proj.bias": "model-00003-of-00004.safetensors", + "visual.blocks.7.mlp.up_proj.weight": "model-00001-of-00004.safetensors", + "visual.blocks.7.norm1.weight": "model-00001-of-00004.safetensors", + "visual.blocks.7.norm2.weight": "model-00001-of-00004.safetensors", + "visual.blocks.8.attn.proj.bias": "model-00001-of-00004.safetensors", + "visual.blocks.8.attn.proj.weight": "model-00003-of-00004.safetensors", + "visual.blocks.8.attn.qkv.bias": "model-00002-of-00004.safetensors", + "visual.blocks.8.attn.qkv.weight": "model-00003-of-00004.safetensors", + "visual.blocks.8.mlp.down_proj.bias": "model-00002-of-00004.safetensors", + "visual.blocks.8.mlp.down_proj.weight": "model-00003-of-00004.safetensors", + "visual.blocks.8.mlp.gate_proj.bias": "model-00001-of-00004.safetensors", + "visual.blocks.8.mlp.gate_proj.weight": "model-00004-of-00004.safetensors", + "visual.blocks.8.mlp.up_proj.bias": "model-00004-of-00004.safetensors", + "visual.blocks.8.mlp.up_proj.weight": "model-00002-of-00004.safetensors", + "visual.blocks.8.norm1.weight": "model-00001-of-00004.safetensors", + "visual.blocks.8.norm2.weight": "model-00002-of-00004.safetensors", + "visual.blocks.9.attn.proj.bias": "model-00002-of-00004.safetensors", + "visual.blocks.9.attn.proj.weight": "model-00001-of-00004.safetensors", + "visual.blocks.9.attn.qkv.bias": "model-00002-of-00004.safetensors", + "visual.blocks.9.attn.qkv.weight": "model-00002-of-00004.safetensors", + "visual.blocks.9.mlp.down_proj.bias": "model-00002-of-00004.safetensors", + "visual.blocks.9.mlp.down_proj.weight": "model-00004-of-00004.safetensors", + "visual.blocks.9.mlp.gate_proj.bias": "model-00004-of-00004.safetensors", + "visual.blocks.9.mlp.gate_proj.weight": "model-00001-of-00004.safetensors", + "visual.blocks.9.mlp.up_proj.bias": "model-00001-of-00004.safetensors", + "visual.blocks.9.mlp.up_proj.weight": "model-00002-of-00004.safetensors", + "visual.blocks.9.norm1.weight": "model-00001-of-00004.safetensors", + "visual.blocks.9.norm2.weight": "model-00002-of-00004.safetensors", + "visual.merger.ln_q.weight": "model-00002-of-00004.safetensors", + "visual.merger.mlp.0.bias": "model-00001-of-00004.safetensors", + "visual.merger.mlp.0.weight": "model-00002-of-00004.safetensors", + "visual.merger.mlp.2.bias": "model-00001-of-00004.safetensors", + "visual.merger.mlp.2.weight": "model-00002-of-00004.safetensors", + "visual.patch_embed.proj.weight": "model-00001-of-00004.safetensors" + } +} diff --git a/posttrain_models/Video-o3_SFT_RL/preprocessor_config.json b/posttrain_models/Video-o3_SFT_RL/preprocessor_config.json new file mode 100644 index 0000000000000000000000000000000000000000..dad4fd31ef14174034bf8d5451f9aad60d1a9337 --- /dev/null +++ b/posttrain_models/Video-o3_SFT_RL/preprocessor_config.json @@ -0,0 +1,39 @@ +{ + "crop_size": null, + "data_format": "channels_first", + "default_to_square": true, + "device": null, + "disable_grouping": null, + "do_center_crop": null, + "do_convert_rgb": true, + "do_normalize": true, + "do_pad": null, + "do_rescale": true, + "do_resize": true, + "image_mean": [ + 0.48145466, + 0.4578275, + 0.40821073 + ], + "image_processor_type": "Qwen2VLImageProcessorFast", + "image_std": [ + 0.26862954, + 0.26130258, + 0.27577711 + ], + "input_data_format": null, + "max_pixels": 12845056, + "merge_size": 2, + "min_pixels": 3136, + "pad_size": null, + "patch_size": 14, + "processor_class": "Qwen2_5_VLProcessor", + "resample": 3, + "rescale_factor": 0.00392156862745098, + "return_tensors": null, + "size": { + "longest_edge": 12845056, + "shortest_edge": 3136 + }, + "temporal_patch_size": 2 +} diff --git a/posttrain_models/Video-o3_SFT_RL/special_tokens_map.json b/posttrain_models/Video-o3_SFT_RL/special_tokens_map.json new file mode 100644 index 0000000000000000000000000000000000000000..ac23c0aaa2434523c494330aeb79c58395378103 --- /dev/null +++ b/posttrain_models/Video-o3_SFT_RL/special_tokens_map.json @@ -0,0 +1,31 @@ +{ + "additional_special_tokens": [ + "<|im_start|>", + "<|im_end|>", + "<|object_ref_start|>", + "<|object_ref_end|>", + "<|box_start|>", + "<|box_end|>", + "<|quad_start|>", + "<|quad_end|>", + "<|vision_start|>", + "<|vision_end|>", + "<|vision_pad|>", + "<|image_pad|>", + "<|video_pad|>" + ], + "eos_token": { + "content": "<|im_end|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false + }, + "pad_token": { + "content": "<|endoftext|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false + } +} diff --git a/posttrain_models/Video-o3_SFT_RL/tokenizer_config.json b/posttrain_models/Video-o3_SFT_RL/tokenizer_config.json new file mode 100644 index 0000000000000000000000000000000000000000..230f071b2d4625056383a7486679127e53fb7d4d --- /dev/null +++ b/posttrain_models/Video-o3_SFT_RL/tokenizer_config.json @@ -0,0 +1,209 @@ +{ + "add_bos_token": false, + "add_prefix_space": false, + "added_tokens_decoder": { + "151643": { + "content": "<|endoftext|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "151644": { + "content": "<|im_start|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "151645": { + "content": "<|im_end|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "151646": { + "content": "<|object_ref_start|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "151647": { + "content": "<|object_ref_end|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "151648": { + "content": "<|box_start|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "151649": { + "content": "<|box_end|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "151650": { + "content": "<|quad_start|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "151651": { + "content": "<|quad_end|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "151652": { + "content": "<|vision_start|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "151653": { + "content": "<|vision_end|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "151654": { + "content": "<|vision_pad|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "151655": { + "content": "<|image_pad|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "151656": { + "content": "<|video_pad|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "151657": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "151658": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "151659": { + "content": "<|fim_prefix|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "151660": { + "content": "<|fim_middle|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "151661": { + "content": "<|fim_suffix|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "151662": { + "content": "<|fim_pad|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "151663": { + "content": "<|repo_name|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "151664": { + "content": "<|file_sep|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + } + }, + "additional_special_tokens": [ + "<|im_start|>", + "<|im_end|>", + "<|object_ref_start|>", + "<|object_ref_end|>", + "<|box_start|>", + "<|box_end|>", + "<|quad_start|>", + "<|quad_end|>", + "<|vision_start|>", + "<|vision_end|>", + "<|vision_pad|>", + "<|image_pad|>", + "<|video_pad|>" + ], + "bos_token": null, + "clean_up_tokenization_spaces": false, + "eos_token": "<|im_end|>", + "errors": "replace", + "extra_special_tokens": {}, + "model_max_length": 131072, + "pad_token": "<|endoftext|>", + "padding_side": "right", + "processor_class": "Qwen2_5_VLProcessor", + "split_special_tokens": false, + "tokenizer_class": "Qwen2Tokenizer", + "unk_token": null +} diff --git a/posttrain_models/Video-o3_SFT_RL/video_preprocessor_config.json b/posttrain_models/Video-o3_SFT_RL/video_preprocessor_config.json new file mode 100644 index 0000000000000000000000000000000000000000..b005795bb1ffbae674c90c8f143a01d89a076e5c --- /dev/null +++ b/posttrain_models/Video-o3_SFT_RL/video_preprocessor_config.json @@ -0,0 +1,93 @@ +{ + "_valid_kwargs_names": [ + "do_convert_rgb", + "do_resize", + "size", + "size_divisor", + "default_to_square", + "resample", + "do_rescale", + "rescale_factor", + "do_normalize", + "image_mean", + "image_std", + "do_pad", + "do_center_crop", + "crop_size", + "data_format", + "input_data_format", + "device", + "min_pixels", + "max_pixels", + "patch_size", + "temporal_patch_size", + "merge_size" + ], + "crop_size": null, + "data_format": "channels_first", + "default_to_square": true, + "device": null, + "do_center_crop": null, + "do_convert_rgb": true, + "do_normalize": true, + "do_pad": null, + "do_rescale": true, + "do_resize": true, + "do_sample_frames": false, + "fps": null, + "image_mean": [ + 0.48145466, + 0.4578275, + 0.40821073 + ], + "image_std": [ + 0.26862954, + 0.26130258, + 0.27577711 + ], + "input_data_format": null, + "max_frames": 768, + "max_pixels": 12845056, + "merge_size": 2, + "min_frames": 4, + "min_pixels": 3136, + "model_valid_processing_keys": [ + "do_convert_rgb", + "do_resize", + "size", + "size_divisor", + "default_to_square", + "resample", + "do_rescale", + "rescale_factor", + "do_normalize", + "image_mean", + "image_std", + "do_pad", + "do_center_crop", + "crop_size", + "data_format", + "input_data_format", + "device", + "min_pixels", + "max_pixels", + "patch_size", + "temporal_patch_size", + "merge_size" + ], + "num_frames": null, + "pad_size": null, + "patch_size": 14, + "processor_class": "Qwen2_5_VLProcessor", + "resample": 3, + "rescale_factor": 0.00392156862745098, + "return_metadata": false, + "size": { + "longest_edge": 12845056, + "shortest_edge": 3136 + }, + "size_divisor": null, + "temporal_patch_size": 2, + "video_metadata": null, + "video_processor_type": "Qwen2VLVideoProcessor" +} diff --git a/posttrain_models/_source_cache_and_symlinks.tar.gz b/posttrain_models/_source_cache_and_symlinks.tar.gz new file mode 100644 index 0000000000000000000000000000000000000000..0af1c2fab8d99884212f21ddd84fb1be658c518f --- /dev/null +++ b/posttrain_models/_source_cache_and_symlinks.tar.gz @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a745156ace519c96933dbfee4516ab4b4ab9e78aa5fa399832a8bdb3a9ea35bc +size 10319 diff --git a/posttrain_models/qwen3_5_35b_a3b_base_highlight_batch1245_epoch4/args.json b/posttrain_models/qwen3_5_35b_a3b_base_highlight_batch1245_epoch4/args.json new file mode 100644 index 0000000000000000000000000000000000000000..99623dd80e3398e11296749c2daf861d8dff0018 --- /dev/null +++ b/posttrain_models/qwen3_5_35b_a3b_base_highlight_batch1245_epoch4/args.json @@ -0,0 +1,398 @@ +{ + "use_ray": false, + "ray_exp_name": null, + "device_groups": null, + "model": "checkpoints/Qwen3.5-35B-A3B", + "model_type": "qwen3_5_moe", + "model_revision": null, + "task_type": "causal_lm", + "torch_dtype": "bfloat16", + "attn_impl": null, + "experts_impl": null, + "new_special_tokens": [], + "num_labels": null, + "problem_type": null, + "rope_scaling": null, + "device_map": null, + "max_memory": {}, + "max_model_len": null, + "local_repo_path": null, + "init_strategy": null, + "template": "qwen3_5", + "system": null, + "max_length": 8192, + "truncation_strategy": "delete", + "max_pixels": null, + "agent_template": null, + "norm_bbox": null, + "use_chat_template": true, + "padding_side": "right", + "padding_free": true, + "loss_scale": "ignore_empty_think", + "sequence_parallel_size": 1, + "template_backend": "swift", + "response_prefix": "", + "enable_thinking": false, + "add_non_thinking_prefix": false, + "dataset": [ + "batch1", + "batch2", + "batch4_part1", + "batch4_part2", + "batch4_part3", + "batch5_part1", + "batch5_part2", + "batch5_part3" + ], + "val_dataset": [], + "cached_dataset": [], + "cached_val_dataset": [], + "split_dataset_ratio": 0.0, + "data_seed": 42, + "dataset_num_proc": 2, + "load_from_cache_file": false, + "dataset_shuffle": true, + "val_dataset_shuffle": false, + "streaming": false, + "interleave_prob": null, + "stopping_strategy": "first_exhausted", + "shuffle_buffer_size": 1000, + "download_mode": "reuse_dataset_if_exists", + "columns": {}, + "strict": false, + "remove_unused_columns": true, + "model_name": null, + "model_author": null, + "custom_dataset_info": [], + "quant_method": null, + "quant_bits": null, + "hqq_axis": null, + "bnb_4bit_compute_dtype": "bfloat16", + "bnb_4bit_quant_type": "nf4", + "bnb_4bit_use_double_quant": true, + "bnb_4bit_quant_storage": null, + "max_new_tokens": null, + "temperature": 0.9, + "top_k": 50, + "top_p": 0.9, + "repetition_penalty": 1.0, + "num_beams": 1, + "stream": false, + "stop_words": [], + "logprobs": false, + "top_logprobs": null, + "structured_outputs_regex": null, + "tuner_backend": "peft", + "tuner_type": "lora", + "train_type": "lora", + "adapters": [], + "external_plugins": [ + "ms-swift/base_highlight_video_dataset.py" + ], + "custom_register_path": [ + "ms-swift/base_highlight_video_dataset.py" + ], + "seed": 42, + "model_kwargs": {}, + "load_args": false, + "load_data_args": false, + "packing": false, + "packing_length": null, + "packing_num_proc": 1, + "lazy_tokenize": true, + "use_hf": false, + "hub_token": null, + "ddp_timeout": 18000000, + "ddp_backend": "nccl", + "ignore_args_error": false, + "use_swift_lora": false, + "freeze_llm": false, + "freeze_vit": true, + "freeze_aligner": true, + "freeze_parameters": [], + "freeze_parameters_regex": null, + "freeze_parameters_ratio": 0.0, + "trainable_parameters": [], + "trainable_parameters_regex": null, + "target_modules": [ + "all-linear" + ], + "target_regex": null, + "modules_to_save": [], + "lora_rank": 8, + "lora_alpha": 32, + "lora_dropout": 0.05, + "lora_bias": "none", + "lora_dtype": null, + "use_rslora": false, + "rlhf_type": null, + "loss_type": null, + "mcore_ref_model": null, + "mcore_ref_adapter": null, + "beta": null, + "rpo_alpha": null, + "reference_free": false, + "label_smoothing": 0.0, + "f_divergence_type": "reverse_kl", + "desirable_weight": 1.0, + "undesirable_weight": 1.0, + "calculate_KL": null, + "center_rewards_coefficient": null, + "teacher_model": null, + "teacher_model_type": null, + "teacher_model_revision": null, + "teacher_model_server": null, + "gkd_logits_topk": null, + "lmbda": 0.5, + "seq_kd": false, + "offload_teacher_model": false, + "sft_alpha": 0.0, + "generation_batch_size": null, + "steps_per_generation": null, + "num_generations": 8, + "num_generations_eval": null, + "max_completion_length": 512, + "importance_sampling_level": "token", + "tau_pos": 1.0, + "tau_neg": 1.05, + "epsilon": 0.2, + "epsilon_high": null, + "delta": null, + "use_vllm": true, + "vllm_mode": null, + "vllm_enable_prefix_caching": true, + "vllm_gpu_memory_utilization": 0.9, + "vllm_tensor_parallel_size": 1, + "vllm_max_model_len": null, + "vllm_enforce_eager": false, + "vllm_limit_mm_per_prompt": null, + "vllm_disable_cascade_attn": false, + "vllm_max_num_seqs": null, + "vllm_mm_processor_cache_gb": null, + "vllm_engine_kwargs": null, + "sleep_level": 0, + "offload_optimizer": false, + "offload_model": false, + "offload_bridge": false, + "vllm_server_base_url": null, + "vllm_server_host": null, + "vllm_server_port": [ + 8000 + ], + "vllm_server_timeout": 240.0, + "vllm_server_group_port": null, + "reward_funcs": [], + "reward_weights": null, + "cosine_min_len_value_wrong": -0.5, + "cosine_max_len_value_wrong": 0.0, + "cosine_min_len_value_correct": 1.0, + "cosine_max_len_value_correct": 0.5, + "cosine_max_len": null, + "repetition_n_grams": 3, + "repetition_max_penalty": -1.0, + "soft_max_length": null, + "soft_cache_length": null, + "dynamic_sample": false, + "max_resample_times": 3, + "overlong_filter": false, + "scale_rewards": "group", + "advantage_estimator": "grpo", + "kl_in_reward": false, + "wandb_log_unique_prompts": null, + "log_completions": false, + "rollout_importance_sampling_mode": null, + "rollout_importance_sampling_threshold": 2.0, + "log_rollout_offpolicy_metrics": false, + "off_policy_sequence_mask_delta": null, + "log_entropy": false, + "top_entropy_quantile": 1.0, + "reward_model": null, + "reward_model_plugin": null, + "sync_ref_model": false, + "ref_model_sync_steps": 512, + "ref_model_mixup_alpha": 0.6, + "async_generate": false, + "move_model_batches": null, + "multi_turn_scheduler": null, + "max_turns": null, + "completion_length_limit_scope": "per_round", + "vllm_server_pass_dataset": false, + "num_iterations": 1, + "micro_batch_size": 8, + "global_batch_size": 64, + "recompute_granularity": "full", + "recompute_method": "uniform", + "recompute_num_layers": 1, + "recompute_modules": [ + "core_attn" + ], + "train_iters": null, + "num_train_epochs": 4, + "masked_softmax_fusion": true, + "bias_dropout_fusion": true, + "bias_activation_fusion": true, + "apply_rope_fusion": false, + "gradient_accumulation_fusion": false, + "cross_entropy_loss_fusion": true, + "cross_entropy_fusion_impl": "te", + "calculate_per_token_loss": true, + "attention_backend": "", + "optimizer": "adam", + "optimizer_cpu_offload": false, + "optimizer_offload_fraction": 1.0, + "use_precision_aware_optimizer": false, + "main_grads_dtype": "float32", + "main_params_dtype": "float32", + "exp_avg_dtype": "float32", + "exp_avg_sq_dtype": "float32", + "manual_gc": false, + "manual_gc_steps": 0, + "manual_gc_eval": true, + "train_dataloader_shuffle": true, + "dataloader_num_workers": 1, + "dataloader_pin_memory": true, + "dataloader_persistent_workers": true, + "dataloader_prefetch_factor": 2, + "data_sharding": false, + "group_by_length": false, + "te_rng_tracker": false, + "data_parallel_random_init": false, + "mlp_padding_free": false, + "lr_warmup_init": 0.0, + "lr": 0.0001, + "lr_decay_style": "cosine", + "lr_decay_iters": null, + "lr_warmup_iters": 0, + "lr_warmup_fraction": 0.05, + "min_lr": 1e-05, + "lr_wsd_decay_style": "exponential", + "lr_wsd_decay_iters": null, + "weight_decay": 0.1, + "weight_decay_incr_style": "constant", + "start_weight_decay": 0.1, + "end_weight_decay": 0.1, + "clip_grad": 1.0, + "adam_beta1": 0.9, + "adam_beta2": 0.95, + "adam_eps": 1e-08, + "sgd_momentum": 0.9, + "output_dir": "/mnt/bn/strategy-mllm-train/user/tanyue/models/posttrain_models/qwen3_5_35b_a3b_base_highlight_batch1245_epoch4", + "save_steps": 500, + "no_save_optim": true, + "no_save_rng": true, + "mcore_model": null, + "mcore_adapter": null, + "no_load_optim": false, + "no_load_rng": false, + "finetune": true, + "perform_initialization": false, + "use_cpu_initialization": false, + "async_save": false, + "save_total_limit": null, + "metric_for_best_model": "loss", + "greater_is_better": false, + "use_persistent_ckpt_worker": false, + "dist_ckpt_save_pre_mcore_014": false, + "dist_ckpt_optim_fully_reshardable": false, + "distrib_optim_fully_reshardable_mem_efficient": false, + "local_rank": 0, + "use_distributed_optimizer": true, + "tensor_model_parallel_size": 1, + "pipeline_model_parallel_size": 2, + "decoder_first_pipeline_num_layers": null, + "decoder_last_pipeline_num_layers": null, + "account_for_embedding_in_pipeline_split": false, + "account_for_loss_in_pipeline_split": false, + "overlap_p2p_comm": false, + "align_param_gather": false, + "sequence_parallel": false, + "context_parallel_size": 1, + "tp_comm_overlap": false, + "overlap_grad_reduce": false, + "overlap_param_gather": false, + "overlap_param_gather_with_optimizer_step": false, + "align_grad_reduce": true, + "virtual_pipeline_model_parallel_size": null, + "microbatch_group_size_per_vp_stage": null, + "pipeline_model_parallel_layout": null, + "expert_model_parallel_size": 4, + "expert_tensor_parallel_size": 1, + "report_to": [ + "wandb" + ], + "logging_steps": 5, + "tensorboard_dir": "/mnt/bn/strategy-mllm-train/user/tanyue/models/posttrain_models/qwen3_5_35b_a3b_base_highlight_batch1245_epoch4/runs", + "tensorboard_queue_size": 50, + "wandb_project": "highlight_base_highlight_qwen3_5", + "wandb_exp_name": "qwen3_5_35b_a3b_base_highlight_batch1245_epoch4", + "swanlab_project": "megatron-swift", + "swanlab_exp_name": null, + "eval_iters": -1, + "eval_steps": 100000, + "fp8_format": null, + "fp8_recipe": "delayed", + "fp8_amax_history_len": 1024, + "fp8_amax_compute_algo": "max", + "fp8_param_gather": false, + "fp16": false, + "bf16": true, + "apply_query_key_layer_scaling": false, + "attention_softmax_in_fp32": true, + "accumulate_allreduce_grads_in_fp32": true, + "moe_router_load_balancing_type": null, + "moe_router_dtype": "fp32", + "moe_token_dispatcher_type": null, + "moe_enable_deepep": false, + "moe_grouped_gemm": true, + "moe_permute_fusion": true, + "moe_aux_loss_coeff": 1e-06, + "moe_z_loss_coeff": null, + "moe_shared_expert_overlap": true, + "moe_layer_recompute": false, + "moe_expert_capacity_factor": null, + "moe_pad_expert_input_to_capacity": false, + "moe_token_drop_policy": null, + "mtp_num_layers": null, + "mtp_loss_scaling_factor": 0.1, + "save_safetensors": true, + "ref_model": null, + "ref_adapters": [], + "merge_lora": true, + "max_shard_size": "5GB", + "vit_gradient_checkpointing": false, + "vit_lr": null, + "aligner_lr": null, + "gradient_checkpointing_kwargs": null, + "check_model": true, + "apply_wd_to_qk_layernorm": false, + "enable_dft_loss": false, + "enable_channel_loss": false, + "save_strategy": "steps", + "callbacks": [ + "print", + "default_flow", + "wandb" + ], + "add_version": false, + "create_checkpoint_symlink": false, + "swift_version": "4.0.1", + "ckpt_dir": null, + "rank": 0, + "global_world_size": 8, + "local_world_size": 8, + "model_suffix": "Qwen3.5-35B-A3B", + "model_info": "ModelInfo(model_type='qwen3_5_moe', model_dir='/opt/tiger/tanyue/highlight_tools/checkpoints/Qwen3.5-35B-A3B', torch_dtype=torch.bfloat16, max_model_len=262144, quant_method=None, quant_bits=None, rope_scaling=None, is_moe_model=True, is_multimodal=True, config=None, task_type='causal_lm', num_labels=None)", + "model_meta": "ModelMeta(model_type='qwen3_5_moe', model_groups=[ModelGroup(models=[Model(ms_model_id='Qwen/Qwen3.5-35B-A3B-Base', hf_model_id='Qwen/Qwen3.5-35B-A3B-Base', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen3.5-35B-A3B', hf_model_id='Qwen/Qwen3.5-35B-A3B', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen3.5-122B-A10B', hf_model_id='Qwen/Qwen3.5-122B-A10B', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen3.5-397B-A17B', hf_model_id='Qwen/Qwen3.5-397B-A17B', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen3.5-35B-A3B-FP8', hf_model_id='Qwen/Qwen3.5-35B-A3B-FP8', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen3.5-122B-A10B-FP8', hf_model_id='Qwen/Qwen3.5-122B-A10B-FP8', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen3.5-397B-A17B-FP8', hf_model_id='Qwen/Qwen3.5-397B-A17B-FP8', model_path=None, ms_revision=None, hf_revision=None)], template='qwen3_5', ignore_patterns=None, requires=None, tags=[])], loader=, template='qwen3_5', model_arch=MultiModelKeys(arch_name='qwen2_vl', embedding=None, module_list=None, lm_head=None, q_proj=None, k_proj=None, v_proj=None, o_proj=None, attention=None, mlp=None, down_proj=None, qkv_proj=None, qk_proj=None, qa_proj=None, qb_proj=None, kv_proj=None, kva_proj=None, kvb_proj=None, language_model=['model.language_model', 'lm_head'], aligner=['model.visual.merger'], vision_tower=['model.visual'], generator=[]), architectures=['Qwen3_5MoeForConditionalGeneration'], additional_saved_files=[], torch_dtype=None, is_multimodal=True, is_reward=False, task_type=None, ignore_patterns=None, requires=['transformers>=5.2.0', 'qwen_vl_utils>=0.0.14', 'decord'], tags=[])", + "model_dir": "/opt/tiger/tanyue/highlight_tools/checkpoints/Qwen3.5-35B-A3B", + "template_meta": "QwenTemplateMeta(template_type='qwen3_5', prefix=[], prompt=['<|im_start|>user\\n{{QUERY}}<|im_end|>\\n<|im_start|>assistant\\n'], chat_sep=['<|im_end|>\\n'], suffix=['<|im_end|>\\n'], template_cls=, system_prefix=['<|im_start|>system\\n{{SYSTEM}}<|im_end|>\\n'], default_system=None, auto_add_bos=False, stop_words=['<|endoftext|>'], agent_template='qwen3_5', is_thinking=True, thinking_prefix='\\n', non_thinking_prefix='\\n\\n\\n\\n', history_thinking_prefix='')", + "_val_dataset_exists": false, + "hub": "", + "seq_length": 8192, + "is_multimodal": true, + "megatron_model_meta": "MegatronModelMeta(megatron_model_type='qwen3_5', model_types=['qwen3_5', 'qwen3_5_moe'], bridge_cls=, visual_cls=, is_multimodal=True, loader=)", + "fp8": null, + "params_dtype": "bfloat16", + "world_size": 8, + "data_parallel_size": 4, + "num_microbatches": 2 +} \ No newline at end of file diff --git a/posttrain_models/qwen3_5_35b_a3b_base_highlight_batch1245_epoch4/checkpoint-1000-merged/args.json b/posttrain_models/qwen3_5_35b_a3b_base_highlight_batch1245_epoch4/checkpoint-1000-merged/args.json new file mode 100644 index 0000000000000000000000000000000000000000..99623dd80e3398e11296749c2daf861d8dff0018 --- /dev/null +++ b/posttrain_models/qwen3_5_35b_a3b_base_highlight_batch1245_epoch4/checkpoint-1000-merged/args.json @@ -0,0 +1,398 @@ +{ + "use_ray": false, + "ray_exp_name": null, + "device_groups": null, + "model": "checkpoints/Qwen3.5-35B-A3B", + "model_type": "qwen3_5_moe", + "model_revision": null, + "task_type": "causal_lm", + "torch_dtype": "bfloat16", + "attn_impl": null, + "experts_impl": null, + "new_special_tokens": [], + "num_labels": null, + "problem_type": null, + "rope_scaling": null, + "device_map": null, + "max_memory": {}, + "max_model_len": null, + "local_repo_path": null, + "init_strategy": null, + "template": "qwen3_5", + "system": null, + "max_length": 8192, + "truncation_strategy": "delete", + "max_pixels": null, + "agent_template": null, + "norm_bbox": null, + "use_chat_template": true, + "padding_side": "right", + "padding_free": true, + "loss_scale": "ignore_empty_think", + "sequence_parallel_size": 1, + "template_backend": "swift", + "response_prefix": "", + "enable_thinking": false, + "add_non_thinking_prefix": false, + "dataset": [ + "batch1", + "batch2", + "batch4_part1", + "batch4_part2", + "batch4_part3", + "batch5_part1", + "batch5_part2", + "batch5_part3" + ], + "val_dataset": [], + "cached_dataset": [], + "cached_val_dataset": [], + "split_dataset_ratio": 0.0, + "data_seed": 42, + "dataset_num_proc": 2, + "load_from_cache_file": false, + "dataset_shuffle": true, + "val_dataset_shuffle": false, + "streaming": false, + "interleave_prob": null, + "stopping_strategy": "first_exhausted", + "shuffle_buffer_size": 1000, + "download_mode": "reuse_dataset_if_exists", + "columns": {}, + "strict": false, + "remove_unused_columns": true, + "model_name": null, + "model_author": null, + "custom_dataset_info": [], + "quant_method": null, + "quant_bits": null, + "hqq_axis": null, + "bnb_4bit_compute_dtype": "bfloat16", + "bnb_4bit_quant_type": "nf4", + "bnb_4bit_use_double_quant": true, + "bnb_4bit_quant_storage": null, + "max_new_tokens": null, + "temperature": 0.9, + "top_k": 50, + "top_p": 0.9, + "repetition_penalty": 1.0, + "num_beams": 1, + "stream": false, + "stop_words": [], + "logprobs": false, + "top_logprobs": null, + "structured_outputs_regex": null, + "tuner_backend": "peft", + "tuner_type": "lora", + "train_type": "lora", + "adapters": [], + "external_plugins": [ + "ms-swift/base_highlight_video_dataset.py" + ], + "custom_register_path": [ + "ms-swift/base_highlight_video_dataset.py" + ], + "seed": 42, + "model_kwargs": {}, + "load_args": false, + "load_data_args": false, + "packing": false, + "packing_length": null, + "packing_num_proc": 1, + "lazy_tokenize": true, + "use_hf": false, + "hub_token": null, + "ddp_timeout": 18000000, + "ddp_backend": "nccl", + "ignore_args_error": false, + "use_swift_lora": false, + "freeze_llm": false, + "freeze_vit": true, + "freeze_aligner": true, + "freeze_parameters": [], + "freeze_parameters_regex": null, + "freeze_parameters_ratio": 0.0, + "trainable_parameters": [], + "trainable_parameters_regex": null, + "target_modules": [ + "all-linear" + ], + "target_regex": null, + "modules_to_save": [], + "lora_rank": 8, + "lora_alpha": 32, + "lora_dropout": 0.05, + "lora_bias": "none", + "lora_dtype": null, + "use_rslora": false, + "rlhf_type": null, + "loss_type": null, + "mcore_ref_model": null, + "mcore_ref_adapter": null, + "beta": null, + "rpo_alpha": null, + "reference_free": false, + "label_smoothing": 0.0, + "f_divergence_type": "reverse_kl", + "desirable_weight": 1.0, + "undesirable_weight": 1.0, + "calculate_KL": null, + "center_rewards_coefficient": null, + "teacher_model": null, + "teacher_model_type": null, + "teacher_model_revision": null, + "teacher_model_server": null, + "gkd_logits_topk": null, + "lmbda": 0.5, + "seq_kd": false, + "offload_teacher_model": false, + "sft_alpha": 0.0, + "generation_batch_size": null, + "steps_per_generation": null, + "num_generations": 8, + "num_generations_eval": null, + "max_completion_length": 512, + "importance_sampling_level": "token", + "tau_pos": 1.0, + "tau_neg": 1.05, + "epsilon": 0.2, + "epsilon_high": null, + "delta": null, + "use_vllm": true, + "vllm_mode": null, + "vllm_enable_prefix_caching": true, + "vllm_gpu_memory_utilization": 0.9, + "vllm_tensor_parallel_size": 1, + "vllm_max_model_len": null, + "vllm_enforce_eager": false, + "vllm_limit_mm_per_prompt": null, + "vllm_disable_cascade_attn": false, + "vllm_max_num_seqs": null, + "vllm_mm_processor_cache_gb": null, + "vllm_engine_kwargs": null, + "sleep_level": 0, + "offload_optimizer": false, + "offload_model": false, + "offload_bridge": false, + "vllm_server_base_url": null, + "vllm_server_host": null, + "vllm_server_port": [ + 8000 + ], + "vllm_server_timeout": 240.0, + "vllm_server_group_port": null, + "reward_funcs": [], + "reward_weights": null, + "cosine_min_len_value_wrong": -0.5, + "cosine_max_len_value_wrong": 0.0, + "cosine_min_len_value_correct": 1.0, + "cosine_max_len_value_correct": 0.5, + "cosine_max_len": null, + "repetition_n_grams": 3, + "repetition_max_penalty": -1.0, + "soft_max_length": null, + "soft_cache_length": null, + "dynamic_sample": false, + "max_resample_times": 3, + "overlong_filter": false, + "scale_rewards": "group", + "advantage_estimator": "grpo", + "kl_in_reward": false, + "wandb_log_unique_prompts": null, + "log_completions": false, + "rollout_importance_sampling_mode": null, + "rollout_importance_sampling_threshold": 2.0, + "log_rollout_offpolicy_metrics": false, + "off_policy_sequence_mask_delta": null, + "log_entropy": false, + "top_entropy_quantile": 1.0, + "reward_model": null, + "reward_model_plugin": null, + "sync_ref_model": false, + "ref_model_sync_steps": 512, + "ref_model_mixup_alpha": 0.6, + "async_generate": false, + "move_model_batches": null, + "multi_turn_scheduler": null, + "max_turns": null, + "completion_length_limit_scope": "per_round", + "vllm_server_pass_dataset": false, + "num_iterations": 1, + "micro_batch_size": 8, + "global_batch_size": 64, + "recompute_granularity": "full", + "recompute_method": "uniform", + "recompute_num_layers": 1, + "recompute_modules": [ + "core_attn" + ], + "train_iters": null, + "num_train_epochs": 4, + "masked_softmax_fusion": true, + "bias_dropout_fusion": true, + "bias_activation_fusion": true, + "apply_rope_fusion": false, + "gradient_accumulation_fusion": false, + "cross_entropy_loss_fusion": true, + "cross_entropy_fusion_impl": "te", + "calculate_per_token_loss": true, + "attention_backend": "", + "optimizer": "adam", + "optimizer_cpu_offload": false, + "optimizer_offload_fraction": 1.0, + "use_precision_aware_optimizer": false, + "main_grads_dtype": "float32", + "main_params_dtype": "float32", + "exp_avg_dtype": "float32", + "exp_avg_sq_dtype": "float32", + "manual_gc": false, + "manual_gc_steps": 0, + "manual_gc_eval": true, + "train_dataloader_shuffle": true, + "dataloader_num_workers": 1, + "dataloader_pin_memory": true, + "dataloader_persistent_workers": true, + "dataloader_prefetch_factor": 2, + "data_sharding": false, + "group_by_length": false, + "te_rng_tracker": false, + "data_parallel_random_init": false, + "mlp_padding_free": false, + "lr_warmup_init": 0.0, + "lr": 0.0001, + "lr_decay_style": "cosine", + "lr_decay_iters": null, + "lr_warmup_iters": 0, + "lr_warmup_fraction": 0.05, + "min_lr": 1e-05, + "lr_wsd_decay_style": "exponential", + "lr_wsd_decay_iters": null, + "weight_decay": 0.1, + "weight_decay_incr_style": "constant", + "start_weight_decay": 0.1, + "end_weight_decay": 0.1, + "clip_grad": 1.0, + "adam_beta1": 0.9, + "adam_beta2": 0.95, + "adam_eps": 1e-08, + "sgd_momentum": 0.9, + "output_dir": "/mnt/bn/strategy-mllm-train/user/tanyue/models/posttrain_models/qwen3_5_35b_a3b_base_highlight_batch1245_epoch4", + "save_steps": 500, + "no_save_optim": true, + "no_save_rng": true, + "mcore_model": null, + "mcore_adapter": null, + "no_load_optim": false, + "no_load_rng": false, + "finetune": true, + "perform_initialization": false, + "use_cpu_initialization": false, + "async_save": false, + "save_total_limit": null, + "metric_for_best_model": "loss", + "greater_is_better": false, + "use_persistent_ckpt_worker": false, + "dist_ckpt_save_pre_mcore_014": false, + "dist_ckpt_optim_fully_reshardable": false, + "distrib_optim_fully_reshardable_mem_efficient": false, + "local_rank": 0, + "use_distributed_optimizer": true, + "tensor_model_parallel_size": 1, + "pipeline_model_parallel_size": 2, + "decoder_first_pipeline_num_layers": null, + "decoder_last_pipeline_num_layers": null, + "account_for_embedding_in_pipeline_split": false, + "account_for_loss_in_pipeline_split": false, + "overlap_p2p_comm": false, + "align_param_gather": false, + "sequence_parallel": false, + "context_parallel_size": 1, + "tp_comm_overlap": false, + "overlap_grad_reduce": false, + "overlap_param_gather": false, + "overlap_param_gather_with_optimizer_step": false, + "align_grad_reduce": true, + "virtual_pipeline_model_parallel_size": null, + "microbatch_group_size_per_vp_stage": null, + "pipeline_model_parallel_layout": null, + "expert_model_parallel_size": 4, + "expert_tensor_parallel_size": 1, + "report_to": [ + "wandb" + ], + "logging_steps": 5, + "tensorboard_dir": "/mnt/bn/strategy-mllm-train/user/tanyue/models/posttrain_models/qwen3_5_35b_a3b_base_highlight_batch1245_epoch4/runs", + "tensorboard_queue_size": 50, + "wandb_project": "highlight_base_highlight_qwen3_5", + "wandb_exp_name": "qwen3_5_35b_a3b_base_highlight_batch1245_epoch4", + "swanlab_project": "megatron-swift", + "swanlab_exp_name": null, + "eval_iters": -1, + "eval_steps": 100000, + "fp8_format": null, + "fp8_recipe": "delayed", + "fp8_amax_history_len": 1024, + "fp8_amax_compute_algo": "max", + "fp8_param_gather": false, + "fp16": false, + "bf16": true, + "apply_query_key_layer_scaling": false, + "attention_softmax_in_fp32": true, + "accumulate_allreduce_grads_in_fp32": true, + "moe_router_load_balancing_type": null, + "moe_router_dtype": "fp32", + "moe_token_dispatcher_type": null, + "moe_enable_deepep": false, + "moe_grouped_gemm": true, + "moe_permute_fusion": true, + "moe_aux_loss_coeff": 1e-06, + "moe_z_loss_coeff": null, + "moe_shared_expert_overlap": true, + "moe_layer_recompute": false, + "moe_expert_capacity_factor": null, + "moe_pad_expert_input_to_capacity": false, + "moe_token_drop_policy": null, + "mtp_num_layers": null, + "mtp_loss_scaling_factor": 0.1, + "save_safetensors": true, + "ref_model": null, + "ref_adapters": [], + "merge_lora": true, + "max_shard_size": "5GB", + "vit_gradient_checkpointing": false, + "vit_lr": null, + "aligner_lr": null, + "gradient_checkpointing_kwargs": null, + "check_model": true, + "apply_wd_to_qk_layernorm": false, + "enable_dft_loss": false, + "enable_channel_loss": false, + "save_strategy": "steps", + "callbacks": [ + "print", + "default_flow", + "wandb" + ], + "add_version": false, + "create_checkpoint_symlink": false, + "swift_version": "4.0.1", + "ckpt_dir": null, + "rank": 0, + "global_world_size": 8, + "local_world_size": 8, + "model_suffix": "Qwen3.5-35B-A3B", + "model_info": "ModelInfo(model_type='qwen3_5_moe', model_dir='/opt/tiger/tanyue/highlight_tools/checkpoints/Qwen3.5-35B-A3B', torch_dtype=torch.bfloat16, max_model_len=262144, quant_method=None, quant_bits=None, rope_scaling=None, is_moe_model=True, is_multimodal=True, config=None, task_type='causal_lm', num_labels=None)", + "model_meta": "ModelMeta(model_type='qwen3_5_moe', model_groups=[ModelGroup(models=[Model(ms_model_id='Qwen/Qwen3.5-35B-A3B-Base', hf_model_id='Qwen/Qwen3.5-35B-A3B-Base', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen3.5-35B-A3B', hf_model_id='Qwen/Qwen3.5-35B-A3B', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen3.5-122B-A10B', hf_model_id='Qwen/Qwen3.5-122B-A10B', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen3.5-397B-A17B', hf_model_id='Qwen/Qwen3.5-397B-A17B', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen3.5-35B-A3B-FP8', hf_model_id='Qwen/Qwen3.5-35B-A3B-FP8', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen3.5-122B-A10B-FP8', hf_model_id='Qwen/Qwen3.5-122B-A10B-FP8', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen3.5-397B-A17B-FP8', hf_model_id='Qwen/Qwen3.5-397B-A17B-FP8', model_path=None, ms_revision=None, hf_revision=None)], template='qwen3_5', ignore_patterns=None, requires=None, tags=[])], loader=, template='qwen3_5', model_arch=MultiModelKeys(arch_name='qwen2_vl', embedding=None, module_list=None, lm_head=None, q_proj=None, k_proj=None, v_proj=None, o_proj=None, attention=None, mlp=None, down_proj=None, qkv_proj=None, qk_proj=None, qa_proj=None, qb_proj=None, kv_proj=None, kva_proj=None, kvb_proj=None, language_model=['model.language_model', 'lm_head'], aligner=['model.visual.merger'], vision_tower=['model.visual'], generator=[]), architectures=['Qwen3_5MoeForConditionalGeneration'], additional_saved_files=[], torch_dtype=None, is_multimodal=True, is_reward=False, task_type=None, ignore_patterns=None, requires=['transformers>=5.2.0', 'qwen_vl_utils>=0.0.14', 'decord'], tags=[])", + "model_dir": "/opt/tiger/tanyue/highlight_tools/checkpoints/Qwen3.5-35B-A3B", + "template_meta": "QwenTemplateMeta(template_type='qwen3_5', prefix=[], prompt=['<|im_start|>user\\n{{QUERY}}<|im_end|>\\n<|im_start|>assistant\\n'], chat_sep=['<|im_end|>\\n'], suffix=['<|im_end|>\\n'], template_cls=, system_prefix=['<|im_start|>system\\n{{SYSTEM}}<|im_end|>\\n'], default_system=None, auto_add_bos=False, stop_words=['<|endoftext|>'], agent_template='qwen3_5', is_thinking=True, thinking_prefix='\\n', non_thinking_prefix='\\n\\n\\n\\n', history_thinking_prefix='')", + "_val_dataset_exists": false, + "hub": "", + "seq_length": 8192, + "is_multimodal": true, + "megatron_model_meta": "MegatronModelMeta(megatron_model_type='qwen3_5', model_types=['qwen3_5', 'qwen3_5_moe'], bridge_cls=, visual_cls=, is_multimodal=True, loader=)", + "fp8": null, + "params_dtype": "bfloat16", + "world_size": 8, + "data_parallel_size": 4, + "num_microbatches": 2 +} \ No newline at end of file diff --git a/posttrain_models/qwen3_5_35b_a3b_base_highlight_batch1245_epoch4/checkpoint-1000-merged/chat_template.jinja b/posttrain_models/qwen3_5_35b_a3b_base_highlight_batch1245_epoch4/checkpoint-1000-merged/chat_template.jinja new file mode 100644 index 0000000000000000000000000000000000000000..a585dec894e63da457d9440ec6aa7caa16d20860 --- /dev/null +++ b/posttrain_models/qwen3_5_35b_a3b_base_highlight_batch1245_epoch4/checkpoint-1000-merged/chat_template.jinja @@ -0,0 +1,154 @@ +{%- set image_count = namespace(value=0) %} +{%- set video_count = namespace(value=0) %} +{%- macro render_content(content, do_vision_count, is_system_content=false) %} + {%- if content is string %} + {{- content }} + {%- elif content is iterable and content is not mapping %} + {%- for item in content %} + {%- if 'image' in item or 'image_url' in item or item.type == 'image' %} + {%- if is_system_content %} + {{- raise_exception('System message cannot contain images.') }} + {%- endif %} + {%- if do_vision_count %} + {%- set image_count.value = image_count.value + 1 %} + {%- endif %} + {%- if add_vision_id %} + {{- 'Picture ' ~ image_count.value ~ ': ' }} + {%- endif %} + {{- '<|vision_start|><|image_pad|><|vision_end|>' }} + {%- elif 'video' in item or item.type == 'video' %} + {%- if is_system_content %} + {{- raise_exception('System message cannot contain videos.') }} + {%- endif %} + {%- if do_vision_count %} + {%- set video_count.value = video_count.value + 1 %} + {%- endif %} + {%- if add_vision_id %} + {{- 'Video ' ~ video_count.value ~ ': ' }} + {%- endif %} + {{- '<|vision_start|><|video_pad|><|vision_end|>' }} + {%- elif 'text' in item %} + {{- item.text }} + {%- else %} + {{- raise_exception('Unexpected item type in content.') }} + {%- endif %} + {%- endfor %} + {%- elif content is none or content is undefined %} + {{- '' }} + {%- else %} + {{- raise_exception('Unexpected content type.') }} + {%- endif %} +{%- endmacro %} +{%- if not messages %} + {{- raise_exception('No messages provided.') }} +{%- endif %} +{%- if tools and tools is iterable and tools is not mapping %} + {{- '<|im_start|>system\n' }} + {{- "# Tools\n\nYou have access to the following functions:\n\n" }} + {%- for tool in tools %} + {{- "\n" }} + {{- tool | tojson }} + {%- endfor %} + {{- "\n" }} + {{- '\n\nIf you choose to call a function ONLY reply in the following format with NO suffix:\n\n\n\n\nvalue_1\n\n\nThis is the value for the second parameter\nthat can span\nmultiple lines\n\n\n\n\n\nReminder:\n- Function calls MUST follow the specified format: an inner block must be nested within XML tags\n- Required parameters MUST be specified\n- You may provide optional reasoning for your function call in natural language BEFORE the function call, but NOT after\n- If there is no function call available, answer the question like normal with your current knowledge and do not tell the user about function calls\n' }} + {%- if messages[0].role == 'system' %} + {%- set content = render_content(messages[0].content, false, true)|trim %} + {%- if content %} + {{- '\n\n' + content }} + {%- endif %} + {%- endif %} + {{- '<|im_end|>\n' }} +{%- else %} + {%- if messages[0].role == 'system' %} + {%- set content = render_content(messages[0].content, false, true)|trim %} + {{- '<|im_start|>system\n' + content + '<|im_end|>\n' }} + {%- endif %} +{%- endif %} +{%- set ns = namespace(multi_step_tool=true, last_query_index=messages|length - 1) %} +{%- for message in messages[::-1] %} + {%- set index = (messages|length - 1) - loop.index0 %} + {%- if ns.multi_step_tool and message.role == "user" %} + {%- set content = render_content(message.content, false)|trim %} + {%- if not(content.startswith('') and content.endswith('')) %} + {%- set ns.multi_step_tool = false %} + {%- set ns.last_query_index = index %} + {%- endif %} + {%- endif %} +{%- endfor %} +{%- if ns.multi_step_tool %} + {{- raise_exception('No user query found in messages.') }} +{%- endif %} +{%- for message in messages %} + {%- set content = render_content(message.content, true)|trim %} + {%- if message.role == "system" %} + {%- if not loop.first %} + {{- raise_exception('System message must be at the beginning.') }} + {%- endif %} + {%- elif message.role == "user" %} + {{- '<|im_start|>' + message.role + '\n' + content + '<|im_end|>' + '\n' }} + {%- elif message.role == "assistant" %} + {%- set reasoning_content = '' %} + {%- if message.reasoning_content is string %} + {%- set reasoning_content = message.reasoning_content %} + {%- else %} + {%- if '' in content %} + {%- set reasoning_content = content.split('')[0].rstrip('\n').split('')[-1].lstrip('\n') %} + {%- set content = content.split('')[-1].lstrip('\n') %} + {%- endif %} + {%- endif %} + {%- set reasoning_content = reasoning_content|trim %} + {%- if loop.index0 > ns.last_query_index %} + {{- '<|im_start|>' + message.role + '\n\n' + reasoning_content + '\n\n\n' + content }} + {%- else %} + {{- '<|im_start|>' + message.role + '\n' + content }} + {%- endif %} + {%- if message.tool_calls and message.tool_calls is iterable and message.tool_calls is not mapping %} + {%- for tool_call in message.tool_calls %} + {%- if tool_call.function is defined %} + {%- set tool_call = tool_call.function %} + {%- endif %} + {%- if loop.first %} + {%- if content|trim %} + {{- '\n\n\n\n' }} + {%- else %} + {{- '\n\n' }} + {%- endif %} + {%- else %} + {{- '\n\n\n' }} + {%- endif %} + {%- if tool_call.arguments is defined %} + {%- for args_name, args_value in tool_call.arguments|items %} + {{- '\n' }} + {%- set args_value = args_value | tojson | safe if args_value is mapping or (args_value is sequence and args_value is not string) else args_value | string %} + {{- args_value }} + {{- '\n\n' }} + {%- endfor %} + {%- endif %} + {{- '\n' }} + {%- endfor %} + {%- endif %} + {{- '<|im_end|>\n' }} + {%- elif message.role == "tool" %} + {%- if loop.previtem and loop.previtem.role != "tool" %} + {{- '<|im_start|>user' }} + {%- endif %} + {{- '\n\n' }} + {{- content }} + {{- '\n' }} + {%- if not loop.last and loop.nextitem.role != "tool" %} + {{- '<|im_end|>\n' }} + {%- elif loop.last %} + {{- '<|im_end|>\n' }} + {%- endif %} + {%- else %} + {{- raise_exception('Unexpected message role.') }} + {%- endif %} +{%- endfor %} +{%- if add_generation_prompt %} + {{- '<|im_start|>assistant\n' }} + {%- if enable_thinking is defined and enable_thinking is false %} + {{- '\n\n\n\n' }} + {%- else %} + {{- '\n' }} + {%- endif %} +{%- endif %} \ No newline at end of file diff --git a/posttrain_models/qwen3_5_35b_a3b_base_highlight_batch1245_epoch4/checkpoint-1000-merged/config.json b/posttrain_models/qwen3_5_35b_a3b_base_highlight_batch1245_epoch4/checkpoint-1000-merged/config.json new file mode 100644 index 0000000000000000000000000000000000000000..8643d6d3f2e157c74392e7d91f431a66f9261762 --- /dev/null +++ b/posttrain_models/qwen3_5_35b_a3b_base_highlight_batch1245_epoch4/checkpoint-1000-merged/config.json @@ -0,0 +1,123 @@ +{ + "architectures": [ + "Qwen3_5MoeForConditionalGeneration" + ], + "dtype": "bfloat16", + "hidden_size": 2048, + "image_token_id": 248056, + "model_type": "qwen3_5_moe", + "text_config": { + "attention_bias": false, + "attention_dropout": 0.0, + "attn_output_gate": true, + "bos_token_id": null, + "dtype": "bfloat16", + "eos_token_id": 248044, + "full_attention_interval": 4, + "head_dim": 256, + "hidden_act": "silu", + "hidden_size": 2048, + "initializer_range": 0.02, + "layer_types": [ + "linear_attention", + "linear_attention", + "linear_attention", + "full_attention", + "linear_attention", + "linear_attention", + "linear_attention", + "full_attention", + "linear_attention", + "linear_attention", + "linear_attention", + "full_attention", + "linear_attention", + "linear_attention", + "linear_attention", + "full_attention", + "linear_attention", + "linear_attention", + "linear_attention", + "full_attention", + "linear_attention", + "linear_attention", + "linear_attention", + "full_attention", + "linear_attention", + "linear_attention", + "linear_attention", + "full_attention", + "linear_attention", + "linear_attention", + "linear_attention", + "full_attention", + "linear_attention", + "linear_attention", + "linear_attention", + "full_attention", + "linear_attention", + "linear_attention", + "linear_attention", + "full_attention" + ], + "linear_conv_kernel_dim": 4, + "linear_key_head_dim": 128, + "linear_num_key_heads": 16, + "linear_num_value_heads": 32, + "linear_value_head_dim": 128, + "mamba_ssm_dtype": "float32", + "max_position_embeddings": 262144, + "mlp_only_layers": [], + "model_type": "qwen3_5_moe_text", + "moe_intermediate_size": 512, + "mtp_num_hidden_layers": 1, + "mtp_use_dedicated_embeddings": false, + "num_attention_heads": 16, + "num_experts": 256, + "num_experts_per_tok": 8, + "num_hidden_layers": 40, + "num_key_value_heads": 2, + "output_router_logits": false, + "pad_token_id": null, + "partial_rotary_factor": 0.25, + "rms_norm_eps": 1e-06, + "rope_parameters": { + "mrope_interleaved": true, + "mrope_section": [ + 11, + 11, + 10 + ], + "partial_rotary_factor": 0.25, + "rope_theta": 10000000, + "rope_type": "default" + }, + "router_aux_loss_coef": 0.001, + "shared_expert_intermediate_size": 512, + "tie_word_embeddings": false, + "use_cache": true, + "vocab_size": 248320 + }, + "tie_word_embeddings": false, + "transformers_version": "5.2.0", + "video_token_id": 248057, + "vision_config": { + "deepstack_visual_indexes": [], + "depth": 27, + "dtype": "bfloat16", + "hidden_act": "gelu_pytorch_tanh", + "hidden_size": 1152, + "in_channels": 3, + "initializer_range": 0.02, + "intermediate_size": 4304, + "model_type": "qwen3_5_moe", + "num_heads": 16, + "num_position_embeddings": 2304, + "out_hidden_size": 2048, + "patch_size": 16, + "spatial_merge_size": 2, + "temporal_patch_size": 2 + }, + "vision_end_token_id": 248054, + "vision_start_token_id": 248053 +} diff --git a/posttrain_models/qwen3_5_35b_a3b_base_highlight_batch1245_epoch4/checkpoint-1000-merged/iter_0001000/common.pt b/posttrain_models/qwen3_5_35b_a3b_base_highlight_batch1245_epoch4/checkpoint-1000-merged/iter_0001000/common.pt new file mode 100644 index 0000000000000000000000000000000000000000..99dbccd0498abbffbe49b8e61dcbc056638de261 --- /dev/null +++ b/posttrain_models/qwen3_5_35b_a3b_base_highlight_batch1245_epoch4/checkpoint-1000-merged/iter_0001000/common.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c3d9304fd899960cfa6473c11c2d329a51fb7a11a6211086c06e929c8b6c2033 +size 15975 diff --git a/posttrain_models/qwen3_5_35b_a3b_base_highlight_batch1245_epoch4/checkpoint-1000-merged/latest_checkpointed_iteration.txt b/posttrain_models/qwen3_5_35b_a3b_base_highlight_batch1245_epoch4/checkpoint-1000-merged/latest_checkpointed_iteration.txt new file mode 100644 index 0000000000000000000000000000000000000000..e37d32abba426c06b752a5e53f48c595c84e9270 --- /dev/null +++ b/posttrain_models/qwen3_5_35b_a3b_base_highlight_batch1245_epoch4/checkpoint-1000-merged/latest_checkpointed_iteration.txt @@ -0,0 +1 @@ +1000 \ No newline at end of file diff --git a/posttrain_models/qwen3_5_35b_a3b_base_highlight_batch1245_epoch4/checkpoint-1000-merged/preprocessor_config.json b/posttrain_models/qwen3_5_35b_a3b_base_highlight_batch1245_epoch4/checkpoint-1000-merged/preprocessor_config.json new file mode 100644 index 0000000000000000000000000000000000000000..2ea84a437d448ff71b08df68fdd949d5cc4ebb64 --- /dev/null +++ b/posttrain_models/qwen3_5_35b_a3b_base_highlight_batch1245_epoch4/checkpoint-1000-merged/preprocessor_config.json @@ -0,0 +1,21 @@ +{ + "size": { + "longest_edge": 16777216, + "shortest_edge": 65536 + }, + "patch_size": 16, + "temporal_patch_size": 2, + "merge_size": 2, + "image_mean": [ + 0.5, + 0.5, + 0.5 + ], + "image_std": [ + 0.5, + 0.5, + 0.5 + ], + "processor_class": "Qwen3VLProcessor", + "image_processor_type": "Qwen2VLImageProcessorFast" +} \ No newline at end of file diff --git a/posttrain_models/qwen3_5_35b_a3b_base_highlight_batch1245_epoch4/checkpoint-1000-merged/processor_config.json b/posttrain_models/qwen3_5_35b_a3b_base_highlight_batch1245_epoch4/checkpoint-1000-merged/processor_config.json new file mode 100644 index 0000000000000000000000000000000000000000..7ad6acdf4203f22b7b990e36ccc3a1fe38563d5e --- /dev/null +++ b/posttrain_models/qwen3_5_35b_a3b_base_highlight_batch1245_epoch4/checkpoint-1000-merged/processor_config.json @@ -0,0 +1,63 @@ +{ + "image_processor": { + "data_format": "channels_first", + "do_convert_rgb": true, + "do_normalize": true, + "do_rescale": true, + "do_resize": true, + "image_mean": [ + 0.5, + 0.5, + 0.5 + ], + "image_processor_type": "Qwen2VLImageProcessorFast", + "image_std": [ + 0.5, + 0.5, + 0.5 + ], + "merge_size": 2, + "patch_size": 16, + "resample": 3, + "rescale_factor": 0.00392156862745098, + "size": { + "longest_edge": 16777216, + "shortest_edge": 65536 + }, + "temporal_patch_size": 2 + }, + "processor_class": "Qwen3VLProcessor", + "video_processor": { + "data_format": "channels_first", + "default_to_square": true, + "do_convert_rgb": true, + "do_normalize": true, + "do_rescale": true, + "do_resize": true, + "do_sample_frames": true, + "fps": 2, + "image_mean": [ + 0.5, + 0.5, + 0.5 + ], + "image_std": [ + 0.5, + 0.5, + 0.5 + ], + "max_frames": 768, + "merge_size": 2, + "min_frames": 4, + "patch_size": 16, + "resample": 3, + "rescale_factor": 0.00392156862745098, + "return_metadata": false, + "size": { + "longest_edge": 25165824, + "shortest_edge": 4096 + }, + "temporal_patch_size": 2, + "video_processor_type": "Qwen3VLVideoProcessor" + } +} diff --git a/posttrain_models/qwen3_5_35b_a3b_base_highlight_batch1245_epoch4/checkpoint-1000-merged/tokenizer_config.json b/posttrain_models/qwen3_5_35b_a3b_base_highlight_batch1245_epoch4/checkpoint-1000-merged/tokenizer_config.json new file mode 100644 index 0000000000000000000000000000000000000000..a068e2468cff426a9b105006e74e044030a6faf4 --- /dev/null +++ b/posttrain_models/qwen3_5_35b_a3b_base_highlight_batch1245_epoch4/checkpoint-1000-merged/tokenizer_config.json @@ -0,0 +1,32 @@ +{ + "add_prefix_space": false, + "audio_bos_token": "<|audio_start|>", + "audio_eos_token": "<|audio_end|>", + "audio_token": "<|audio_pad|>", + "backend": "tokenizers", + "bos_token": null, + "clean_up_tokenization_spaces": false, + "eos_token": "<|im_end|>", + "errors": "replace", + "image_token": "<|image_pad|>", + "is_local": true, + "model_max_length": 262144, + "model_specific_special_tokens": { + "audio_bos_token": "<|audio_start|>", + "audio_eos_token": "<|audio_end|>", + "audio_token": "<|audio_pad|>", + "image_token": "<|image_pad|>", + "video_token": "<|video_pad|>", + "vision_bos_token": "<|vision_start|>", + "vision_eos_token": "<|vision_end|>" + }, + "pad_token": "<|endoftext|>", + "pretokenize_regex": "(?i:'s|'t|'re|'ve|'m|'ll|'d)|[^\\r\\n\\p{L}\\p{N}]?[\\p{L}\\p{M}]+|\\p{N}| ?[^\\s\\p{L}\\p{M}\\p{N}]+[\\r\\n]*|\\s*[\\r\\n]+|\\s+(?!\\S)|\\s+", + "processor_class": "Qwen3VLProcessor", + "split_special_tokens": false, + "tokenizer_class": "TokenizersBackend", + "unk_token": null, + "video_token": "<|video_pad|>", + "vision_bos_token": "<|vision_start|>", + "vision_eos_token": "<|vision_end|>" +} diff --git a/posttrain_models/qwen3_5_35b_a3b_base_highlight_batch1245_epoch4/checkpoint-1000/args.json b/posttrain_models/qwen3_5_35b_a3b_base_highlight_batch1245_epoch4/checkpoint-1000/args.json new file mode 100644 index 0000000000000000000000000000000000000000..99623dd80e3398e11296749c2daf861d8dff0018 --- /dev/null +++ b/posttrain_models/qwen3_5_35b_a3b_base_highlight_batch1245_epoch4/checkpoint-1000/args.json @@ -0,0 +1,398 @@ +{ + "use_ray": false, + "ray_exp_name": null, + "device_groups": null, + "model": "checkpoints/Qwen3.5-35B-A3B", + "model_type": "qwen3_5_moe", + "model_revision": null, + "task_type": "causal_lm", + "torch_dtype": "bfloat16", + "attn_impl": null, + "experts_impl": null, + "new_special_tokens": [], + "num_labels": null, + "problem_type": null, + "rope_scaling": null, + "device_map": null, + "max_memory": {}, + "max_model_len": null, + "local_repo_path": null, + "init_strategy": null, + "template": "qwen3_5", + "system": null, + "max_length": 8192, + "truncation_strategy": "delete", + "max_pixels": null, + "agent_template": null, + "norm_bbox": null, + "use_chat_template": true, + "padding_side": "right", + "padding_free": true, + "loss_scale": "ignore_empty_think", + "sequence_parallel_size": 1, + "template_backend": "swift", + "response_prefix": "", + "enable_thinking": false, + "add_non_thinking_prefix": false, + "dataset": [ + "batch1", + "batch2", + "batch4_part1", + "batch4_part2", + "batch4_part3", + "batch5_part1", + "batch5_part2", + "batch5_part3" + ], + "val_dataset": [], + "cached_dataset": [], + "cached_val_dataset": [], + "split_dataset_ratio": 0.0, + "data_seed": 42, + "dataset_num_proc": 2, + "load_from_cache_file": false, + "dataset_shuffle": true, + "val_dataset_shuffle": false, + "streaming": false, + "interleave_prob": null, + "stopping_strategy": "first_exhausted", + "shuffle_buffer_size": 1000, + "download_mode": "reuse_dataset_if_exists", + "columns": {}, + "strict": false, + "remove_unused_columns": true, + "model_name": null, + "model_author": null, + "custom_dataset_info": [], + "quant_method": null, + "quant_bits": null, + "hqq_axis": null, + "bnb_4bit_compute_dtype": "bfloat16", + "bnb_4bit_quant_type": "nf4", + "bnb_4bit_use_double_quant": true, + "bnb_4bit_quant_storage": null, + "max_new_tokens": null, + "temperature": 0.9, + "top_k": 50, + "top_p": 0.9, + "repetition_penalty": 1.0, + "num_beams": 1, + "stream": false, + "stop_words": [], + "logprobs": false, + "top_logprobs": null, + "structured_outputs_regex": null, + "tuner_backend": "peft", + "tuner_type": "lora", + "train_type": "lora", + "adapters": [], + "external_plugins": [ + "ms-swift/base_highlight_video_dataset.py" + ], + "custom_register_path": [ + "ms-swift/base_highlight_video_dataset.py" + ], + "seed": 42, + "model_kwargs": {}, + "load_args": false, + "load_data_args": false, + "packing": false, + "packing_length": null, + "packing_num_proc": 1, + "lazy_tokenize": true, + "use_hf": false, + "hub_token": null, + "ddp_timeout": 18000000, + "ddp_backend": "nccl", + "ignore_args_error": false, + "use_swift_lora": false, + "freeze_llm": false, + "freeze_vit": true, + "freeze_aligner": true, + "freeze_parameters": [], + "freeze_parameters_regex": null, + "freeze_parameters_ratio": 0.0, + "trainable_parameters": [], + "trainable_parameters_regex": null, + "target_modules": [ + "all-linear" + ], + "target_regex": null, + "modules_to_save": [], + "lora_rank": 8, + "lora_alpha": 32, + "lora_dropout": 0.05, + "lora_bias": "none", + "lora_dtype": null, + "use_rslora": false, + "rlhf_type": null, + "loss_type": null, + "mcore_ref_model": null, + "mcore_ref_adapter": null, + "beta": null, + "rpo_alpha": null, + "reference_free": false, + "label_smoothing": 0.0, + "f_divergence_type": "reverse_kl", + "desirable_weight": 1.0, + "undesirable_weight": 1.0, + "calculate_KL": null, + "center_rewards_coefficient": null, + "teacher_model": null, + "teacher_model_type": null, + "teacher_model_revision": null, + "teacher_model_server": null, + "gkd_logits_topk": null, + "lmbda": 0.5, + "seq_kd": false, + "offload_teacher_model": false, + "sft_alpha": 0.0, + "generation_batch_size": null, + "steps_per_generation": null, + "num_generations": 8, + "num_generations_eval": null, + "max_completion_length": 512, + "importance_sampling_level": "token", + "tau_pos": 1.0, + "tau_neg": 1.05, + "epsilon": 0.2, + "epsilon_high": null, + "delta": null, + "use_vllm": true, + "vllm_mode": null, + "vllm_enable_prefix_caching": true, + "vllm_gpu_memory_utilization": 0.9, + "vllm_tensor_parallel_size": 1, + "vllm_max_model_len": null, + "vllm_enforce_eager": false, + "vllm_limit_mm_per_prompt": null, + "vllm_disable_cascade_attn": false, + "vllm_max_num_seqs": null, + "vllm_mm_processor_cache_gb": null, + "vllm_engine_kwargs": null, + "sleep_level": 0, + "offload_optimizer": false, + "offload_model": false, + "offload_bridge": false, + "vllm_server_base_url": null, + "vllm_server_host": null, + "vllm_server_port": [ + 8000 + ], + "vllm_server_timeout": 240.0, + "vllm_server_group_port": null, + "reward_funcs": [], + "reward_weights": null, + "cosine_min_len_value_wrong": -0.5, + "cosine_max_len_value_wrong": 0.0, + "cosine_min_len_value_correct": 1.0, + "cosine_max_len_value_correct": 0.5, + "cosine_max_len": null, + "repetition_n_grams": 3, + "repetition_max_penalty": -1.0, + "soft_max_length": null, + "soft_cache_length": null, + "dynamic_sample": false, + "max_resample_times": 3, + "overlong_filter": false, + "scale_rewards": "group", + "advantage_estimator": "grpo", + "kl_in_reward": false, + "wandb_log_unique_prompts": null, + "log_completions": false, + "rollout_importance_sampling_mode": null, + "rollout_importance_sampling_threshold": 2.0, + "log_rollout_offpolicy_metrics": false, + "off_policy_sequence_mask_delta": null, + "log_entropy": false, + "top_entropy_quantile": 1.0, + "reward_model": null, + "reward_model_plugin": null, + "sync_ref_model": false, + "ref_model_sync_steps": 512, + "ref_model_mixup_alpha": 0.6, + "async_generate": false, + "move_model_batches": null, + "multi_turn_scheduler": null, + "max_turns": null, + "completion_length_limit_scope": "per_round", + "vllm_server_pass_dataset": false, + "num_iterations": 1, + "micro_batch_size": 8, + "global_batch_size": 64, + "recompute_granularity": "full", + "recompute_method": "uniform", + "recompute_num_layers": 1, + "recompute_modules": [ + "core_attn" + ], + "train_iters": null, + "num_train_epochs": 4, + "masked_softmax_fusion": true, + "bias_dropout_fusion": true, + "bias_activation_fusion": true, + "apply_rope_fusion": false, + "gradient_accumulation_fusion": false, + "cross_entropy_loss_fusion": true, + "cross_entropy_fusion_impl": "te", + "calculate_per_token_loss": true, + "attention_backend": "", + "optimizer": "adam", + "optimizer_cpu_offload": false, + "optimizer_offload_fraction": 1.0, + "use_precision_aware_optimizer": false, + "main_grads_dtype": "float32", + "main_params_dtype": "float32", + "exp_avg_dtype": "float32", + "exp_avg_sq_dtype": "float32", + "manual_gc": false, + "manual_gc_steps": 0, + "manual_gc_eval": true, + "train_dataloader_shuffle": true, + "dataloader_num_workers": 1, + "dataloader_pin_memory": true, + "dataloader_persistent_workers": true, + "dataloader_prefetch_factor": 2, + "data_sharding": false, + "group_by_length": false, + "te_rng_tracker": false, + "data_parallel_random_init": false, + "mlp_padding_free": false, + "lr_warmup_init": 0.0, + "lr": 0.0001, + "lr_decay_style": "cosine", + "lr_decay_iters": null, + "lr_warmup_iters": 0, + "lr_warmup_fraction": 0.05, + "min_lr": 1e-05, + "lr_wsd_decay_style": "exponential", + "lr_wsd_decay_iters": null, + "weight_decay": 0.1, + "weight_decay_incr_style": "constant", + "start_weight_decay": 0.1, + "end_weight_decay": 0.1, + "clip_grad": 1.0, + "adam_beta1": 0.9, + "adam_beta2": 0.95, + "adam_eps": 1e-08, + "sgd_momentum": 0.9, + "output_dir": "/mnt/bn/strategy-mllm-train/user/tanyue/models/posttrain_models/qwen3_5_35b_a3b_base_highlight_batch1245_epoch4", + "save_steps": 500, + "no_save_optim": true, + "no_save_rng": true, + "mcore_model": null, + "mcore_adapter": null, + "no_load_optim": false, + "no_load_rng": false, + "finetune": true, + "perform_initialization": false, + "use_cpu_initialization": false, + "async_save": false, + "save_total_limit": null, + "metric_for_best_model": "loss", + "greater_is_better": false, + "use_persistent_ckpt_worker": false, + "dist_ckpt_save_pre_mcore_014": false, + "dist_ckpt_optim_fully_reshardable": false, + "distrib_optim_fully_reshardable_mem_efficient": false, + "local_rank": 0, + "use_distributed_optimizer": true, + "tensor_model_parallel_size": 1, + "pipeline_model_parallel_size": 2, + "decoder_first_pipeline_num_layers": null, + "decoder_last_pipeline_num_layers": null, + "account_for_embedding_in_pipeline_split": false, + "account_for_loss_in_pipeline_split": false, + "overlap_p2p_comm": false, + "align_param_gather": false, + "sequence_parallel": false, + "context_parallel_size": 1, + "tp_comm_overlap": false, + "overlap_grad_reduce": false, + "overlap_param_gather": false, + "overlap_param_gather_with_optimizer_step": false, + "align_grad_reduce": true, + "virtual_pipeline_model_parallel_size": null, + "microbatch_group_size_per_vp_stage": null, + "pipeline_model_parallel_layout": null, + "expert_model_parallel_size": 4, + "expert_tensor_parallel_size": 1, + "report_to": [ + "wandb" + ], + "logging_steps": 5, + "tensorboard_dir": "/mnt/bn/strategy-mllm-train/user/tanyue/models/posttrain_models/qwen3_5_35b_a3b_base_highlight_batch1245_epoch4/runs", + "tensorboard_queue_size": 50, + "wandb_project": "highlight_base_highlight_qwen3_5", + "wandb_exp_name": "qwen3_5_35b_a3b_base_highlight_batch1245_epoch4", + "swanlab_project": "megatron-swift", + "swanlab_exp_name": null, + "eval_iters": -1, + "eval_steps": 100000, + "fp8_format": null, + "fp8_recipe": "delayed", + "fp8_amax_history_len": 1024, + "fp8_amax_compute_algo": "max", + "fp8_param_gather": false, + "fp16": false, + "bf16": true, + "apply_query_key_layer_scaling": false, + "attention_softmax_in_fp32": true, + "accumulate_allreduce_grads_in_fp32": true, + "moe_router_load_balancing_type": null, + "moe_router_dtype": "fp32", + "moe_token_dispatcher_type": null, + "moe_enable_deepep": false, + "moe_grouped_gemm": true, + "moe_permute_fusion": true, + "moe_aux_loss_coeff": 1e-06, + "moe_z_loss_coeff": null, + "moe_shared_expert_overlap": true, + "moe_layer_recompute": false, + "moe_expert_capacity_factor": null, + "moe_pad_expert_input_to_capacity": false, + "moe_token_drop_policy": null, + "mtp_num_layers": null, + "mtp_loss_scaling_factor": 0.1, + "save_safetensors": true, + "ref_model": null, + "ref_adapters": [], + "merge_lora": true, + "max_shard_size": "5GB", + "vit_gradient_checkpointing": false, + "vit_lr": null, + "aligner_lr": null, + "gradient_checkpointing_kwargs": null, + "check_model": true, + "apply_wd_to_qk_layernorm": false, + "enable_dft_loss": false, + "enable_channel_loss": false, + "save_strategy": "steps", + "callbacks": [ + "print", + "default_flow", + "wandb" + ], + "add_version": false, + "create_checkpoint_symlink": false, + "swift_version": "4.0.1", + "ckpt_dir": null, + "rank": 0, + "global_world_size": 8, + "local_world_size": 8, + "model_suffix": "Qwen3.5-35B-A3B", + "model_info": "ModelInfo(model_type='qwen3_5_moe', model_dir='/opt/tiger/tanyue/highlight_tools/checkpoints/Qwen3.5-35B-A3B', torch_dtype=torch.bfloat16, max_model_len=262144, quant_method=None, quant_bits=None, rope_scaling=None, is_moe_model=True, is_multimodal=True, config=None, task_type='causal_lm', num_labels=None)", + "model_meta": "ModelMeta(model_type='qwen3_5_moe', model_groups=[ModelGroup(models=[Model(ms_model_id='Qwen/Qwen3.5-35B-A3B-Base', hf_model_id='Qwen/Qwen3.5-35B-A3B-Base', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen3.5-35B-A3B', hf_model_id='Qwen/Qwen3.5-35B-A3B', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen3.5-122B-A10B', hf_model_id='Qwen/Qwen3.5-122B-A10B', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen3.5-397B-A17B', hf_model_id='Qwen/Qwen3.5-397B-A17B', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen3.5-35B-A3B-FP8', hf_model_id='Qwen/Qwen3.5-35B-A3B-FP8', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen3.5-122B-A10B-FP8', hf_model_id='Qwen/Qwen3.5-122B-A10B-FP8', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen3.5-397B-A17B-FP8', hf_model_id='Qwen/Qwen3.5-397B-A17B-FP8', model_path=None, ms_revision=None, hf_revision=None)], template='qwen3_5', ignore_patterns=None, requires=None, tags=[])], loader=, template='qwen3_5', model_arch=MultiModelKeys(arch_name='qwen2_vl', embedding=None, module_list=None, lm_head=None, q_proj=None, k_proj=None, v_proj=None, o_proj=None, attention=None, mlp=None, down_proj=None, qkv_proj=None, qk_proj=None, qa_proj=None, qb_proj=None, kv_proj=None, kva_proj=None, kvb_proj=None, language_model=['model.language_model', 'lm_head'], aligner=['model.visual.merger'], vision_tower=['model.visual'], generator=[]), architectures=['Qwen3_5MoeForConditionalGeneration'], additional_saved_files=[], torch_dtype=None, is_multimodal=True, is_reward=False, task_type=None, ignore_patterns=None, requires=['transformers>=5.2.0', 'qwen_vl_utils>=0.0.14', 'decord'], tags=[])", + "model_dir": "/opt/tiger/tanyue/highlight_tools/checkpoints/Qwen3.5-35B-A3B", + "template_meta": "QwenTemplateMeta(template_type='qwen3_5', prefix=[], prompt=['<|im_start|>user\\n{{QUERY}}<|im_end|>\\n<|im_start|>assistant\\n'], chat_sep=['<|im_end|>\\n'], suffix=['<|im_end|>\\n'], template_cls=, system_prefix=['<|im_start|>system\\n{{SYSTEM}}<|im_end|>\\n'], default_system=None, auto_add_bos=False, stop_words=['<|endoftext|>'], agent_template='qwen3_5', is_thinking=True, thinking_prefix='\\n', non_thinking_prefix='\\n\\n\\n\\n', history_thinking_prefix='')", + "_val_dataset_exists": false, + "hub": "", + "seq_length": 8192, + "is_multimodal": true, + "megatron_model_meta": "MegatronModelMeta(megatron_model_type='qwen3_5', model_types=['qwen3_5', 'qwen3_5_moe'], bridge_cls=, visual_cls=, is_multimodal=True, loader=)", + "fp8": null, + "params_dtype": "bfloat16", + "world_size": 8, + "data_parallel_size": 4, + "num_microbatches": 2 +} \ No newline at end of file diff --git a/posttrain_models/qwen3_5_35b_a3b_base_highlight_batch1245_epoch4/checkpoint-1000/iter_0001000/.metadata b/posttrain_models/qwen3_5_35b_a3b_base_highlight_batch1245_epoch4/checkpoint-1000/iter_0001000/.metadata new file mode 100644 index 0000000000000000000000000000000000000000..d076f86b6c71a4db2a839020d4e8a6c97c7b7a69 Binary files /dev/null and b/posttrain_models/qwen3_5_35b_a3b_base_highlight_batch1245_epoch4/checkpoint-1000/iter_0001000/.metadata differ diff --git a/posttrain_models/qwen3_5_35b_a3b_base_highlight_batch1245_epoch4/checkpoint-1000/iter_0001000/common.pt b/posttrain_models/qwen3_5_35b_a3b_base_highlight_batch1245_epoch4/checkpoint-1000/iter_0001000/common.pt new file mode 100644 index 0000000000000000000000000000000000000000..99dbccd0498abbffbe49b8e61dcbc056638de261 --- /dev/null +++ b/posttrain_models/qwen3_5_35b_a3b_base_highlight_batch1245_epoch4/checkpoint-1000/iter_0001000/common.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c3d9304fd899960cfa6473c11c2d329a51fb7a11a6211086c06e929c8b6c2033 +size 15975 diff --git a/posttrain_models/qwen3_5_35b_a3b_base_highlight_batch1245_epoch4/checkpoint-1000/iter_0001000/metadata.json b/posttrain_models/qwen3_5_35b_a3b_base_highlight_batch1245_epoch4/checkpoint-1000/iter_0001000/metadata.json new file mode 100644 index 0000000000000000000000000000000000000000..c9dbde0bcf4d2d993122dcc7d6bcb1eef8b6fb77 --- /dev/null +++ b/posttrain_models/qwen3_5_35b_a3b_base_highlight_batch1245_epoch4/checkpoint-1000/iter_0001000/metadata.json @@ -0,0 +1 @@ +{"sharded_backend": "torch_dist", "sharded_backend_version": 1, "common_backend": "torch", "common_backend_version": 1} \ No newline at end of file diff --git a/posttrain_models/qwen3_5_35b_a3b_base_highlight_batch1245_epoch4/checkpoint-1000/latest_checkpointed_iteration.txt b/posttrain_models/qwen3_5_35b_a3b_base_highlight_batch1245_epoch4/checkpoint-1000/latest_checkpointed_iteration.txt new file mode 100644 index 0000000000000000000000000000000000000000..e37d32abba426c06b752a5e53f48c595c84e9270 --- /dev/null +++ b/posttrain_models/qwen3_5_35b_a3b_base_highlight_batch1245_epoch4/checkpoint-1000/latest_checkpointed_iteration.txt @@ -0,0 +1 @@ +1000 \ No newline at end of file diff --git a/posttrain_models/qwen3_5_35b_a3b_base_highlight_batch1245_epoch4/checkpoint-1500-merged/args.json b/posttrain_models/qwen3_5_35b_a3b_base_highlight_batch1245_epoch4/checkpoint-1500-merged/args.json new file mode 100644 index 0000000000000000000000000000000000000000..99623dd80e3398e11296749c2daf861d8dff0018 --- /dev/null +++ b/posttrain_models/qwen3_5_35b_a3b_base_highlight_batch1245_epoch4/checkpoint-1500-merged/args.json @@ -0,0 +1,398 @@ +{ + "use_ray": false, + "ray_exp_name": null, + "device_groups": null, + "model": "checkpoints/Qwen3.5-35B-A3B", + "model_type": "qwen3_5_moe", + "model_revision": null, + "task_type": "causal_lm", + "torch_dtype": "bfloat16", + "attn_impl": null, + "experts_impl": null, + "new_special_tokens": [], + "num_labels": null, + "problem_type": null, + "rope_scaling": null, + "device_map": null, + "max_memory": {}, + "max_model_len": null, + "local_repo_path": null, + "init_strategy": null, + "template": "qwen3_5", + "system": null, + "max_length": 8192, + "truncation_strategy": "delete", + "max_pixels": null, + "agent_template": null, + "norm_bbox": null, + "use_chat_template": true, + "padding_side": "right", + "padding_free": true, + "loss_scale": "ignore_empty_think", + "sequence_parallel_size": 1, + "template_backend": "swift", + "response_prefix": "", + "enable_thinking": false, + "add_non_thinking_prefix": false, + "dataset": [ + "batch1", + "batch2", + "batch4_part1", + "batch4_part2", + "batch4_part3", + "batch5_part1", + "batch5_part2", + "batch5_part3" + ], + "val_dataset": [], + "cached_dataset": [], + "cached_val_dataset": [], + "split_dataset_ratio": 0.0, + "data_seed": 42, + "dataset_num_proc": 2, + "load_from_cache_file": false, + "dataset_shuffle": true, + "val_dataset_shuffle": false, + "streaming": false, + "interleave_prob": null, + "stopping_strategy": "first_exhausted", + "shuffle_buffer_size": 1000, + "download_mode": "reuse_dataset_if_exists", + "columns": {}, + "strict": false, + "remove_unused_columns": true, + "model_name": null, + "model_author": null, + "custom_dataset_info": [], + "quant_method": null, + "quant_bits": null, + "hqq_axis": null, + "bnb_4bit_compute_dtype": "bfloat16", + "bnb_4bit_quant_type": "nf4", + "bnb_4bit_use_double_quant": true, + "bnb_4bit_quant_storage": null, + "max_new_tokens": null, + "temperature": 0.9, + "top_k": 50, + "top_p": 0.9, + "repetition_penalty": 1.0, + "num_beams": 1, + "stream": false, + "stop_words": [], + "logprobs": false, + "top_logprobs": null, + "structured_outputs_regex": null, + "tuner_backend": "peft", + "tuner_type": "lora", + "train_type": "lora", + "adapters": [], + "external_plugins": [ + "ms-swift/base_highlight_video_dataset.py" + ], + "custom_register_path": [ + "ms-swift/base_highlight_video_dataset.py" + ], + "seed": 42, + "model_kwargs": {}, + "load_args": false, + "load_data_args": false, + "packing": false, + "packing_length": null, + "packing_num_proc": 1, + "lazy_tokenize": true, + "use_hf": false, + "hub_token": null, + "ddp_timeout": 18000000, + "ddp_backend": "nccl", + "ignore_args_error": false, + "use_swift_lora": false, + "freeze_llm": false, + "freeze_vit": true, + "freeze_aligner": true, + "freeze_parameters": [], + "freeze_parameters_regex": null, + "freeze_parameters_ratio": 0.0, + "trainable_parameters": [], + "trainable_parameters_regex": null, + "target_modules": [ + "all-linear" + ], + "target_regex": null, + "modules_to_save": [], + "lora_rank": 8, + "lora_alpha": 32, + "lora_dropout": 0.05, + "lora_bias": "none", + "lora_dtype": null, + "use_rslora": false, + "rlhf_type": null, + "loss_type": null, + "mcore_ref_model": null, + "mcore_ref_adapter": null, + "beta": null, + "rpo_alpha": null, + "reference_free": false, + "label_smoothing": 0.0, + "f_divergence_type": "reverse_kl", + "desirable_weight": 1.0, + "undesirable_weight": 1.0, + "calculate_KL": null, + "center_rewards_coefficient": null, + "teacher_model": null, + "teacher_model_type": null, + "teacher_model_revision": null, + "teacher_model_server": null, + "gkd_logits_topk": null, + "lmbda": 0.5, + "seq_kd": false, + "offload_teacher_model": false, + "sft_alpha": 0.0, + "generation_batch_size": null, + "steps_per_generation": null, + "num_generations": 8, + "num_generations_eval": null, + "max_completion_length": 512, + "importance_sampling_level": "token", + "tau_pos": 1.0, + "tau_neg": 1.05, + "epsilon": 0.2, + "epsilon_high": null, + "delta": null, + "use_vllm": true, + "vllm_mode": null, + "vllm_enable_prefix_caching": true, + "vllm_gpu_memory_utilization": 0.9, + "vllm_tensor_parallel_size": 1, + "vllm_max_model_len": null, + "vllm_enforce_eager": false, + "vllm_limit_mm_per_prompt": null, + "vllm_disable_cascade_attn": false, + "vllm_max_num_seqs": null, + "vllm_mm_processor_cache_gb": null, + "vllm_engine_kwargs": null, + "sleep_level": 0, + "offload_optimizer": false, + "offload_model": false, + "offload_bridge": false, + "vllm_server_base_url": null, + "vllm_server_host": null, + "vllm_server_port": [ + 8000 + ], + "vllm_server_timeout": 240.0, + "vllm_server_group_port": null, + "reward_funcs": [], + "reward_weights": null, + "cosine_min_len_value_wrong": -0.5, + "cosine_max_len_value_wrong": 0.0, + "cosine_min_len_value_correct": 1.0, + "cosine_max_len_value_correct": 0.5, + "cosine_max_len": null, + "repetition_n_grams": 3, + "repetition_max_penalty": -1.0, + "soft_max_length": null, + "soft_cache_length": null, + "dynamic_sample": false, + "max_resample_times": 3, + "overlong_filter": false, + "scale_rewards": "group", + "advantage_estimator": "grpo", + "kl_in_reward": false, + "wandb_log_unique_prompts": null, + "log_completions": false, + "rollout_importance_sampling_mode": null, + "rollout_importance_sampling_threshold": 2.0, + "log_rollout_offpolicy_metrics": false, + "off_policy_sequence_mask_delta": null, + "log_entropy": false, + "top_entropy_quantile": 1.0, + "reward_model": null, + "reward_model_plugin": null, + "sync_ref_model": false, + "ref_model_sync_steps": 512, + "ref_model_mixup_alpha": 0.6, + "async_generate": false, + "move_model_batches": null, + "multi_turn_scheduler": null, + "max_turns": null, + "completion_length_limit_scope": "per_round", + "vllm_server_pass_dataset": false, + "num_iterations": 1, + "micro_batch_size": 8, + "global_batch_size": 64, + "recompute_granularity": "full", + "recompute_method": "uniform", + "recompute_num_layers": 1, + "recompute_modules": [ + "core_attn" + ], + "train_iters": null, + "num_train_epochs": 4, + "masked_softmax_fusion": true, + "bias_dropout_fusion": true, + "bias_activation_fusion": true, + "apply_rope_fusion": false, + "gradient_accumulation_fusion": false, + "cross_entropy_loss_fusion": true, + "cross_entropy_fusion_impl": "te", + "calculate_per_token_loss": true, + "attention_backend": "", + "optimizer": "adam", + "optimizer_cpu_offload": false, + "optimizer_offload_fraction": 1.0, + "use_precision_aware_optimizer": false, + "main_grads_dtype": "float32", + "main_params_dtype": "float32", + "exp_avg_dtype": "float32", + "exp_avg_sq_dtype": "float32", + "manual_gc": false, + "manual_gc_steps": 0, + "manual_gc_eval": true, + "train_dataloader_shuffle": true, + "dataloader_num_workers": 1, + "dataloader_pin_memory": true, + "dataloader_persistent_workers": true, + "dataloader_prefetch_factor": 2, + "data_sharding": false, + "group_by_length": false, + "te_rng_tracker": false, + "data_parallel_random_init": false, + "mlp_padding_free": false, + "lr_warmup_init": 0.0, + "lr": 0.0001, + "lr_decay_style": "cosine", + "lr_decay_iters": null, + "lr_warmup_iters": 0, + "lr_warmup_fraction": 0.05, + "min_lr": 1e-05, + "lr_wsd_decay_style": "exponential", + "lr_wsd_decay_iters": null, + "weight_decay": 0.1, + "weight_decay_incr_style": "constant", + "start_weight_decay": 0.1, + "end_weight_decay": 0.1, + "clip_grad": 1.0, + "adam_beta1": 0.9, + "adam_beta2": 0.95, + "adam_eps": 1e-08, + "sgd_momentum": 0.9, + "output_dir": "/mnt/bn/strategy-mllm-train/user/tanyue/models/posttrain_models/qwen3_5_35b_a3b_base_highlight_batch1245_epoch4", + "save_steps": 500, + "no_save_optim": true, + "no_save_rng": true, + "mcore_model": null, + "mcore_adapter": null, + "no_load_optim": false, + "no_load_rng": false, + "finetune": true, + "perform_initialization": false, + "use_cpu_initialization": false, + "async_save": false, + "save_total_limit": null, + "metric_for_best_model": "loss", + "greater_is_better": false, + "use_persistent_ckpt_worker": false, + "dist_ckpt_save_pre_mcore_014": false, + "dist_ckpt_optim_fully_reshardable": false, + "distrib_optim_fully_reshardable_mem_efficient": false, + "local_rank": 0, + "use_distributed_optimizer": true, + "tensor_model_parallel_size": 1, + "pipeline_model_parallel_size": 2, + "decoder_first_pipeline_num_layers": null, + "decoder_last_pipeline_num_layers": null, + "account_for_embedding_in_pipeline_split": false, + "account_for_loss_in_pipeline_split": false, + "overlap_p2p_comm": false, + "align_param_gather": false, + "sequence_parallel": false, + "context_parallel_size": 1, + "tp_comm_overlap": false, + "overlap_grad_reduce": false, + "overlap_param_gather": false, + "overlap_param_gather_with_optimizer_step": false, + "align_grad_reduce": true, + "virtual_pipeline_model_parallel_size": null, + "microbatch_group_size_per_vp_stage": null, + "pipeline_model_parallel_layout": null, + "expert_model_parallel_size": 4, + "expert_tensor_parallel_size": 1, + "report_to": [ + "wandb" + ], + "logging_steps": 5, + "tensorboard_dir": "/mnt/bn/strategy-mllm-train/user/tanyue/models/posttrain_models/qwen3_5_35b_a3b_base_highlight_batch1245_epoch4/runs", + "tensorboard_queue_size": 50, + "wandb_project": "highlight_base_highlight_qwen3_5", + "wandb_exp_name": "qwen3_5_35b_a3b_base_highlight_batch1245_epoch4", + "swanlab_project": "megatron-swift", + "swanlab_exp_name": null, + "eval_iters": -1, + "eval_steps": 100000, + "fp8_format": null, + "fp8_recipe": "delayed", + "fp8_amax_history_len": 1024, + "fp8_amax_compute_algo": "max", + "fp8_param_gather": false, + "fp16": false, + "bf16": true, + "apply_query_key_layer_scaling": false, + "attention_softmax_in_fp32": true, + "accumulate_allreduce_grads_in_fp32": true, + "moe_router_load_balancing_type": null, + "moe_router_dtype": "fp32", + "moe_token_dispatcher_type": null, + "moe_enable_deepep": false, + "moe_grouped_gemm": true, + "moe_permute_fusion": true, + "moe_aux_loss_coeff": 1e-06, + "moe_z_loss_coeff": null, + "moe_shared_expert_overlap": true, + "moe_layer_recompute": false, + "moe_expert_capacity_factor": null, + "moe_pad_expert_input_to_capacity": false, + "moe_token_drop_policy": null, + "mtp_num_layers": null, + "mtp_loss_scaling_factor": 0.1, + "save_safetensors": true, + "ref_model": null, + "ref_adapters": [], + "merge_lora": true, + "max_shard_size": "5GB", + "vit_gradient_checkpointing": false, + "vit_lr": null, + "aligner_lr": null, + "gradient_checkpointing_kwargs": null, + "check_model": true, + "apply_wd_to_qk_layernorm": false, + "enable_dft_loss": false, + "enable_channel_loss": false, + "save_strategy": "steps", + "callbacks": [ + "print", + "default_flow", + "wandb" + ], + "add_version": false, + "create_checkpoint_symlink": false, + "swift_version": "4.0.1", + "ckpt_dir": null, + "rank": 0, + "global_world_size": 8, + "local_world_size": 8, + "model_suffix": "Qwen3.5-35B-A3B", + "model_info": "ModelInfo(model_type='qwen3_5_moe', model_dir='/opt/tiger/tanyue/highlight_tools/checkpoints/Qwen3.5-35B-A3B', torch_dtype=torch.bfloat16, max_model_len=262144, quant_method=None, quant_bits=None, rope_scaling=None, is_moe_model=True, is_multimodal=True, config=None, task_type='causal_lm', num_labels=None)", + "model_meta": "ModelMeta(model_type='qwen3_5_moe', model_groups=[ModelGroup(models=[Model(ms_model_id='Qwen/Qwen3.5-35B-A3B-Base', hf_model_id='Qwen/Qwen3.5-35B-A3B-Base', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen3.5-35B-A3B', hf_model_id='Qwen/Qwen3.5-35B-A3B', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen3.5-122B-A10B', hf_model_id='Qwen/Qwen3.5-122B-A10B', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen3.5-397B-A17B', hf_model_id='Qwen/Qwen3.5-397B-A17B', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen3.5-35B-A3B-FP8', hf_model_id='Qwen/Qwen3.5-35B-A3B-FP8', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen3.5-122B-A10B-FP8', hf_model_id='Qwen/Qwen3.5-122B-A10B-FP8', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen3.5-397B-A17B-FP8', hf_model_id='Qwen/Qwen3.5-397B-A17B-FP8', model_path=None, ms_revision=None, hf_revision=None)], template='qwen3_5', ignore_patterns=None, requires=None, tags=[])], loader=, template='qwen3_5', model_arch=MultiModelKeys(arch_name='qwen2_vl', embedding=None, module_list=None, lm_head=None, q_proj=None, k_proj=None, v_proj=None, o_proj=None, attention=None, mlp=None, down_proj=None, qkv_proj=None, qk_proj=None, qa_proj=None, qb_proj=None, kv_proj=None, kva_proj=None, kvb_proj=None, language_model=['model.language_model', 'lm_head'], aligner=['model.visual.merger'], vision_tower=['model.visual'], generator=[]), architectures=['Qwen3_5MoeForConditionalGeneration'], additional_saved_files=[], torch_dtype=None, is_multimodal=True, is_reward=False, task_type=None, ignore_patterns=None, requires=['transformers>=5.2.0', 'qwen_vl_utils>=0.0.14', 'decord'], tags=[])", + "model_dir": "/opt/tiger/tanyue/highlight_tools/checkpoints/Qwen3.5-35B-A3B", + "template_meta": "QwenTemplateMeta(template_type='qwen3_5', prefix=[], prompt=['<|im_start|>user\\n{{QUERY}}<|im_end|>\\n<|im_start|>assistant\\n'], chat_sep=['<|im_end|>\\n'], suffix=['<|im_end|>\\n'], template_cls=, system_prefix=['<|im_start|>system\\n{{SYSTEM}}<|im_end|>\\n'], default_system=None, auto_add_bos=False, stop_words=['<|endoftext|>'], agent_template='qwen3_5', is_thinking=True, thinking_prefix='\\n', non_thinking_prefix='\\n\\n\\n\\n', history_thinking_prefix='')", + "_val_dataset_exists": false, + "hub": "", + "seq_length": 8192, + "is_multimodal": true, + "megatron_model_meta": "MegatronModelMeta(megatron_model_type='qwen3_5', model_types=['qwen3_5', 'qwen3_5_moe'], bridge_cls=, visual_cls=, is_multimodal=True, loader=)", + "fp8": null, + "params_dtype": "bfloat16", + "world_size": 8, + "data_parallel_size": 4, + "num_microbatches": 2 +} \ No newline at end of file diff --git a/posttrain_models/qwen3_5_35b_a3b_base_highlight_batch1245_epoch4/checkpoint-1500-merged/chat_template.jinja b/posttrain_models/qwen3_5_35b_a3b_base_highlight_batch1245_epoch4/checkpoint-1500-merged/chat_template.jinja new file mode 100644 index 0000000000000000000000000000000000000000..a585dec894e63da457d9440ec6aa7caa16d20860 --- /dev/null +++ b/posttrain_models/qwen3_5_35b_a3b_base_highlight_batch1245_epoch4/checkpoint-1500-merged/chat_template.jinja @@ -0,0 +1,154 @@ +{%- set image_count = namespace(value=0) %} +{%- set video_count = namespace(value=0) %} +{%- macro render_content(content, do_vision_count, is_system_content=false) %} + {%- if content is string %} + {{- content }} + {%- elif content is iterable and content is not mapping %} + {%- for item in content %} + {%- if 'image' in item or 'image_url' in item or item.type == 'image' %} + {%- if is_system_content %} + {{- raise_exception('System message cannot contain images.') }} + {%- endif %} + {%- if do_vision_count %} + {%- set image_count.value = image_count.value + 1 %} + {%- endif %} + {%- if add_vision_id %} + {{- 'Picture ' ~ image_count.value ~ ': ' }} + {%- endif %} + {{- '<|vision_start|><|image_pad|><|vision_end|>' }} + {%- elif 'video' in item or item.type == 'video' %} + {%- if is_system_content %} + {{- raise_exception('System message cannot contain videos.') }} + {%- endif %} + {%- if do_vision_count %} + {%- set video_count.value = video_count.value + 1 %} + {%- endif %} + {%- if add_vision_id %} + {{- 'Video ' ~ video_count.value ~ ': ' }} + {%- endif %} + {{- '<|vision_start|><|video_pad|><|vision_end|>' }} + {%- elif 'text' in item %} + {{- item.text }} + {%- else %} + {{- raise_exception('Unexpected item type in content.') }} + {%- endif %} + {%- endfor %} + {%- elif content is none or content is undefined %} + {{- '' }} + {%- else %} + {{- raise_exception('Unexpected content type.') }} + {%- endif %} +{%- endmacro %} +{%- if not messages %} + {{- raise_exception('No messages provided.') }} +{%- endif %} +{%- if tools and tools is iterable and tools is not mapping %} + {{- '<|im_start|>system\n' }} + {{- "# Tools\n\nYou have access to the following functions:\n\n" }} + {%- for tool in tools %} + {{- "\n" }} + {{- tool | tojson }} + {%- endfor %} + {{- "\n" }} + {{- '\n\nIf you choose to call a function ONLY reply in the following format with NO suffix:\n\n\n\n\nvalue_1\n\n\nThis is the value for the second parameter\nthat can span\nmultiple lines\n\n\n\n\n\nReminder:\n- Function calls MUST follow the specified format: an inner block must be nested within XML tags\n- Required parameters MUST be specified\n- You may provide optional reasoning for your function call in natural language BEFORE the function call, but NOT after\n- If there is no function call available, answer the question like normal with your current knowledge and do not tell the user about function calls\n' }} + {%- if messages[0].role == 'system' %} + {%- set content = render_content(messages[0].content, false, true)|trim %} + {%- if content %} + {{- '\n\n' + content }} + {%- endif %} + {%- endif %} + {{- '<|im_end|>\n' }} +{%- else %} + {%- if messages[0].role == 'system' %} + {%- set content = render_content(messages[0].content, false, true)|trim %} + {{- '<|im_start|>system\n' + content + '<|im_end|>\n' }} + {%- endif %} +{%- endif %} +{%- set ns = namespace(multi_step_tool=true, last_query_index=messages|length - 1) %} +{%- for message in messages[::-1] %} + {%- set index = (messages|length - 1) - loop.index0 %} + {%- if ns.multi_step_tool and message.role == "user" %} + {%- set content = render_content(message.content, false)|trim %} + {%- if not(content.startswith('') and content.endswith('')) %} + {%- set ns.multi_step_tool = false %} + {%- set ns.last_query_index = index %} + {%- endif %} + {%- endif %} +{%- endfor %} +{%- if ns.multi_step_tool %} + {{- raise_exception('No user query found in messages.') }} +{%- endif %} +{%- for message in messages %} + {%- set content = render_content(message.content, true)|trim %} + {%- if message.role == "system" %} + {%- if not loop.first %} + {{- raise_exception('System message must be at the beginning.') }} + {%- endif %} + {%- elif message.role == "user" %} + {{- '<|im_start|>' + message.role + '\n' + content + '<|im_end|>' + '\n' }} + {%- elif message.role == "assistant" %} + {%- set reasoning_content = '' %} + {%- if message.reasoning_content is string %} + {%- set reasoning_content = message.reasoning_content %} + {%- else %} + {%- if '' in content %} + {%- set reasoning_content = content.split('')[0].rstrip('\n').split('')[-1].lstrip('\n') %} + {%- set content = content.split('')[-1].lstrip('\n') %} + {%- endif %} + {%- endif %} + {%- set reasoning_content = reasoning_content|trim %} + {%- if loop.index0 > ns.last_query_index %} + {{- '<|im_start|>' + message.role + '\n\n' + reasoning_content + '\n\n\n' + content }} + {%- else %} + {{- '<|im_start|>' + message.role + '\n' + content }} + {%- endif %} + {%- if message.tool_calls and message.tool_calls is iterable and message.tool_calls is not mapping %} + {%- for tool_call in message.tool_calls %} + {%- if tool_call.function is defined %} + {%- set tool_call = tool_call.function %} + {%- endif %} + {%- if loop.first %} + {%- if content|trim %} + {{- '\n\n\n\n' }} + {%- else %} + {{- '\n\n' }} + {%- endif %} + {%- else %} + {{- '\n\n\n' }} + {%- endif %} + {%- if tool_call.arguments is defined %} + {%- for args_name, args_value in tool_call.arguments|items %} + {{- '\n' }} + {%- set args_value = args_value | tojson | safe if args_value is mapping or (args_value is sequence and args_value is not string) else args_value | string %} + {{- args_value }} + {{- '\n\n' }} + {%- endfor %} + {%- endif %} + {{- '\n' }} + {%- endfor %} + {%- endif %} + {{- '<|im_end|>\n' }} + {%- elif message.role == "tool" %} + {%- if loop.previtem and loop.previtem.role != "tool" %} + {{- '<|im_start|>user' }} + {%- endif %} + {{- '\n\n' }} + {{- content }} + {{- '\n' }} + {%- if not loop.last and loop.nextitem.role != "tool" %} + {{- '<|im_end|>\n' }} + {%- elif loop.last %} + {{- '<|im_end|>\n' }} + {%- endif %} + {%- else %} + {{- raise_exception('Unexpected message role.') }} + {%- endif %} +{%- endfor %} +{%- if add_generation_prompt %} + {{- '<|im_start|>assistant\n' }} + {%- if enable_thinking is defined and enable_thinking is false %} + {{- '\n\n\n\n' }} + {%- else %} + {{- '\n' }} + {%- endif %} +{%- endif %} \ No newline at end of file diff --git a/posttrain_models/qwen3_5_35b_a3b_base_highlight_batch1245_epoch4/checkpoint-1500-merged/config.json b/posttrain_models/qwen3_5_35b_a3b_base_highlight_batch1245_epoch4/checkpoint-1500-merged/config.json new file mode 100644 index 0000000000000000000000000000000000000000..8643d6d3f2e157c74392e7d91f431a66f9261762 --- /dev/null +++ b/posttrain_models/qwen3_5_35b_a3b_base_highlight_batch1245_epoch4/checkpoint-1500-merged/config.json @@ -0,0 +1,123 @@ +{ + "architectures": [ + "Qwen3_5MoeForConditionalGeneration" + ], + "dtype": "bfloat16", + "hidden_size": 2048, + "image_token_id": 248056, + "model_type": "qwen3_5_moe", + "text_config": { + "attention_bias": false, + "attention_dropout": 0.0, + "attn_output_gate": true, + "bos_token_id": null, + "dtype": "bfloat16", + "eos_token_id": 248044, + "full_attention_interval": 4, + "head_dim": 256, + "hidden_act": "silu", + "hidden_size": 2048, + "initializer_range": 0.02, + "layer_types": [ + "linear_attention", + "linear_attention", + "linear_attention", + "full_attention", + "linear_attention", + "linear_attention", + "linear_attention", + "full_attention", + "linear_attention", + "linear_attention", + "linear_attention", + "full_attention", + "linear_attention", + "linear_attention", + "linear_attention", + "full_attention", + "linear_attention", + "linear_attention", + "linear_attention", + "full_attention", + "linear_attention", + "linear_attention", + "linear_attention", + "full_attention", + "linear_attention", + "linear_attention", + "linear_attention", + "full_attention", + "linear_attention", + "linear_attention", + "linear_attention", + "full_attention", + "linear_attention", + "linear_attention", + "linear_attention", + "full_attention", + "linear_attention", + "linear_attention", + "linear_attention", + "full_attention" + ], + "linear_conv_kernel_dim": 4, + "linear_key_head_dim": 128, + "linear_num_key_heads": 16, + "linear_num_value_heads": 32, + "linear_value_head_dim": 128, + "mamba_ssm_dtype": "float32", + "max_position_embeddings": 262144, + "mlp_only_layers": [], + "model_type": "qwen3_5_moe_text", + "moe_intermediate_size": 512, + "mtp_num_hidden_layers": 1, + "mtp_use_dedicated_embeddings": false, + "num_attention_heads": 16, + "num_experts": 256, + "num_experts_per_tok": 8, + "num_hidden_layers": 40, + "num_key_value_heads": 2, + "output_router_logits": false, + "pad_token_id": null, + "partial_rotary_factor": 0.25, + "rms_norm_eps": 1e-06, + "rope_parameters": { + "mrope_interleaved": true, + "mrope_section": [ + 11, + 11, + 10 + ], + "partial_rotary_factor": 0.25, + "rope_theta": 10000000, + "rope_type": "default" + }, + "router_aux_loss_coef": 0.001, + "shared_expert_intermediate_size": 512, + "tie_word_embeddings": false, + "use_cache": true, + "vocab_size": 248320 + }, + "tie_word_embeddings": false, + "transformers_version": "5.2.0", + "video_token_id": 248057, + "vision_config": { + "deepstack_visual_indexes": [], + "depth": 27, + "dtype": "bfloat16", + "hidden_act": "gelu_pytorch_tanh", + "hidden_size": 1152, + "in_channels": 3, + "initializer_range": 0.02, + "intermediate_size": 4304, + "model_type": "qwen3_5_moe", + "num_heads": 16, + "num_position_embeddings": 2304, + "out_hidden_size": 2048, + "patch_size": 16, + "spatial_merge_size": 2, + "temporal_patch_size": 2 + }, + "vision_end_token_id": 248054, + "vision_start_token_id": 248053 +} diff --git a/posttrain_models/qwen3_5_35b_a3b_base_highlight_batch1245_epoch4/checkpoint-1500-merged/iter_0001500/common.pt b/posttrain_models/qwen3_5_35b_a3b_base_highlight_batch1245_epoch4/checkpoint-1500-merged/iter_0001500/common.pt new file mode 100644 index 0000000000000000000000000000000000000000..0c5d00601028e55feea5cbc55c0a8b3b7c137ac4 --- /dev/null +++ b/posttrain_models/qwen3_5_35b_a3b_base_highlight_batch1245_epoch4/checkpoint-1500-merged/iter_0001500/common.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3c521bbb30467e08142381a06dcbc55112ff4e7f53b6e04db81dcece7a5182a2 +size 15975 diff --git a/posttrain_models/qwen3_5_35b_a3b_base_highlight_batch1245_epoch4/checkpoint-1500-merged/latest_checkpointed_iteration.txt b/posttrain_models/qwen3_5_35b_a3b_base_highlight_batch1245_epoch4/checkpoint-1500-merged/latest_checkpointed_iteration.txt new file mode 100644 index 0000000000000000000000000000000000000000..37021f4a27201e7257a4ac7a992b0b4ebba61a01 --- /dev/null +++ b/posttrain_models/qwen3_5_35b_a3b_base_highlight_batch1245_epoch4/checkpoint-1500-merged/latest_checkpointed_iteration.txt @@ -0,0 +1 @@ +1500 \ No newline at end of file diff --git a/posttrain_models/qwen3_5_35b_a3b_base_highlight_batch1245_epoch4/checkpoint-1500-merged/preprocessor_config.json b/posttrain_models/qwen3_5_35b_a3b_base_highlight_batch1245_epoch4/checkpoint-1500-merged/preprocessor_config.json new file mode 100644 index 0000000000000000000000000000000000000000..2ea84a437d448ff71b08df68fdd949d5cc4ebb64 --- /dev/null +++ b/posttrain_models/qwen3_5_35b_a3b_base_highlight_batch1245_epoch4/checkpoint-1500-merged/preprocessor_config.json @@ -0,0 +1,21 @@ +{ + "size": { + "longest_edge": 16777216, + "shortest_edge": 65536 + }, + "patch_size": 16, + "temporal_patch_size": 2, + "merge_size": 2, + "image_mean": [ + 0.5, + 0.5, + 0.5 + ], + "image_std": [ + 0.5, + 0.5, + 0.5 + ], + "processor_class": "Qwen3VLProcessor", + "image_processor_type": "Qwen2VLImageProcessorFast" +} \ No newline at end of file diff --git a/posttrain_models/qwen3_5_35b_a3b_base_highlight_batch1245_epoch4/checkpoint-1500-merged/processor_config.json b/posttrain_models/qwen3_5_35b_a3b_base_highlight_batch1245_epoch4/checkpoint-1500-merged/processor_config.json new file mode 100644 index 0000000000000000000000000000000000000000..7ad6acdf4203f22b7b990e36ccc3a1fe38563d5e --- /dev/null +++ b/posttrain_models/qwen3_5_35b_a3b_base_highlight_batch1245_epoch4/checkpoint-1500-merged/processor_config.json @@ -0,0 +1,63 @@ +{ + "image_processor": { + "data_format": "channels_first", + "do_convert_rgb": true, + "do_normalize": true, + "do_rescale": true, + "do_resize": true, + "image_mean": [ + 0.5, + 0.5, + 0.5 + ], + "image_processor_type": "Qwen2VLImageProcessorFast", + "image_std": [ + 0.5, + 0.5, + 0.5 + ], + "merge_size": 2, + "patch_size": 16, + "resample": 3, + "rescale_factor": 0.00392156862745098, + "size": { + "longest_edge": 16777216, + "shortest_edge": 65536 + }, + "temporal_patch_size": 2 + }, + "processor_class": "Qwen3VLProcessor", + "video_processor": { + "data_format": "channels_first", + "default_to_square": true, + "do_convert_rgb": true, + "do_normalize": true, + "do_rescale": true, + "do_resize": true, + "do_sample_frames": true, + "fps": 2, + "image_mean": [ + 0.5, + 0.5, + 0.5 + ], + "image_std": [ + 0.5, + 0.5, + 0.5 + ], + "max_frames": 768, + "merge_size": 2, + "min_frames": 4, + "patch_size": 16, + "resample": 3, + "rescale_factor": 0.00392156862745098, + "return_metadata": false, + "size": { + "longest_edge": 25165824, + "shortest_edge": 4096 + }, + "temporal_patch_size": 2, + "video_processor_type": "Qwen3VLVideoProcessor" + } +} diff --git a/posttrain_models/qwen3_5_35b_a3b_base_highlight_batch1245_epoch4/checkpoint-1500-merged/tokenizer_config.json b/posttrain_models/qwen3_5_35b_a3b_base_highlight_batch1245_epoch4/checkpoint-1500-merged/tokenizer_config.json new file mode 100644 index 0000000000000000000000000000000000000000..a068e2468cff426a9b105006e74e044030a6faf4 --- /dev/null +++ b/posttrain_models/qwen3_5_35b_a3b_base_highlight_batch1245_epoch4/checkpoint-1500-merged/tokenizer_config.json @@ -0,0 +1,32 @@ +{ + "add_prefix_space": false, + "audio_bos_token": "<|audio_start|>", + "audio_eos_token": "<|audio_end|>", + "audio_token": "<|audio_pad|>", + "backend": "tokenizers", + "bos_token": null, + "clean_up_tokenization_spaces": false, + "eos_token": "<|im_end|>", + "errors": "replace", + "image_token": "<|image_pad|>", + "is_local": true, + "model_max_length": 262144, + "model_specific_special_tokens": { + "audio_bos_token": "<|audio_start|>", + "audio_eos_token": "<|audio_end|>", + "audio_token": "<|audio_pad|>", + "image_token": "<|image_pad|>", + "video_token": "<|video_pad|>", + "vision_bos_token": "<|vision_start|>", + "vision_eos_token": "<|vision_end|>" + }, + "pad_token": "<|endoftext|>", + "pretokenize_regex": "(?i:'s|'t|'re|'ve|'m|'ll|'d)|[^\\r\\n\\p{L}\\p{N}]?[\\p{L}\\p{M}]+|\\p{N}| ?[^\\s\\p{L}\\p{M}\\p{N}]+[\\r\\n]*|\\s*[\\r\\n]+|\\s+(?!\\S)|\\s+", + "processor_class": "Qwen3VLProcessor", + "split_special_tokens": false, + "tokenizer_class": "TokenizersBackend", + "unk_token": null, + "video_token": "<|video_pad|>", + "vision_bos_token": "<|vision_start|>", + "vision_eos_token": "<|vision_end|>" +} diff --git a/posttrain_models/qwen3_5_35b_a3b_base_highlight_batch1245_epoch4/checkpoint-1500/args.json b/posttrain_models/qwen3_5_35b_a3b_base_highlight_batch1245_epoch4/checkpoint-1500/args.json new file mode 100644 index 0000000000000000000000000000000000000000..99623dd80e3398e11296749c2daf861d8dff0018 --- /dev/null +++ b/posttrain_models/qwen3_5_35b_a3b_base_highlight_batch1245_epoch4/checkpoint-1500/args.json @@ -0,0 +1,398 @@ +{ + "use_ray": false, + "ray_exp_name": null, + "device_groups": null, + "model": "checkpoints/Qwen3.5-35B-A3B", + "model_type": "qwen3_5_moe", + "model_revision": null, + "task_type": "causal_lm", + "torch_dtype": "bfloat16", + "attn_impl": null, + "experts_impl": null, + "new_special_tokens": [], + "num_labels": null, + "problem_type": null, + "rope_scaling": null, + "device_map": null, + "max_memory": {}, + "max_model_len": null, + "local_repo_path": null, + "init_strategy": null, + "template": "qwen3_5", + "system": null, + "max_length": 8192, + "truncation_strategy": "delete", + "max_pixels": null, + "agent_template": null, + "norm_bbox": null, + "use_chat_template": true, + "padding_side": "right", + "padding_free": true, + "loss_scale": "ignore_empty_think", + "sequence_parallel_size": 1, + "template_backend": "swift", + "response_prefix": "", + "enable_thinking": false, + "add_non_thinking_prefix": false, + "dataset": [ + "batch1", + "batch2", + "batch4_part1", + "batch4_part2", + "batch4_part3", + "batch5_part1", + "batch5_part2", + "batch5_part3" + ], + "val_dataset": [], + "cached_dataset": [], + "cached_val_dataset": [], + "split_dataset_ratio": 0.0, + "data_seed": 42, + "dataset_num_proc": 2, + "load_from_cache_file": false, + "dataset_shuffle": true, + "val_dataset_shuffle": false, + "streaming": false, + "interleave_prob": null, + "stopping_strategy": "first_exhausted", + "shuffle_buffer_size": 1000, + "download_mode": "reuse_dataset_if_exists", + "columns": {}, + "strict": false, + "remove_unused_columns": true, + "model_name": null, + "model_author": null, + "custom_dataset_info": [], + "quant_method": null, + "quant_bits": null, + "hqq_axis": null, + "bnb_4bit_compute_dtype": "bfloat16", + "bnb_4bit_quant_type": "nf4", + "bnb_4bit_use_double_quant": true, + "bnb_4bit_quant_storage": null, + "max_new_tokens": null, + "temperature": 0.9, + "top_k": 50, + "top_p": 0.9, + "repetition_penalty": 1.0, + "num_beams": 1, + "stream": false, + "stop_words": [], + "logprobs": false, + "top_logprobs": null, + "structured_outputs_regex": null, + "tuner_backend": "peft", + "tuner_type": "lora", + "train_type": "lora", + "adapters": [], + "external_plugins": [ + "ms-swift/base_highlight_video_dataset.py" + ], + "custom_register_path": [ + "ms-swift/base_highlight_video_dataset.py" + ], + "seed": 42, + "model_kwargs": {}, + "load_args": false, + "load_data_args": false, + "packing": false, + "packing_length": null, + "packing_num_proc": 1, + "lazy_tokenize": true, + "use_hf": false, + "hub_token": null, + "ddp_timeout": 18000000, + "ddp_backend": "nccl", + "ignore_args_error": false, + "use_swift_lora": false, + "freeze_llm": false, + "freeze_vit": true, + "freeze_aligner": true, + "freeze_parameters": [], + "freeze_parameters_regex": null, + "freeze_parameters_ratio": 0.0, + "trainable_parameters": [], + "trainable_parameters_regex": null, + "target_modules": [ + "all-linear" + ], + "target_regex": null, + "modules_to_save": [], + "lora_rank": 8, + "lora_alpha": 32, + "lora_dropout": 0.05, + "lora_bias": "none", + "lora_dtype": null, + "use_rslora": false, + "rlhf_type": null, + "loss_type": null, + "mcore_ref_model": null, + "mcore_ref_adapter": null, + "beta": null, + "rpo_alpha": null, + "reference_free": false, + "label_smoothing": 0.0, + "f_divergence_type": "reverse_kl", + "desirable_weight": 1.0, + "undesirable_weight": 1.0, + "calculate_KL": null, + "center_rewards_coefficient": null, + "teacher_model": null, + "teacher_model_type": null, + "teacher_model_revision": null, + "teacher_model_server": null, + "gkd_logits_topk": null, + "lmbda": 0.5, + "seq_kd": false, + "offload_teacher_model": false, + "sft_alpha": 0.0, + "generation_batch_size": null, + "steps_per_generation": null, + "num_generations": 8, + "num_generations_eval": null, + "max_completion_length": 512, + "importance_sampling_level": "token", + "tau_pos": 1.0, + "tau_neg": 1.05, + "epsilon": 0.2, + "epsilon_high": null, + "delta": null, + "use_vllm": true, + "vllm_mode": null, + "vllm_enable_prefix_caching": true, + "vllm_gpu_memory_utilization": 0.9, + "vllm_tensor_parallel_size": 1, + "vllm_max_model_len": null, + "vllm_enforce_eager": false, + "vllm_limit_mm_per_prompt": null, + "vllm_disable_cascade_attn": false, + "vllm_max_num_seqs": null, + "vllm_mm_processor_cache_gb": null, + "vllm_engine_kwargs": null, + "sleep_level": 0, + "offload_optimizer": false, + "offload_model": false, + "offload_bridge": false, + "vllm_server_base_url": null, + "vllm_server_host": null, + "vllm_server_port": [ + 8000 + ], + "vllm_server_timeout": 240.0, + "vllm_server_group_port": null, + "reward_funcs": [], + "reward_weights": null, + "cosine_min_len_value_wrong": -0.5, + "cosine_max_len_value_wrong": 0.0, + "cosine_min_len_value_correct": 1.0, + "cosine_max_len_value_correct": 0.5, + "cosine_max_len": null, + "repetition_n_grams": 3, + "repetition_max_penalty": -1.0, + "soft_max_length": null, + "soft_cache_length": null, + "dynamic_sample": false, + "max_resample_times": 3, + "overlong_filter": false, + "scale_rewards": "group", + "advantage_estimator": "grpo", + "kl_in_reward": false, + "wandb_log_unique_prompts": null, + "log_completions": false, + "rollout_importance_sampling_mode": null, + "rollout_importance_sampling_threshold": 2.0, + "log_rollout_offpolicy_metrics": false, + "off_policy_sequence_mask_delta": null, + "log_entropy": false, + "top_entropy_quantile": 1.0, + "reward_model": null, + "reward_model_plugin": null, + "sync_ref_model": false, + "ref_model_sync_steps": 512, + "ref_model_mixup_alpha": 0.6, + "async_generate": false, + "move_model_batches": null, + "multi_turn_scheduler": null, + "max_turns": null, + "completion_length_limit_scope": "per_round", + "vllm_server_pass_dataset": false, + "num_iterations": 1, + "micro_batch_size": 8, + "global_batch_size": 64, + "recompute_granularity": "full", + "recompute_method": "uniform", + "recompute_num_layers": 1, + "recompute_modules": [ + "core_attn" + ], + "train_iters": null, + "num_train_epochs": 4, + "masked_softmax_fusion": true, + "bias_dropout_fusion": true, + "bias_activation_fusion": true, + "apply_rope_fusion": false, + "gradient_accumulation_fusion": false, + "cross_entropy_loss_fusion": true, + "cross_entropy_fusion_impl": "te", + "calculate_per_token_loss": true, + "attention_backend": "", + "optimizer": "adam", + "optimizer_cpu_offload": false, + "optimizer_offload_fraction": 1.0, + "use_precision_aware_optimizer": false, + "main_grads_dtype": "float32", + "main_params_dtype": "float32", + "exp_avg_dtype": "float32", + "exp_avg_sq_dtype": "float32", + "manual_gc": false, + "manual_gc_steps": 0, + "manual_gc_eval": true, + "train_dataloader_shuffle": true, + "dataloader_num_workers": 1, + "dataloader_pin_memory": true, + "dataloader_persistent_workers": true, + "dataloader_prefetch_factor": 2, + "data_sharding": false, + "group_by_length": false, + "te_rng_tracker": false, + "data_parallel_random_init": false, + "mlp_padding_free": false, + "lr_warmup_init": 0.0, + "lr": 0.0001, + "lr_decay_style": "cosine", + "lr_decay_iters": null, + "lr_warmup_iters": 0, + "lr_warmup_fraction": 0.05, + "min_lr": 1e-05, + "lr_wsd_decay_style": "exponential", + "lr_wsd_decay_iters": null, + "weight_decay": 0.1, + "weight_decay_incr_style": "constant", + "start_weight_decay": 0.1, + "end_weight_decay": 0.1, + "clip_grad": 1.0, + "adam_beta1": 0.9, + "adam_beta2": 0.95, + "adam_eps": 1e-08, + "sgd_momentum": 0.9, + "output_dir": "/mnt/bn/strategy-mllm-train/user/tanyue/models/posttrain_models/qwen3_5_35b_a3b_base_highlight_batch1245_epoch4", + "save_steps": 500, + "no_save_optim": true, + "no_save_rng": true, + "mcore_model": null, + "mcore_adapter": null, + "no_load_optim": false, + "no_load_rng": false, + "finetune": true, + "perform_initialization": false, + "use_cpu_initialization": false, + "async_save": false, + "save_total_limit": null, + "metric_for_best_model": "loss", + "greater_is_better": false, + "use_persistent_ckpt_worker": false, + "dist_ckpt_save_pre_mcore_014": false, + "dist_ckpt_optim_fully_reshardable": false, + "distrib_optim_fully_reshardable_mem_efficient": false, + "local_rank": 0, + "use_distributed_optimizer": true, + "tensor_model_parallel_size": 1, + "pipeline_model_parallel_size": 2, + "decoder_first_pipeline_num_layers": null, + "decoder_last_pipeline_num_layers": null, + "account_for_embedding_in_pipeline_split": false, + "account_for_loss_in_pipeline_split": false, + "overlap_p2p_comm": false, + "align_param_gather": false, + "sequence_parallel": false, + "context_parallel_size": 1, + "tp_comm_overlap": false, + "overlap_grad_reduce": false, + "overlap_param_gather": false, + "overlap_param_gather_with_optimizer_step": false, + "align_grad_reduce": true, + "virtual_pipeline_model_parallel_size": null, + "microbatch_group_size_per_vp_stage": null, + "pipeline_model_parallel_layout": null, + "expert_model_parallel_size": 4, + "expert_tensor_parallel_size": 1, + "report_to": [ + "wandb" + ], + "logging_steps": 5, + "tensorboard_dir": "/mnt/bn/strategy-mllm-train/user/tanyue/models/posttrain_models/qwen3_5_35b_a3b_base_highlight_batch1245_epoch4/runs", + "tensorboard_queue_size": 50, + "wandb_project": "highlight_base_highlight_qwen3_5", + "wandb_exp_name": "qwen3_5_35b_a3b_base_highlight_batch1245_epoch4", + "swanlab_project": "megatron-swift", + "swanlab_exp_name": null, + "eval_iters": -1, + "eval_steps": 100000, + "fp8_format": null, + "fp8_recipe": "delayed", + "fp8_amax_history_len": 1024, + "fp8_amax_compute_algo": "max", + "fp8_param_gather": false, + "fp16": false, + "bf16": true, + "apply_query_key_layer_scaling": false, + "attention_softmax_in_fp32": true, + "accumulate_allreduce_grads_in_fp32": true, + "moe_router_load_balancing_type": null, + "moe_router_dtype": "fp32", + "moe_token_dispatcher_type": null, + "moe_enable_deepep": false, + "moe_grouped_gemm": true, + "moe_permute_fusion": true, + "moe_aux_loss_coeff": 1e-06, + "moe_z_loss_coeff": null, + "moe_shared_expert_overlap": true, + "moe_layer_recompute": false, + "moe_expert_capacity_factor": null, + "moe_pad_expert_input_to_capacity": false, + "moe_token_drop_policy": null, + "mtp_num_layers": null, + "mtp_loss_scaling_factor": 0.1, + "save_safetensors": true, + "ref_model": null, + "ref_adapters": [], + "merge_lora": true, + "max_shard_size": "5GB", + "vit_gradient_checkpointing": false, + "vit_lr": null, + "aligner_lr": null, + "gradient_checkpointing_kwargs": null, + "check_model": true, + "apply_wd_to_qk_layernorm": false, + "enable_dft_loss": false, + "enable_channel_loss": false, + "save_strategy": "steps", + "callbacks": [ + "print", + "default_flow", + "wandb" + ], + "add_version": false, + "create_checkpoint_symlink": false, + "swift_version": "4.0.1", + "ckpt_dir": null, + "rank": 0, + "global_world_size": 8, + "local_world_size": 8, + "model_suffix": "Qwen3.5-35B-A3B", + "model_info": "ModelInfo(model_type='qwen3_5_moe', model_dir='/opt/tiger/tanyue/highlight_tools/checkpoints/Qwen3.5-35B-A3B', torch_dtype=torch.bfloat16, max_model_len=262144, quant_method=None, quant_bits=None, rope_scaling=None, is_moe_model=True, is_multimodal=True, config=None, task_type='causal_lm', num_labels=None)", + "model_meta": "ModelMeta(model_type='qwen3_5_moe', model_groups=[ModelGroup(models=[Model(ms_model_id='Qwen/Qwen3.5-35B-A3B-Base', hf_model_id='Qwen/Qwen3.5-35B-A3B-Base', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen3.5-35B-A3B', hf_model_id='Qwen/Qwen3.5-35B-A3B', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen3.5-122B-A10B', hf_model_id='Qwen/Qwen3.5-122B-A10B', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen3.5-397B-A17B', hf_model_id='Qwen/Qwen3.5-397B-A17B', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen3.5-35B-A3B-FP8', hf_model_id='Qwen/Qwen3.5-35B-A3B-FP8', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen3.5-122B-A10B-FP8', hf_model_id='Qwen/Qwen3.5-122B-A10B-FP8', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen3.5-397B-A17B-FP8', hf_model_id='Qwen/Qwen3.5-397B-A17B-FP8', model_path=None, ms_revision=None, hf_revision=None)], template='qwen3_5', ignore_patterns=None, requires=None, tags=[])], loader=, template='qwen3_5', model_arch=MultiModelKeys(arch_name='qwen2_vl', embedding=None, module_list=None, lm_head=None, q_proj=None, k_proj=None, v_proj=None, o_proj=None, attention=None, mlp=None, down_proj=None, qkv_proj=None, qk_proj=None, qa_proj=None, qb_proj=None, kv_proj=None, kva_proj=None, kvb_proj=None, language_model=['model.language_model', 'lm_head'], aligner=['model.visual.merger'], vision_tower=['model.visual'], generator=[]), architectures=['Qwen3_5MoeForConditionalGeneration'], additional_saved_files=[], torch_dtype=None, is_multimodal=True, is_reward=False, task_type=None, ignore_patterns=None, requires=['transformers>=5.2.0', 'qwen_vl_utils>=0.0.14', 'decord'], tags=[])", + "model_dir": "/opt/tiger/tanyue/highlight_tools/checkpoints/Qwen3.5-35B-A3B", + "template_meta": "QwenTemplateMeta(template_type='qwen3_5', prefix=[], prompt=['<|im_start|>user\\n{{QUERY}}<|im_end|>\\n<|im_start|>assistant\\n'], chat_sep=['<|im_end|>\\n'], suffix=['<|im_end|>\\n'], template_cls=, system_prefix=['<|im_start|>system\\n{{SYSTEM}}<|im_end|>\\n'], default_system=None, auto_add_bos=False, stop_words=['<|endoftext|>'], agent_template='qwen3_5', is_thinking=True, thinking_prefix='\\n', non_thinking_prefix='\\n\\n\\n\\n', history_thinking_prefix='')", + "_val_dataset_exists": false, + "hub": "", + "seq_length": 8192, + "is_multimodal": true, + "megatron_model_meta": "MegatronModelMeta(megatron_model_type='qwen3_5', model_types=['qwen3_5', 'qwen3_5_moe'], bridge_cls=, visual_cls=, is_multimodal=True, loader=)", + "fp8": null, + "params_dtype": "bfloat16", + "world_size": 8, + "data_parallel_size": 4, + "num_microbatches": 2 +} \ No newline at end of file diff --git a/posttrain_models/qwen3_5_35b_a3b_base_highlight_batch1245_epoch4/checkpoint-1500/iter_0001500/.metadata b/posttrain_models/qwen3_5_35b_a3b_base_highlight_batch1245_epoch4/checkpoint-1500/iter_0001500/.metadata new file mode 100644 index 0000000000000000000000000000000000000000..9868694c37a01df8d3caec92414898abb503e547 Binary files /dev/null and b/posttrain_models/qwen3_5_35b_a3b_base_highlight_batch1245_epoch4/checkpoint-1500/iter_0001500/.metadata differ diff --git a/posttrain_models/qwen3_5_35b_a3b_base_highlight_batch1245_epoch4/checkpoint-1500/iter_0001500/common.pt b/posttrain_models/qwen3_5_35b_a3b_base_highlight_batch1245_epoch4/checkpoint-1500/iter_0001500/common.pt new file mode 100644 index 0000000000000000000000000000000000000000..0c5d00601028e55feea5cbc55c0a8b3b7c137ac4 --- /dev/null +++ b/posttrain_models/qwen3_5_35b_a3b_base_highlight_batch1245_epoch4/checkpoint-1500/iter_0001500/common.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3c521bbb30467e08142381a06dcbc55112ff4e7f53b6e04db81dcece7a5182a2 +size 15975 diff --git a/posttrain_models/qwen3_5_35b_a3b_base_highlight_batch1245_epoch4/checkpoint-1500/iter_0001500/metadata.json b/posttrain_models/qwen3_5_35b_a3b_base_highlight_batch1245_epoch4/checkpoint-1500/iter_0001500/metadata.json new file mode 100644 index 0000000000000000000000000000000000000000..c9dbde0bcf4d2d993122dcc7d6bcb1eef8b6fb77 --- /dev/null +++ b/posttrain_models/qwen3_5_35b_a3b_base_highlight_batch1245_epoch4/checkpoint-1500/iter_0001500/metadata.json @@ -0,0 +1 @@ +{"sharded_backend": "torch_dist", "sharded_backend_version": 1, "common_backend": "torch", "common_backend_version": 1} \ No newline at end of file diff --git a/posttrain_models/qwen3_5_35b_a3b_base_highlight_batch1245_epoch4/checkpoint-1500/latest_checkpointed_iteration.txt b/posttrain_models/qwen3_5_35b_a3b_base_highlight_batch1245_epoch4/checkpoint-1500/latest_checkpointed_iteration.txt new file mode 100644 index 0000000000000000000000000000000000000000..37021f4a27201e7257a4ac7a992b0b4ebba61a01 --- /dev/null +++ b/posttrain_models/qwen3_5_35b_a3b_base_highlight_batch1245_epoch4/checkpoint-1500/latest_checkpointed_iteration.txt @@ -0,0 +1 @@ +1500 \ No newline at end of file diff --git a/posttrain_models/qwen3_5_35b_a3b_base_highlight_batch1245_epoch4/checkpoint-2000-merged/args.json b/posttrain_models/qwen3_5_35b_a3b_base_highlight_batch1245_epoch4/checkpoint-2000-merged/args.json new file mode 100644 index 0000000000000000000000000000000000000000..99623dd80e3398e11296749c2daf861d8dff0018 --- /dev/null +++ b/posttrain_models/qwen3_5_35b_a3b_base_highlight_batch1245_epoch4/checkpoint-2000-merged/args.json @@ -0,0 +1,398 @@ +{ + "use_ray": false, + "ray_exp_name": null, + "device_groups": null, + "model": "checkpoints/Qwen3.5-35B-A3B", + "model_type": "qwen3_5_moe", + "model_revision": null, + "task_type": "causal_lm", + "torch_dtype": "bfloat16", + "attn_impl": null, + "experts_impl": null, + "new_special_tokens": [], + "num_labels": null, + "problem_type": null, + "rope_scaling": null, + "device_map": null, + "max_memory": {}, + "max_model_len": null, + "local_repo_path": null, + "init_strategy": null, + "template": "qwen3_5", + "system": null, + "max_length": 8192, + "truncation_strategy": "delete", + "max_pixels": null, + "agent_template": null, + "norm_bbox": null, + "use_chat_template": true, + "padding_side": "right", + "padding_free": true, + "loss_scale": "ignore_empty_think", + "sequence_parallel_size": 1, + "template_backend": "swift", + "response_prefix": "", + "enable_thinking": false, + "add_non_thinking_prefix": false, + "dataset": [ + "batch1", + "batch2", + "batch4_part1", + "batch4_part2", + "batch4_part3", + "batch5_part1", + "batch5_part2", + "batch5_part3" + ], + "val_dataset": [], + "cached_dataset": [], + "cached_val_dataset": [], + "split_dataset_ratio": 0.0, + "data_seed": 42, + "dataset_num_proc": 2, + "load_from_cache_file": false, + "dataset_shuffle": true, + "val_dataset_shuffle": false, + "streaming": false, + "interleave_prob": null, + "stopping_strategy": "first_exhausted", + "shuffle_buffer_size": 1000, + "download_mode": "reuse_dataset_if_exists", + "columns": {}, + "strict": false, + "remove_unused_columns": true, + "model_name": null, + "model_author": null, + "custom_dataset_info": [], + "quant_method": null, + "quant_bits": null, + "hqq_axis": null, + "bnb_4bit_compute_dtype": "bfloat16", + "bnb_4bit_quant_type": "nf4", + "bnb_4bit_use_double_quant": true, + "bnb_4bit_quant_storage": null, + "max_new_tokens": null, + "temperature": 0.9, + "top_k": 50, + "top_p": 0.9, + "repetition_penalty": 1.0, + "num_beams": 1, + "stream": false, + "stop_words": [], + "logprobs": false, + "top_logprobs": null, + "structured_outputs_regex": null, + "tuner_backend": "peft", + "tuner_type": "lora", + "train_type": "lora", + "adapters": [], + "external_plugins": [ + "ms-swift/base_highlight_video_dataset.py" + ], + "custom_register_path": [ + "ms-swift/base_highlight_video_dataset.py" + ], + "seed": 42, + "model_kwargs": {}, + "load_args": false, + "load_data_args": false, + "packing": false, + "packing_length": null, + "packing_num_proc": 1, + "lazy_tokenize": true, + "use_hf": false, + "hub_token": null, + "ddp_timeout": 18000000, + "ddp_backend": "nccl", + "ignore_args_error": false, + "use_swift_lora": false, + "freeze_llm": false, + "freeze_vit": true, + "freeze_aligner": true, + "freeze_parameters": [], + "freeze_parameters_regex": null, + "freeze_parameters_ratio": 0.0, + "trainable_parameters": [], + "trainable_parameters_regex": null, + "target_modules": [ + "all-linear" + ], + "target_regex": null, + "modules_to_save": [], + "lora_rank": 8, + "lora_alpha": 32, + "lora_dropout": 0.05, + "lora_bias": "none", + "lora_dtype": null, + "use_rslora": false, + "rlhf_type": null, + "loss_type": null, + "mcore_ref_model": null, + "mcore_ref_adapter": null, + "beta": null, + "rpo_alpha": null, + "reference_free": false, + "label_smoothing": 0.0, + "f_divergence_type": "reverse_kl", + "desirable_weight": 1.0, + "undesirable_weight": 1.0, + "calculate_KL": null, + "center_rewards_coefficient": null, + "teacher_model": null, + "teacher_model_type": null, + "teacher_model_revision": null, + "teacher_model_server": null, + "gkd_logits_topk": null, + "lmbda": 0.5, + "seq_kd": false, + "offload_teacher_model": false, + "sft_alpha": 0.0, + "generation_batch_size": null, + "steps_per_generation": null, + "num_generations": 8, + "num_generations_eval": null, + "max_completion_length": 512, + "importance_sampling_level": "token", + "tau_pos": 1.0, + "tau_neg": 1.05, + "epsilon": 0.2, + "epsilon_high": null, + "delta": null, + "use_vllm": true, + "vllm_mode": null, + "vllm_enable_prefix_caching": true, + "vllm_gpu_memory_utilization": 0.9, + "vllm_tensor_parallel_size": 1, + "vllm_max_model_len": null, + "vllm_enforce_eager": false, + "vllm_limit_mm_per_prompt": null, + "vllm_disable_cascade_attn": false, + "vllm_max_num_seqs": null, + "vllm_mm_processor_cache_gb": null, + "vllm_engine_kwargs": null, + "sleep_level": 0, + "offload_optimizer": false, + "offload_model": false, + "offload_bridge": false, + "vllm_server_base_url": null, + "vllm_server_host": null, + "vllm_server_port": [ + 8000 + ], + "vllm_server_timeout": 240.0, + "vllm_server_group_port": null, + "reward_funcs": [], + "reward_weights": null, + "cosine_min_len_value_wrong": -0.5, + "cosine_max_len_value_wrong": 0.0, + "cosine_min_len_value_correct": 1.0, + "cosine_max_len_value_correct": 0.5, + "cosine_max_len": null, + "repetition_n_grams": 3, + "repetition_max_penalty": -1.0, + "soft_max_length": null, + "soft_cache_length": null, + "dynamic_sample": false, + "max_resample_times": 3, + "overlong_filter": false, + "scale_rewards": "group", + "advantage_estimator": "grpo", + "kl_in_reward": false, + "wandb_log_unique_prompts": null, + "log_completions": false, + "rollout_importance_sampling_mode": null, + "rollout_importance_sampling_threshold": 2.0, + "log_rollout_offpolicy_metrics": false, + "off_policy_sequence_mask_delta": null, + "log_entropy": false, + "top_entropy_quantile": 1.0, + "reward_model": null, + "reward_model_plugin": null, + "sync_ref_model": false, + "ref_model_sync_steps": 512, + "ref_model_mixup_alpha": 0.6, + "async_generate": false, + "move_model_batches": null, + "multi_turn_scheduler": null, + "max_turns": null, + "completion_length_limit_scope": "per_round", + "vllm_server_pass_dataset": false, + "num_iterations": 1, + "micro_batch_size": 8, + "global_batch_size": 64, + "recompute_granularity": "full", + "recompute_method": "uniform", + "recompute_num_layers": 1, + "recompute_modules": [ + "core_attn" + ], + "train_iters": null, + "num_train_epochs": 4, + "masked_softmax_fusion": true, + "bias_dropout_fusion": true, + "bias_activation_fusion": true, + "apply_rope_fusion": false, + "gradient_accumulation_fusion": false, + "cross_entropy_loss_fusion": true, + "cross_entropy_fusion_impl": "te", + "calculate_per_token_loss": true, + "attention_backend": "", + "optimizer": "adam", + "optimizer_cpu_offload": false, + "optimizer_offload_fraction": 1.0, + "use_precision_aware_optimizer": false, + "main_grads_dtype": "float32", + "main_params_dtype": "float32", + "exp_avg_dtype": "float32", + "exp_avg_sq_dtype": "float32", + "manual_gc": false, + "manual_gc_steps": 0, + "manual_gc_eval": true, + "train_dataloader_shuffle": true, + "dataloader_num_workers": 1, + "dataloader_pin_memory": true, + "dataloader_persistent_workers": true, + "dataloader_prefetch_factor": 2, + "data_sharding": false, + "group_by_length": false, + "te_rng_tracker": false, + "data_parallel_random_init": false, + "mlp_padding_free": false, + "lr_warmup_init": 0.0, + "lr": 0.0001, + "lr_decay_style": "cosine", + "lr_decay_iters": null, + "lr_warmup_iters": 0, + "lr_warmup_fraction": 0.05, + "min_lr": 1e-05, + "lr_wsd_decay_style": "exponential", + "lr_wsd_decay_iters": null, + "weight_decay": 0.1, + "weight_decay_incr_style": "constant", + "start_weight_decay": 0.1, + "end_weight_decay": 0.1, + "clip_grad": 1.0, + "adam_beta1": 0.9, + "adam_beta2": 0.95, + "adam_eps": 1e-08, + "sgd_momentum": 0.9, + "output_dir": "/mnt/bn/strategy-mllm-train/user/tanyue/models/posttrain_models/qwen3_5_35b_a3b_base_highlight_batch1245_epoch4", + "save_steps": 500, + "no_save_optim": true, + "no_save_rng": true, + "mcore_model": null, + "mcore_adapter": null, + "no_load_optim": false, + "no_load_rng": false, + "finetune": true, + "perform_initialization": false, + "use_cpu_initialization": false, + "async_save": false, + "save_total_limit": null, + "metric_for_best_model": "loss", + "greater_is_better": false, + "use_persistent_ckpt_worker": false, + "dist_ckpt_save_pre_mcore_014": false, + "dist_ckpt_optim_fully_reshardable": false, + "distrib_optim_fully_reshardable_mem_efficient": false, + "local_rank": 0, + "use_distributed_optimizer": true, + "tensor_model_parallel_size": 1, + "pipeline_model_parallel_size": 2, + "decoder_first_pipeline_num_layers": null, + "decoder_last_pipeline_num_layers": null, + "account_for_embedding_in_pipeline_split": false, + "account_for_loss_in_pipeline_split": false, + "overlap_p2p_comm": false, + "align_param_gather": false, + "sequence_parallel": false, + "context_parallel_size": 1, + "tp_comm_overlap": false, + "overlap_grad_reduce": false, + "overlap_param_gather": false, + "overlap_param_gather_with_optimizer_step": false, + "align_grad_reduce": true, + "virtual_pipeline_model_parallel_size": null, + "microbatch_group_size_per_vp_stage": null, + "pipeline_model_parallel_layout": null, + "expert_model_parallel_size": 4, + "expert_tensor_parallel_size": 1, + "report_to": [ + "wandb" + ], + "logging_steps": 5, + "tensorboard_dir": "/mnt/bn/strategy-mllm-train/user/tanyue/models/posttrain_models/qwen3_5_35b_a3b_base_highlight_batch1245_epoch4/runs", + "tensorboard_queue_size": 50, + "wandb_project": "highlight_base_highlight_qwen3_5", + "wandb_exp_name": "qwen3_5_35b_a3b_base_highlight_batch1245_epoch4", + "swanlab_project": "megatron-swift", + "swanlab_exp_name": null, + "eval_iters": -1, + "eval_steps": 100000, + "fp8_format": null, + "fp8_recipe": "delayed", + "fp8_amax_history_len": 1024, + "fp8_amax_compute_algo": "max", + "fp8_param_gather": false, + "fp16": false, + "bf16": true, + "apply_query_key_layer_scaling": false, + "attention_softmax_in_fp32": true, + "accumulate_allreduce_grads_in_fp32": true, + "moe_router_load_balancing_type": null, + "moe_router_dtype": "fp32", + "moe_token_dispatcher_type": null, + "moe_enable_deepep": false, + "moe_grouped_gemm": true, + "moe_permute_fusion": true, + "moe_aux_loss_coeff": 1e-06, + "moe_z_loss_coeff": null, + "moe_shared_expert_overlap": true, + "moe_layer_recompute": false, + "moe_expert_capacity_factor": null, + "moe_pad_expert_input_to_capacity": false, + "moe_token_drop_policy": null, + "mtp_num_layers": null, + "mtp_loss_scaling_factor": 0.1, + "save_safetensors": true, + "ref_model": null, + "ref_adapters": [], + "merge_lora": true, + "max_shard_size": "5GB", + "vit_gradient_checkpointing": false, + "vit_lr": null, + "aligner_lr": null, + "gradient_checkpointing_kwargs": null, + "check_model": true, + "apply_wd_to_qk_layernorm": false, + "enable_dft_loss": false, + "enable_channel_loss": false, + "save_strategy": "steps", + "callbacks": [ + "print", + "default_flow", + "wandb" + ], + "add_version": false, + "create_checkpoint_symlink": false, + "swift_version": "4.0.1", + "ckpt_dir": null, + "rank": 0, + "global_world_size": 8, + "local_world_size": 8, + "model_suffix": "Qwen3.5-35B-A3B", + "model_info": "ModelInfo(model_type='qwen3_5_moe', model_dir='/opt/tiger/tanyue/highlight_tools/checkpoints/Qwen3.5-35B-A3B', torch_dtype=torch.bfloat16, max_model_len=262144, quant_method=None, quant_bits=None, rope_scaling=None, is_moe_model=True, is_multimodal=True, config=None, task_type='causal_lm', num_labels=None)", + "model_meta": "ModelMeta(model_type='qwen3_5_moe', model_groups=[ModelGroup(models=[Model(ms_model_id='Qwen/Qwen3.5-35B-A3B-Base', hf_model_id='Qwen/Qwen3.5-35B-A3B-Base', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen3.5-35B-A3B', hf_model_id='Qwen/Qwen3.5-35B-A3B', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen3.5-122B-A10B', hf_model_id='Qwen/Qwen3.5-122B-A10B', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen3.5-397B-A17B', hf_model_id='Qwen/Qwen3.5-397B-A17B', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen3.5-35B-A3B-FP8', hf_model_id='Qwen/Qwen3.5-35B-A3B-FP8', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen3.5-122B-A10B-FP8', hf_model_id='Qwen/Qwen3.5-122B-A10B-FP8', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen3.5-397B-A17B-FP8', hf_model_id='Qwen/Qwen3.5-397B-A17B-FP8', model_path=None, ms_revision=None, hf_revision=None)], template='qwen3_5', ignore_patterns=None, requires=None, tags=[])], loader=, template='qwen3_5', model_arch=MultiModelKeys(arch_name='qwen2_vl', embedding=None, module_list=None, lm_head=None, q_proj=None, k_proj=None, v_proj=None, o_proj=None, attention=None, mlp=None, down_proj=None, qkv_proj=None, qk_proj=None, qa_proj=None, qb_proj=None, kv_proj=None, kva_proj=None, kvb_proj=None, language_model=['model.language_model', 'lm_head'], aligner=['model.visual.merger'], vision_tower=['model.visual'], generator=[]), architectures=['Qwen3_5MoeForConditionalGeneration'], additional_saved_files=[], torch_dtype=None, is_multimodal=True, is_reward=False, task_type=None, ignore_patterns=None, requires=['transformers>=5.2.0', 'qwen_vl_utils>=0.0.14', 'decord'], tags=[])", + "model_dir": "/opt/tiger/tanyue/highlight_tools/checkpoints/Qwen3.5-35B-A3B", + "template_meta": "QwenTemplateMeta(template_type='qwen3_5', prefix=[], prompt=['<|im_start|>user\\n{{QUERY}}<|im_end|>\\n<|im_start|>assistant\\n'], chat_sep=['<|im_end|>\\n'], suffix=['<|im_end|>\\n'], template_cls=, system_prefix=['<|im_start|>system\\n{{SYSTEM}}<|im_end|>\\n'], default_system=None, auto_add_bos=False, stop_words=['<|endoftext|>'], agent_template='qwen3_5', is_thinking=True, thinking_prefix='\\n', non_thinking_prefix='\\n\\n\\n\\n', history_thinking_prefix='')", + "_val_dataset_exists": false, + "hub": "", + "seq_length": 8192, + "is_multimodal": true, + "megatron_model_meta": "MegatronModelMeta(megatron_model_type='qwen3_5', model_types=['qwen3_5', 'qwen3_5_moe'], bridge_cls=, visual_cls=, is_multimodal=True, loader=)", + "fp8": null, + "params_dtype": "bfloat16", + "world_size": 8, + "data_parallel_size": 4, + "num_microbatches": 2 +} \ No newline at end of file diff --git a/posttrain_models/qwen3_5_35b_a3b_base_highlight_batch1245_epoch4/checkpoint-2000-merged/chat_template.jinja b/posttrain_models/qwen3_5_35b_a3b_base_highlight_batch1245_epoch4/checkpoint-2000-merged/chat_template.jinja new file mode 100644 index 0000000000000000000000000000000000000000..a585dec894e63da457d9440ec6aa7caa16d20860 --- /dev/null +++ b/posttrain_models/qwen3_5_35b_a3b_base_highlight_batch1245_epoch4/checkpoint-2000-merged/chat_template.jinja @@ -0,0 +1,154 @@ +{%- set image_count = namespace(value=0) %} +{%- set video_count = namespace(value=0) %} +{%- macro render_content(content, do_vision_count, is_system_content=false) %} + {%- if content is string %} + {{- content }} + {%- elif content is iterable and content is not mapping %} + {%- for item in content %} + {%- if 'image' in item or 'image_url' in item or item.type == 'image' %} + {%- if is_system_content %} + {{- raise_exception('System message cannot contain images.') }} + {%- endif %} + {%- if do_vision_count %} + {%- set image_count.value = image_count.value + 1 %} + {%- endif %} + {%- if add_vision_id %} + {{- 'Picture ' ~ image_count.value ~ ': ' }} + {%- endif %} + {{- '<|vision_start|><|image_pad|><|vision_end|>' }} + {%- elif 'video' in item or item.type == 'video' %} + {%- if is_system_content %} + {{- raise_exception('System message cannot contain videos.') }} + {%- endif %} + {%- if do_vision_count %} + {%- set video_count.value = video_count.value + 1 %} + {%- endif %} + {%- if add_vision_id %} + {{- 'Video ' ~ video_count.value ~ ': ' }} + {%- endif %} + {{- '<|vision_start|><|video_pad|><|vision_end|>' }} + {%- elif 'text' in item %} + {{- item.text }} + {%- else %} + {{- raise_exception('Unexpected item type in content.') }} + {%- endif %} + {%- endfor %} + {%- elif content is none or content is undefined %} + {{- '' }} + {%- else %} + {{- raise_exception('Unexpected content type.') }} + {%- endif %} +{%- endmacro %} +{%- if not messages %} + {{- raise_exception('No messages provided.') }} +{%- endif %} +{%- if tools and tools is iterable and tools is not mapping %} + {{- '<|im_start|>system\n' }} + {{- "# Tools\n\nYou have access to the following functions:\n\n" }} + {%- for tool in tools %} + {{- "\n" }} + {{- tool | tojson }} + {%- endfor %} + {{- "\n" }} + {{- '\n\nIf you choose to call a function ONLY reply in the following format with NO suffix:\n\n\n\n\nvalue_1\n\n\nThis is the value for the second parameter\nthat can span\nmultiple lines\n\n\n\n\n\nReminder:\n- Function calls MUST follow the specified format: an inner block must be nested within XML tags\n- Required parameters MUST be specified\n- You may provide optional reasoning for your function call in natural language BEFORE the function call, but NOT after\n- If there is no function call available, answer the question like normal with your current knowledge and do not tell the user about function calls\n' }} + {%- if messages[0].role == 'system' %} + {%- set content = render_content(messages[0].content, false, true)|trim %} + {%- if content %} + {{- '\n\n' + content }} + {%- endif %} + {%- endif %} + {{- '<|im_end|>\n' }} +{%- else %} + {%- if messages[0].role == 'system' %} + {%- set content = render_content(messages[0].content, false, true)|trim %} + {{- '<|im_start|>system\n' + content + '<|im_end|>\n' }} + {%- endif %} +{%- endif %} +{%- set ns = namespace(multi_step_tool=true, last_query_index=messages|length - 1) %} +{%- for message in messages[::-1] %} + {%- set index = (messages|length - 1) - loop.index0 %} + {%- if ns.multi_step_tool and message.role == "user" %} + {%- set content = render_content(message.content, false)|trim %} + {%- if not(content.startswith('') and content.endswith('')) %} + {%- set ns.multi_step_tool = false %} + {%- set ns.last_query_index = index %} + {%- endif %} + {%- endif %} +{%- endfor %} +{%- if ns.multi_step_tool %} + {{- raise_exception('No user query found in messages.') }} +{%- endif %} +{%- for message in messages %} + {%- set content = render_content(message.content, true)|trim %} + {%- if message.role == "system" %} + {%- if not loop.first %} + {{- raise_exception('System message must be at the beginning.') }} + {%- endif %} + {%- elif message.role == "user" %} + {{- '<|im_start|>' + message.role + '\n' + content + '<|im_end|>' + '\n' }} + {%- elif message.role == "assistant" %} + {%- set reasoning_content = '' %} + {%- if message.reasoning_content is string %} + {%- set reasoning_content = message.reasoning_content %} + {%- else %} + {%- if '' in content %} + {%- set reasoning_content = content.split('')[0].rstrip('\n').split('')[-1].lstrip('\n') %} + {%- set content = content.split('')[-1].lstrip('\n') %} + {%- endif %} + {%- endif %} + {%- set reasoning_content = reasoning_content|trim %} + {%- if loop.index0 > ns.last_query_index %} + {{- '<|im_start|>' + message.role + '\n\n' + reasoning_content + '\n\n\n' + content }} + {%- else %} + {{- '<|im_start|>' + message.role + '\n' + content }} + {%- endif %} + {%- if message.tool_calls and message.tool_calls is iterable and message.tool_calls is not mapping %} + {%- for tool_call in message.tool_calls %} + {%- if tool_call.function is defined %} + {%- set tool_call = tool_call.function %} + {%- endif %} + {%- if loop.first %} + {%- if content|trim %} + {{- '\n\n\n\n' }} + {%- else %} + {{- '\n\n' }} + {%- endif %} + {%- else %} + {{- '\n\n\n' }} + {%- endif %} + {%- if tool_call.arguments is defined %} + {%- for args_name, args_value in tool_call.arguments|items %} + {{- '\n' }} + {%- set args_value = args_value | tojson | safe if args_value is mapping or (args_value is sequence and args_value is not string) else args_value | string %} + {{- args_value }} + {{- '\n\n' }} + {%- endfor %} + {%- endif %} + {{- '\n' }} + {%- endfor %} + {%- endif %} + {{- '<|im_end|>\n' }} + {%- elif message.role == "tool" %} + {%- if loop.previtem and loop.previtem.role != "tool" %} + {{- '<|im_start|>user' }} + {%- endif %} + {{- '\n\n' }} + {{- content }} + {{- '\n' }} + {%- if not loop.last and loop.nextitem.role != "tool" %} + {{- '<|im_end|>\n' }} + {%- elif loop.last %} + {{- '<|im_end|>\n' }} + {%- endif %} + {%- else %} + {{- raise_exception('Unexpected message role.') }} + {%- endif %} +{%- endfor %} +{%- if add_generation_prompt %} + {{- '<|im_start|>assistant\n' }} + {%- if enable_thinking is defined and enable_thinking is false %} + {{- '\n\n\n\n' }} + {%- else %} + {{- '\n' }} + {%- endif %} +{%- endif %} \ No newline at end of file diff --git a/posttrain_models/qwen3_5_35b_a3b_base_highlight_batch1245_epoch4/checkpoint-2000-merged/config.json b/posttrain_models/qwen3_5_35b_a3b_base_highlight_batch1245_epoch4/checkpoint-2000-merged/config.json new file mode 100644 index 0000000000000000000000000000000000000000..8643d6d3f2e157c74392e7d91f431a66f9261762 --- /dev/null +++ b/posttrain_models/qwen3_5_35b_a3b_base_highlight_batch1245_epoch4/checkpoint-2000-merged/config.json @@ -0,0 +1,123 @@ +{ + "architectures": [ + "Qwen3_5MoeForConditionalGeneration" + ], + "dtype": "bfloat16", + "hidden_size": 2048, + "image_token_id": 248056, + "model_type": "qwen3_5_moe", + "text_config": { + "attention_bias": false, + "attention_dropout": 0.0, + "attn_output_gate": true, + "bos_token_id": null, + "dtype": "bfloat16", + "eos_token_id": 248044, + "full_attention_interval": 4, + "head_dim": 256, + "hidden_act": "silu", + "hidden_size": 2048, + "initializer_range": 0.02, + "layer_types": [ + "linear_attention", + "linear_attention", + "linear_attention", + "full_attention", + "linear_attention", + "linear_attention", + "linear_attention", + "full_attention", + "linear_attention", + "linear_attention", + "linear_attention", + "full_attention", + "linear_attention", + "linear_attention", + "linear_attention", + "full_attention", + "linear_attention", + "linear_attention", + "linear_attention", + "full_attention", + "linear_attention", + "linear_attention", + "linear_attention", + "full_attention", + "linear_attention", + "linear_attention", + "linear_attention", + "full_attention", + "linear_attention", + "linear_attention", + "linear_attention", + "full_attention", + "linear_attention", + "linear_attention", + "linear_attention", + "full_attention", + "linear_attention", + "linear_attention", + "linear_attention", + "full_attention" + ], + "linear_conv_kernel_dim": 4, + "linear_key_head_dim": 128, + "linear_num_key_heads": 16, + "linear_num_value_heads": 32, + "linear_value_head_dim": 128, + "mamba_ssm_dtype": "float32", + "max_position_embeddings": 262144, + "mlp_only_layers": [], + "model_type": "qwen3_5_moe_text", + "moe_intermediate_size": 512, + "mtp_num_hidden_layers": 1, + "mtp_use_dedicated_embeddings": false, + "num_attention_heads": 16, + "num_experts": 256, + "num_experts_per_tok": 8, + "num_hidden_layers": 40, + "num_key_value_heads": 2, + "output_router_logits": false, + "pad_token_id": null, + "partial_rotary_factor": 0.25, + "rms_norm_eps": 1e-06, + "rope_parameters": { + "mrope_interleaved": true, + "mrope_section": [ + 11, + 11, + 10 + ], + "partial_rotary_factor": 0.25, + "rope_theta": 10000000, + "rope_type": "default" + }, + "router_aux_loss_coef": 0.001, + "shared_expert_intermediate_size": 512, + "tie_word_embeddings": false, + "use_cache": true, + "vocab_size": 248320 + }, + "tie_word_embeddings": false, + "transformers_version": "5.2.0", + "video_token_id": 248057, + "vision_config": { + "deepstack_visual_indexes": [], + "depth": 27, + "dtype": "bfloat16", + "hidden_act": "gelu_pytorch_tanh", + "hidden_size": 1152, + "in_channels": 3, + "initializer_range": 0.02, + "intermediate_size": 4304, + "model_type": "qwen3_5_moe", + "num_heads": 16, + "num_position_embeddings": 2304, + "out_hidden_size": 2048, + "patch_size": 16, + "spatial_merge_size": 2, + "temporal_patch_size": 2 + }, + "vision_end_token_id": 248054, + "vision_start_token_id": 248053 +} diff --git a/posttrain_models/qwen3_5_35b_a3b_base_highlight_batch1245_epoch4/checkpoint-2000-merged/iter_0002000/common.pt b/posttrain_models/qwen3_5_35b_a3b_base_highlight_batch1245_epoch4/checkpoint-2000-merged/iter_0002000/common.pt new file mode 100644 index 0000000000000000000000000000000000000000..8d0ea215c437f7ef0b166193dba82a3d129d6c3a --- /dev/null +++ b/posttrain_models/qwen3_5_35b_a3b_base_highlight_batch1245_epoch4/checkpoint-2000-merged/iter_0002000/common.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b4457d7ba55b2d8952598c2ddc62634c839368969b368f1a7dba325273ee965e +size 15975 diff --git a/posttrain_models/qwen3_5_35b_a3b_base_highlight_batch1245_epoch4/checkpoint-2000-merged/latest_checkpointed_iteration.txt b/posttrain_models/qwen3_5_35b_a3b_base_highlight_batch1245_epoch4/checkpoint-2000-merged/latest_checkpointed_iteration.txt new file mode 100644 index 0000000000000000000000000000000000000000..9463411b62f21b7ed88bbe711c958b3b66153330 --- /dev/null +++ b/posttrain_models/qwen3_5_35b_a3b_base_highlight_batch1245_epoch4/checkpoint-2000-merged/latest_checkpointed_iteration.txt @@ -0,0 +1 @@ +2000 \ No newline at end of file diff --git a/posttrain_models/qwen3_5_35b_a3b_base_highlight_batch1245_epoch4/checkpoint-2000-merged/preprocessor_config.json b/posttrain_models/qwen3_5_35b_a3b_base_highlight_batch1245_epoch4/checkpoint-2000-merged/preprocessor_config.json new file mode 100644 index 0000000000000000000000000000000000000000..2ea84a437d448ff71b08df68fdd949d5cc4ebb64 --- /dev/null +++ b/posttrain_models/qwen3_5_35b_a3b_base_highlight_batch1245_epoch4/checkpoint-2000-merged/preprocessor_config.json @@ -0,0 +1,21 @@ +{ + "size": { + "longest_edge": 16777216, + "shortest_edge": 65536 + }, + "patch_size": 16, + "temporal_patch_size": 2, + "merge_size": 2, + "image_mean": [ + 0.5, + 0.5, + 0.5 + ], + "image_std": [ + 0.5, + 0.5, + 0.5 + ], + "processor_class": "Qwen3VLProcessor", + "image_processor_type": "Qwen2VLImageProcessorFast" +} \ No newline at end of file diff --git a/posttrain_models/qwen3_5_35b_a3b_base_highlight_batch1245_epoch4/checkpoint-2000-merged/processor_config.json b/posttrain_models/qwen3_5_35b_a3b_base_highlight_batch1245_epoch4/checkpoint-2000-merged/processor_config.json new file mode 100644 index 0000000000000000000000000000000000000000..7ad6acdf4203f22b7b990e36ccc3a1fe38563d5e --- /dev/null +++ b/posttrain_models/qwen3_5_35b_a3b_base_highlight_batch1245_epoch4/checkpoint-2000-merged/processor_config.json @@ -0,0 +1,63 @@ +{ + "image_processor": { + "data_format": "channels_first", + "do_convert_rgb": true, + "do_normalize": true, + "do_rescale": true, + "do_resize": true, + "image_mean": [ + 0.5, + 0.5, + 0.5 + ], + "image_processor_type": "Qwen2VLImageProcessorFast", + "image_std": [ + 0.5, + 0.5, + 0.5 + ], + "merge_size": 2, + "patch_size": 16, + "resample": 3, + "rescale_factor": 0.00392156862745098, + "size": { + "longest_edge": 16777216, + "shortest_edge": 65536 + }, + "temporal_patch_size": 2 + }, + "processor_class": "Qwen3VLProcessor", + "video_processor": { + "data_format": "channels_first", + "default_to_square": true, + "do_convert_rgb": true, + "do_normalize": true, + "do_rescale": true, + "do_resize": true, + "do_sample_frames": true, + "fps": 2, + "image_mean": [ + 0.5, + 0.5, + 0.5 + ], + "image_std": [ + 0.5, + 0.5, + 0.5 + ], + "max_frames": 768, + "merge_size": 2, + "min_frames": 4, + "patch_size": 16, + "resample": 3, + "rescale_factor": 0.00392156862745098, + "return_metadata": false, + "size": { + "longest_edge": 25165824, + "shortest_edge": 4096 + }, + "temporal_patch_size": 2, + "video_processor_type": "Qwen3VLVideoProcessor" + } +} diff --git a/posttrain_models/qwen3_5_35b_a3b_base_highlight_batch1245_epoch4/checkpoint-2000-merged/tokenizer_config.json b/posttrain_models/qwen3_5_35b_a3b_base_highlight_batch1245_epoch4/checkpoint-2000-merged/tokenizer_config.json new file mode 100644 index 0000000000000000000000000000000000000000..a068e2468cff426a9b105006e74e044030a6faf4 --- /dev/null +++ b/posttrain_models/qwen3_5_35b_a3b_base_highlight_batch1245_epoch4/checkpoint-2000-merged/tokenizer_config.json @@ -0,0 +1,32 @@ +{ + "add_prefix_space": false, + "audio_bos_token": "<|audio_start|>", + "audio_eos_token": "<|audio_end|>", + "audio_token": "<|audio_pad|>", + "backend": "tokenizers", + "bos_token": null, + "clean_up_tokenization_spaces": false, + "eos_token": "<|im_end|>", + "errors": "replace", + "image_token": "<|image_pad|>", + "is_local": true, + "model_max_length": 262144, + "model_specific_special_tokens": { + "audio_bos_token": "<|audio_start|>", + "audio_eos_token": "<|audio_end|>", + "audio_token": "<|audio_pad|>", + "image_token": "<|image_pad|>", + "video_token": "<|video_pad|>", + "vision_bos_token": "<|vision_start|>", + "vision_eos_token": "<|vision_end|>" + }, + "pad_token": "<|endoftext|>", + "pretokenize_regex": "(?i:'s|'t|'re|'ve|'m|'ll|'d)|[^\\r\\n\\p{L}\\p{N}]?[\\p{L}\\p{M}]+|\\p{N}| ?[^\\s\\p{L}\\p{M}\\p{N}]+[\\r\\n]*|\\s*[\\r\\n]+|\\s+(?!\\S)|\\s+", + "processor_class": "Qwen3VLProcessor", + "split_special_tokens": false, + "tokenizer_class": "TokenizersBackend", + "unk_token": null, + "video_token": "<|video_pad|>", + "vision_bos_token": "<|vision_start|>", + "vision_eos_token": "<|vision_end|>" +} diff --git a/posttrain_models/qwen3_5_35b_a3b_base_highlight_batch1245_epoch4/checkpoint-2000/args.json b/posttrain_models/qwen3_5_35b_a3b_base_highlight_batch1245_epoch4/checkpoint-2000/args.json new file mode 100644 index 0000000000000000000000000000000000000000..99623dd80e3398e11296749c2daf861d8dff0018 --- /dev/null +++ b/posttrain_models/qwen3_5_35b_a3b_base_highlight_batch1245_epoch4/checkpoint-2000/args.json @@ -0,0 +1,398 @@ +{ + "use_ray": false, + "ray_exp_name": null, + "device_groups": null, + "model": "checkpoints/Qwen3.5-35B-A3B", + "model_type": "qwen3_5_moe", + "model_revision": null, + "task_type": "causal_lm", + "torch_dtype": "bfloat16", + "attn_impl": null, + "experts_impl": null, + "new_special_tokens": [], + "num_labels": null, + "problem_type": null, + "rope_scaling": null, + "device_map": null, + "max_memory": {}, + "max_model_len": null, + "local_repo_path": null, + "init_strategy": null, + "template": "qwen3_5", + "system": null, + "max_length": 8192, + "truncation_strategy": "delete", + "max_pixels": null, + "agent_template": null, + "norm_bbox": null, + "use_chat_template": true, + "padding_side": "right", + "padding_free": true, + "loss_scale": "ignore_empty_think", + "sequence_parallel_size": 1, + "template_backend": "swift", + "response_prefix": "", + "enable_thinking": false, + "add_non_thinking_prefix": false, + "dataset": [ + "batch1", + "batch2", + "batch4_part1", + "batch4_part2", + "batch4_part3", + "batch5_part1", + "batch5_part2", + "batch5_part3" + ], + "val_dataset": [], + "cached_dataset": [], + "cached_val_dataset": [], + "split_dataset_ratio": 0.0, + "data_seed": 42, + "dataset_num_proc": 2, + "load_from_cache_file": false, + "dataset_shuffle": true, + "val_dataset_shuffle": false, + "streaming": false, + "interleave_prob": null, + "stopping_strategy": "first_exhausted", + "shuffle_buffer_size": 1000, + "download_mode": "reuse_dataset_if_exists", + "columns": {}, + "strict": false, + "remove_unused_columns": true, + "model_name": null, + "model_author": null, + "custom_dataset_info": [], + "quant_method": null, + "quant_bits": null, + "hqq_axis": null, + "bnb_4bit_compute_dtype": "bfloat16", + "bnb_4bit_quant_type": "nf4", + "bnb_4bit_use_double_quant": true, + "bnb_4bit_quant_storage": null, + "max_new_tokens": null, + "temperature": 0.9, + "top_k": 50, + "top_p": 0.9, + "repetition_penalty": 1.0, + "num_beams": 1, + "stream": false, + "stop_words": [], + "logprobs": false, + "top_logprobs": null, + "structured_outputs_regex": null, + "tuner_backend": "peft", + "tuner_type": "lora", + "train_type": "lora", + "adapters": [], + "external_plugins": [ + "ms-swift/base_highlight_video_dataset.py" + ], + "custom_register_path": [ + "ms-swift/base_highlight_video_dataset.py" + ], + "seed": 42, + "model_kwargs": {}, + "load_args": false, + "load_data_args": false, + "packing": false, + "packing_length": null, + "packing_num_proc": 1, + "lazy_tokenize": true, + "use_hf": false, + "hub_token": null, + "ddp_timeout": 18000000, + "ddp_backend": "nccl", + "ignore_args_error": false, + "use_swift_lora": false, + "freeze_llm": false, + "freeze_vit": true, + "freeze_aligner": true, + "freeze_parameters": [], + "freeze_parameters_regex": null, + "freeze_parameters_ratio": 0.0, + "trainable_parameters": [], + "trainable_parameters_regex": null, + "target_modules": [ + "all-linear" + ], + "target_regex": null, + "modules_to_save": [], + "lora_rank": 8, + "lora_alpha": 32, + "lora_dropout": 0.05, + "lora_bias": "none", + "lora_dtype": null, + "use_rslora": false, + "rlhf_type": null, + "loss_type": null, + "mcore_ref_model": null, + "mcore_ref_adapter": null, + "beta": null, + "rpo_alpha": null, + "reference_free": false, + "label_smoothing": 0.0, + "f_divergence_type": "reverse_kl", + "desirable_weight": 1.0, + "undesirable_weight": 1.0, + "calculate_KL": null, + "center_rewards_coefficient": null, + "teacher_model": null, + "teacher_model_type": null, + "teacher_model_revision": null, + "teacher_model_server": null, + "gkd_logits_topk": null, + "lmbda": 0.5, + "seq_kd": false, + "offload_teacher_model": false, + "sft_alpha": 0.0, + "generation_batch_size": null, + "steps_per_generation": null, + "num_generations": 8, + "num_generations_eval": null, + "max_completion_length": 512, + "importance_sampling_level": "token", + "tau_pos": 1.0, + "tau_neg": 1.05, + "epsilon": 0.2, + "epsilon_high": null, + "delta": null, + "use_vllm": true, + "vllm_mode": null, + "vllm_enable_prefix_caching": true, + "vllm_gpu_memory_utilization": 0.9, + "vllm_tensor_parallel_size": 1, + "vllm_max_model_len": null, + "vllm_enforce_eager": false, + "vllm_limit_mm_per_prompt": null, + "vllm_disable_cascade_attn": false, + "vllm_max_num_seqs": null, + "vllm_mm_processor_cache_gb": null, + "vllm_engine_kwargs": null, + "sleep_level": 0, + "offload_optimizer": false, + "offload_model": false, + "offload_bridge": false, + "vllm_server_base_url": null, + "vllm_server_host": null, + "vllm_server_port": [ + 8000 + ], + "vllm_server_timeout": 240.0, + "vllm_server_group_port": null, + "reward_funcs": [], + "reward_weights": null, + "cosine_min_len_value_wrong": -0.5, + "cosine_max_len_value_wrong": 0.0, + "cosine_min_len_value_correct": 1.0, + "cosine_max_len_value_correct": 0.5, + "cosine_max_len": null, + "repetition_n_grams": 3, + "repetition_max_penalty": -1.0, + "soft_max_length": null, + "soft_cache_length": null, + "dynamic_sample": false, + "max_resample_times": 3, + "overlong_filter": false, + "scale_rewards": "group", + "advantage_estimator": "grpo", + "kl_in_reward": false, + "wandb_log_unique_prompts": null, + "log_completions": false, + "rollout_importance_sampling_mode": null, + "rollout_importance_sampling_threshold": 2.0, + "log_rollout_offpolicy_metrics": false, + "off_policy_sequence_mask_delta": null, + "log_entropy": false, + "top_entropy_quantile": 1.0, + "reward_model": null, + "reward_model_plugin": null, + "sync_ref_model": false, + "ref_model_sync_steps": 512, + "ref_model_mixup_alpha": 0.6, + "async_generate": false, + "move_model_batches": null, + "multi_turn_scheduler": null, + "max_turns": null, + "completion_length_limit_scope": "per_round", + "vllm_server_pass_dataset": false, + "num_iterations": 1, + "micro_batch_size": 8, + "global_batch_size": 64, + "recompute_granularity": "full", + "recompute_method": "uniform", + "recompute_num_layers": 1, + "recompute_modules": [ + "core_attn" + ], + "train_iters": null, + "num_train_epochs": 4, + "masked_softmax_fusion": true, + "bias_dropout_fusion": true, + "bias_activation_fusion": true, + "apply_rope_fusion": false, + "gradient_accumulation_fusion": false, + "cross_entropy_loss_fusion": true, + "cross_entropy_fusion_impl": "te", + "calculate_per_token_loss": true, + "attention_backend": "", + "optimizer": "adam", + "optimizer_cpu_offload": false, + "optimizer_offload_fraction": 1.0, + "use_precision_aware_optimizer": false, + "main_grads_dtype": "float32", + "main_params_dtype": "float32", + "exp_avg_dtype": "float32", + "exp_avg_sq_dtype": "float32", + "manual_gc": false, + "manual_gc_steps": 0, + "manual_gc_eval": true, + "train_dataloader_shuffle": true, + "dataloader_num_workers": 1, + "dataloader_pin_memory": true, + "dataloader_persistent_workers": true, + "dataloader_prefetch_factor": 2, + "data_sharding": false, + "group_by_length": false, + "te_rng_tracker": false, + "data_parallel_random_init": false, + "mlp_padding_free": false, + "lr_warmup_init": 0.0, + "lr": 0.0001, + "lr_decay_style": "cosine", + "lr_decay_iters": null, + "lr_warmup_iters": 0, + "lr_warmup_fraction": 0.05, + "min_lr": 1e-05, + "lr_wsd_decay_style": "exponential", + "lr_wsd_decay_iters": null, + "weight_decay": 0.1, + "weight_decay_incr_style": "constant", + "start_weight_decay": 0.1, + "end_weight_decay": 0.1, + "clip_grad": 1.0, + "adam_beta1": 0.9, + "adam_beta2": 0.95, + "adam_eps": 1e-08, + "sgd_momentum": 0.9, + "output_dir": "/mnt/bn/strategy-mllm-train/user/tanyue/models/posttrain_models/qwen3_5_35b_a3b_base_highlight_batch1245_epoch4", + "save_steps": 500, + "no_save_optim": true, + "no_save_rng": true, + "mcore_model": null, + "mcore_adapter": null, + "no_load_optim": false, + "no_load_rng": false, + "finetune": true, + "perform_initialization": false, + "use_cpu_initialization": false, + "async_save": false, + "save_total_limit": null, + "metric_for_best_model": "loss", + "greater_is_better": false, + "use_persistent_ckpt_worker": false, + "dist_ckpt_save_pre_mcore_014": false, + "dist_ckpt_optim_fully_reshardable": false, + "distrib_optim_fully_reshardable_mem_efficient": false, + "local_rank": 0, + "use_distributed_optimizer": true, + "tensor_model_parallel_size": 1, + "pipeline_model_parallel_size": 2, + "decoder_first_pipeline_num_layers": null, + "decoder_last_pipeline_num_layers": null, + "account_for_embedding_in_pipeline_split": false, + "account_for_loss_in_pipeline_split": false, + "overlap_p2p_comm": false, + "align_param_gather": false, + "sequence_parallel": false, + "context_parallel_size": 1, + "tp_comm_overlap": false, + "overlap_grad_reduce": false, + "overlap_param_gather": false, + "overlap_param_gather_with_optimizer_step": false, + "align_grad_reduce": true, + "virtual_pipeline_model_parallel_size": null, + "microbatch_group_size_per_vp_stage": null, + "pipeline_model_parallel_layout": null, + "expert_model_parallel_size": 4, + "expert_tensor_parallel_size": 1, + "report_to": [ + "wandb" + ], + "logging_steps": 5, + "tensorboard_dir": "/mnt/bn/strategy-mllm-train/user/tanyue/models/posttrain_models/qwen3_5_35b_a3b_base_highlight_batch1245_epoch4/runs", + "tensorboard_queue_size": 50, + "wandb_project": "highlight_base_highlight_qwen3_5", + "wandb_exp_name": "qwen3_5_35b_a3b_base_highlight_batch1245_epoch4", + "swanlab_project": "megatron-swift", + "swanlab_exp_name": null, + "eval_iters": -1, + "eval_steps": 100000, + "fp8_format": null, + "fp8_recipe": "delayed", + "fp8_amax_history_len": 1024, + "fp8_amax_compute_algo": "max", + "fp8_param_gather": false, + "fp16": false, + "bf16": true, + "apply_query_key_layer_scaling": false, + "attention_softmax_in_fp32": true, + "accumulate_allreduce_grads_in_fp32": true, + "moe_router_load_balancing_type": null, + "moe_router_dtype": "fp32", + "moe_token_dispatcher_type": null, + "moe_enable_deepep": false, + "moe_grouped_gemm": true, + "moe_permute_fusion": true, + "moe_aux_loss_coeff": 1e-06, + "moe_z_loss_coeff": null, + "moe_shared_expert_overlap": true, + "moe_layer_recompute": false, + "moe_expert_capacity_factor": null, + "moe_pad_expert_input_to_capacity": false, + "moe_token_drop_policy": null, + "mtp_num_layers": null, + "mtp_loss_scaling_factor": 0.1, + "save_safetensors": true, + "ref_model": null, + "ref_adapters": [], + "merge_lora": true, + "max_shard_size": "5GB", + "vit_gradient_checkpointing": false, + "vit_lr": null, + "aligner_lr": null, + "gradient_checkpointing_kwargs": null, + "check_model": true, + "apply_wd_to_qk_layernorm": false, + "enable_dft_loss": false, + "enable_channel_loss": false, + "save_strategy": "steps", + "callbacks": [ + "print", + "default_flow", + "wandb" + ], + "add_version": false, + "create_checkpoint_symlink": false, + "swift_version": "4.0.1", + "ckpt_dir": null, + "rank": 0, + "global_world_size": 8, + "local_world_size": 8, + "model_suffix": "Qwen3.5-35B-A3B", + "model_info": "ModelInfo(model_type='qwen3_5_moe', model_dir='/opt/tiger/tanyue/highlight_tools/checkpoints/Qwen3.5-35B-A3B', torch_dtype=torch.bfloat16, max_model_len=262144, quant_method=None, quant_bits=None, rope_scaling=None, is_moe_model=True, is_multimodal=True, config=None, task_type='causal_lm', num_labels=None)", + "model_meta": "ModelMeta(model_type='qwen3_5_moe', model_groups=[ModelGroup(models=[Model(ms_model_id='Qwen/Qwen3.5-35B-A3B-Base', hf_model_id='Qwen/Qwen3.5-35B-A3B-Base', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen3.5-35B-A3B', hf_model_id='Qwen/Qwen3.5-35B-A3B', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen3.5-122B-A10B', hf_model_id='Qwen/Qwen3.5-122B-A10B', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen3.5-397B-A17B', hf_model_id='Qwen/Qwen3.5-397B-A17B', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen3.5-35B-A3B-FP8', hf_model_id='Qwen/Qwen3.5-35B-A3B-FP8', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen3.5-122B-A10B-FP8', hf_model_id='Qwen/Qwen3.5-122B-A10B-FP8', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen3.5-397B-A17B-FP8', hf_model_id='Qwen/Qwen3.5-397B-A17B-FP8', model_path=None, ms_revision=None, hf_revision=None)], template='qwen3_5', ignore_patterns=None, requires=None, tags=[])], loader=, template='qwen3_5', model_arch=MultiModelKeys(arch_name='qwen2_vl', embedding=None, module_list=None, lm_head=None, q_proj=None, k_proj=None, v_proj=None, o_proj=None, attention=None, mlp=None, down_proj=None, qkv_proj=None, qk_proj=None, qa_proj=None, qb_proj=None, kv_proj=None, kva_proj=None, kvb_proj=None, language_model=['model.language_model', 'lm_head'], aligner=['model.visual.merger'], vision_tower=['model.visual'], generator=[]), architectures=['Qwen3_5MoeForConditionalGeneration'], additional_saved_files=[], torch_dtype=None, is_multimodal=True, is_reward=False, task_type=None, ignore_patterns=None, requires=['transformers>=5.2.0', 'qwen_vl_utils>=0.0.14', 'decord'], tags=[])", + "model_dir": "/opt/tiger/tanyue/highlight_tools/checkpoints/Qwen3.5-35B-A3B", + "template_meta": "QwenTemplateMeta(template_type='qwen3_5', prefix=[], prompt=['<|im_start|>user\\n{{QUERY}}<|im_end|>\\n<|im_start|>assistant\\n'], chat_sep=['<|im_end|>\\n'], suffix=['<|im_end|>\\n'], template_cls=, system_prefix=['<|im_start|>system\\n{{SYSTEM}}<|im_end|>\\n'], default_system=None, auto_add_bos=False, stop_words=['<|endoftext|>'], agent_template='qwen3_5', is_thinking=True, thinking_prefix='\\n', non_thinking_prefix='\\n\\n\\n\\n', history_thinking_prefix='')", + "_val_dataset_exists": false, + "hub": "", + "seq_length": 8192, + "is_multimodal": true, + "megatron_model_meta": "MegatronModelMeta(megatron_model_type='qwen3_5', model_types=['qwen3_5', 'qwen3_5_moe'], bridge_cls=, visual_cls=, is_multimodal=True, loader=)", + "fp8": null, + "params_dtype": "bfloat16", + "world_size": 8, + "data_parallel_size": 4, + "num_microbatches": 2 +} \ No newline at end of file diff --git a/posttrain_models/qwen3_5_35b_a3b_base_highlight_batch1245_epoch4/checkpoint-2000/iter_0002000/.metadata b/posttrain_models/qwen3_5_35b_a3b_base_highlight_batch1245_epoch4/checkpoint-2000/iter_0002000/.metadata new file mode 100644 index 0000000000000000000000000000000000000000..2631a0db736d1307c0d14e4388759ad553d401a0 Binary files /dev/null and b/posttrain_models/qwen3_5_35b_a3b_base_highlight_batch1245_epoch4/checkpoint-2000/iter_0002000/.metadata differ diff --git a/posttrain_models/qwen3_5_35b_a3b_base_highlight_batch1245_epoch4/checkpoint-2000/iter_0002000/common.pt b/posttrain_models/qwen3_5_35b_a3b_base_highlight_batch1245_epoch4/checkpoint-2000/iter_0002000/common.pt new file mode 100644 index 0000000000000000000000000000000000000000..8d0ea215c437f7ef0b166193dba82a3d129d6c3a --- /dev/null +++ b/posttrain_models/qwen3_5_35b_a3b_base_highlight_batch1245_epoch4/checkpoint-2000/iter_0002000/common.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b4457d7ba55b2d8952598c2ddc62634c839368969b368f1a7dba325273ee965e +size 15975 diff --git a/posttrain_models/qwen3_5_35b_a3b_base_highlight_batch1245_epoch4/checkpoint-2000/iter_0002000/metadata.json b/posttrain_models/qwen3_5_35b_a3b_base_highlight_batch1245_epoch4/checkpoint-2000/iter_0002000/metadata.json new file mode 100644 index 0000000000000000000000000000000000000000..c9dbde0bcf4d2d993122dcc7d6bcb1eef8b6fb77 --- /dev/null +++ b/posttrain_models/qwen3_5_35b_a3b_base_highlight_batch1245_epoch4/checkpoint-2000/iter_0002000/metadata.json @@ -0,0 +1 @@ +{"sharded_backend": "torch_dist", "sharded_backend_version": 1, "common_backend": "torch", "common_backend_version": 1} \ No newline at end of file diff --git a/posttrain_models/qwen3_5_35b_a3b_base_highlight_batch1245_epoch4/checkpoint-2000/latest_checkpointed_iteration.txt b/posttrain_models/qwen3_5_35b_a3b_base_highlight_batch1245_epoch4/checkpoint-2000/latest_checkpointed_iteration.txt new file mode 100644 index 0000000000000000000000000000000000000000..9463411b62f21b7ed88bbe711c958b3b66153330 --- /dev/null +++ b/posttrain_models/qwen3_5_35b_a3b_base_highlight_batch1245_epoch4/checkpoint-2000/latest_checkpointed_iteration.txt @@ -0,0 +1 @@ +2000 \ No newline at end of file diff --git a/posttrain_models/qwen3_5_35b_a3b_base_highlight_batch1245_epoch4/checkpoint-2500-merged/args.json b/posttrain_models/qwen3_5_35b_a3b_base_highlight_batch1245_epoch4/checkpoint-2500-merged/args.json new file mode 100644 index 0000000000000000000000000000000000000000..99623dd80e3398e11296749c2daf861d8dff0018 --- /dev/null +++ b/posttrain_models/qwen3_5_35b_a3b_base_highlight_batch1245_epoch4/checkpoint-2500-merged/args.json @@ -0,0 +1,398 @@ +{ + "use_ray": false, + "ray_exp_name": null, + "device_groups": null, + "model": "checkpoints/Qwen3.5-35B-A3B", + "model_type": "qwen3_5_moe", + "model_revision": null, + "task_type": "causal_lm", + "torch_dtype": "bfloat16", + "attn_impl": null, + "experts_impl": null, + "new_special_tokens": [], + "num_labels": null, + "problem_type": null, + "rope_scaling": null, + "device_map": null, + "max_memory": {}, + "max_model_len": null, + "local_repo_path": null, + "init_strategy": null, + "template": "qwen3_5", + "system": null, + "max_length": 8192, + "truncation_strategy": "delete", + "max_pixels": null, + "agent_template": null, + "norm_bbox": null, + "use_chat_template": true, + "padding_side": "right", + "padding_free": true, + "loss_scale": "ignore_empty_think", + "sequence_parallel_size": 1, + "template_backend": "swift", + "response_prefix": "", + "enable_thinking": false, + "add_non_thinking_prefix": false, + "dataset": [ + "batch1", + "batch2", + "batch4_part1", + "batch4_part2", + "batch4_part3", + "batch5_part1", + "batch5_part2", + "batch5_part3" + ], + "val_dataset": [], + "cached_dataset": [], + "cached_val_dataset": [], + "split_dataset_ratio": 0.0, + "data_seed": 42, + "dataset_num_proc": 2, + "load_from_cache_file": false, + "dataset_shuffle": true, + "val_dataset_shuffle": false, + "streaming": false, + "interleave_prob": null, + "stopping_strategy": "first_exhausted", + "shuffle_buffer_size": 1000, + "download_mode": "reuse_dataset_if_exists", + "columns": {}, + "strict": false, + "remove_unused_columns": true, + "model_name": null, + "model_author": null, + "custom_dataset_info": [], + "quant_method": null, + "quant_bits": null, + "hqq_axis": null, + "bnb_4bit_compute_dtype": "bfloat16", + "bnb_4bit_quant_type": "nf4", + "bnb_4bit_use_double_quant": true, + "bnb_4bit_quant_storage": null, + "max_new_tokens": null, + "temperature": 0.9, + "top_k": 50, + "top_p": 0.9, + "repetition_penalty": 1.0, + "num_beams": 1, + "stream": false, + "stop_words": [], + "logprobs": false, + "top_logprobs": null, + "structured_outputs_regex": null, + "tuner_backend": "peft", + "tuner_type": "lora", + "train_type": "lora", + "adapters": [], + "external_plugins": [ + "ms-swift/base_highlight_video_dataset.py" + ], + "custom_register_path": [ + "ms-swift/base_highlight_video_dataset.py" + ], + "seed": 42, + "model_kwargs": {}, + "load_args": false, + "load_data_args": false, + "packing": false, + "packing_length": null, + "packing_num_proc": 1, + "lazy_tokenize": true, + "use_hf": false, + "hub_token": null, + "ddp_timeout": 18000000, + "ddp_backend": "nccl", + "ignore_args_error": false, + "use_swift_lora": false, + "freeze_llm": false, + "freeze_vit": true, + "freeze_aligner": true, + "freeze_parameters": [], + "freeze_parameters_regex": null, + "freeze_parameters_ratio": 0.0, + "trainable_parameters": [], + "trainable_parameters_regex": null, + "target_modules": [ + "all-linear" + ], + "target_regex": null, + "modules_to_save": [], + "lora_rank": 8, + "lora_alpha": 32, + "lora_dropout": 0.05, + "lora_bias": "none", + "lora_dtype": null, + "use_rslora": false, + "rlhf_type": null, + "loss_type": null, + "mcore_ref_model": null, + "mcore_ref_adapter": null, + "beta": null, + "rpo_alpha": null, + "reference_free": false, + "label_smoothing": 0.0, + "f_divergence_type": "reverse_kl", + "desirable_weight": 1.0, + "undesirable_weight": 1.0, + "calculate_KL": null, + "center_rewards_coefficient": null, + "teacher_model": null, + "teacher_model_type": null, + "teacher_model_revision": null, + "teacher_model_server": null, + "gkd_logits_topk": null, + "lmbda": 0.5, + "seq_kd": false, + "offload_teacher_model": false, + "sft_alpha": 0.0, + "generation_batch_size": null, + "steps_per_generation": null, + "num_generations": 8, + "num_generations_eval": null, + "max_completion_length": 512, + "importance_sampling_level": "token", + "tau_pos": 1.0, + "tau_neg": 1.05, + "epsilon": 0.2, + "epsilon_high": null, + "delta": null, + "use_vllm": true, + "vllm_mode": null, + "vllm_enable_prefix_caching": true, + "vllm_gpu_memory_utilization": 0.9, + "vllm_tensor_parallel_size": 1, + "vllm_max_model_len": null, + "vllm_enforce_eager": false, + "vllm_limit_mm_per_prompt": null, + "vllm_disable_cascade_attn": false, + "vllm_max_num_seqs": null, + "vllm_mm_processor_cache_gb": null, + "vllm_engine_kwargs": null, + "sleep_level": 0, + "offload_optimizer": false, + "offload_model": false, + "offload_bridge": false, + "vllm_server_base_url": null, + "vllm_server_host": null, + "vllm_server_port": [ + 8000 + ], + "vllm_server_timeout": 240.0, + "vllm_server_group_port": null, + "reward_funcs": [], + "reward_weights": null, + "cosine_min_len_value_wrong": -0.5, + "cosine_max_len_value_wrong": 0.0, + "cosine_min_len_value_correct": 1.0, + "cosine_max_len_value_correct": 0.5, + "cosine_max_len": null, + "repetition_n_grams": 3, + "repetition_max_penalty": -1.0, + "soft_max_length": null, + "soft_cache_length": null, + "dynamic_sample": false, + "max_resample_times": 3, + "overlong_filter": false, + "scale_rewards": "group", + "advantage_estimator": "grpo", + "kl_in_reward": false, + "wandb_log_unique_prompts": null, + "log_completions": false, + "rollout_importance_sampling_mode": null, + "rollout_importance_sampling_threshold": 2.0, + "log_rollout_offpolicy_metrics": false, + "off_policy_sequence_mask_delta": null, + "log_entropy": false, + "top_entropy_quantile": 1.0, + "reward_model": null, + "reward_model_plugin": null, + "sync_ref_model": false, + "ref_model_sync_steps": 512, + "ref_model_mixup_alpha": 0.6, + "async_generate": false, + "move_model_batches": null, + "multi_turn_scheduler": null, + "max_turns": null, + "completion_length_limit_scope": "per_round", + "vllm_server_pass_dataset": false, + "num_iterations": 1, + "micro_batch_size": 8, + "global_batch_size": 64, + "recompute_granularity": "full", + "recompute_method": "uniform", + "recompute_num_layers": 1, + "recompute_modules": [ + "core_attn" + ], + "train_iters": null, + "num_train_epochs": 4, + "masked_softmax_fusion": true, + "bias_dropout_fusion": true, + "bias_activation_fusion": true, + "apply_rope_fusion": false, + "gradient_accumulation_fusion": false, + "cross_entropy_loss_fusion": true, + "cross_entropy_fusion_impl": "te", + "calculate_per_token_loss": true, + "attention_backend": "", + "optimizer": "adam", + "optimizer_cpu_offload": false, + "optimizer_offload_fraction": 1.0, + "use_precision_aware_optimizer": false, + "main_grads_dtype": "float32", + "main_params_dtype": "float32", + "exp_avg_dtype": "float32", + "exp_avg_sq_dtype": "float32", + "manual_gc": false, + "manual_gc_steps": 0, + "manual_gc_eval": true, + "train_dataloader_shuffle": true, + "dataloader_num_workers": 1, + "dataloader_pin_memory": true, + "dataloader_persistent_workers": true, + "dataloader_prefetch_factor": 2, + "data_sharding": false, + "group_by_length": false, + "te_rng_tracker": false, + "data_parallel_random_init": false, + "mlp_padding_free": false, + "lr_warmup_init": 0.0, + "lr": 0.0001, + "lr_decay_style": "cosine", + "lr_decay_iters": null, + "lr_warmup_iters": 0, + "lr_warmup_fraction": 0.05, + "min_lr": 1e-05, + "lr_wsd_decay_style": "exponential", + "lr_wsd_decay_iters": null, + "weight_decay": 0.1, + "weight_decay_incr_style": "constant", + "start_weight_decay": 0.1, + "end_weight_decay": 0.1, + "clip_grad": 1.0, + "adam_beta1": 0.9, + "adam_beta2": 0.95, + "adam_eps": 1e-08, + "sgd_momentum": 0.9, + "output_dir": "/mnt/bn/strategy-mllm-train/user/tanyue/models/posttrain_models/qwen3_5_35b_a3b_base_highlight_batch1245_epoch4", + "save_steps": 500, + "no_save_optim": true, + "no_save_rng": true, + "mcore_model": null, + "mcore_adapter": null, + "no_load_optim": false, + "no_load_rng": false, + "finetune": true, + "perform_initialization": false, + "use_cpu_initialization": false, + "async_save": false, + "save_total_limit": null, + "metric_for_best_model": "loss", + "greater_is_better": false, + "use_persistent_ckpt_worker": false, + "dist_ckpt_save_pre_mcore_014": false, + "dist_ckpt_optim_fully_reshardable": false, + "distrib_optim_fully_reshardable_mem_efficient": false, + "local_rank": 0, + "use_distributed_optimizer": true, + "tensor_model_parallel_size": 1, + "pipeline_model_parallel_size": 2, + "decoder_first_pipeline_num_layers": null, + "decoder_last_pipeline_num_layers": null, + "account_for_embedding_in_pipeline_split": false, + "account_for_loss_in_pipeline_split": false, + "overlap_p2p_comm": false, + "align_param_gather": false, + "sequence_parallel": false, + "context_parallel_size": 1, + "tp_comm_overlap": false, + "overlap_grad_reduce": false, + "overlap_param_gather": false, + "overlap_param_gather_with_optimizer_step": false, + "align_grad_reduce": true, + "virtual_pipeline_model_parallel_size": null, + "microbatch_group_size_per_vp_stage": null, + "pipeline_model_parallel_layout": null, + "expert_model_parallel_size": 4, + "expert_tensor_parallel_size": 1, + "report_to": [ + "wandb" + ], + "logging_steps": 5, + "tensorboard_dir": "/mnt/bn/strategy-mllm-train/user/tanyue/models/posttrain_models/qwen3_5_35b_a3b_base_highlight_batch1245_epoch4/runs", + "tensorboard_queue_size": 50, + "wandb_project": "highlight_base_highlight_qwen3_5", + "wandb_exp_name": "qwen3_5_35b_a3b_base_highlight_batch1245_epoch4", + "swanlab_project": "megatron-swift", + "swanlab_exp_name": null, + "eval_iters": -1, + "eval_steps": 100000, + "fp8_format": null, + "fp8_recipe": "delayed", + "fp8_amax_history_len": 1024, + "fp8_amax_compute_algo": "max", + "fp8_param_gather": false, + "fp16": false, + "bf16": true, + "apply_query_key_layer_scaling": false, + "attention_softmax_in_fp32": true, + "accumulate_allreduce_grads_in_fp32": true, + "moe_router_load_balancing_type": null, + "moe_router_dtype": "fp32", + "moe_token_dispatcher_type": null, + "moe_enable_deepep": false, + "moe_grouped_gemm": true, + "moe_permute_fusion": true, + "moe_aux_loss_coeff": 1e-06, + "moe_z_loss_coeff": null, + "moe_shared_expert_overlap": true, + "moe_layer_recompute": false, + "moe_expert_capacity_factor": null, + "moe_pad_expert_input_to_capacity": false, + "moe_token_drop_policy": null, + "mtp_num_layers": null, + "mtp_loss_scaling_factor": 0.1, + "save_safetensors": true, + "ref_model": null, + "ref_adapters": [], + "merge_lora": true, + "max_shard_size": "5GB", + "vit_gradient_checkpointing": false, + "vit_lr": null, + "aligner_lr": null, + "gradient_checkpointing_kwargs": null, + "check_model": true, + "apply_wd_to_qk_layernorm": false, + "enable_dft_loss": false, + "enable_channel_loss": false, + "save_strategy": "steps", + "callbacks": [ + "print", + "default_flow", + "wandb" + ], + "add_version": false, + "create_checkpoint_symlink": false, + "swift_version": "4.0.1", + "ckpt_dir": null, + "rank": 0, + "global_world_size": 8, + "local_world_size": 8, + "model_suffix": "Qwen3.5-35B-A3B", + "model_info": "ModelInfo(model_type='qwen3_5_moe', model_dir='/opt/tiger/tanyue/highlight_tools/checkpoints/Qwen3.5-35B-A3B', torch_dtype=torch.bfloat16, max_model_len=262144, quant_method=None, quant_bits=None, rope_scaling=None, is_moe_model=True, is_multimodal=True, config=None, task_type='causal_lm', num_labels=None)", + "model_meta": "ModelMeta(model_type='qwen3_5_moe', model_groups=[ModelGroup(models=[Model(ms_model_id='Qwen/Qwen3.5-35B-A3B-Base', hf_model_id='Qwen/Qwen3.5-35B-A3B-Base', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen3.5-35B-A3B', hf_model_id='Qwen/Qwen3.5-35B-A3B', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen3.5-122B-A10B', hf_model_id='Qwen/Qwen3.5-122B-A10B', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen3.5-397B-A17B', hf_model_id='Qwen/Qwen3.5-397B-A17B', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen3.5-35B-A3B-FP8', hf_model_id='Qwen/Qwen3.5-35B-A3B-FP8', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen3.5-122B-A10B-FP8', hf_model_id='Qwen/Qwen3.5-122B-A10B-FP8', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen3.5-397B-A17B-FP8', hf_model_id='Qwen/Qwen3.5-397B-A17B-FP8', model_path=None, ms_revision=None, hf_revision=None)], template='qwen3_5', ignore_patterns=None, requires=None, tags=[])], loader=, template='qwen3_5', model_arch=MultiModelKeys(arch_name='qwen2_vl', embedding=None, module_list=None, lm_head=None, q_proj=None, k_proj=None, v_proj=None, o_proj=None, attention=None, mlp=None, down_proj=None, qkv_proj=None, qk_proj=None, qa_proj=None, qb_proj=None, kv_proj=None, kva_proj=None, kvb_proj=None, language_model=['model.language_model', 'lm_head'], aligner=['model.visual.merger'], vision_tower=['model.visual'], generator=[]), architectures=['Qwen3_5MoeForConditionalGeneration'], additional_saved_files=[], torch_dtype=None, is_multimodal=True, is_reward=False, task_type=None, ignore_patterns=None, requires=['transformers>=5.2.0', 'qwen_vl_utils>=0.0.14', 'decord'], tags=[])", + "model_dir": "/opt/tiger/tanyue/highlight_tools/checkpoints/Qwen3.5-35B-A3B", + "template_meta": "QwenTemplateMeta(template_type='qwen3_5', prefix=[], prompt=['<|im_start|>user\\n{{QUERY}}<|im_end|>\\n<|im_start|>assistant\\n'], chat_sep=['<|im_end|>\\n'], suffix=['<|im_end|>\\n'], template_cls=, system_prefix=['<|im_start|>system\\n{{SYSTEM}}<|im_end|>\\n'], default_system=None, auto_add_bos=False, stop_words=['<|endoftext|>'], agent_template='qwen3_5', is_thinking=True, thinking_prefix='\\n', non_thinking_prefix='\\n\\n\\n\\n', history_thinking_prefix='')", + "_val_dataset_exists": false, + "hub": "", + "seq_length": 8192, + "is_multimodal": true, + "megatron_model_meta": "MegatronModelMeta(megatron_model_type='qwen3_5', model_types=['qwen3_5', 'qwen3_5_moe'], bridge_cls=, visual_cls=, is_multimodal=True, loader=)", + "fp8": null, + "params_dtype": "bfloat16", + "world_size": 8, + "data_parallel_size": 4, + "num_microbatches": 2 +} \ No newline at end of file diff --git a/posttrain_models/qwen3_5_35b_a3b_base_highlight_batch1245_epoch4/checkpoint-2500-merged/chat_template.jinja b/posttrain_models/qwen3_5_35b_a3b_base_highlight_batch1245_epoch4/checkpoint-2500-merged/chat_template.jinja new file mode 100644 index 0000000000000000000000000000000000000000..a585dec894e63da457d9440ec6aa7caa16d20860 --- /dev/null +++ b/posttrain_models/qwen3_5_35b_a3b_base_highlight_batch1245_epoch4/checkpoint-2500-merged/chat_template.jinja @@ -0,0 +1,154 @@ +{%- set image_count = namespace(value=0) %} +{%- set video_count = namespace(value=0) %} +{%- macro render_content(content, do_vision_count, is_system_content=false) %} + {%- if content is string %} + {{- content }} + {%- elif content is iterable and content is not mapping %} + {%- for item in content %} + {%- if 'image' in item or 'image_url' in item or item.type == 'image' %} + {%- if is_system_content %} + {{- raise_exception('System message cannot contain images.') }} + {%- endif %} + {%- if do_vision_count %} + {%- set image_count.value = image_count.value + 1 %} + {%- endif %} + {%- if add_vision_id %} + {{- 'Picture ' ~ image_count.value ~ ': ' }} + {%- endif %} + {{- '<|vision_start|><|image_pad|><|vision_end|>' }} + {%- elif 'video' in item or item.type == 'video' %} + {%- if is_system_content %} + {{- raise_exception('System message cannot contain videos.') }} + {%- endif %} + {%- if do_vision_count %} + {%- set video_count.value = video_count.value + 1 %} + {%- endif %} + {%- if add_vision_id %} + {{- 'Video ' ~ video_count.value ~ ': ' }} + {%- endif %} + {{- '<|vision_start|><|video_pad|><|vision_end|>' }} + {%- elif 'text' in item %} + {{- item.text }} + {%- else %} + {{- raise_exception('Unexpected item type in content.') }} + {%- endif %} + {%- endfor %} + {%- elif content is none or content is undefined %} + {{- '' }} + {%- else %} + {{- raise_exception('Unexpected content type.') }} + {%- endif %} +{%- endmacro %} +{%- if not messages %} + {{- raise_exception('No messages provided.') }} +{%- endif %} +{%- if tools and tools is iterable and tools is not mapping %} + {{- '<|im_start|>system\n' }} + {{- "# Tools\n\nYou have access to the following functions:\n\n" }} + {%- for tool in tools %} + {{- "\n" }} + {{- tool | tojson }} + {%- endfor %} + {{- "\n" }} + {{- '\n\nIf you choose to call a function ONLY reply in the following format with NO suffix:\n\n\n\n\nvalue_1\n\n\nThis is the value for the second parameter\nthat can span\nmultiple lines\n\n\n\n\n\nReminder:\n- Function calls MUST follow the specified format: an inner block must be nested within XML tags\n- Required parameters MUST be specified\n- You may provide optional reasoning for your function call in natural language BEFORE the function call, but NOT after\n- If there is no function call available, answer the question like normal with your current knowledge and do not tell the user about function calls\n' }} + {%- if messages[0].role == 'system' %} + {%- set content = render_content(messages[0].content, false, true)|trim %} + {%- if content %} + {{- '\n\n' + content }} + {%- endif %} + {%- endif %} + {{- '<|im_end|>\n' }} +{%- else %} + {%- if messages[0].role == 'system' %} + {%- set content = render_content(messages[0].content, false, true)|trim %} + {{- '<|im_start|>system\n' + content + '<|im_end|>\n' }} + {%- endif %} +{%- endif %} +{%- set ns = namespace(multi_step_tool=true, last_query_index=messages|length - 1) %} +{%- for message in messages[::-1] %} + {%- set index = (messages|length - 1) - loop.index0 %} + {%- if ns.multi_step_tool and message.role == "user" %} + {%- set content = render_content(message.content, false)|trim %} + {%- if not(content.startswith('') and content.endswith('')) %} + {%- set ns.multi_step_tool = false %} + {%- set ns.last_query_index = index %} + {%- endif %} + {%- endif %} +{%- endfor %} +{%- if ns.multi_step_tool %} + {{- raise_exception('No user query found in messages.') }} +{%- endif %} +{%- for message in messages %} + {%- set content = render_content(message.content, true)|trim %} + {%- if message.role == "system" %} + {%- if not loop.first %} + {{- raise_exception('System message must be at the beginning.') }} + {%- endif %} + {%- elif message.role == "user" %} + {{- '<|im_start|>' + message.role + '\n' + content + '<|im_end|>' + '\n' }} + {%- elif message.role == "assistant" %} + {%- set reasoning_content = '' %} + {%- if message.reasoning_content is string %} + {%- set reasoning_content = message.reasoning_content %} + {%- else %} + {%- if '' in content %} + {%- set reasoning_content = content.split('')[0].rstrip('\n').split('')[-1].lstrip('\n') %} + {%- set content = content.split('')[-1].lstrip('\n') %} + {%- endif %} + {%- endif %} + {%- set reasoning_content = reasoning_content|trim %} + {%- if loop.index0 > ns.last_query_index %} + {{- '<|im_start|>' + message.role + '\n\n' + reasoning_content + '\n\n\n' + content }} + {%- else %} + {{- '<|im_start|>' + message.role + '\n' + content }} + {%- endif %} + {%- if message.tool_calls and message.tool_calls is iterable and message.tool_calls is not mapping %} + {%- for tool_call in message.tool_calls %} + {%- if tool_call.function is defined %} + {%- set tool_call = tool_call.function %} + {%- endif %} + {%- if loop.first %} + {%- if content|trim %} + {{- '\n\n\n\n' }} + {%- else %} + {{- '\n\n' }} + {%- endif %} + {%- else %} + {{- '\n\n\n' }} + {%- endif %} + {%- if tool_call.arguments is defined %} + {%- for args_name, args_value in tool_call.arguments|items %} + {{- '\n' }} + {%- set args_value = args_value | tojson | safe if args_value is mapping or (args_value is sequence and args_value is not string) else args_value | string %} + {{- args_value }} + {{- '\n\n' }} + {%- endfor %} + {%- endif %} + {{- '\n' }} + {%- endfor %} + {%- endif %} + {{- '<|im_end|>\n' }} + {%- elif message.role == "tool" %} + {%- if loop.previtem and loop.previtem.role != "tool" %} + {{- '<|im_start|>user' }} + {%- endif %} + {{- '\n\n' }} + {{- content }} + {{- '\n' }} + {%- if not loop.last and loop.nextitem.role != "tool" %} + {{- '<|im_end|>\n' }} + {%- elif loop.last %} + {{- '<|im_end|>\n' }} + {%- endif %} + {%- else %} + {{- raise_exception('Unexpected message role.') }} + {%- endif %} +{%- endfor %} +{%- if add_generation_prompt %} + {{- '<|im_start|>assistant\n' }} + {%- if enable_thinking is defined and enable_thinking is false %} + {{- '\n\n\n\n' }} + {%- else %} + {{- '\n' }} + {%- endif %} +{%- endif %} \ No newline at end of file diff --git a/posttrain_models/qwen3_5_35b_a3b_base_highlight_batch1245_epoch4/checkpoint-2500-merged/config.json b/posttrain_models/qwen3_5_35b_a3b_base_highlight_batch1245_epoch4/checkpoint-2500-merged/config.json new file mode 100644 index 0000000000000000000000000000000000000000..8643d6d3f2e157c74392e7d91f431a66f9261762 --- /dev/null +++ b/posttrain_models/qwen3_5_35b_a3b_base_highlight_batch1245_epoch4/checkpoint-2500-merged/config.json @@ -0,0 +1,123 @@ +{ + "architectures": [ + "Qwen3_5MoeForConditionalGeneration" + ], + "dtype": "bfloat16", + "hidden_size": 2048, + "image_token_id": 248056, + "model_type": "qwen3_5_moe", + "text_config": { + "attention_bias": false, + "attention_dropout": 0.0, + "attn_output_gate": true, + "bos_token_id": null, + "dtype": "bfloat16", + "eos_token_id": 248044, + "full_attention_interval": 4, + "head_dim": 256, + "hidden_act": "silu", + "hidden_size": 2048, + "initializer_range": 0.02, + "layer_types": [ + "linear_attention", + "linear_attention", + "linear_attention", + "full_attention", + "linear_attention", + "linear_attention", + "linear_attention", + "full_attention", + "linear_attention", + "linear_attention", + "linear_attention", + "full_attention", + "linear_attention", + "linear_attention", + "linear_attention", + "full_attention", + "linear_attention", + "linear_attention", + "linear_attention", + "full_attention", + "linear_attention", + "linear_attention", + "linear_attention", + "full_attention", + "linear_attention", + "linear_attention", + "linear_attention", + "full_attention", + "linear_attention", + "linear_attention", + "linear_attention", + "full_attention", + "linear_attention", + "linear_attention", + "linear_attention", + "full_attention", + "linear_attention", + "linear_attention", + "linear_attention", + "full_attention" + ], + "linear_conv_kernel_dim": 4, + "linear_key_head_dim": 128, + "linear_num_key_heads": 16, + "linear_num_value_heads": 32, + "linear_value_head_dim": 128, + "mamba_ssm_dtype": "float32", + "max_position_embeddings": 262144, + "mlp_only_layers": [], + "model_type": "qwen3_5_moe_text", + "moe_intermediate_size": 512, + "mtp_num_hidden_layers": 1, + "mtp_use_dedicated_embeddings": false, + "num_attention_heads": 16, + "num_experts": 256, + "num_experts_per_tok": 8, + "num_hidden_layers": 40, + "num_key_value_heads": 2, + "output_router_logits": false, + "pad_token_id": null, + "partial_rotary_factor": 0.25, + "rms_norm_eps": 1e-06, + "rope_parameters": { + "mrope_interleaved": true, + "mrope_section": [ + 11, + 11, + 10 + ], + "partial_rotary_factor": 0.25, + "rope_theta": 10000000, + "rope_type": "default" + }, + "router_aux_loss_coef": 0.001, + "shared_expert_intermediate_size": 512, + "tie_word_embeddings": false, + "use_cache": true, + "vocab_size": 248320 + }, + "tie_word_embeddings": false, + "transformers_version": "5.2.0", + "video_token_id": 248057, + "vision_config": { + "deepstack_visual_indexes": [], + "depth": 27, + "dtype": "bfloat16", + "hidden_act": "gelu_pytorch_tanh", + "hidden_size": 1152, + "in_channels": 3, + "initializer_range": 0.02, + "intermediate_size": 4304, + "model_type": "qwen3_5_moe", + "num_heads": 16, + "num_position_embeddings": 2304, + "out_hidden_size": 2048, + "patch_size": 16, + "spatial_merge_size": 2, + "temporal_patch_size": 2 + }, + "vision_end_token_id": 248054, + "vision_start_token_id": 248053 +} diff --git a/posttrain_models/qwen3_5_35b_a3b_base_highlight_batch1245_epoch4/checkpoint-2500-merged/iter_0002500/common.pt b/posttrain_models/qwen3_5_35b_a3b_base_highlight_batch1245_epoch4/checkpoint-2500-merged/iter_0002500/common.pt new file mode 100644 index 0000000000000000000000000000000000000000..a83f630c6ef4400a6bed933a5bd15efcc6bc4c77 --- /dev/null +++ b/posttrain_models/qwen3_5_35b_a3b_base_highlight_batch1245_epoch4/checkpoint-2500-merged/iter_0002500/common.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3f147316675d2dc08483cb28178cb2f0abcf20b7b2c1e0c60544a7f6ef0289ea +size 15975 diff --git a/posttrain_models/qwen3_5_35b_a3b_base_highlight_batch1245_epoch4/checkpoint-2500-merged/latest_checkpointed_iteration.txt b/posttrain_models/qwen3_5_35b_a3b_base_highlight_batch1245_epoch4/checkpoint-2500-merged/latest_checkpointed_iteration.txt new file mode 100644 index 0000000000000000000000000000000000000000..17ddab537cd985db11969e021ace6f6072bb6012 --- /dev/null +++ b/posttrain_models/qwen3_5_35b_a3b_base_highlight_batch1245_epoch4/checkpoint-2500-merged/latest_checkpointed_iteration.txt @@ -0,0 +1 @@ +2500 \ No newline at end of file diff --git a/posttrain_models/qwen3_5_35b_a3b_base_highlight_batch1245_epoch4/checkpoint-2500-merged/preprocessor_config.json b/posttrain_models/qwen3_5_35b_a3b_base_highlight_batch1245_epoch4/checkpoint-2500-merged/preprocessor_config.json new file mode 100644 index 0000000000000000000000000000000000000000..2ea84a437d448ff71b08df68fdd949d5cc4ebb64 --- /dev/null +++ b/posttrain_models/qwen3_5_35b_a3b_base_highlight_batch1245_epoch4/checkpoint-2500-merged/preprocessor_config.json @@ -0,0 +1,21 @@ +{ + "size": { + "longest_edge": 16777216, + "shortest_edge": 65536 + }, + "patch_size": 16, + "temporal_patch_size": 2, + "merge_size": 2, + "image_mean": [ + 0.5, + 0.5, + 0.5 + ], + "image_std": [ + 0.5, + 0.5, + 0.5 + ], + "processor_class": "Qwen3VLProcessor", + "image_processor_type": "Qwen2VLImageProcessorFast" +} \ No newline at end of file diff --git a/posttrain_models/qwen3_5_35b_a3b_base_highlight_batch1245_epoch4/checkpoint-2500-merged/processor_config.json b/posttrain_models/qwen3_5_35b_a3b_base_highlight_batch1245_epoch4/checkpoint-2500-merged/processor_config.json new file mode 100644 index 0000000000000000000000000000000000000000..7ad6acdf4203f22b7b990e36ccc3a1fe38563d5e --- /dev/null +++ b/posttrain_models/qwen3_5_35b_a3b_base_highlight_batch1245_epoch4/checkpoint-2500-merged/processor_config.json @@ -0,0 +1,63 @@ +{ + "image_processor": { + "data_format": "channels_first", + "do_convert_rgb": true, + "do_normalize": true, + "do_rescale": true, + "do_resize": true, + "image_mean": [ + 0.5, + 0.5, + 0.5 + ], + "image_processor_type": "Qwen2VLImageProcessorFast", + "image_std": [ + 0.5, + 0.5, + 0.5 + ], + "merge_size": 2, + "patch_size": 16, + "resample": 3, + "rescale_factor": 0.00392156862745098, + "size": { + "longest_edge": 16777216, + "shortest_edge": 65536 + }, + "temporal_patch_size": 2 + }, + "processor_class": "Qwen3VLProcessor", + "video_processor": { + "data_format": "channels_first", + "default_to_square": true, + "do_convert_rgb": true, + "do_normalize": true, + "do_rescale": true, + "do_resize": true, + "do_sample_frames": true, + "fps": 2, + "image_mean": [ + 0.5, + 0.5, + 0.5 + ], + "image_std": [ + 0.5, + 0.5, + 0.5 + ], + "max_frames": 768, + "merge_size": 2, + "min_frames": 4, + "patch_size": 16, + "resample": 3, + "rescale_factor": 0.00392156862745098, + "return_metadata": false, + "size": { + "longest_edge": 25165824, + "shortest_edge": 4096 + }, + "temporal_patch_size": 2, + "video_processor_type": "Qwen3VLVideoProcessor" + } +} diff --git a/posttrain_models/qwen3_5_35b_a3b_base_highlight_batch1245_epoch4/checkpoint-2500-merged/tokenizer_config.json b/posttrain_models/qwen3_5_35b_a3b_base_highlight_batch1245_epoch4/checkpoint-2500-merged/tokenizer_config.json new file mode 100644 index 0000000000000000000000000000000000000000..a068e2468cff426a9b105006e74e044030a6faf4 --- /dev/null +++ b/posttrain_models/qwen3_5_35b_a3b_base_highlight_batch1245_epoch4/checkpoint-2500-merged/tokenizer_config.json @@ -0,0 +1,32 @@ +{ + "add_prefix_space": false, + "audio_bos_token": "<|audio_start|>", + "audio_eos_token": "<|audio_end|>", + "audio_token": "<|audio_pad|>", + "backend": "tokenizers", + "bos_token": null, + "clean_up_tokenization_spaces": false, + "eos_token": "<|im_end|>", + "errors": "replace", + "image_token": "<|image_pad|>", + "is_local": true, + "model_max_length": 262144, + "model_specific_special_tokens": { + "audio_bos_token": "<|audio_start|>", + "audio_eos_token": "<|audio_end|>", + "audio_token": "<|audio_pad|>", + "image_token": "<|image_pad|>", + "video_token": "<|video_pad|>", + "vision_bos_token": "<|vision_start|>", + "vision_eos_token": "<|vision_end|>" + }, + "pad_token": "<|endoftext|>", + "pretokenize_regex": "(?i:'s|'t|'re|'ve|'m|'ll|'d)|[^\\r\\n\\p{L}\\p{N}]?[\\p{L}\\p{M}]+|\\p{N}| ?[^\\s\\p{L}\\p{M}\\p{N}]+[\\r\\n]*|\\s*[\\r\\n]+|\\s+(?!\\S)|\\s+", + "processor_class": "Qwen3VLProcessor", + "split_special_tokens": false, + "tokenizer_class": "TokenizersBackend", + "unk_token": null, + "video_token": "<|video_pad|>", + "vision_bos_token": "<|vision_start|>", + "vision_eos_token": "<|vision_end|>" +} diff --git a/posttrain_models/qwen3_5_35b_a3b_base_highlight_batch1245_epoch4/checkpoint-2500/args.json b/posttrain_models/qwen3_5_35b_a3b_base_highlight_batch1245_epoch4/checkpoint-2500/args.json new file mode 100644 index 0000000000000000000000000000000000000000..99623dd80e3398e11296749c2daf861d8dff0018 --- /dev/null +++ b/posttrain_models/qwen3_5_35b_a3b_base_highlight_batch1245_epoch4/checkpoint-2500/args.json @@ -0,0 +1,398 @@ +{ + "use_ray": false, + "ray_exp_name": null, + "device_groups": null, + "model": "checkpoints/Qwen3.5-35B-A3B", + "model_type": "qwen3_5_moe", + "model_revision": null, + "task_type": "causal_lm", + "torch_dtype": "bfloat16", + "attn_impl": null, + "experts_impl": null, + "new_special_tokens": [], + "num_labels": null, + "problem_type": null, + "rope_scaling": null, + "device_map": null, + "max_memory": {}, + "max_model_len": null, + "local_repo_path": null, + "init_strategy": null, + "template": "qwen3_5", + "system": null, + "max_length": 8192, + "truncation_strategy": "delete", + "max_pixels": null, + "agent_template": null, + "norm_bbox": null, + "use_chat_template": true, + "padding_side": "right", + "padding_free": true, + "loss_scale": "ignore_empty_think", + "sequence_parallel_size": 1, + "template_backend": "swift", + "response_prefix": "", + "enable_thinking": false, + "add_non_thinking_prefix": false, + "dataset": [ + "batch1", + "batch2", + "batch4_part1", + "batch4_part2", + "batch4_part3", + "batch5_part1", + "batch5_part2", + "batch5_part3" + ], + "val_dataset": [], + "cached_dataset": [], + "cached_val_dataset": [], + "split_dataset_ratio": 0.0, + "data_seed": 42, + "dataset_num_proc": 2, + "load_from_cache_file": false, + "dataset_shuffle": true, + "val_dataset_shuffle": false, + "streaming": false, + "interleave_prob": null, + "stopping_strategy": "first_exhausted", + "shuffle_buffer_size": 1000, + "download_mode": "reuse_dataset_if_exists", + "columns": {}, + "strict": false, + "remove_unused_columns": true, + "model_name": null, + "model_author": null, + "custom_dataset_info": [], + "quant_method": null, + "quant_bits": null, + "hqq_axis": null, + "bnb_4bit_compute_dtype": "bfloat16", + "bnb_4bit_quant_type": "nf4", + "bnb_4bit_use_double_quant": true, + "bnb_4bit_quant_storage": null, + "max_new_tokens": null, + "temperature": 0.9, + "top_k": 50, + "top_p": 0.9, + "repetition_penalty": 1.0, + "num_beams": 1, + "stream": false, + "stop_words": [], + "logprobs": false, + "top_logprobs": null, + "structured_outputs_regex": null, + "tuner_backend": "peft", + "tuner_type": "lora", + "train_type": "lora", + "adapters": [], + "external_plugins": [ + "ms-swift/base_highlight_video_dataset.py" + ], + "custom_register_path": [ + "ms-swift/base_highlight_video_dataset.py" + ], + "seed": 42, + "model_kwargs": {}, + "load_args": false, + "load_data_args": false, + "packing": false, + "packing_length": null, + "packing_num_proc": 1, + "lazy_tokenize": true, + "use_hf": false, + "hub_token": null, + "ddp_timeout": 18000000, + "ddp_backend": "nccl", + "ignore_args_error": false, + "use_swift_lora": false, + "freeze_llm": false, + "freeze_vit": true, + "freeze_aligner": true, + "freeze_parameters": [], + "freeze_parameters_regex": null, + "freeze_parameters_ratio": 0.0, + "trainable_parameters": [], + "trainable_parameters_regex": null, + "target_modules": [ + "all-linear" + ], + "target_regex": null, + "modules_to_save": [], + "lora_rank": 8, + "lora_alpha": 32, + "lora_dropout": 0.05, + "lora_bias": "none", + "lora_dtype": null, + "use_rslora": false, + "rlhf_type": null, + "loss_type": null, + "mcore_ref_model": null, + "mcore_ref_adapter": null, + "beta": null, + "rpo_alpha": null, + "reference_free": false, + "label_smoothing": 0.0, + "f_divergence_type": "reverse_kl", + "desirable_weight": 1.0, + "undesirable_weight": 1.0, + "calculate_KL": null, + "center_rewards_coefficient": null, + "teacher_model": null, + "teacher_model_type": null, + "teacher_model_revision": null, + "teacher_model_server": null, + "gkd_logits_topk": null, + "lmbda": 0.5, + "seq_kd": false, + "offload_teacher_model": false, + "sft_alpha": 0.0, + "generation_batch_size": null, + "steps_per_generation": null, + "num_generations": 8, + "num_generations_eval": null, + "max_completion_length": 512, + "importance_sampling_level": "token", + "tau_pos": 1.0, + "tau_neg": 1.05, + "epsilon": 0.2, + "epsilon_high": null, + "delta": null, + "use_vllm": true, + "vllm_mode": null, + "vllm_enable_prefix_caching": true, + "vllm_gpu_memory_utilization": 0.9, + "vllm_tensor_parallel_size": 1, + "vllm_max_model_len": null, + "vllm_enforce_eager": false, + "vllm_limit_mm_per_prompt": null, + "vllm_disable_cascade_attn": false, + "vllm_max_num_seqs": null, + "vllm_mm_processor_cache_gb": null, + "vllm_engine_kwargs": null, + "sleep_level": 0, + "offload_optimizer": false, + "offload_model": false, + "offload_bridge": false, + "vllm_server_base_url": null, + "vllm_server_host": null, + "vllm_server_port": [ + 8000 + ], + "vllm_server_timeout": 240.0, + "vllm_server_group_port": null, + "reward_funcs": [], + "reward_weights": null, + "cosine_min_len_value_wrong": -0.5, + "cosine_max_len_value_wrong": 0.0, + "cosine_min_len_value_correct": 1.0, + "cosine_max_len_value_correct": 0.5, + "cosine_max_len": null, + "repetition_n_grams": 3, + "repetition_max_penalty": -1.0, + "soft_max_length": null, + "soft_cache_length": null, + "dynamic_sample": false, + "max_resample_times": 3, + "overlong_filter": false, + "scale_rewards": "group", + "advantage_estimator": "grpo", + "kl_in_reward": false, + "wandb_log_unique_prompts": null, + "log_completions": false, + "rollout_importance_sampling_mode": null, + "rollout_importance_sampling_threshold": 2.0, + "log_rollout_offpolicy_metrics": false, + "off_policy_sequence_mask_delta": null, + "log_entropy": false, + "top_entropy_quantile": 1.0, + "reward_model": null, + "reward_model_plugin": null, + "sync_ref_model": false, + "ref_model_sync_steps": 512, + "ref_model_mixup_alpha": 0.6, + "async_generate": false, + "move_model_batches": null, + "multi_turn_scheduler": null, + "max_turns": null, + "completion_length_limit_scope": "per_round", + "vllm_server_pass_dataset": false, + "num_iterations": 1, + "micro_batch_size": 8, + "global_batch_size": 64, + "recompute_granularity": "full", + "recompute_method": "uniform", + "recompute_num_layers": 1, + "recompute_modules": [ + "core_attn" + ], + "train_iters": null, + "num_train_epochs": 4, + "masked_softmax_fusion": true, + "bias_dropout_fusion": true, + "bias_activation_fusion": true, + "apply_rope_fusion": false, + "gradient_accumulation_fusion": false, + "cross_entropy_loss_fusion": true, + "cross_entropy_fusion_impl": "te", + "calculate_per_token_loss": true, + "attention_backend": "", + "optimizer": "adam", + "optimizer_cpu_offload": false, + "optimizer_offload_fraction": 1.0, + "use_precision_aware_optimizer": false, + "main_grads_dtype": "float32", + "main_params_dtype": "float32", + "exp_avg_dtype": "float32", + "exp_avg_sq_dtype": "float32", + "manual_gc": false, + "manual_gc_steps": 0, + "manual_gc_eval": true, + "train_dataloader_shuffle": true, + "dataloader_num_workers": 1, + "dataloader_pin_memory": true, + "dataloader_persistent_workers": true, + "dataloader_prefetch_factor": 2, + "data_sharding": false, + "group_by_length": false, + "te_rng_tracker": false, + "data_parallel_random_init": false, + "mlp_padding_free": false, + "lr_warmup_init": 0.0, + "lr": 0.0001, + "lr_decay_style": "cosine", + "lr_decay_iters": null, + "lr_warmup_iters": 0, + "lr_warmup_fraction": 0.05, + "min_lr": 1e-05, + "lr_wsd_decay_style": "exponential", + "lr_wsd_decay_iters": null, + "weight_decay": 0.1, + "weight_decay_incr_style": "constant", + "start_weight_decay": 0.1, + "end_weight_decay": 0.1, + "clip_grad": 1.0, + "adam_beta1": 0.9, + "adam_beta2": 0.95, + "adam_eps": 1e-08, + "sgd_momentum": 0.9, + "output_dir": "/mnt/bn/strategy-mllm-train/user/tanyue/models/posttrain_models/qwen3_5_35b_a3b_base_highlight_batch1245_epoch4", + "save_steps": 500, + "no_save_optim": true, + "no_save_rng": true, + "mcore_model": null, + "mcore_adapter": null, + "no_load_optim": false, + "no_load_rng": false, + "finetune": true, + "perform_initialization": false, + "use_cpu_initialization": false, + "async_save": false, + "save_total_limit": null, + "metric_for_best_model": "loss", + "greater_is_better": false, + "use_persistent_ckpt_worker": false, + "dist_ckpt_save_pre_mcore_014": false, + "dist_ckpt_optim_fully_reshardable": false, + "distrib_optim_fully_reshardable_mem_efficient": false, + "local_rank": 0, + "use_distributed_optimizer": true, + "tensor_model_parallel_size": 1, + "pipeline_model_parallel_size": 2, + "decoder_first_pipeline_num_layers": null, + "decoder_last_pipeline_num_layers": null, + "account_for_embedding_in_pipeline_split": false, + "account_for_loss_in_pipeline_split": false, + "overlap_p2p_comm": false, + "align_param_gather": false, + "sequence_parallel": false, + "context_parallel_size": 1, + "tp_comm_overlap": false, + "overlap_grad_reduce": false, + "overlap_param_gather": false, + "overlap_param_gather_with_optimizer_step": false, + "align_grad_reduce": true, + "virtual_pipeline_model_parallel_size": null, + "microbatch_group_size_per_vp_stage": null, + "pipeline_model_parallel_layout": null, + "expert_model_parallel_size": 4, + "expert_tensor_parallel_size": 1, + "report_to": [ + "wandb" + ], + "logging_steps": 5, + "tensorboard_dir": "/mnt/bn/strategy-mllm-train/user/tanyue/models/posttrain_models/qwen3_5_35b_a3b_base_highlight_batch1245_epoch4/runs", + "tensorboard_queue_size": 50, + "wandb_project": "highlight_base_highlight_qwen3_5", + "wandb_exp_name": "qwen3_5_35b_a3b_base_highlight_batch1245_epoch4", + "swanlab_project": "megatron-swift", + "swanlab_exp_name": null, + "eval_iters": -1, + "eval_steps": 100000, + "fp8_format": null, + "fp8_recipe": "delayed", + "fp8_amax_history_len": 1024, + "fp8_amax_compute_algo": "max", + "fp8_param_gather": false, + "fp16": false, + "bf16": true, + "apply_query_key_layer_scaling": false, + "attention_softmax_in_fp32": true, + "accumulate_allreduce_grads_in_fp32": true, + "moe_router_load_balancing_type": null, + "moe_router_dtype": "fp32", + "moe_token_dispatcher_type": null, + "moe_enable_deepep": false, + "moe_grouped_gemm": true, + "moe_permute_fusion": true, + "moe_aux_loss_coeff": 1e-06, + "moe_z_loss_coeff": null, + "moe_shared_expert_overlap": true, + "moe_layer_recompute": false, + "moe_expert_capacity_factor": null, + "moe_pad_expert_input_to_capacity": false, + "moe_token_drop_policy": null, + "mtp_num_layers": null, + "mtp_loss_scaling_factor": 0.1, + "save_safetensors": true, + "ref_model": null, + "ref_adapters": [], + "merge_lora": true, + "max_shard_size": "5GB", + "vit_gradient_checkpointing": false, + "vit_lr": null, + "aligner_lr": null, + "gradient_checkpointing_kwargs": null, + "check_model": true, + "apply_wd_to_qk_layernorm": false, + "enable_dft_loss": false, + "enable_channel_loss": false, + "save_strategy": "steps", + "callbacks": [ + "print", + "default_flow", + "wandb" + ], + "add_version": false, + "create_checkpoint_symlink": false, + "swift_version": "4.0.1", + "ckpt_dir": null, + "rank": 0, + "global_world_size": 8, + "local_world_size": 8, + "model_suffix": "Qwen3.5-35B-A3B", + "model_info": "ModelInfo(model_type='qwen3_5_moe', model_dir='/opt/tiger/tanyue/highlight_tools/checkpoints/Qwen3.5-35B-A3B', torch_dtype=torch.bfloat16, max_model_len=262144, quant_method=None, quant_bits=None, rope_scaling=None, is_moe_model=True, is_multimodal=True, config=None, task_type='causal_lm', num_labels=None)", + "model_meta": "ModelMeta(model_type='qwen3_5_moe', model_groups=[ModelGroup(models=[Model(ms_model_id='Qwen/Qwen3.5-35B-A3B-Base', hf_model_id='Qwen/Qwen3.5-35B-A3B-Base', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen3.5-35B-A3B', hf_model_id='Qwen/Qwen3.5-35B-A3B', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen3.5-122B-A10B', hf_model_id='Qwen/Qwen3.5-122B-A10B', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen3.5-397B-A17B', hf_model_id='Qwen/Qwen3.5-397B-A17B', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen3.5-35B-A3B-FP8', hf_model_id='Qwen/Qwen3.5-35B-A3B-FP8', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen3.5-122B-A10B-FP8', hf_model_id='Qwen/Qwen3.5-122B-A10B-FP8', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen3.5-397B-A17B-FP8', hf_model_id='Qwen/Qwen3.5-397B-A17B-FP8', model_path=None, ms_revision=None, hf_revision=None)], template='qwen3_5', ignore_patterns=None, requires=None, tags=[])], loader=, template='qwen3_5', model_arch=MultiModelKeys(arch_name='qwen2_vl', embedding=None, module_list=None, lm_head=None, q_proj=None, k_proj=None, v_proj=None, o_proj=None, attention=None, mlp=None, down_proj=None, qkv_proj=None, qk_proj=None, qa_proj=None, qb_proj=None, kv_proj=None, kva_proj=None, kvb_proj=None, language_model=['model.language_model', 'lm_head'], aligner=['model.visual.merger'], vision_tower=['model.visual'], generator=[]), architectures=['Qwen3_5MoeForConditionalGeneration'], additional_saved_files=[], torch_dtype=None, is_multimodal=True, is_reward=False, task_type=None, ignore_patterns=None, requires=['transformers>=5.2.0', 'qwen_vl_utils>=0.0.14', 'decord'], tags=[])", + "model_dir": "/opt/tiger/tanyue/highlight_tools/checkpoints/Qwen3.5-35B-A3B", + "template_meta": "QwenTemplateMeta(template_type='qwen3_5', prefix=[], prompt=['<|im_start|>user\\n{{QUERY}}<|im_end|>\\n<|im_start|>assistant\\n'], chat_sep=['<|im_end|>\\n'], suffix=['<|im_end|>\\n'], template_cls=, system_prefix=['<|im_start|>system\\n{{SYSTEM}}<|im_end|>\\n'], default_system=None, auto_add_bos=False, stop_words=['<|endoftext|>'], agent_template='qwen3_5', is_thinking=True, thinking_prefix='\\n', non_thinking_prefix='\\n\\n\\n\\n', history_thinking_prefix='')", + "_val_dataset_exists": false, + "hub": "", + "seq_length": 8192, + "is_multimodal": true, + "megatron_model_meta": "MegatronModelMeta(megatron_model_type='qwen3_5', model_types=['qwen3_5', 'qwen3_5_moe'], bridge_cls=, visual_cls=, is_multimodal=True, loader=)", + "fp8": null, + "params_dtype": "bfloat16", + "world_size": 8, + "data_parallel_size": 4, + "num_microbatches": 2 +} \ No newline at end of file diff --git a/posttrain_models/qwen3_5_35b_a3b_base_highlight_batch1245_epoch4/checkpoint-2500/iter_0002500/.metadata b/posttrain_models/qwen3_5_35b_a3b_base_highlight_batch1245_epoch4/checkpoint-2500/iter_0002500/.metadata new file mode 100644 index 0000000000000000000000000000000000000000..9510406fe6689a0e1457b809f54a427cf4258d2a Binary files /dev/null and b/posttrain_models/qwen3_5_35b_a3b_base_highlight_batch1245_epoch4/checkpoint-2500/iter_0002500/.metadata differ diff --git a/posttrain_models/qwen3_5_35b_a3b_base_highlight_batch1245_epoch4/checkpoint-2500/iter_0002500/common.pt b/posttrain_models/qwen3_5_35b_a3b_base_highlight_batch1245_epoch4/checkpoint-2500/iter_0002500/common.pt new file mode 100644 index 0000000000000000000000000000000000000000..a83f630c6ef4400a6bed933a5bd15efcc6bc4c77 --- /dev/null +++ b/posttrain_models/qwen3_5_35b_a3b_base_highlight_batch1245_epoch4/checkpoint-2500/iter_0002500/common.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3f147316675d2dc08483cb28178cb2f0abcf20b7b2c1e0c60544a7f6ef0289ea +size 15975 diff --git a/posttrain_models/qwen3_5_35b_a3b_base_highlight_batch1245_epoch4/checkpoint-2500/iter_0002500/metadata.json b/posttrain_models/qwen3_5_35b_a3b_base_highlight_batch1245_epoch4/checkpoint-2500/iter_0002500/metadata.json new file mode 100644 index 0000000000000000000000000000000000000000..c9dbde0bcf4d2d993122dcc7d6bcb1eef8b6fb77 --- /dev/null +++ b/posttrain_models/qwen3_5_35b_a3b_base_highlight_batch1245_epoch4/checkpoint-2500/iter_0002500/metadata.json @@ -0,0 +1 @@ +{"sharded_backend": "torch_dist", "sharded_backend_version": 1, "common_backend": "torch", "common_backend_version": 1} \ No newline at end of file diff --git a/posttrain_models/qwen3_5_35b_a3b_base_highlight_batch1245_epoch4/checkpoint-2500/latest_checkpointed_iteration.txt b/posttrain_models/qwen3_5_35b_a3b_base_highlight_batch1245_epoch4/checkpoint-2500/latest_checkpointed_iteration.txt new file mode 100644 index 0000000000000000000000000000000000000000..17ddab537cd985db11969e021ace6f6072bb6012 --- /dev/null +++ b/posttrain_models/qwen3_5_35b_a3b_base_highlight_batch1245_epoch4/checkpoint-2500/latest_checkpointed_iteration.txt @@ -0,0 +1 @@ +2500 \ No newline at end of file diff --git a/posttrain_models/qwen3_5_35b_a3b_base_highlight_batch1245_epoch4/checkpoint-2862-merged/args.json b/posttrain_models/qwen3_5_35b_a3b_base_highlight_batch1245_epoch4/checkpoint-2862-merged/args.json new file mode 100644 index 0000000000000000000000000000000000000000..99623dd80e3398e11296749c2daf861d8dff0018 --- /dev/null +++ b/posttrain_models/qwen3_5_35b_a3b_base_highlight_batch1245_epoch4/checkpoint-2862-merged/args.json @@ -0,0 +1,398 @@ +{ + "use_ray": false, + "ray_exp_name": null, + "device_groups": null, + "model": "checkpoints/Qwen3.5-35B-A3B", + "model_type": "qwen3_5_moe", + "model_revision": null, + "task_type": "causal_lm", + "torch_dtype": "bfloat16", + "attn_impl": null, + "experts_impl": null, + "new_special_tokens": [], + "num_labels": null, + "problem_type": null, + "rope_scaling": null, + "device_map": null, + "max_memory": {}, + "max_model_len": null, + "local_repo_path": null, + "init_strategy": null, + "template": "qwen3_5", + "system": null, + "max_length": 8192, + "truncation_strategy": "delete", + "max_pixels": null, + "agent_template": null, + "norm_bbox": null, + "use_chat_template": true, + "padding_side": "right", + "padding_free": true, + "loss_scale": "ignore_empty_think", + "sequence_parallel_size": 1, + "template_backend": "swift", + "response_prefix": "", + "enable_thinking": false, + "add_non_thinking_prefix": false, + "dataset": [ + "batch1", + "batch2", + "batch4_part1", + "batch4_part2", + "batch4_part3", + "batch5_part1", + "batch5_part2", + "batch5_part3" + ], + "val_dataset": [], + "cached_dataset": [], + "cached_val_dataset": [], + "split_dataset_ratio": 0.0, + "data_seed": 42, + "dataset_num_proc": 2, + "load_from_cache_file": false, + "dataset_shuffle": true, + "val_dataset_shuffle": false, + "streaming": false, + "interleave_prob": null, + "stopping_strategy": "first_exhausted", + "shuffle_buffer_size": 1000, + "download_mode": "reuse_dataset_if_exists", + "columns": {}, + "strict": false, + "remove_unused_columns": true, + "model_name": null, + "model_author": null, + "custom_dataset_info": [], + "quant_method": null, + "quant_bits": null, + "hqq_axis": null, + "bnb_4bit_compute_dtype": "bfloat16", + "bnb_4bit_quant_type": "nf4", + "bnb_4bit_use_double_quant": true, + "bnb_4bit_quant_storage": null, + "max_new_tokens": null, + "temperature": 0.9, + "top_k": 50, + "top_p": 0.9, + "repetition_penalty": 1.0, + "num_beams": 1, + "stream": false, + "stop_words": [], + "logprobs": false, + "top_logprobs": null, + "structured_outputs_regex": null, + "tuner_backend": "peft", + "tuner_type": "lora", + "train_type": "lora", + "adapters": [], + "external_plugins": [ + "ms-swift/base_highlight_video_dataset.py" + ], + "custom_register_path": [ + "ms-swift/base_highlight_video_dataset.py" + ], + "seed": 42, + "model_kwargs": {}, + "load_args": false, + "load_data_args": false, + "packing": false, + "packing_length": null, + "packing_num_proc": 1, + "lazy_tokenize": true, + "use_hf": false, + "hub_token": null, + "ddp_timeout": 18000000, + "ddp_backend": "nccl", + "ignore_args_error": false, + "use_swift_lora": false, + "freeze_llm": false, + "freeze_vit": true, + "freeze_aligner": true, + "freeze_parameters": [], + "freeze_parameters_regex": null, + "freeze_parameters_ratio": 0.0, + "trainable_parameters": [], + "trainable_parameters_regex": null, + "target_modules": [ + "all-linear" + ], + "target_regex": null, + "modules_to_save": [], + "lora_rank": 8, + "lora_alpha": 32, + "lora_dropout": 0.05, + "lora_bias": "none", + "lora_dtype": null, + "use_rslora": false, + "rlhf_type": null, + "loss_type": null, + "mcore_ref_model": null, + "mcore_ref_adapter": null, + "beta": null, + "rpo_alpha": null, + "reference_free": false, + "label_smoothing": 0.0, + "f_divergence_type": "reverse_kl", + "desirable_weight": 1.0, + "undesirable_weight": 1.0, + "calculate_KL": null, + "center_rewards_coefficient": null, + "teacher_model": null, + "teacher_model_type": null, + "teacher_model_revision": null, + "teacher_model_server": null, + "gkd_logits_topk": null, + "lmbda": 0.5, + "seq_kd": false, + "offload_teacher_model": false, + "sft_alpha": 0.0, + "generation_batch_size": null, + "steps_per_generation": null, + "num_generations": 8, + "num_generations_eval": null, + "max_completion_length": 512, + "importance_sampling_level": "token", + "tau_pos": 1.0, + "tau_neg": 1.05, + "epsilon": 0.2, + "epsilon_high": null, + "delta": null, + "use_vllm": true, + "vllm_mode": null, + "vllm_enable_prefix_caching": true, + "vllm_gpu_memory_utilization": 0.9, + "vllm_tensor_parallel_size": 1, + "vllm_max_model_len": null, + "vllm_enforce_eager": false, + "vllm_limit_mm_per_prompt": null, + "vllm_disable_cascade_attn": false, + "vllm_max_num_seqs": null, + "vllm_mm_processor_cache_gb": null, + "vllm_engine_kwargs": null, + "sleep_level": 0, + "offload_optimizer": false, + "offload_model": false, + "offload_bridge": false, + "vllm_server_base_url": null, + "vllm_server_host": null, + "vllm_server_port": [ + 8000 + ], + "vllm_server_timeout": 240.0, + "vllm_server_group_port": null, + "reward_funcs": [], + "reward_weights": null, + "cosine_min_len_value_wrong": -0.5, + "cosine_max_len_value_wrong": 0.0, + "cosine_min_len_value_correct": 1.0, + "cosine_max_len_value_correct": 0.5, + "cosine_max_len": null, + "repetition_n_grams": 3, + "repetition_max_penalty": -1.0, + "soft_max_length": null, + "soft_cache_length": null, + "dynamic_sample": false, + "max_resample_times": 3, + "overlong_filter": false, + "scale_rewards": "group", + "advantage_estimator": "grpo", + "kl_in_reward": false, + "wandb_log_unique_prompts": null, + "log_completions": false, + "rollout_importance_sampling_mode": null, + "rollout_importance_sampling_threshold": 2.0, + "log_rollout_offpolicy_metrics": false, + "off_policy_sequence_mask_delta": null, + "log_entropy": false, + "top_entropy_quantile": 1.0, + "reward_model": null, + "reward_model_plugin": null, + "sync_ref_model": false, + "ref_model_sync_steps": 512, + "ref_model_mixup_alpha": 0.6, + "async_generate": false, + "move_model_batches": null, + "multi_turn_scheduler": null, + "max_turns": null, + "completion_length_limit_scope": "per_round", + "vllm_server_pass_dataset": false, + "num_iterations": 1, + "micro_batch_size": 8, + "global_batch_size": 64, + "recompute_granularity": "full", + "recompute_method": "uniform", + "recompute_num_layers": 1, + "recompute_modules": [ + "core_attn" + ], + "train_iters": null, + "num_train_epochs": 4, + "masked_softmax_fusion": true, + "bias_dropout_fusion": true, + "bias_activation_fusion": true, + "apply_rope_fusion": false, + "gradient_accumulation_fusion": false, + "cross_entropy_loss_fusion": true, + "cross_entropy_fusion_impl": "te", + "calculate_per_token_loss": true, + "attention_backend": "", + "optimizer": "adam", + "optimizer_cpu_offload": false, + "optimizer_offload_fraction": 1.0, + "use_precision_aware_optimizer": false, + "main_grads_dtype": "float32", + "main_params_dtype": "float32", + "exp_avg_dtype": "float32", + "exp_avg_sq_dtype": "float32", + "manual_gc": false, + "manual_gc_steps": 0, + "manual_gc_eval": true, + "train_dataloader_shuffle": true, + "dataloader_num_workers": 1, + "dataloader_pin_memory": true, + "dataloader_persistent_workers": true, + "dataloader_prefetch_factor": 2, + "data_sharding": false, + "group_by_length": false, + "te_rng_tracker": false, + "data_parallel_random_init": false, + "mlp_padding_free": false, + "lr_warmup_init": 0.0, + "lr": 0.0001, + "lr_decay_style": "cosine", + "lr_decay_iters": null, + "lr_warmup_iters": 0, + "lr_warmup_fraction": 0.05, + "min_lr": 1e-05, + "lr_wsd_decay_style": "exponential", + "lr_wsd_decay_iters": null, + "weight_decay": 0.1, + "weight_decay_incr_style": "constant", + "start_weight_decay": 0.1, + "end_weight_decay": 0.1, + "clip_grad": 1.0, + "adam_beta1": 0.9, + "adam_beta2": 0.95, + "adam_eps": 1e-08, + "sgd_momentum": 0.9, + "output_dir": "/mnt/bn/strategy-mllm-train/user/tanyue/models/posttrain_models/qwen3_5_35b_a3b_base_highlight_batch1245_epoch4", + "save_steps": 500, + "no_save_optim": true, + "no_save_rng": true, + "mcore_model": null, + "mcore_adapter": null, + "no_load_optim": false, + "no_load_rng": false, + "finetune": true, + "perform_initialization": false, + "use_cpu_initialization": false, + "async_save": false, + "save_total_limit": null, + "metric_for_best_model": "loss", + "greater_is_better": false, + "use_persistent_ckpt_worker": false, + "dist_ckpt_save_pre_mcore_014": false, + "dist_ckpt_optim_fully_reshardable": false, + "distrib_optim_fully_reshardable_mem_efficient": false, + "local_rank": 0, + "use_distributed_optimizer": true, + "tensor_model_parallel_size": 1, + "pipeline_model_parallel_size": 2, + "decoder_first_pipeline_num_layers": null, + "decoder_last_pipeline_num_layers": null, + "account_for_embedding_in_pipeline_split": false, + "account_for_loss_in_pipeline_split": false, + "overlap_p2p_comm": false, + "align_param_gather": false, + "sequence_parallel": false, + "context_parallel_size": 1, + "tp_comm_overlap": false, + "overlap_grad_reduce": false, + "overlap_param_gather": false, + "overlap_param_gather_with_optimizer_step": false, + "align_grad_reduce": true, + "virtual_pipeline_model_parallel_size": null, + "microbatch_group_size_per_vp_stage": null, + "pipeline_model_parallel_layout": null, + "expert_model_parallel_size": 4, + "expert_tensor_parallel_size": 1, + "report_to": [ + "wandb" + ], + "logging_steps": 5, + "tensorboard_dir": "/mnt/bn/strategy-mllm-train/user/tanyue/models/posttrain_models/qwen3_5_35b_a3b_base_highlight_batch1245_epoch4/runs", + "tensorboard_queue_size": 50, + "wandb_project": "highlight_base_highlight_qwen3_5", + "wandb_exp_name": "qwen3_5_35b_a3b_base_highlight_batch1245_epoch4", + "swanlab_project": "megatron-swift", + "swanlab_exp_name": null, + "eval_iters": -1, + "eval_steps": 100000, + "fp8_format": null, + "fp8_recipe": "delayed", + "fp8_amax_history_len": 1024, + "fp8_amax_compute_algo": "max", + "fp8_param_gather": false, + "fp16": false, + "bf16": true, + "apply_query_key_layer_scaling": false, + "attention_softmax_in_fp32": true, + "accumulate_allreduce_grads_in_fp32": true, + "moe_router_load_balancing_type": null, + "moe_router_dtype": "fp32", + "moe_token_dispatcher_type": null, + "moe_enable_deepep": false, + "moe_grouped_gemm": true, + "moe_permute_fusion": true, + "moe_aux_loss_coeff": 1e-06, + "moe_z_loss_coeff": null, + "moe_shared_expert_overlap": true, + "moe_layer_recompute": false, + "moe_expert_capacity_factor": null, + "moe_pad_expert_input_to_capacity": false, + "moe_token_drop_policy": null, + "mtp_num_layers": null, + "mtp_loss_scaling_factor": 0.1, + "save_safetensors": true, + "ref_model": null, + "ref_adapters": [], + "merge_lora": true, + "max_shard_size": "5GB", + "vit_gradient_checkpointing": false, + "vit_lr": null, + "aligner_lr": null, + "gradient_checkpointing_kwargs": null, + "check_model": true, + "apply_wd_to_qk_layernorm": false, + "enable_dft_loss": false, + "enable_channel_loss": false, + "save_strategy": "steps", + "callbacks": [ + "print", + "default_flow", + "wandb" + ], + "add_version": false, + "create_checkpoint_symlink": false, + "swift_version": "4.0.1", + "ckpt_dir": null, + "rank": 0, + "global_world_size": 8, + "local_world_size": 8, + "model_suffix": "Qwen3.5-35B-A3B", + "model_info": "ModelInfo(model_type='qwen3_5_moe', model_dir='/opt/tiger/tanyue/highlight_tools/checkpoints/Qwen3.5-35B-A3B', torch_dtype=torch.bfloat16, max_model_len=262144, quant_method=None, quant_bits=None, rope_scaling=None, is_moe_model=True, is_multimodal=True, config=None, task_type='causal_lm', num_labels=None)", + "model_meta": "ModelMeta(model_type='qwen3_5_moe', model_groups=[ModelGroup(models=[Model(ms_model_id='Qwen/Qwen3.5-35B-A3B-Base', hf_model_id='Qwen/Qwen3.5-35B-A3B-Base', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen3.5-35B-A3B', hf_model_id='Qwen/Qwen3.5-35B-A3B', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen3.5-122B-A10B', hf_model_id='Qwen/Qwen3.5-122B-A10B', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen3.5-397B-A17B', hf_model_id='Qwen/Qwen3.5-397B-A17B', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen3.5-35B-A3B-FP8', hf_model_id='Qwen/Qwen3.5-35B-A3B-FP8', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen3.5-122B-A10B-FP8', hf_model_id='Qwen/Qwen3.5-122B-A10B-FP8', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen3.5-397B-A17B-FP8', hf_model_id='Qwen/Qwen3.5-397B-A17B-FP8', model_path=None, ms_revision=None, hf_revision=None)], template='qwen3_5', ignore_patterns=None, requires=None, tags=[])], loader=, template='qwen3_5', model_arch=MultiModelKeys(arch_name='qwen2_vl', embedding=None, module_list=None, lm_head=None, q_proj=None, k_proj=None, v_proj=None, o_proj=None, attention=None, mlp=None, down_proj=None, qkv_proj=None, qk_proj=None, qa_proj=None, qb_proj=None, kv_proj=None, kva_proj=None, kvb_proj=None, language_model=['model.language_model', 'lm_head'], aligner=['model.visual.merger'], vision_tower=['model.visual'], generator=[]), architectures=['Qwen3_5MoeForConditionalGeneration'], additional_saved_files=[], torch_dtype=None, is_multimodal=True, is_reward=False, task_type=None, ignore_patterns=None, requires=['transformers>=5.2.0', 'qwen_vl_utils>=0.0.14', 'decord'], tags=[])", + "model_dir": "/opt/tiger/tanyue/highlight_tools/checkpoints/Qwen3.5-35B-A3B", + "template_meta": "QwenTemplateMeta(template_type='qwen3_5', prefix=[], prompt=['<|im_start|>user\\n{{QUERY}}<|im_end|>\\n<|im_start|>assistant\\n'], chat_sep=['<|im_end|>\\n'], suffix=['<|im_end|>\\n'], template_cls=, system_prefix=['<|im_start|>system\\n{{SYSTEM}}<|im_end|>\\n'], default_system=None, auto_add_bos=False, stop_words=['<|endoftext|>'], agent_template='qwen3_5', is_thinking=True, thinking_prefix='\\n', non_thinking_prefix='\\n\\n\\n\\n', history_thinking_prefix='')", + "_val_dataset_exists": false, + "hub": "", + "seq_length": 8192, + "is_multimodal": true, + "megatron_model_meta": "MegatronModelMeta(megatron_model_type='qwen3_5', model_types=['qwen3_5', 'qwen3_5_moe'], bridge_cls=, visual_cls=, is_multimodal=True, loader=)", + "fp8": null, + "params_dtype": "bfloat16", + "world_size": 8, + "data_parallel_size": 4, + "num_microbatches": 2 +} \ No newline at end of file diff --git a/posttrain_models/qwen3_5_35b_a3b_base_highlight_batch1245_epoch4/checkpoint-2862-merged/chat_template.jinja b/posttrain_models/qwen3_5_35b_a3b_base_highlight_batch1245_epoch4/checkpoint-2862-merged/chat_template.jinja new file mode 100644 index 0000000000000000000000000000000000000000..a585dec894e63da457d9440ec6aa7caa16d20860 --- /dev/null +++ b/posttrain_models/qwen3_5_35b_a3b_base_highlight_batch1245_epoch4/checkpoint-2862-merged/chat_template.jinja @@ -0,0 +1,154 @@ +{%- set image_count = namespace(value=0) %} +{%- set video_count = namespace(value=0) %} +{%- macro render_content(content, do_vision_count, is_system_content=false) %} + {%- if content is string %} + {{- content }} + {%- elif content is iterable and content is not mapping %} + {%- for item in content %} + {%- if 'image' in item or 'image_url' in item or item.type == 'image' %} + {%- if is_system_content %} + {{- raise_exception('System message cannot contain images.') }} + {%- endif %} + {%- if do_vision_count %} + {%- set image_count.value = image_count.value + 1 %} + {%- endif %} + {%- if add_vision_id %} + {{- 'Picture ' ~ image_count.value ~ ': ' }} + {%- endif %} + {{- '<|vision_start|><|image_pad|><|vision_end|>' }} + {%- elif 'video' in item or item.type == 'video' %} + {%- if is_system_content %} + {{- raise_exception('System message cannot contain videos.') }} + {%- endif %} + {%- if do_vision_count %} + {%- set video_count.value = video_count.value + 1 %} + {%- endif %} + {%- if add_vision_id %} + {{- 'Video ' ~ video_count.value ~ ': ' }} + {%- endif %} + {{- '<|vision_start|><|video_pad|><|vision_end|>' }} + {%- elif 'text' in item %} + {{- item.text }} + {%- else %} + {{- raise_exception('Unexpected item type in content.') }} + {%- endif %} + {%- endfor %} + {%- elif content is none or content is undefined %} + {{- '' }} + {%- else %} + {{- raise_exception('Unexpected content type.') }} + {%- endif %} +{%- endmacro %} +{%- if not messages %} + {{- raise_exception('No messages provided.') }} +{%- endif %} +{%- if tools and tools is iterable and tools is not mapping %} + {{- '<|im_start|>system\n' }} + {{- "# Tools\n\nYou have access to the following functions:\n\n" }} + {%- for tool in tools %} + {{- "\n" }} + {{- tool | tojson }} + {%- endfor %} + {{- "\n" }} + {{- '\n\nIf you choose to call a function ONLY reply in the following format with NO suffix:\n\n\n\n\nvalue_1\n\n\nThis is the value for the second parameter\nthat can span\nmultiple lines\n\n\n\n\n\nReminder:\n- Function calls MUST follow the specified format: an inner block must be nested within XML tags\n- Required parameters MUST be specified\n- You may provide optional reasoning for your function call in natural language BEFORE the function call, but NOT after\n- If there is no function call available, answer the question like normal with your current knowledge and do not tell the user about function calls\n' }} + {%- if messages[0].role == 'system' %} + {%- set content = render_content(messages[0].content, false, true)|trim %} + {%- if content %} + {{- '\n\n' + content }} + {%- endif %} + {%- endif %} + {{- '<|im_end|>\n' }} +{%- else %} + {%- if messages[0].role == 'system' %} + {%- set content = render_content(messages[0].content, false, true)|trim %} + {{- '<|im_start|>system\n' + content + '<|im_end|>\n' }} + {%- endif %} +{%- endif %} +{%- set ns = namespace(multi_step_tool=true, last_query_index=messages|length - 1) %} +{%- for message in messages[::-1] %} + {%- set index = (messages|length - 1) - loop.index0 %} + {%- if ns.multi_step_tool and message.role == "user" %} + {%- set content = render_content(message.content, false)|trim %} + {%- if not(content.startswith('') and content.endswith('')) %} + {%- set ns.multi_step_tool = false %} + {%- set ns.last_query_index = index %} + {%- endif %} + {%- endif %} +{%- endfor %} +{%- if ns.multi_step_tool %} + {{- raise_exception('No user query found in messages.') }} +{%- endif %} +{%- for message in messages %} + {%- set content = render_content(message.content, true)|trim %} + {%- if message.role == "system" %} + {%- if not loop.first %} + {{- raise_exception('System message must be at the beginning.') }} + {%- endif %} + {%- elif message.role == "user" %} + {{- '<|im_start|>' + message.role + '\n' + content + '<|im_end|>' + '\n' }} + {%- elif message.role == "assistant" %} + {%- set reasoning_content = '' %} + {%- if message.reasoning_content is string %} + {%- set reasoning_content = message.reasoning_content %} + {%- else %} + {%- if '' in content %} + {%- set reasoning_content = content.split('')[0].rstrip('\n').split('')[-1].lstrip('\n') %} + {%- set content = content.split('')[-1].lstrip('\n') %} + {%- endif %} + {%- endif %} + {%- set reasoning_content = reasoning_content|trim %} + {%- if loop.index0 > ns.last_query_index %} + {{- '<|im_start|>' + message.role + '\n\n' + reasoning_content + '\n\n\n' + content }} + {%- else %} + {{- '<|im_start|>' + message.role + '\n' + content }} + {%- endif %} + {%- if message.tool_calls and message.tool_calls is iterable and message.tool_calls is not mapping %} + {%- for tool_call in message.tool_calls %} + {%- if tool_call.function is defined %} + {%- set tool_call = tool_call.function %} + {%- endif %} + {%- if loop.first %} + {%- if content|trim %} + {{- '\n\n\n\n' }} + {%- else %} + {{- '\n\n' }} + {%- endif %} + {%- else %} + {{- '\n\n\n' }} + {%- endif %} + {%- if tool_call.arguments is defined %} + {%- for args_name, args_value in tool_call.arguments|items %} + {{- '\n' }} + {%- set args_value = args_value | tojson | safe if args_value is mapping or (args_value is sequence and args_value is not string) else args_value | string %} + {{- args_value }} + {{- '\n\n' }} + {%- endfor %} + {%- endif %} + {{- '\n' }} + {%- endfor %} + {%- endif %} + {{- '<|im_end|>\n' }} + {%- elif message.role == "tool" %} + {%- if loop.previtem and loop.previtem.role != "tool" %} + {{- '<|im_start|>user' }} + {%- endif %} + {{- '\n\n' }} + {{- content }} + {{- '\n' }} + {%- if not loop.last and loop.nextitem.role != "tool" %} + {{- '<|im_end|>\n' }} + {%- elif loop.last %} + {{- '<|im_end|>\n' }} + {%- endif %} + {%- else %} + {{- raise_exception('Unexpected message role.') }} + {%- endif %} +{%- endfor %} +{%- if add_generation_prompt %} + {{- '<|im_start|>assistant\n' }} + {%- if enable_thinking is defined and enable_thinking is false %} + {{- '\n\n\n\n' }} + {%- else %} + {{- '\n' }} + {%- endif %} +{%- endif %} \ No newline at end of file diff --git a/posttrain_models/qwen3_5_35b_a3b_base_highlight_batch1245_epoch4/checkpoint-2862-merged/config.json b/posttrain_models/qwen3_5_35b_a3b_base_highlight_batch1245_epoch4/checkpoint-2862-merged/config.json new file mode 100644 index 0000000000000000000000000000000000000000..8643d6d3f2e157c74392e7d91f431a66f9261762 --- /dev/null +++ b/posttrain_models/qwen3_5_35b_a3b_base_highlight_batch1245_epoch4/checkpoint-2862-merged/config.json @@ -0,0 +1,123 @@ +{ + "architectures": [ + "Qwen3_5MoeForConditionalGeneration" + ], + "dtype": "bfloat16", + "hidden_size": 2048, + "image_token_id": 248056, + "model_type": "qwen3_5_moe", + "text_config": { + "attention_bias": false, + "attention_dropout": 0.0, + "attn_output_gate": true, + "bos_token_id": null, + "dtype": "bfloat16", + "eos_token_id": 248044, + "full_attention_interval": 4, + "head_dim": 256, + "hidden_act": "silu", + "hidden_size": 2048, + "initializer_range": 0.02, + "layer_types": [ + "linear_attention", + "linear_attention", + "linear_attention", + "full_attention", + "linear_attention", + "linear_attention", + "linear_attention", + "full_attention", + "linear_attention", + "linear_attention", + "linear_attention", + "full_attention", + "linear_attention", + "linear_attention", + "linear_attention", + "full_attention", + "linear_attention", + "linear_attention", + "linear_attention", + "full_attention", + "linear_attention", + "linear_attention", + "linear_attention", + "full_attention", + "linear_attention", + "linear_attention", + "linear_attention", + "full_attention", + "linear_attention", + "linear_attention", + "linear_attention", + "full_attention", + "linear_attention", + "linear_attention", + "linear_attention", + "full_attention", + "linear_attention", + "linear_attention", + "linear_attention", + "full_attention" + ], + "linear_conv_kernel_dim": 4, + "linear_key_head_dim": 128, + "linear_num_key_heads": 16, + "linear_num_value_heads": 32, + "linear_value_head_dim": 128, + "mamba_ssm_dtype": "float32", + "max_position_embeddings": 262144, + "mlp_only_layers": [], + "model_type": "qwen3_5_moe_text", + "moe_intermediate_size": 512, + "mtp_num_hidden_layers": 1, + "mtp_use_dedicated_embeddings": false, + "num_attention_heads": 16, + "num_experts": 256, + "num_experts_per_tok": 8, + "num_hidden_layers": 40, + "num_key_value_heads": 2, + "output_router_logits": false, + "pad_token_id": null, + "partial_rotary_factor": 0.25, + "rms_norm_eps": 1e-06, + "rope_parameters": { + "mrope_interleaved": true, + "mrope_section": [ + 11, + 11, + 10 + ], + "partial_rotary_factor": 0.25, + "rope_theta": 10000000, + "rope_type": "default" + }, + "router_aux_loss_coef": 0.001, + "shared_expert_intermediate_size": 512, + "tie_word_embeddings": false, + "use_cache": true, + "vocab_size": 248320 + }, + "tie_word_embeddings": false, + "transformers_version": "5.2.0", + "video_token_id": 248057, + "vision_config": { + "deepstack_visual_indexes": [], + "depth": 27, + "dtype": "bfloat16", + "hidden_act": "gelu_pytorch_tanh", + "hidden_size": 1152, + "in_channels": 3, + "initializer_range": 0.02, + "intermediate_size": 4304, + "model_type": "qwen3_5_moe", + "num_heads": 16, + "num_position_embeddings": 2304, + "out_hidden_size": 2048, + "patch_size": 16, + "spatial_merge_size": 2, + "temporal_patch_size": 2 + }, + "vision_end_token_id": 248054, + "vision_start_token_id": 248053 +} diff --git a/posttrain_models/qwen3_5_35b_a3b_base_highlight_batch1245_epoch4/checkpoint-2862-merged/iter_0002862/common.pt b/posttrain_models/qwen3_5_35b_a3b_base_highlight_batch1245_epoch4/checkpoint-2862-merged/iter_0002862/common.pt new file mode 100644 index 0000000000000000000000000000000000000000..897de42f6bb27c59efb6ddf21fe4abfab02df788 --- /dev/null +++ b/posttrain_models/qwen3_5_35b_a3b_base_highlight_batch1245_epoch4/checkpoint-2862-merged/iter_0002862/common.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:23245c3a2cde4ece1c0ccdc9bf90bf18e1050db4205c848bdd8bf71f836c32e3 +size 15975 diff --git a/posttrain_models/qwen3_5_35b_a3b_base_highlight_batch1245_epoch4/checkpoint-2862-merged/latest_checkpointed_iteration.txt b/posttrain_models/qwen3_5_35b_a3b_base_highlight_batch1245_epoch4/checkpoint-2862-merged/latest_checkpointed_iteration.txt new file mode 100644 index 0000000000000000000000000000000000000000..8eb976760c5085c5dcf96ebc64ce73480e96e06d --- /dev/null +++ b/posttrain_models/qwen3_5_35b_a3b_base_highlight_batch1245_epoch4/checkpoint-2862-merged/latest_checkpointed_iteration.txt @@ -0,0 +1 @@ +2862 \ No newline at end of file diff --git a/posttrain_models/qwen3_5_35b_a3b_base_highlight_batch1245_epoch4/checkpoint-2862-merged/preprocessor_config.json b/posttrain_models/qwen3_5_35b_a3b_base_highlight_batch1245_epoch4/checkpoint-2862-merged/preprocessor_config.json new file mode 100644 index 0000000000000000000000000000000000000000..2ea84a437d448ff71b08df68fdd949d5cc4ebb64 --- /dev/null +++ b/posttrain_models/qwen3_5_35b_a3b_base_highlight_batch1245_epoch4/checkpoint-2862-merged/preprocessor_config.json @@ -0,0 +1,21 @@ +{ + "size": { + "longest_edge": 16777216, + "shortest_edge": 65536 + }, + "patch_size": 16, + "temporal_patch_size": 2, + "merge_size": 2, + "image_mean": [ + 0.5, + 0.5, + 0.5 + ], + "image_std": [ + 0.5, + 0.5, + 0.5 + ], + "processor_class": "Qwen3VLProcessor", + "image_processor_type": "Qwen2VLImageProcessorFast" +} \ No newline at end of file diff --git a/posttrain_models/qwen3_5_35b_a3b_base_highlight_batch1245_epoch4/checkpoint-2862-merged/processor_config.json b/posttrain_models/qwen3_5_35b_a3b_base_highlight_batch1245_epoch4/checkpoint-2862-merged/processor_config.json new file mode 100644 index 0000000000000000000000000000000000000000..7ad6acdf4203f22b7b990e36ccc3a1fe38563d5e --- /dev/null +++ b/posttrain_models/qwen3_5_35b_a3b_base_highlight_batch1245_epoch4/checkpoint-2862-merged/processor_config.json @@ -0,0 +1,63 @@ +{ + "image_processor": { + "data_format": "channels_first", + "do_convert_rgb": true, + "do_normalize": true, + "do_rescale": true, + "do_resize": true, + "image_mean": [ + 0.5, + 0.5, + 0.5 + ], + "image_processor_type": "Qwen2VLImageProcessorFast", + "image_std": [ + 0.5, + 0.5, + 0.5 + ], + "merge_size": 2, + "patch_size": 16, + "resample": 3, + "rescale_factor": 0.00392156862745098, + "size": { + "longest_edge": 16777216, + "shortest_edge": 65536 + }, + "temporal_patch_size": 2 + }, + "processor_class": "Qwen3VLProcessor", + "video_processor": { + "data_format": "channels_first", + "default_to_square": true, + "do_convert_rgb": true, + "do_normalize": true, + "do_rescale": true, + "do_resize": true, + "do_sample_frames": true, + "fps": 2, + "image_mean": [ + 0.5, + 0.5, + 0.5 + ], + "image_std": [ + 0.5, + 0.5, + 0.5 + ], + "max_frames": 768, + "merge_size": 2, + "min_frames": 4, + "patch_size": 16, + "resample": 3, + "rescale_factor": 0.00392156862745098, + "return_metadata": false, + "size": { + "longest_edge": 25165824, + "shortest_edge": 4096 + }, + "temporal_patch_size": 2, + "video_processor_type": "Qwen3VLVideoProcessor" + } +} diff --git a/posttrain_models/qwen3_5_35b_a3b_base_highlight_batch1245_epoch4/checkpoint-2862-merged/tokenizer_config.json b/posttrain_models/qwen3_5_35b_a3b_base_highlight_batch1245_epoch4/checkpoint-2862-merged/tokenizer_config.json new file mode 100644 index 0000000000000000000000000000000000000000..a068e2468cff426a9b105006e74e044030a6faf4 --- /dev/null +++ b/posttrain_models/qwen3_5_35b_a3b_base_highlight_batch1245_epoch4/checkpoint-2862-merged/tokenizer_config.json @@ -0,0 +1,32 @@ +{ + "add_prefix_space": false, + "audio_bos_token": "<|audio_start|>", + "audio_eos_token": "<|audio_end|>", + "audio_token": "<|audio_pad|>", + "backend": "tokenizers", + "bos_token": null, + "clean_up_tokenization_spaces": false, + "eos_token": "<|im_end|>", + "errors": "replace", + "image_token": "<|image_pad|>", + "is_local": true, + "model_max_length": 262144, + "model_specific_special_tokens": { + "audio_bos_token": "<|audio_start|>", + "audio_eos_token": "<|audio_end|>", + "audio_token": "<|audio_pad|>", + "image_token": "<|image_pad|>", + "video_token": "<|video_pad|>", + "vision_bos_token": "<|vision_start|>", + "vision_eos_token": "<|vision_end|>" + }, + "pad_token": "<|endoftext|>", + "pretokenize_regex": "(?i:'s|'t|'re|'ve|'m|'ll|'d)|[^\\r\\n\\p{L}\\p{N}]?[\\p{L}\\p{M}]+|\\p{N}| ?[^\\s\\p{L}\\p{M}\\p{N}]+[\\r\\n]*|\\s*[\\r\\n]+|\\s+(?!\\S)|\\s+", + "processor_class": "Qwen3VLProcessor", + "split_special_tokens": false, + "tokenizer_class": "TokenizersBackend", + "unk_token": null, + "video_token": "<|video_pad|>", + "vision_bos_token": "<|vision_start|>", + "vision_eos_token": "<|vision_end|>" +} diff --git a/posttrain_models/qwen3_5_35b_a3b_base_highlight_batch1245_epoch4/checkpoint-2862/args.json b/posttrain_models/qwen3_5_35b_a3b_base_highlight_batch1245_epoch4/checkpoint-2862/args.json new file mode 100644 index 0000000000000000000000000000000000000000..99623dd80e3398e11296749c2daf861d8dff0018 --- /dev/null +++ b/posttrain_models/qwen3_5_35b_a3b_base_highlight_batch1245_epoch4/checkpoint-2862/args.json @@ -0,0 +1,398 @@ +{ + "use_ray": false, + "ray_exp_name": null, + "device_groups": null, + "model": "checkpoints/Qwen3.5-35B-A3B", + "model_type": "qwen3_5_moe", + "model_revision": null, + "task_type": "causal_lm", + "torch_dtype": "bfloat16", + "attn_impl": null, + "experts_impl": null, + "new_special_tokens": [], + "num_labels": null, + "problem_type": null, + "rope_scaling": null, + "device_map": null, + "max_memory": {}, + "max_model_len": null, + "local_repo_path": null, + "init_strategy": null, + "template": "qwen3_5", + "system": null, + "max_length": 8192, + "truncation_strategy": "delete", + "max_pixels": null, + "agent_template": null, + "norm_bbox": null, + "use_chat_template": true, + "padding_side": "right", + "padding_free": true, + "loss_scale": "ignore_empty_think", + "sequence_parallel_size": 1, + "template_backend": "swift", + "response_prefix": "", + "enable_thinking": false, + "add_non_thinking_prefix": false, + "dataset": [ + "batch1", + "batch2", + "batch4_part1", + "batch4_part2", + "batch4_part3", + "batch5_part1", + "batch5_part2", + "batch5_part3" + ], + "val_dataset": [], + "cached_dataset": [], + "cached_val_dataset": [], + "split_dataset_ratio": 0.0, + "data_seed": 42, + "dataset_num_proc": 2, + "load_from_cache_file": false, + "dataset_shuffle": true, + "val_dataset_shuffle": false, + "streaming": false, + "interleave_prob": null, + "stopping_strategy": "first_exhausted", + "shuffle_buffer_size": 1000, + "download_mode": "reuse_dataset_if_exists", + "columns": {}, + "strict": false, + "remove_unused_columns": true, + "model_name": null, + "model_author": null, + "custom_dataset_info": [], + "quant_method": null, + "quant_bits": null, + "hqq_axis": null, + "bnb_4bit_compute_dtype": "bfloat16", + "bnb_4bit_quant_type": "nf4", + "bnb_4bit_use_double_quant": true, + "bnb_4bit_quant_storage": null, + "max_new_tokens": null, + "temperature": 0.9, + "top_k": 50, + "top_p": 0.9, + "repetition_penalty": 1.0, + "num_beams": 1, + "stream": false, + "stop_words": [], + "logprobs": false, + "top_logprobs": null, + "structured_outputs_regex": null, + "tuner_backend": "peft", + "tuner_type": "lora", + "train_type": "lora", + "adapters": [], + "external_plugins": [ + "ms-swift/base_highlight_video_dataset.py" + ], + "custom_register_path": [ + "ms-swift/base_highlight_video_dataset.py" + ], + "seed": 42, + "model_kwargs": {}, + "load_args": false, + "load_data_args": false, + "packing": false, + "packing_length": null, + "packing_num_proc": 1, + "lazy_tokenize": true, + "use_hf": false, + "hub_token": null, + "ddp_timeout": 18000000, + "ddp_backend": "nccl", + "ignore_args_error": false, + "use_swift_lora": false, + "freeze_llm": false, + "freeze_vit": true, + "freeze_aligner": true, + "freeze_parameters": [], + "freeze_parameters_regex": null, + "freeze_parameters_ratio": 0.0, + "trainable_parameters": [], + "trainable_parameters_regex": null, + "target_modules": [ + "all-linear" + ], + "target_regex": null, + "modules_to_save": [], + "lora_rank": 8, + "lora_alpha": 32, + "lora_dropout": 0.05, + "lora_bias": "none", + "lora_dtype": null, + "use_rslora": false, + "rlhf_type": null, + "loss_type": null, + "mcore_ref_model": null, + "mcore_ref_adapter": null, + "beta": null, + "rpo_alpha": null, + "reference_free": false, + "label_smoothing": 0.0, + "f_divergence_type": "reverse_kl", + "desirable_weight": 1.0, + "undesirable_weight": 1.0, + "calculate_KL": null, + "center_rewards_coefficient": null, + "teacher_model": null, + "teacher_model_type": null, + "teacher_model_revision": null, + "teacher_model_server": null, + "gkd_logits_topk": null, + "lmbda": 0.5, + "seq_kd": false, + "offload_teacher_model": false, + "sft_alpha": 0.0, + "generation_batch_size": null, + "steps_per_generation": null, + "num_generations": 8, + "num_generations_eval": null, + "max_completion_length": 512, + "importance_sampling_level": "token", + "tau_pos": 1.0, + "tau_neg": 1.05, + "epsilon": 0.2, + "epsilon_high": null, + "delta": null, + "use_vllm": true, + "vllm_mode": null, + "vllm_enable_prefix_caching": true, + "vllm_gpu_memory_utilization": 0.9, + "vllm_tensor_parallel_size": 1, + "vllm_max_model_len": null, + "vllm_enforce_eager": false, + "vllm_limit_mm_per_prompt": null, + "vllm_disable_cascade_attn": false, + "vllm_max_num_seqs": null, + "vllm_mm_processor_cache_gb": null, + "vllm_engine_kwargs": null, + "sleep_level": 0, + "offload_optimizer": false, + "offload_model": false, + "offload_bridge": false, + "vllm_server_base_url": null, + "vllm_server_host": null, + "vllm_server_port": [ + 8000 + ], + "vllm_server_timeout": 240.0, + "vllm_server_group_port": null, + "reward_funcs": [], + "reward_weights": null, + "cosine_min_len_value_wrong": -0.5, + "cosine_max_len_value_wrong": 0.0, + "cosine_min_len_value_correct": 1.0, + "cosine_max_len_value_correct": 0.5, + "cosine_max_len": null, + "repetition_n_grams": 3, + "repetition_max_penalty": -1.0, + "soft_max_length": null, + "soft_cache_length": null, + "dynamic_sample": false, + "max_resample_times": 3, + "overlong_filter": false, + "scale_rewards": "group", + "advantage_estimator": "grpo", + "kl_in_reward": false, + "wandb_log_unique_prompts": null, + "log_completions": false, + "rollout_importance_sampling_mode": null, + "rollout_importance_sampling_threshold": 2.0, + "log_rollout_offpolicy_metrics": false, + "off_policy_sequence_mask_delta": null, + "log_entropy": false, + "top_entropy_quantile": 1.0, + "reward_model": null, + "reward_model_plugin": null, + "sync_ref_model": false, + "ref_model_sync_steps": 512, + "ref_model_mixup_alpha": 0.6, + "async_generate": false, + "move_model_batches": null, + "multi_turn_scheduler": null, + "max_turns": null, + "completion_length_limit_scope": "per_round", + "vllm_server_pass_dataset": false, + "num_iterations": 1, + "micro_batch_size": 8, + "global_batch_size": 64, + "recompute_granularity": "full", + "recompute_method": "uniform", + "recompute_num_layers": 1, + "recompute_modules": [ + "core_attn" + ], + "train_iters": null, + "num_train_epochs": 4, + "masked_softmax_fusion": true, + "bias_dropout_fusion": true, + "bias_activation_fusion": true, + "apply_rope_fusion": false, + "gradient_accumulation_fusion": false, + "cross_entropy_loss_fusion": true, + "cross_entropy_fusion_impl": "te", + "calculate_per_token_loss": true, + "attention_backend": "", + "optimizer": "adam", + "optimizer_cpu_offload": false, + "optimizer_offload_fraction": 1.0, + "use_precision_aware_optimizer": false, + "main_grads_dtype": "float32", + "main_params_dtype": "float32", + "exp_avg_dtype": "float32", + "exp_avg_sq_dtype": "float32", + "manual_gc": false, + "manual_gc_steps": 0, + "manual_gc_eval": true, + "train_dataloader_shuffle": true, + "dataloader_num_workers": 1, + "dataloader_pin_memory": true, + "dataloader_persistent_workers": true, + "dataloader_prefetch_factor": 2, + "data_sharding": false, + "group_by_length": false, + "te_rng_tracker": false, + "data_parallel_random_init": false, + "mlp_padding_free": false, + "lr_warmup_init": 0.0, + "lr": 0.0001, + "lr_decay_style": "cosine", + "lr_decay_iters": null, + "lr_warmup_iters": 0, + "lr_warmup_fraction": 0.05, + "min_lr": 1e-05, + "lr_wsd_decay_style": "exponential", + "lr_wsd_decay_iters": null, + "weight_decay": 0.1, + "weight_decay_incr_style": "constant", + "start_weight_decay": 0.1, + "end_weight_decay": 0.1, + "clip_grad": 1.0, + "adam_beta1": 0.9, + "adam_beta2": 0.95, + "adam_eps": 1e-08, + "sgd_momentum": 0.9, + "output_dir": "/mnt/bn/strategy-mllm-train/user/tanyue/models/posttrain_models/qwen3_5_35b_a3b_base_highlight_batch1245_epoch4", + "save_steps": 500, + "no_save_optim": true, + "no_save_rng": true, + "mcore_model": null, + "mcore_adapter": null, + "no_load_optim": false, + "no_load_rng": false, + "finetune": true, + "perform_initialization": false, + "use_cpu_initialization": false, + "async_save": false, + "save_total_limit": null, + "metric_for_best_model": "loss", + "greater_is_better": false, + "use_persistent_ckpt_worker": false, + "dist_ckpt_save_pre_mcore_014": false, + "dist_ckpt_optim_fully_reshardable": false, + "distrib_optim_fully_reshardable_mem_efficient": false, + "local_rank": 0, + "use_distributed_optimizer": true, + "tensor_model_parallel_size": 1, + "pipeline_model_parallel_size": 2, + "decoder_first_pipeline_num_layers": null, + "decoder_last_pipeline_num_layers": null, + "account_for_embedding_in_pipeline_split": false, + "account_for_loss_in_pipeline_split": false, + "overlap_p2p_comm": false, + "align_param_gather": false, + "sequence_parallel": false, + "context_parallel_size": 1, + "tp_comm_overlap": false, + "overlap_grad_reduce": false, + "overlap_param_gather": false, + "overlap_param_gather_with_optimizer_step": false, + "align_grad_reduce": true, + "virtual_pipeline_model_parallel_size": null, + "microbatch_group_size_per_vp_stage": null, + "pipeline_model_parallel_layout": null, + "expert_model_parallel_size": 4, + "expert_tensor_parallel_size": 1, + "report_to": [ + "wandb" + ], + "logging_steps": 5, + "tensorboard_dir": "/mnt/bn/strategy-mllm-train/user/tanyue/models/posttrain_models/qwen3_5_35b_a3b_base_highlight_batch1245_epoch4/runs", + "tensorboard_queue_size": 50, + "wandb_project": "highlight_base_highlight_qwen3_5", + "wandb_exp_name": "qwen3_5_35b_a3b_base_highlight_batch1245_epoch4", + "swanlab_project": "megatron-swift", + "swanlab_exp_name": null, + "eval_iters": -1, + "eval_steps": 100000, + "fp8_format": null, + "fp8_recipe": "delayed", + "fp8_amax_history_len": 1024, + "fp8_amax_compute_algo": "max", + "fp8_param_gather": false, + "fp16": false, + "bf16": true, + "apply_query_key_layer_scaling": false, + "attention_softmax_in_fp32": true, + "accumulate_allreduce_grads_in_fp32": true, + "moe_router_load_balancing_type": null, + "moe_router_dtype": "fp32", + "moe_token_dispatcher_type": null, + "moe_enable_deepep": false, + "moe_grouped_gemm": true, + "moe_permute_fusion": true, + "moe_aux_loss_coeff": 1e-06, + "moe_z_loss_coeff": null, + "moe_shared_expert_overlap": true, + "moe_layer_recompute": false, + "moe_expert_capacity_factor": null, + "moe_pad_expert_input_to_capacity": false, + "moe_token_drop_policy": null, + "mtp_num_layers": null, + "mtp_loss_scaling_factor": 0.1, + "save_safetensors": true, + "ref_model": null, + "ref_adapters": [], + "merge_lora": true, + "max_shard_size": "5GB", + "vit_gradient_checkpointing": false, + "vit_lr": null, + "aligner_lr": null, + "gradient_checkpointing_kwargs": null, + "check_model": true, + "apply_wd_to_qk_layernorm": false, + "enable_dft_loss": false, + "enable_channel_loss": false, + "save_strategy": "steps", + "callbacks": [ + "print", + "default_flow", + "wandb" + ], + "add_version": false, + "create_checkpoint_symlink": false, + "swift_version": "4.0.1", + "ckpt_dir": null, + "rank": 0, + "global_world_size": 8, + "local_world_size": 8, + "model_suffix": "Qwen3.5-35B-A3B", + "model_info": "ModelInfo(model_type='qwen3_5_moe', model_dir='/opt/tiger/tanyue/highlight_tools/checkpoints/Qwen3.5-35B-A3B', torch_dtype=torch.bfloat16, max_model_len=262144, quant_method=None, quant_bits=None, rope_scaling=None, is_moe_model=True, is_multimodal=True, config=None, task_type='causal_lm', num_labels=None)", + "model_meta": "ModelMeta(model_type='qwen3_5_moe', model_groups=[ModelGroup(models=[Model(ms_model_id='Qwen/Qwen3.5-35B-A3B-Base', hf_model_id='Qwen/Qwen3.5-35B-A3B-Base', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen3.5-35B-A3B', hf_model_id='Qwen/Qwen3.5-35B-A3B', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen3.5-122B-A10B', hf_model_id='Qwen/Qwen3.5-122B-A10B', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen3.5-397B-A17B', hf_model_id='Qwen/Qwen3.5-397B-A17B', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen3.5-35B-A3B-FP8', hf_model_id='Qwen/Qwen3.5-35B-A3B-FP8', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen3.5-122B-A10B-FP8', hf_model_id='Qwen/Qwen3.5-122B-A10B-FP8', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen3.5-397B-A17B-FP8', hf_model_id='Qwen/Qwen3.5-397B-A17B-FP8', model_path=None, ms_revision=None, hf_revision=None)], template='qwen3_5', ignore_patterns=None, requires=None, tags=[])], loader=, template='qwen3_5', model_arch=MultiModelKeys(arch_name='qwen2_vl', embedding=None, module_list=None, lm_head=None, q_proj=None, k_proj=None, v_proj=None, o_proj=None, attention=None, mlp=None, down_proj=None, qkv_proj=None, qk_proj=None, qa_proj=None, qb_proj=None, kv_proj=None, kva_proj=None, kvb_proj=None, language_model=['model.language_model', 'lm_head'], aligner=['model.visual.merger'], vision_tower=['model.visual'], generator=[]), architectures=['Qwen3_5MoeForConditionalGeneration'], additional_saved_files=[], torch_dtype=None, is_multimodal=True, is_reward=False, task_type=None, ignore_patterns=None, requires=['transformers>=5.2.0', 'qwen_vl_utils>=0.0.14', 'decord'], tags=[])", + "model_dir": "/opt/tiger/tanyue/highlight_tools/checkpoints/Qwen3.5-35B-A3B", + "template_meta": "QwenTemplateMeta(template_type='qwen3_5', prefix=[], prompt=['<|im_start|>user\\n{{QUERY}}<|im_end|>\\n<|im_start|>assistant\\n'], chat_sep=['<|im_end|>\\n'], suffix=['<|im_end|>\\n'], template_cls=, system_prefix=['<|im_start|>system\\n{{SYSTEM}}<|im_end|>\\n'], default_system=None, auto_add_bos=False, stop_words=['<|endoftext|>'], agent_template='qwen3_5', is_thinking=True, thinking_prefix='\\n', non_thinking_prefix='\\n\\n\\n\\n', history_thinking_prefix='')", + "_val_dataset_exists": false, + "hub": "", + "seq_length": 8192, + "is_multimodal": true, + "megatron_model_meta": "MegatronModelMeta(megatron_model_type='qwen3_5', model_types=['qwen3_5', 'qwen3_5_moe'], bridge_cls=, visual_cls=, is_multimodal=True, loader=)", + "fp8": null, + "params_dtype": "bfloat16", + "world_size": 8, + "data_parallel_size": 4, + "num_microbatches": 2 +} \ No newline at end of file diff --git a/posttrain_models/qwen3_5_35b_a3b_base_highlight_batch1245_epoch4/checkpoint-2862/iter_0002862/.metadata b/posttrain_models/qwen3_5_35b_a3b_base_highlight_batch1245_epoch4/checkpoint-2862/iter_0002862/.metadata new file mode 100644 index 0000000000000000000000000000000000000000..30501a8bd9f75f401b23bebd375c0e529907af64 Binary files /dev/null and b/posttrain_models/qwen3_5_35b_a3b_base_highlight_batch1245_epoch4/checkpoint-2862/iter_0002862/.metadata differ diff --git a/posttrain_models/qwen3_5_35b_a3b_base_highlight_batch1245_epoch4/checkpoint-2862/iter_0002862/common.pt b/posttrain_models/qwen3_5_35b_a3b_base_highlight_batch1245_epoch4/checkpoint-2862/iter_0002862/common.pt new file mode 100644 index 0000000000000000000000000000000000000000..897de42f6bb27c59efb6ddf21fe4abfab02df788 --- /dev/null +++ b/posttrain_models/qwen3_5_35b_a3b_base_highlight_batch1245_epoch4/checkpoint-2862/iter_0002862/common.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:23245c3a2cde4ece1c0ccdc9bf90bf18e1050db4205c848bdd8bf71f836c32e3 +size 15975 diff --git a/posttrain_models/qwen3_5_35b_a3b_base_highlight_batch1245_epoch4/checkpoint-2862/iter_0002862/metadata.json b/posttrain_models/qwen3_5_35b_a3b_base_highlight_batch1245_epoch4/checkpoint-2862/iter_0002862/metadata.json new file mode 100644 index 0000000000000000000000000000000000000000..c9dbde0bcf4d2d993122dcc7d6bcb1eef8b6fb77 --- /dev/null +++ b/posttrain_models/qwen3_5_35b_a3b_base_highlight_batch1245_epoch4/checkpoint-2862/iter_0002862/metadata.json @@ -0,0 +1 @@ +{"sharded_backend": "torch_dist", "sharded_backend_version": 1, "common_backend": "torch", "common_backend_version": 1} \ No newline at end of file diff --git a/posttrain_models/qwen3_5_35b_a3b_base_highlight_batch1245_epoch4/checkpoint-2862/latest_checkpointed_iteration.txt b/posttrain_models/qwen3_5_35b_a3b_base_highlight_batch1245_epoch4/checkpoint-2862/latest_checkpointed_iteration.txt new file mode 100644 index 0000000000000000000000000000000000000000..8eb976760c5085c5dcf96ebc64ce73480e96e06d --- /dev/null +++ b/posttrain_models/qwen3_5_35b_a3b_base_highlight_batch1245_epoch4/checkpoint-2862/latest_checkpointed_iteration.txt @@ -0,0 +1 @@ +2862 \ No newline at end of file diff --git a/posttrain_models/qwen3_5_35b_a3b_base_highlight_batch1245_epoch4/checkpoint-500-merged/args.json b/posttrain_models/qwen3_5_35b_a3b_base_highlight_batch1245_epoch4/checkpoint-500-merged/args.json new file mode 100644 index 0000000000000000000000000000000000000000..99623dd80e3398e11296749c2daf861d8dff0018 --- /dev/null +++ b/posttrain_models/qwen3_5_35b_a3b_base_highlight_batch1245_epoch4/checkpoint-500-merged/args.json @@ -0,0 +1,398 @@ +{ + "use_ray": false, + "ray_exp_name": null, + "device_groups": null, + "model": "checkpoints/Qwen3.5-35B-A3B", + "model_type": "qwen3_5_moe", + "model_revision": null, + "task_type": "causal_lm", + "torch_dtype": "bfloat16", + "attn_impl": null, + "experts_impl": null, + "new_special_tokens": [], + "num_labels": null, + "problem_type": null, + "rope_scaling": null, + "device_map": null, + "max_memory": {}, + "max_model_len": null, + "local_repo_path": null, + "init_strategy": null, + "template": "qwen3_5", + "system": null, + "max_length": 8192, + "truncation_strategy": "delete", + "max_pixels": null, + "agent_template": null, + "norm_bbox": null, + "use_chat_template": true, + "padding_side": "right", + "padding_free": true, + "loss_scale": "ignore_empty_think", + "sequence_parallel_size": 1, + "template_backend": "swift", + "response_prefix": "", + "enable_thinking": false, + "add_non_thinking_prefix": false, + "dataset": [ + "batch1", + "batch2", + "batch4_part1", + "batch4_part2", + "batch4_part3", + "batch5_part1", + "batch5_part2", + "batch5_part3" + ], + "val_dataset": [], + "cached_dataset": [], + "cached_val_dataset": [], + "split_dataset_ratio": 0.0, + "data_seed": 42, + "dataset_num_proc": 2, + "load_from_cache_file": false, + "dataset_shuffle": true, + "val_dataset_shuffle": false, + "streaming": false, + "interleave_prob": null, + "stopping_strategy": "first_exhausted", + "shuffle_buffer_size": 1000, + "download_mode": "reuse_dataset_if_exists", + "columns": {}, + "strict": false, + "remove_unused_columns": true, + "model_name": null, + "model_author": null, + "custom_dataset_info": [], + "quant_method": null, + "quant_bits": null, + "hqq_axis": null, + "bnb_4bit_compute_dtype": "bfloat16", + "bnb_4bit_quant_type": "nf4", + "bnb_4bit_use_double_quant": true, + "bnb_4bit_quant_storage": null, + "max_new_tokens": null, + "temperature": 0.9, + "top_k": 50, + "top_p": 0.9, + "repetition_penalty": 1.0, + "num_beams": 1, + "stream": false, + "stop_words": [], + "logprobs": false, + "top_logprobs": null, + "structured_outputs_regex": null, + "tuner_backend": "peft", + "tuner_type": "lora", + "train_type": "lora", + "adapters": [], + "external_plugins": [ + "ms-swift/base_highlight_video_dataset.py" + ], + "custom_register_path": [ + "ms-swift/base_highlight_video_dataset.py" + ], + "seed": 42, + "model_kwargs": {}, + "load_args": false, + "load_data_args": false, + "packing": false, + "packing_length": null, + "packing_num_proc": 1, + "lazy_tokenize": true, + "use_hf": false, + "hub_token": null, + "ddp_timeout": 18000000, + "ddp_backend": "nccl", + "ignore_args_error": false, + "use_swift_lora": false, + "freeze_llm": false, + "freeze_vit": true, + "freeze_aligner": true, + "freeze_parameters": [], + "freeze_parameters_regex": null, + "freeze_parameters_ratio": 0.0, + "trainable_parameters": [], + "trainable_parameters_regex": null, + "target_modules": [ + "all-linear" + ], + "target_regex": null, + "modules_to_save": [], + "lora_rank": 8, + "lora_alpha": 32, + "lora_dropout": 0.05, + "lora_bias": "none", + "lora_dtype": null, + "use_rslora": false, + "rlhf_type": null, + "loss_type": null, + "mcore_ref_model": null, + "mcore_ref_adapter": null, + "beta": null, + "rpo_alpha": null, + "reference_free": false, + "label_smoothing": 0.0, + "f_divergence_type": "reverse_kl", + "desirable_weight": 1.0, + "undesirable_weight": 1.0, + "calculate_KL": null, + "center_rewards_coefficient": null, + "teacher_model": null, + "teacher_model_type": null, + "teacher_model_revision": null, + "teacher_model_server": null, + "gkd_logits_topk": null, + "lmbda": 0.5, + "seq_kd": false, + "offload_teacher_model": false, + "sft_alpha": 0.0, + "generation_batch_size": null, + "steps_per_generation": null, + "num_generations": 8, + "num_generations_eval": null, + "max_completion_length": 512, + "importance_sampling_level": "token", + "tau_pos": 1.0, + "tau_neg": 1.05, + "epsilon": 0.2, + "epsilon_high": null, + "delta": null, + "use_vllm": true, + "vllm_mode": null, + "vllm_enable_prefix_caching": true, + "vllm_gpu_memory_utilization": 0.9, + "vllm_tensor_parallel_size": 1, + "vllm_max_model_len": null, + "vllm_enforce_eager": false, + "vllm_limit_mm_per_prompt": null, + "vllm_disable_cascade_attn": false, + "vllm_max_num_seqs": null, + "vllm_mm_processor_cache_gb": null, + "vllm_engine_kwargs": null, + "sleep_level": 0, + "offload_optimizer": false, + "offload_model": false, + "offload_bridge": false, + "vllm_server_base_url": null, + "vllm_server_host": null, + "vllm_server_port": [ + 8000 + ], + "vllm_server_timeout": 240.0, + "vllm_server_group_port": null, + "reward_funcs": [], + "reward_weights": null, + "cosine_min_len_value_wrong": -0.5, + "cosine_max_len_value_wrong": 0.0, + "cosine_min_len_value_correct": 1.0, + "cosine_max_len_value_correct": 0.5, + "cosine_max_len": null, + "repetition_n_grams": 3, + "repetition_max_penalty": -1.0, + "soft_max_length": null, + "soft_cache_length": null, + "dynamic_sample": false, + "max_resample_times": 3, + "overlong_filter": false, + "scale_rewards": "group", + "advantage_estimator": "grpo", + "kl_in_reward": false, + "wandb_log_unique_prompts": null, + "log_completions": false, + "rollout_importance_sampling_mode": null, + "rollout_importance_sampling_threshold": 2.0, + "log_rollout_offpolicy_metrics": false, + "off_policy_sequence_mask_delta": null, + "log_entropy": false, + "top_entropy_quantile": 1.0, + "reward_model": null, + "reward_model_plugin": null, + "sync_ref_model": false, + "ref_model_sync_steps": 512, + "ref_model_mixup_alpha": 0.6, + "async_generate": false, + "move_model_batches": null, + "multi_turn_scheduler": null, + "max_turns": null, + "completion_length_limit_scope": "per_round", + "vllm_server_pass_dataset": false, + "num_iterations": 1, + "micro_batch_size": 8, + "global_batch_size": 64, + "recompute_granularity": "full", + "recompute_method": "uniform", + "recompute_num_layers": 1, + "recompute_modules": [ + "core_attn" + ], + "train_iters": null, + "num_train_epochs": 4, + "masked_softmax_fusion": true, + "bias_dropout_fusion": true, + "bias_activation_fusion": true, + "apply_rope_fusion": false, + "gradient_accumulation_fusion": false, + "cross_entropy_loss_fusion": true, + "cross_entropy_fusion_impl": "te", + "calculate_per_token_loss": true, + "attention_backend": "", + "optimizer": "adam", + "optimizer_cpu_offload": false, + "optimizer_offload_fraction": 1.0, + "use_precision_aware_optimizer": false, + "main_grads_dtype": "float32", + "main_params_dtype": "float32", + "exp_avg_dtype": "float32", + "exp_avg_sq_dtype": "float32", + "manual_gc": false, + "manual_gc_steps": 0, + "manual_gc_eval": true, + "train_dataloader_shuffle": true, + "dataloader_num_workers": 1, + "dataloader_pin_memory": true, + "dataloader_persistent_workers": true, + "dataloader_prefetch_factor": 2, + "data_sharding": false, + "group_by_length": false, + "te_rng_tracker": false, + "data_parallel_random_init": false, + "mlp_padding_free": false, + "lr_warmup_init": 0.0, + "lr": 0.0001, + "lr_decay_style": "cosine", + "lr_decay_iters": null, + "lr_warmup_iters": 0, + "lr_warmup_fraction": 0.05, + "min_lr": 1e-05, + "lr_wsd_decay_style": "exponential", + "lr_wsd_decay_iters": null, + "weight_decay": 0.1, + "weight_decay_incr_style": "constant", + "start_weight_decay": 0.1, + "end_weight_decay": 0.1, + "clip_grad": 1.0, + "adam_beta1": 0.9, + "adam_beta2": 0.95, + "adam_eps": 1e-08, + "sgd_momentum": 0.9, + "output_dir": "/mnt/bn/strategy-mllm-train/user/tanyue/models/posttrain_models/qwen3_5_35b_a3b_base_highlight_batch1245_epoch4", + "save_steps": 500, + "no_save_optim": true, + "no_save_rng": true, + "mcore_model": null, + "mcore_adapter": null, + "no_load_optim": false, + "no_load_rng": false, + "finetune": true, + "perform_initialization": false, + "use_cpu_initialization": false, + "async_save": false, + "save_total_limit": null, + "metric_for_best_model": "loss", + "greater_is_better": false, + "use_persistent_ckpt_worker": false, + "dist_ckpt_save_pre_mcore_014": false, + "dist_ckpt_optim_fully_reshardable": false, + "distrib_optim_fully_reshardable_mem_efficient": false, + "local_rank": 0, + "use_distributed_optimizer": true, + "tensor_model_parallel_size": 1, + "pipeline_model_parallel_size": 2, + "decoder_first_pipeline_num_layers": null, + "decoder_last_pipeline_num_layers": null, + "account_for_embedding_in_pipeline_split": false, + "account_for_loss_in_pipeline_split": false, + "overlap_p2p_comm": false, + "align_param_gather": false, + "sequence_parallel": false, + "context_parallel_size": 1, + "tp_comm_overlap": false, + "overlap_grad_reduce": false, + "overlap_param_gather": false, + "overlap_param_gather_with_optimizer_step": false, + "align_grad_reduce": true, + "virtual_pipeline_model_parallel_size": null, + "microbatch_group_size_per_vp_stage": null, + "pipeline_model_parallel_layout": null, + "expert_model_parallel_size": 4, + "expert_tensor_parallel_size": 1, + "report_to": [ + "wandb" + ], + "logging_steps": 5, + "tensorboard_dir": "/mnt/bn/strategy-mllm-train/user/tanyue/models/posttrain_models/qwen3_5_35b_a3b_base_highlight_batch1245_epoch4/runs", + "tensorboard_queue_size": 50, + "wandb_project": "highlight_base_highlight_qwen3_5", + "wandb_exp_name": "qwen3_5_35b_a3b_base_highlight_batch1245_epoch4", + "swanlab_project": "megatron-swift", + "swanlab_exp_name": null, + "eval_iters": -1, + "eval_steps": 100000, + "fp8_format": null, + "fp8_recipe": "delayed", + "fp8_amax_history_len": 1024, + "fp8_amax_compute_algo": "max", + "fp8_param_gather": false, + "fp16": false, + "bf16": true, + "apply_query_key_layer_scaling": false, + "attention_softmax_in_fp32": true, + "accumulate_allreduce_grads_in_fp32": true, + "moe_router_load_balancing_type": null, + "moe_router_dtype": "fp32", + "moe_token_dispatcher_type": null, + "moe_enable_deepep": false, + "moe_grouped_gemm": true, + "moe_permute_fusion": true, + "moe_aux_loss_coeff": 1e-06, + "moe_z_loss_coeff": null, + "moe_shared_expert_overlap": true, + "moe_layer_recompute": false, + "moe_expert_capacity_factor": null, + "moe_pad_expert_input_to_capacity": false, + "moe_token_drop_policy": null, + "mtp_num_layers": null, + "mtp_loss_scaling_factor": 0.1, + "save_safetensors": true, + "ref_model": null, + "ref_adapters": [], + "merge_lora": true, + "max_shard_size": "5GB", + "vit_gradient_checkpointing": false, + "vit_lr": null, + "aligner_lr": null, + "gradient_checkpointing_kwargs": null, + "check_model": true, + "apply_wd_to_qk_layernorm": false, + "enable_dft_loss": false, + "enable_channel_loss": false, + "save_strategy": "steps", + "callbacks": [ + "print", + "default_flow", + "wandb" + ], + "add_version": false, + "create_checkpoint_symlink": false, + "swift_version": "4.0.1", + "ckpt_dir": null, + "rank": 0, + "global_world_size": 8, + "local_world_size": 8, + "model_suffix": "Qwen3.5-35B-A3B", + "model_info": "ModelInfo(model_type='qwen3_5_moe', model_dir='/opt/tiger/tanyue/highlight_tools/checkpoints/Qwen3.5-35B-A3B', torch_dtype=torch.bfloat16, max_model_len=262144, quant_method=None, quant_bits=None, rope_scaling=None, is_moe_model=True, is_multimodal=True, config=None, task_type='causal_lm', num_labels=None)", + "model_meta": "ModelMeta(model_type='qwen3_5_moe', model_groups=[ModelGroup(models=[Model(ms_model_id='Qwen/Qwen3.5-35B-A3B-Base', hf_model_id='Qwen/Qwen3.5-35B-A3B-Base', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen3.5-35B-A3B', hf_model_id='Qwen/Qwen3.5-35B-A3B', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen3.5-122B-A10B', hf_model_id='Qwen/Qwen3.5-122B-A10B', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen3.5-397B-A17B', hf_model_id='Qwen/Qwen3.5-397B-A17B', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen3.5-35B-A3B-FP8', hf_model_id='Qwen/Qwen3.5-35B-A3B-FP8', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen3.5-122B-A10B-FP8', hf_model_id='Qwen/Qwen3.5-122B-A10B-FP8', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen3.5-397B-A17B-FP8', hf_model_id='Qwen/Qwen3.5-397B-A17B-FP8', model_path=None, ms_revision=None, hf_revision=None)], template='qwen3_5', ignore_patterns=None, requires=None, tags=[])], loader=, template='qwen3_5', model_arch=MultiModelKeys(arch_name='qwen2_vl', embedding=None, module_list=None, lm_head=None, q_proj=None, k_proj=None, v_proj=None, o_proj=None, attention=None, mlp=None, down_proj=None, qkv_proj=None, qk_proj=None, qa_proj=None, qb_proj=None, kv_proj=None, kva_proj=None, kvb_proj=None, language_model=['model.language_model', 'lm_head'], aligner=['model.visual.merger'], vision_tower=['model.visual'], generator=[]), architectures=['Qwen3_5MoeForConditionalGeneration'], additional_saved_files=[], torch_dtype=None, is_multimodal=True, is_reward=False, task_type=None, ignore_patterns=None, requires=['transformers>=5.2.0', 'qwen_vl_utils>=0.0.14', 'decord'], tags=[])", + "model_dir": "/opt/tiger/tanyue/highlight_tools/checkpoints/Qwen3.5-35B-A3B", + "template_meta": "QwenTemplateMeta(template_type='qwen3_5', prefix=[], prompt=['<|im_start|>user\\n{{QUERY}}<|im_end|>\\n<|im_start|>assistant\\n'], chat_sep=['<|im_end|>\\n'], suffix=['<|im_end|>\\n'], template_cls=, system_prefix=['<|im_start|>system\\n{{SYSTEM}}<|im_end|>\\n'], default_system=None, auto_add_bos=False, stop_words=['<|endoftext|>'], agent_template='qwen3_5', is_thinking=True, thinking_prefix='\\n', non_thinking_prefix='\\n\\n\\n\\n', history_thinking_prefix='')", + "_val_dataset_exists": false, + "hub": "", + "seq_length": 8192, + "is_multimodal": true, + "megatron_model_meta": "MegatronModelMeta(megatron_model_type='qwen3_5', model_types=['qwen3_5', 'qwen3_5_moe'], bridge_cls=, visual_cls=, is_multimodal=True, loader=)", + "fp8": null, + "params_dtype": "bfloat16", + "world_size": 8, + "data_parallel_size": 4, + "num_microbatches": 2 +} \ No newline at end of file diff --git a/posttrain_models/qwen3_5_35b_a3b_base_highlight_batch1245_epoch4/checkpoint-500-merged/chat_template.jinja b/posttrain_models/qwen3_5_35b_a3b_base_highlight_batch1245_epoch4/checkpoint-500-merged/chat_template.jinja new file mode 100644 index 0000000000000000000000000000000000000000..a585dec894e63da457d9440ec6aa7caa16d20860 --- /dev/null +++ b/posttrain_models/qwen3_5_35b_a3b_base_highlight_batch1245_epoch4/checkpoint-500-merged/chat_template.jinja @@ -0,0 +1,154 @@ +{%- set image_count = namespace(value=0) %} +{%- set video_count = namespace(value=0) %} +{%- macro render_content(content, do_vision_count, is_system_content=false) %} + {%- if content is string %} + {{- content }} + {%- elif content is iterable and content is not mapping %} + {%- for item in content %} + {%- if 'image' in item or 'image_url' in item or item.type == 'image' %} + {%- if is_system_content %} + {{- raise_exception('System message cannot contain images.') }} + {%- endif %} + {%- if do_vision_count %} + {%- set image_count.value = image_count.value + 1 %} + {%- endif %} + {%- if add_vision_id %} + {{- 'Picture ' ~ image_count.value ~ ': ' }} + {%- endif %} + {{- '<|vision_start|><|image_pad|><|vision_end|>' }} + {%- elif 'video' in item or item.type == 'video' %} + {%- if is_system_content %} + {{- raise_exception('System message cannot contain videos.') }} + {%- endif %} + {%- if do_vision_count %} + {%- set video_count.value = video_count.value + 1 %} + {%- endif %} + {%- if add_vision_id %} + {{- 'Video ' ~ video_count.value ~ ': ' }} + {%- endif %} + {{- '<|vision_start|><|video_pad|><|vision_end|>' }} + {%- elif 'text' in item %} + {{- item.text }} + {%- else %} + {{- raise_exception('Unexpected item type in content.') }} + {%- endif %} + {%- endfor %} + {%- elif content is none or content is undefined %} + {{- '' }} + {%- else %} + {{- raise_exception('Unexpected content type.') }} + {%- endif %} +{%- endmacro %} +{%- if not messages %} + {{- raise_exception('No messages provided.') }} +{%- endif %} +{%- if tools and tools is iterable and tools is not mapping %} + {{- '<|im_start|>system\n' }} + {{- "# Tools\n\nYou have access to the following functions:\n\n" }} + {%- for tool in tools %} + {{- "\n" }} + {{- tool | tojson }} + {%- endfor %} + {{- "\n" }} + {{- '\n\nIf you choose to call a function ONLY reply in the following format with NO suffix:\n\n\n\n\nvalue_1\n\n\nThis is the value for the second parameter\nthat can span\nmultiple lines\n\n\n\n\n\nReminder:\n- Function calls MUST follow the specified format: an inner block must be nested within XML tags\n- Required parameters MUST be specified\n- You may provide optional reasoning for your function call in natural language BEFORE the function call, but NOT after\n- If there is no function call available, answer the question like normal with your current knowledge and do not tell the user about function calls\n' }} + {%- if messages[0].role == 'system' %} + {%- set content = render_content(messages[0].content, false, true)|trim %} + {%- if content %} + {{- '\n\n' + content }} + {%- endif %} + {%- endif %} + {{- '<|im_end|>\n' }} +{%- else %} + {%- if messages[0].role == 'system' %} + {%- set content = render_content(messages[0].content, false, true)|trim %} + {{- '<|im_start|>system\n' + content + '<|im_end|>\n' }} + {%- endif %} +{%- endif %} +{%- set ns = namespace(multi_step_tool=true, last_query_index=messages|length - 1) %} +{%- for message in messages[::-1] %} + {%- set index = (messages|length - 1) - loop.index0 %} + {%- if ns.multi_step_tool and message.role == "user" %} + {%- set content = render_content(message.content, false)|trim %} + {%- if not(content.startswith('') and content.endswith('')) %} + {%- set ns.multi_step_tool = false %} + {%- set ns.last_query_index = index %} + {%- endif %} + {%- endif %} +{%- endfor %} +{%- if ns.multi_step_tool %} + {{- raise_exception('No user query found in messages.') }} +{%- endif %} +{%- for message in messages %} + {%- set content = render_content(message.content, true)|trim %} + {%- if message.role == "system" %} + {%- if not loop.first %} + {{- raise_exception('System message must be at the beginning.') }} + {%- endif %} + {%- elif message.role == "user" %} + {{- '<|im_start|>' + message.role + '\n' + content + '<|im_end|>' + '\n' }} + {%- elif message.role == "assistant" %} + {%- set reasoning_content = '' %} + {%- if message.reasoning_content is string %} + {%- set reasoning_content = message.reasoning_content %} + {%- else %} + {%- if '' in content %} + {%- set reasoning_content = content.split('')[0].rstrip('\n').split('')[-1].lstrip('\n') %} + {%- set content = content.split('')[-1].lstrip('\n') %} + {%- endif %} + {%- endif %} + {%- set reasoning_content = reasoning_content|trim %} + {%- if loop.index0 > ns.last_query_index %} + {{- '<|im_start|>' + message.role + '\n\n' + reasoning_content + '\n\n\n' + content }} + {%- else %} + {{- '<|im_start|>' + message.role + '\n' + content }} + {%- endif %} + {%- if message.tool_calls and message.tool_calls is iterable and message.tool_calls is not mapping %} + {%- for tool_call in message.tool_calls %} + {%- if tool_call.function is defined %} + {%- set tool_call = tool_call.function %} + {%- endif %} + {%- if loop.first %} + {%- if content|trim %} + {{- '\n\n\n\n' }} + {%- else %} + {{- '\n\n' }} + {%- endif %} + {%- else %} + {{- '\n\n\n' }} + {%- endif %} + {%- if tool_call.arguments is defined %} + {%- for args_name, args_value in tool_call.arguments|items %} + {{- '\n' }} + {%- set args_value = args_value | tojson | safe if args_value is mapping or (args_value is sequence and args_value is not string) else args_value | string %} + {{- args_value }} + {{- '\n\n' }} + {%- endfor %} + {%- endif %} + {{- '\n' }} + {%- endfor %} + {%- endif %} + {{- '<|im_end|>\n' }} + {%- elif message.role == "tool" %} + {%- if loop.previtem and loop.previtem.role != "tool" %} + {{- '<|im_start|>user' }} + {%- endif %} + {{- '\n\n' }} + {{- content }} + {{- '\n' }} + {%- if not loop.last and loop.nextitem.role != "tool" %} + {{- '<|im_end|>\n' }} + {%- elif loop.last %} + {{- '<|im_end|>\n' }} + {%- endif %} + {%- else %} + {{- raise_exception('Unexpected message role.') }} + {%- endif %} +{%- endfor %} +{%- if add_generation_prompt %} + {{- '<|im_start|>assistant\n' }} + {%- if enable_thinking is defined and enable_thinking is false %} + {{- '\n\n\n\n' }} + {%- else %} + {{- '\n' }} + {%- endif %} +{%- endif %} \ No newline at end of file diff --git a/posttrain_models/qwen3_5_35b_a3b_base_highlight_batch1245_epoch4/checkpoint-500-merged/config.json b/posttrain_models/qwen3_5_35b_a3b_base_highlight_batch1245_epoch4/checkpoint-500-merged/config.json new file mode 100644 index 0000000000000000000000000000000000000000..8643d6d3f2e157c74392e7d91f431a66f9261762 --- /dev/null +++ b/posttrain_models/qwen3_5_35b_a3b_base_highlight_batch1245_epoch4/checkpoint-500-merged/config.json @@ -0,0 +1,123 @@ +{ + "architectures": [ + "Qwen3_5MoeForConditionalGeneration" + ], + "dtype": "bfloat16", + "hidden_size": 2048, + "image_token_id": 248056, + "model_type": "qwen3_5_moe", + "text_config": { + "attention_bias": false, + "attention_dropout": 0.0, + "attn_output_gate": true, + "bos_token_id": null, + "dtype": "bfloat16", + "eos_token_id": 248044, + "full_attention_interval": 4, + "head_dim": 256, + "hidden_act": "silu", + "hidden_size": 2048, + "initializer_range": 0.02, + "layer_types": [ + "linear_attention", + "linear_attention", + "linear_attention", + "full_attention", + "linear_attention", + "linear_attention", + "linear_attention", + "full_attention", + "linear_attention", + "linear_attention", + "linear_attention", + "full_attention", + "linear_attention", + "linear_attention", + "linear_attention", + "full_attention", + "linear_attention", + "linear_attention", + "linear_attention", + "full_attention", + "linear_attention", + "linear_attention", + "linear_attention", + "full_attention", + "linear_attention", + "linear_attention", + "linear_attention", + "full_attention", + "linear_attention", + "linear_attention", + "linear_attention", + "full_attention", + "linear_attention", + "linear_attention", + "linear_attention", + "full_attention", + "linear_attention", + "linear_attention", + "linear_attention", + "full_attention" + ], + "linear_conv_kernel_dim": 4, + "linear_key_head_dim": 128, + "linear_num_key_heads": 16, + "linear_num_value_heads": 32, + "linear_value_head_dim": 128, + "mamba_ssm_dtype": "float32", + "max_position_embeddings": 262144, + "mlp_only_layers": [], + "model_type": "qwen3_5_moe_text", + "moe_intermediate_size": 512, + "mtp_num_hidden_layers": 1, + "mtp_use_dedicated_embeddings": false, + "num_attention_heads": 16, + "num_experts": 256, + "num_experts_per_tok": 8, + "num_hidden_layers": 40, + "num_key_value_heads": 2, + "output_router_logits": false, + "pad_token_id": null, + "partial_rotary_factor": 0.25, + "rms_norm_eps": 1e-06, + "rope_parameters": { + "mrope_interleaved": true, + "mrope_section": [ + 11, + 11, + 10 + ], + "partial_rotary_factor": 0.25, + "rope_theta": 10000000, + "rope_type": "default" + }, + "router_aux_loss_coef": 0.001, + "shared_expert_intermediate_size": 512, + "tie_word_embeddings": false, + "use_cache": true, + "vocab_size": 248320 + }, + "tie_word_embeddings": false, + "transformers_version": "5.2.0", + "video_token_id": 248057, + "vision_config": { + "deepstack_visual_indexes": [], + "depth": 27, + "dtype": "bfloat16", + "hidden_act": "gelu_pytorch_tanh", + "hidden_size": 1152, + "in_channels": 3, + "initializer_range": 0.02, + "intermediate_size": 4304, + "model_type": "qwen3_5_moe", + "num_heads": 16, + "num_position_embeddings": 2304, + "out_hidden_size": 2048, + "patch_size": 16, + "spatial_merge_size": 2, + "temporal_patch_size": 2 + }, + "vision_end_token_id": 248054, + "vision_start_token_id": 248053 +} diff --git a/posttrain_models/qwen3_5_35b_a3b_base_highlight_batch1245_epoch4/checkpoint-500-merged/iter_0000500/common.pt b/posttrain_models/qwen3_5_35b_a3b_base_highlight_batch1245_epoch4/checkpoint-500-merged/iter_0000500/common.pt new file mode 100644 index 0000000000000000000000000000000000000000..2c39ebfe639c977c9bab5a0bd3166908d80bd784 --- /dev/null +++ b/posttrain_models/qwen3_5_35b_a3b_base_highlight_batch1245_epoch4/checkpoint-500-merged/iter_0000500/common.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f992eca1cdc1249a4be44a1a4a62c565fc18b8c38531b60bff2753ba9087dd72 +size 15975 diff --git a/posttrain_models/qwen3_5_35b_a3b_base_highlight_batch1245_epoch4/checkpoint-500-merged/latest_checkpointed_iteration.txt b/posttrain_models/qwen3_5_35b_a3b_base_highlight_batch1245_epoch4/checkpoint-500-merged/latest_checkpointed_iteration.txt new file mode 100644 index 0000000000000000000000000000000000000000..eb1f49486af7c892e115f610fa32b505125766fc --- /dev/null +++ b/posttrain_models/qwen3_5_35b_a3b_base_highlight_batch1245_epoch4/checkpoint-500-merged/latest_checkpointed_iteration.txt @@ -0,0 +1 @@ +500 \ No newline at end of file diff --git a/posttrain_models/qwen3_5_35b_a3b_base_highlight_batch1245_epoch4/checkpoint-500-merged/preprocessor_config.json b/posttrain_models/qwen3_5_35b_a3b_base_highlight_batch1245_epoch4/checkpoint-500-merged/preprocessor_config.json new file mode 100644 index 0000000000000000000000000000000000000000..2ea84a437d448ff71b08df68fdd949d5cc4ebb64 --- /dev/null +++ b/posttrain_models/qwen3_5_35b_a3b_base_highlight_batch1245_epoch4/checkpoint-500-merged/preprocessor_config.json @@ -0,0 +1,21 @@ +{ + "size": { + "longest_edge": 16777216, + "shortest_edge": 65536 + }, + "patch_size": 16, + "temporal_patch_size": 2, + "merge_size": 2, + "image_mean": [ + 0.5, + 0.5, + 0.5 + ], + "image_std": [ + 0.5, + 0.5, + 0.5 + ], + "processor_class": "Qwen3VLProcessor", + "image_processor_type": "Qwen2VLImageProcessorFast" +} \ No newline at end of file diff --git a/posttrain_models/qwen3_5_35b_a3b_base_highlight_batch1245_epoch4/checkpoint-500-merged/processor_config.json b/posttrain_models/qwen3_5_35b_a3b_base_highlight_batch1245_epoch4/checkpoint-500-merged/processor_config.json new file mode 100644 index 0000000000000000000000000000000000000000..7ad6acdf4203f22b7b990e36ccc3a1fe38563d5e --- /dev/null +++ b/posttrain_models/qwen3_5_35b_a3b_base_highlight_batch1245_epoch4/checkpoint-500-merged/processor_config.json @@ -0,0 +1,63 @@ +{ + "image_processor": { + "data_format": "channels_first", + "do_convert_rgb": true, + "do_normalize": true, + "do_rescale": true, + "do_resize": true, + "image_mean": [ + 0.5, + 0.5, + 0.5 + ], + "image_processor_type": "Qwen2VLImageProcessorFast", + "image_std": [ + 0.5, + 0.5, + 0.5 + ], + "merge_size": 2, + "patch_size": 16, + "resample": 3, + "rescale_factor": 0.00392156862745098, + "size": { + "longest_edge": 16777216, + "shortest_edge": 65536 + }, + "temporal_patch_size": 2 + }, + "processor_class": "Qwen3VLProcessor", + "video_processor": { + "data_format": "channels_first", + "default_to_square": true, + "do_convert_rgb": true, + "do_normalize": true, + "do_rescale": true, + "do_resize": true, + "do_sample_frames": true, + "fps": 2, + "image_mean": [ + 0.5, + 0.5, + 0.5 + ], + "image_std": [ + 0.5, + 0.5, + 0.5 + ], + "max_frames": 768, + "merge_size": 2, + "min_frames": 4, + "patch_size": 16, + "resample": 3, + "rescale_factor": 0.00392156862745098, + "return_metadata": false, + "size": { + "longest_edge": 25165824, + "shortest_edge": 4096 + }, + "temporal_patch_size": 2, + "video_processor_type": "Qwen3VLVideoProcessor" + } +} diff --git a/posttrain_models/qwen3_5_35b_a3b_base_highlight_batch1245_epoch4/checkpoint-500-merged/tokenizer_config.json b/posttrain_models/qwen3_5_35b_a3b_base_highlight_batch1245_epoch4/checkpoint-500-merged/tokenizer_config.json new file mode 100644 index 0000000000000000000000000000000000000000..a068e2468cff426a9b105006e74e044030a6faf4 --- /dev/null +++ b/posttrain_models/qwen3_5_35b_a3b_base_highlight_batch1245_epoch4/checkpoint-500-merged/tokenizer_config.json @@ -0,0 +1,32 @@ +{ + "add_prefix_space": false, + "audio_bos_token": "<|audio_start|>", + "audio_eos_token": "<|audio_end|>", + "audio_token": "<|audio_pad|>", + "backend": "tokenizers", + "bos_token": null, + "clean_up_tokenization_spaces": false, + "eos_token": "<|im_end|>", + "errors": "replace", + "image_token": "<|image_pad|>", + "is_local": true, + "model_max_length": 262144, + "model_specific_special_tokens": { + "audio_bos_token": "<|audio_start|>", + "audio_eos_token": "<|audio_end|>", + "audio_token": "<|audio_pad|>", + "image_token": "<|image_pad|>", + "video_token": "<|video_pad|>", + "vision_bos_token": "<|vision_start|>", + "vision_eos_token": "<|vision_end|>" + }, + "pad_token": "<|endoftext|>", + "pretokenize_regex": "(?i:'s|'t|'re|'ve|'m|'ll|'d)|[^\\r\\n\\p{L}\\p{N}]?[\\p{L}\\p{M}]+|\\p{N}| ?[^\\s\\p{L}\\p{M}\\p{N}]+[\\r\\n]*|\\s*[\\r\\n]+|\\s+(?!\\S)|\\s+", + "processor_class": "Qwen3VLProcessor", + "split_special_tokens": false, + "tokenizer_class": "TokenizersBackend", + "unk_token": null, + "video_token": "<|video_pad|>", + "vision_bos_token": "<|vision_start|>", + "vision_eos_token": "<|vision_end|>" +} diff --git a/posttrain_models/qwen3_5_35b_a3b_base_highlight_batch1245_epoch4/checkpoint-500/args.json b/posttrain_models/qwen3_5_35b_a3b_base_highlight_batch1245_epoch4/checkpoint-500/args.json new file mode 100644 index 0000000000000000000000000000000000000000..99623dd80e3398e11296749c2daf861d8dff0018 --- /dev/null +++ b/posttrain_models/qwen3_5_35b_a3b_base_highlight_batch1245_epoch4/checkpoint-500/args.json @@ -0,0 +1,398 @@ +{ + "use_ray": false, + "ray_exp_name": null, + "device_groups": null, + "model": "checkpoints/Qwen3.5-35B-A3B", + "model_type": "qwen3_5_moe", + "model_revision": null, + "task_type": "causal_lm", + "torch_dtype": "bfloat16", + "attn_impl": null, + "experts_impl": null, + "new_special_tokens": [], + "num_labels": null, + "problem_type": null, + "rope_scaling": null, + "device_map": null, + "max_memory": {}, + "max_model_len": null, + "local_repo_path": null, + "init_strategy": null, + "template": "qwen3_5", + "system": null, + "max_length": 8192, + "truncation_strategy": "delete", + "max_pixels": null, + "agent_template": null, + "norm_bbox": null, + "use_chat_template": true, + "padding_side": "right", + "padding_free": true, + "loss_scale": "ignore_empty_think", + "sequence_parallel_size": 1, + "template_backend": "swift", + "response_prefix": "", + "enable_thinking": false, + "add_non_thinking_prefix": false, + "dataset": [ + "batch1", + "batch2", + "batch4_part1", + "batch4_part2", + "batch4_part3", + "batch5_part1", + "batch5_part2", + "batch5_part3" + ], + "val_dataset": [], + "cached_dataset": [], + "cached_val_dataset": [], + "split_dataset_ratio": 0.0, + "data_seed": 42, + "dataset_num_proc": 2, + "load_from_cache_file": false, + "dataset_shuffle": true, + "val_dataset_shuffle": false, + "streaming": false, + "interleave_prob": null, + "stopping_strategy": "first_exhausted", + "shuffle_buffer_size": 1000, + "download_mode": "reuse_dataset_if_exists", + "columns": {}, + "strict": false, + "remove_unused_columns": true, + "model_name": null, + "model_author": null, + "custom_dataset_info": [], + "quant_method": null, + "quant_bits": null, + "hqq_axis": null, + "bnb_4bit_compute_dtype": "bfloat16", + "bnb_4bit_quant_type": "nf4", + "bnb_4bit_use_double_quant": true, + "bnb_4bit_quant_storage": null, + "max_new_tokens": null, + "temperature": 0.9, + "top_k": 50, + "top_p": 0.9, + "repetition_penalty": 1.0, + "num_beams": 1, + "stream": false, + "stop_words": [], + "logprobs": false, + "top_logprobs": null, + "structured_outputs_regex": null, + "tuner_backend": "peft", + "tuner_type": "lora", + "train_type": "lora", + "adapters": [], + "external_plugins": [ + "ms-swift/base_highlight_video_dataset.py" + ], + "custom_register_path": [ + "ms-swift/base_highlight_video_dataset.py" + ], + "seed": 42, + "model_kwargs": {}, + "load_args": false, + "load_data_args": false, + "packing": false, + "packing_length": null, + "packing_num_proc": 1, + "lazy_tokenize": true, + "use_hf": false, + "hub_token": null, + "ddp_timeout": 18000000, + "ddp_backend": "nccl", + "ignore_args_error": false, + "use_swift_lora": false, + "freeze_llm": false, + "freeze_vit": true, + "freeze_aligner": true, + "freeze_parameters": [], + "freeze_parameters_regex": null, + "freeze_parameters_ratio": 0.0, + "trainable_parameters": [], + "trainable_parameters_regex": null, + "target_modules": [ + "all-linear" + ], + "target_regex": null, + "modules_to_save": [], + "lora_rank": 8, + "lora_alpha": 32, + "lora_dropout": 0.05, + "lora_bias": "none", + "lora_dtype": null, + "use_rslora": false, + "rlhf_type": null, + "loss_type": null, + "mcore_ref_model": null, + "mcore_ref_adapter": null, + "beta": null, + "rpo_alpha": null, + "reference_free": false, + "label_smoothing": 0.0, + "f_divergence_type": "reverse_kl", + "desirable_weight": 1.0, + "undesirable_weight": 1.0, + "calculate_KL": null, + "center_rewards_coefficient": null, + "teacher_model": null, + "teacher_model_type": null, + "teacher_model_revision": null, + "teacher_model_server": null, + "gkd_logits_topk": null, + "lmbda": 0.5, + "seq_kd": false, + "offload_teacher_model": false, + "sft_alpha": 0.0, + "generation_batch_size": null, + "steps_per_generation": null, + "num_generations": 8, + "num_generations_eval": null, + "max_completion_length": 512, + "importance_sampling_level": "token", + "tau_pos": 1.0, + "tau_neg": 1.05, + "epsilon": 0.2, + "epsilon_high": null, + "delta": null, + "use_vllm": true, + "vllm_mode": null, + "vllm_enable_prefix_caching": true, + "vllm_gpu_memory_utilization": 0.9, + "vllm_tensor_parallel_size": 1, + "vllm_max_model_len": null, + "vllm_enforce_eager": false, + "vllm_limit_mm_per_prompt": null, + "vllm_disable_cascade_attn": false, + "vllm_max_num_seqs": null, + "vllm_mm_processor_cache_gb": null, + "vllm_engine_kwargs": null, + "sleep_level": 0, + "offload_optimizer": false, + "offload_model": false, + "offload_bridge": false, + "vllm_server_base_url": null, + "vllm_server_host": null, + "vllm_server_port": [ + 8000 + ], + "vllm_server_timeout": 240.0, + "vllm_server_group_port": null, + "reward_funcs": [], + "reward_weights": null, + "cosine_min_len_value_wrong": -0.5, + "cosine_max_len_value_wrong": 0.0, + "cosine_min_len_value_correct": 1.0, + "cosine_max_len_value_correct": 0.5, + "cosine_max_len": null, + "repetition_n_grams": 3, + "repetition_max_penalty": -1.0, + "soft_max_length": null, + "soft_cache_length": null, + "dynamic_sample": false, + "max_resample_times": 3, + "overlong_filter": false, + "scale_rewards": "group", + "advantage_estimator": "grpo", + "kl_in_reward": false, + "wandb_log_unique_prompts": null, + "log_completions": false, + "rollout_importance_sampling_mode": null, + "rollout_importance_sampling_threshold": 2.0, + "log_rollout_offpolicy_metrics": false, + "off_policy_sequence_mask_delta": null, + "log_entropy": false, + "top_entropy_quantile": 1.0, + "reward_model": null, + "reward_model_plugin": null, + "sync_ref_model": false, + "ref_model_sync_steps": 512, + "ref_model_mixup_alpha": 0.6, + "async_generate": false, + "move_model_batches": null, + "multi_turn_scheduler": null, + "max_turns": null, + "completion_length_limit_scope": "per_round", + "vllm_server_pass_dataset": false, + "num_iterations": 1, + "micro_batch_size": 8, + "global_batch_size": 64, + "recompute_granularity": "full", + "recompute_method": "uniform", + "recompute_num_layers": 1, + "recompute_modules": [ + "core_attn" + ], + "train_iters": null, + "num_train_epochs": 4, + "masked_softmax_fusion": true, + "bias_dropout_fusion": true, + "bias_activation_fusion": true, + "apply_rope_fusion": false, + "gradient_accumulation_fusion": false, + "cross_entropy_loss_fusion": true, + "cross_entropy_fusion_impl": "te", + "calculate_per_token_loss": true, + "attention_backend": "", + "optimizer": "adam", + "optimizer_cpu_offload": false, + "optimizer_offload_fraction": 1.0, + "use_precision_aware_optimizer": false, + "main_grads_dtype": "float32", + "main_params_dtype": "float32", + "exp_avg_dtype": "float32", + "exp_avg_sq_dtype": "float32", + "manual_gc": false, + "manual_gc_steps": 0, + "manual_gc_eval": true, + "train_dataloader_shuffle": true, + "dataloader_num_workers": 1, + "dataloader_pin_memory": true, + "dataloader_persistent_workers": true, + "dataloader_prefetch_factor": 2, + "data_sharding": false, + "group_by_length": false, + "te_rng_tracker": false, + "data_parallel_random_init": false, + "mlp_padding_free": false, + "lr_warmup_init": 0.0, + "lr": 0.0001, + "lr_decay_style": "cosine", + "lr_decay_iters": null, + "lr_warmup_iters": 0, + "lr_warmup_fraction": 0.05, + "min_lr": 1e-05, + "lr_wsd_decay_style": "exponential", + "lr_wsd_decay_iters": null, + "weight_decay": 0.1, + "weight_decay_incr_style": "constant", + "start_weight_decay": 0.1, + "end_weight_decay": 0.1, + "clip_grad": 1.0, + "adam_beta1": 0.9, + "adam_beta2": 0.95, + "adam_eps": 1e-08, + "sgd_momentum": 0.9, + "output_dir": "/mnt/bn/strategy-mllm-train/user/tanyue/models/posttrain_models/qwen3_5_35b_a3b_base_highlight_batch1245_epoch4", + "save_steps": 500, + "no_save_optim": true, + "no_save_rng": true, + "mcore_model": null, + "mcore_adapter": null, + "no_load_optim": false, + "no_load_rng": false, + "finetune": true, + "perform_initialization": false, + "use_cpu_initialization": false, + "async_save": false, + "save_total_limit": null, + "metric_for_best_model": "loss", + "greater_is_better": false, + "use_persistent_ckpt_worker": false, + "dist_ckpt_save_pre_mcore_014": false, + "dist_ckpt_optim_fully_reshardable": false, + "distrib_optim_fully_reshardable_mem_efficient": false, + "local_rank": 0, + "use_distributed_optimizer": true, + "tensor_model_parallel_size": 1, + "pipeline_model_parallel_size": 2, + "decoder_first_pipeline_num_layers": null, + "decoder_last_pipeline_num_layers": null, + "account_for_embedding_in_pipeline_split": false, + "account_for_loss_in_pipeline_split": false, + "overlap_p2p_comm": false, + "align_param_gather": false, + "sequence_parallel": false, + "context_parallel_size": 1, + "tp_comm_overlap": false, + "overlap_grad_reduce": false, + "overlap_param_gather": false, + "overlap_param_gather_with_optimizer_step": false, + "align_grad_reduce": true, + "virtual_pipeline_model_parallel_size": null, + "microbatch_group_size_per_vp_stage": null, + "pipeline_model_parallel_layout": null, + "expert_model_parallel_size": 4, + "expert_tensor_parallel_size": 1, + "report_to": [ + "wandb" + ], + "logging_steps": 5, + "tensorboard_dir": "/mnt/bn/strategy-mllm-train/user/tanyue/models/posttrain_models/qwen3_5_35b_a3b_base_highlight_batch1245_epoch4/runs", + "tensorboard_queue_size": 50, + "wandb_project": "highlight_base_highlight_qwen3_5", + "wandb_exp_name": "qwen3_5_35b_a3b_base_highlight_batch1245_epoch4", + "swanlab_project": "megatron-swift", + "swanlab_exp_name": null, + "eval_iters": -1, + "eval_steps": 100000, + "fp8_format": null, + "fp8_recipe": "delayed", + "fp8_amax_history_len": 1024, + "fp8_amax_compute_algo": "max", + "fp8_param_gather": false, + "fp16": false, + "bf16": true, + "apply_query_key_layer_scaling": false, + "attention_softmax_in_fp32": true, + "accumulate_allreduce_grads_in_fp32": true, + "moe_router_load_balancing_type": null, + "moe_router_dtype": "fp32", + "moe_token_dispatcher_type": null, + "moe_enable_deepep": false, + "moe_grouped_gemm": true, + "moe_permute_fusion": true, + "moe_aux_loss_coeff": 1e-06, + "moe_z_loss_coeff": null, + "moe_shared_expert_overlap": true, + "moe_layer_recompute": false, + "moe_expert_capacity_factor": null, + "moe_pad_expert_input_to_capacity": false, + "moe_token_drop_policy": null, + "mtp_num_layers": null, + "mtp_loss_scaling_factor": 0.1, + "save_safetensors": true, + "ref_model": null, + "ref_adapters": [], + "merge_lora": true, + "max_shard_size": "5GB", + "vit_gradient_checkpointing": false, + "vit_lr": null, + "aligner_lr": null, + "gradient_checkpointing_kwargs": null, + "check_model": true, + "apply_wd_to_qk_layernorm": false, + "enable_dft_loss": false, + "enable_channel_loss": false, + "save_strategy": "steps", + "callbacks": [ + "print", + "default_flow", + "wandb" + ], + "add_version": false, + "create_checkpoint_symlink": false, + "swift_version": "4.0.1", + "ckpt_dir": null, + "rank": 0, + "global_world_size": 8, + "local_world_size": 8, + "model_suffix": "Qwen3.5-35B-A3B", + "model_info": "ModelInfo(model_type='qwen3_5_moe', model_dir='/opt/tiger/tanyue/highlight_tools/checkpoints/Qwen3.5-35B-A3B', torch_dtype=torch.bfloat16, max_model_len=262144, quant_method=None, quant_bits=None, rope_scaling=None, is_moe_model=True, is_multimodal=True, config=None, task_type='causal_lm', num_labels=None)", + "model_meta": "ModelMeta(model_type='qwen3_5_moe', model_groups=[ModelGroup(models=[Model(ms_model_id='Qwen/Qwen3.5-35B-A3B-Base', hf_model_id='Qwen/Qwen3.5-35B-A3B-Base', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen3.5-35B-A3B', hf_model_id='Qwen/Qwen3.5-35B-A3B', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen3.5-122B-A10B', hf_model_id='Qwen/Qwen3.5-122B-A10B', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen3.5-397B-A17B', hf_model_id='Qwen/Qwen3.5-397B-A17B', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen3.5-35B-A3B-FP8', hf_model_id='Qwen/Qwen3.5-35B-A3B-FP8', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen3.5-122B-A10B-FP8', hf_model_id='Qwen/Qwen3.5-122B-A10B-FP8', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen3.5-397B-A17B-FP8', hf_model_id='Qwen/Qwen3.5-397B-A17B-FP8', model_path=None, ms_revision=None, hf_revision=None)], template='qwen3_5', ignore_patterns=None, requires=None, tags=[])], loader=, template='qwen3_5', model_arch=MultiModelKeys(arch_name='qwen2_vl', embedding=None, module_list=None, lm_head=None, q_proj=None, k_proj=None, v_proj=None, o_proj=None, attention=None, mlp=None, down_proj=None, qkv_proj=None, qk_proj=None, qa_proj=None, qb_proj=None, kv_proj=None, kva_proj=None, kvb_proj=None, language_model=['model.language_model', 'lm_head'], aligner=['model.visual.merger'], vision_tower=['model.visual'], generator=[]), architectures=['Qwen3_5MoeForConditionalGeneration'], additional_saved_files=[], torch_dtype=None, is_multimodal=True, is_reward=False, task_type=None, ignore_patterns=None, requires=['transformers>=5.2.0', 'qwen_vl_utils>=0.0.14', 'decord'], tags=[])", + "model_dir": "/opt/tiger/tanyue/highlight_tools/checkpoints/Qwen3.5-35B-A3B", + "template_meta": "QwenTemplateMeta(template_type='qwen3_5', prefix=[], prompt=['<|im_start|>user\\n{{QUERY}}<|im_end|>\\n<|im_start|>assistant\\n'], chat_sep=['<|im_end|>\\n'], suffix=['<|im_end|>\\n'], template_cls=, system_prefix=['<|im_start|>system\\n{{SYSTEM}}<|im_end|>\\n'], default_system=None, auto_add_bos=False, stop_words=['<|endoftext|>'], agent_template='qwen3_5', is_thinking=True, thinking_prefix='\\n', non_thinking_prefix='\\n\\n\\n\\n', history_thinking_prefix='')", + "_val_dataset_exists": false, + "hub": "", + "seq_length": 8192, + "is_multimodal": true, + "megatron_model_meta": "MegatronModelMeta(megatron_model_type='qwen3_5', model_types=['qwen3_5', 'qwen3_5_moe'], bridge_cls=, visual_cls=, is_multimodal=True, loader=)", + "fp8": null, + "params_dtype": "bfloat16", + "world_size": 8, + "data_parallel_size": 4, + "num_microbatches": 2 +} \ No newline at end of file diff --git a/posttrain_models/qwen3_5_35b_a3b_base_highlight_batch1245_epoch4/checkpoint-500/iter_0000500/.metadata b/posttrain_models/qwen3_5_35b_a3b_base_highlight_batch1245_epoch4/checkpoint-500/iter_0000500/.metadata new file mode 100644 index 0000000000000000000000000000000000000000..de64d7c8e6c0d968d732218b0747d5cace554029 Binary files /dev/null and b/posttrain_models/qwen3_5_35b_a3b_base_highlight_batch1245_epoch4/checkpoint-500/iter_0000500/.metadata differ diff --git a/posttrain_models/qwen3_5_35b_a3b_base_highlight_batch1245_epoch4/checkpoint-500/iter_0000500/common.pt b/posttrain_models/qwen3_5_35b_a3b_base_highlight_batch1245_epoch4/checkpoint-500/iter_0000500/common.pt new file mode 100644 index 0000000000000000000000000000000000000000..2c39ebfe639c977c9bab5a0bd3166908d80bd784 --- /dev/null +++ b/posttrain_models/qwen3_5_35b_a3b_base_highlight_batch1245_epoch4/checkpoint-500/iter_0000500/common.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f992eca1cdc1249a4be44a1a4a62c565fc18b8c38531b60bff2753ba9087dd72 +size 15975 diff --git a/posttrain_models/qwen3_5_35b_a3b_base_highlight_batch1245_epoch4/checkpoint-500/iter_0000500/metadata.json b/posttrain_models/qwen3_5_35b_a3b_base_highlight_batch1245_epoch4/checkpoint-500/iter_0000500/metadata.json new file mode 100644 index 0000000000000000000000000000000000000000..c9dbde0bcf4d2d993122dcc7d6bcb1eef8b6fb77 --- /dev/null +++ b/posttrain_models/qwen3_5_35b_a3b_base_highlight_batch1245_epoch4/checkpoint-500/iter_0000500/metadata.json @@ -0,0 +1 @@ +{"sharded_backend": "torch_dist", "sharded_backend_version": 1, "common_backend": "torch", "common_backend_version": 1} \ No newline at end of file diff --git a/posttrain_models/qwen3_5_35b_a3b_base_highlight_batch1245_epoch4/checkpoint-500/latest_checkpointed_iteration.txt b/posttrain_models/qwen3_5_35b_a3b_base_highlight_batch1245_epoch4/checkpoint-500/latest_checkpointed_iteration.txt new file mode 100644 index 0000000000000000000000000000000000000000..eb1f49486af7c892e115f610fa32b505125766fc --- /dev/null +++ b/posttrain_models/qwen3_5_35b_a3b_base_highlight_batch1245_epoch4/checkpoint-500/latest_checkpointed_iteration.txt @@ -0,0 +1 @@ +500 \ No newline at end of file diff --git a/posttrain_models/qwen3_5_35b_a3b_base_highlight_batch1245_epoch4/logging.jsonl b/posttrain_models/qwen3_5_35b_a3b_base_highlight_batch1245_epoch4/logging.jsonl new file mode 100644 index 0000000000000000000000000000000000000000..62672138124d8c379f052f506201f0b3f1ac85e9 --- /dev/null +++ b/posttrain_models/qwen3_5_35b_a3b_base_highlight_batch1245_epoch4/logging.jsonl @@ -0,0 +1,574 @@ +{"loss": 3.72084546, "grad_norm": 15.82293415, "learning_rate": 7e-07, "load_balancing_loss": 1.21740389, "iteration": "1/2862", "elapsed_time": "2m 9s", "remaining_time": "4d 6h 54m 46s", "memory(GiB)": 60.8, "train_speed(s/it)": 129.49537} +{"loss": 3.70470357, "grad_norm": 14.79979992, "learning_rate": 3.49e-06, "load_balancing_loss": 1.21481693, "iteration": "5/2862", "elapsed_time": "4m 35s", "remaining_time": "1d 19h 35m 9s", "memory(GiB)": 63.25, "train_speed(s/it)": 54.920746} +{"loss": 3.61771059, "grad_norm": 14.84189129, "learning_rate": 6.99e-06, "load_balancing_loss": 1.2126745, "iteration": "10/2862", "elapsed_time": "7m 37s", "remaining_time": "1d 12h 12m 33s", "memory(GiB)": 63.29, "train_speed(s/it)": 45.705665} +{"loss": 2.80221319, "grad_norm": 14.05025482, "learning_rate": 1.048e-05, "load_balancing_loss": 1.21235549, "iteration": "15/2862", "elapsed_time": "10m 48s", "remaining_time": "1d 10h 8m 55s", "memory(GiB)": 64.05, "train_speed(s/it)": 43.180609} +{"loss": 1.64530754, "grad_norm": 6.90338659, "learning_rate": 1.398e-05, "load_balancing_loss": 1.21028161, "iteration": "20/2862", "elapsed_time": "13m 59s", "remaining_time": "1d 9h 7m 17s", "memory(GiB)": 64.05, "train_speed(s/it)": 41.955176} +{"loss": 0.94163609, "grad_norm": 1.92290401, "learning_rate": 1.747e-05, "load_balancing_loss": 1.20901525, "iteration": "25/2862", "elapsed_time": "17m 10s", "remaining_time": "1d 8h 27m 24s", "memory(GiB)": 64.05, "train_speed(s/it)": 41.185811} +{"loss": 0.74593633, "grad_norm": 0.75927627, "learning_rate": 2.096e-05, "load_balancing_loss": 1.20687962, "iteration": "30/2862", "elapsed_time": "20m 17s", "remaining_time": "1d 7h 55m 27s", "memory(GiB)": 64.05, "train_speed(s/it)": 40.581706} +{"loss": 0.72669464, "grad_norm": 0.71038121, "learning_rate": 2.446e-05, "load_balancing_loss": 1.2059598, "iteration": "35/2862", "elapsed_time": "23m 29s", "remaining_time": "1d 7h 37m 9s", "memory(GiB)": 64.05, "train_speed(s/it)": 40.264961} +{"loss": 0.70007342, "grad_norm": 0.92817646, "learning_rate": 2.795e-05, "load_balancing_loss": 1.20950091, "iteration": "40/2862", "elapsed_time": "26m 38s", "remaining_time": "1d 7h 18m 53s", "memory(GiB)": 64.05, "train_speed(s/it)": 39.947775} +{"loss": 0.6879679, "grad_norm": 0.80518758, "learning_rate": 3.145e-05, "load_balancing_loss": 1.2055887, "iteration": "45/2862", "elapsed_time": "29m 47s", "remaining_time": "1d 7h 4m 37s", "memory(GiB)": 64.46, "train_speed(s/it)": 39.715012} +{"loss": 0.69015867, "grad_norm": 0.8453486, "learning_rate": 3.494e-05, "load_balancing_loss": 1.20753765, "iteration": "50/2862", "elapsed_time": "32m 54s", "remaining_time": "1d 6h 50m 31s", "memory(GiB)": 64.46, "train_speed(s/it)": 39.484711} +{"loss": 0.68311226, "grad_norm": 0.96917897, "learning_rate": 3.843e-05, "load_balancing_loss": 1.20891857, "iteration": "55/2862", "elapsed_time": "36m 2s", "remaining_time": "1d 6h 38m 47s", "memory(GiB)": 64.46, "train_speed(s/it)": 39.304287} +{"loss": 0.6644696, "grad_norm": 0.89529657, "learning_rate": 4.193e-05, "load_balancing_loss": 1.21045625, "iteration": "60/2862", "elapsed_time": "39m 10s", "remaining_time": "1d 6h 29m 13s", "memory(GiB)": 64.83, "train_speed(s/it)": 39.169575} +{"loss": 0.65473288, "grad_norm": 1.02400339, "learning_rate": 4.542e-05, "load_balancing_loss": 1.20570183, "iteration": "65/2862", "elapsed_time": "42m 19s", "remaining_time": "1d 6h 20m 41s", "memory(GiB)": 64.83, "train_speed(s/it)": 39.056532} +{"loss": 0.66717309, "grad_norm": 0.85172474, "learning_rate": 4.892e-05, "load_balancing_loss": 1.20175302, "iteration": "70/2862", "elapsed_time": "45m 29s", "remaining_time": "1d 6h 14m 11s", "memory(GiB)": 64.83, "train_speed(s/it)": 38.98665} +{"loss": 0.65584248, "grad_norm": 0.70646632, "learning_rate": 5.241e-05, "load_balancing_loss": 1.20030046, "iteration": "75/2862", "elapsed_time": "48m 41s", "remaining_time": "1d 6h 9m 7s", "memory(GiB)": 64.83, "train_speed(s/it)": 38.947575} +{"loss": 0.66451067, "grad_norm": 0.88214946, "learning_rate": 5.59e-05, "load_balancing_loss": 1.20140147, "iteration": "80/2862", "elapsed_time": "51m 53s", "remaining_time": "1d 6h 4m 8s", "memory(GiB)": 64.83, "train_speed(s/it)": 38.910103} +{"loss": 0.65699643, "grad_norm": 1.40547299, "learning_rate": 5.94e-05, "load_balancing_loss": 1.19697344, "iteration": "85/2862", "elapsed_time": "55m 3s", "remaining_time": "1d 5h 58m 15s", "memory(GiB)": 66.12, "train_speed(s/it)": 38.853016} +{"loss": 0.65363204, "grad_norm": 0.78572667, "learning_rate": 6.289e-05, "load_balancing_loss": 1.1924299, "iteration": "90/2862", "elapsed_time": "58m 9s", "remaining_time": "1d 5h 51m 9s", "memory(GiB)": 66.12, "train_speed(s/it)": 38.769327} +{"loss": 0.66170144, "grad_norm": 0.71814775, "learning_rate": 6.639e-05, "load_balancing_loss": 1.19237542, "iteration": "95/2862", "elapsed_time": "1h 1m 19s", "remaining_time": "1d 5h 45m 56s", "memory(GiB)": 66.12, "train_speed(s/it)": 38.726343} +{"loss": 0.64938277, "grad_norm": 0.81525832, "learning_rate": 6.988e-05, "load_balancing_loss": 1.18511128, "iteration": "100/2862", "elapsed_time": "1h 4m 30s", "remaining_time": "1d 5h 41m 16s", "memory(GiB)": 66.12, "train_speed(s/it)": 38.6953} +{"loss": 0.64875948, "grad_norm": 0.73608881, "learning_rate": 7.338e-05, "load_balancing_loss": 1.18824792, "iteration": "105/2862", "elapsed_time": "1h 7m 38s", "remaining_time": "1d 5h 35m 58s", "memory(GiB)": 66.12, "train_speed(s/it)": 38.650092} +{"loss": 0.65992391, "grad_norm": 0.73273087, "learning_rate": 7.687e-05, "load_balancing_loss": 1.1869961, "iteration": "110/2862", "elapsed_time": "1h 10m 48s", "remaining_time": "1d 5h 31m 15s", "memory(GiB)": 66.14, "train_speed(s/it)": 38.617423} +{"loss": 0.6553123, "grad_norm": 0.65753096, "learning_rate": 8.036e-05, "load_balancing_loss": 1.18260825, "iteration": "115/2862", "elapsed_time": "1h 13m 58s", "remaining_time": "1d 5h 26m 55s", "memory(GiB)": 66.14, "train_speed(s/it)": 38.593071} +{"loss": 0.65578932, "grad_norm": 0.60591197, "learning_rate": 8.386e-05, "load_balancing_loss": 1.17934906, "iteration": "120/2862", "elapsed_time": "1h 17m 8s", "remaining_time": "1d 5h 22m 19s", "memory(GiB)": 66.14, "train_speed(s/it)": 38.562842} +{"loss": 0.64132589, "grad_norm": 0.61141318, "learning_rate": 8.735e-05, "load_balancing_loss": 1.17918277, "iteration": "125/2862", "elapsed_time": "1h 20m 19s", "remaining_time": "1d 5h 18m 45s", "memory(GiB)": 66.14, "train_speed(s/it)": 38.554962} +{"loss": 0.65019208, "grad_norm": 0.61272979, "learning_rate": 9.085e-05, "load_balancing_loss": 1.17443085, "iteration": "130/2862", "elapsed_time": "1h 23m 31s", "remaining_time": "1d 5h 15m 7s", "memory(GiB)": 66.14, "train_speed(s/it)": 38.545903} +{"loss": 0.64436907, "grad_norm": 0.61850095, "learning_rate": 9.434e-05, "load_balancing_loss": 1.17391396, "iteration": "135/2862", "elapsed_time": "1h 26m 42s", "remaining_time": "1d 5h 11m 17s", "memory(GiB)": 66.14, "train_speed(s/it)": 38.532138} +{"loss": 0.63973582, "grad_norm": 0.55788726, "learning_rate": 9.783e-05, "load_balancing_loss": 1.17159009, "iteration": "140/2862", "elapsed_time": "1h 29m 51s", "remaining_time": "1d 5h 7m 0s", "memory(GiB)": 66.14, "train_speed(s/it)": 38.508604} +{"loss": 0.65176833, "grad_norm": 0.57016224, "learning_rate": 0.0001, "load_balancing_loss": 1.17155588, "iteration": "145/2862", "elapsed_time": "1h 33m 2s", "remaining_time": "1d 5h 3m 12s", "memory(GiB)": 66.14, "train_speed(s/it)": 38.495562} +{"loss": 0.64913934, "grad_norm": 0.56914818, "learning_rate": 0.0001, "load_balancing_loss": 1.16695917, "iteration": "150/2862", "elapsed_time": "1h 36m 12s", "remaining_time": "1d 4h 59m 11s", "memory(GiB)": 66.14, "train_speed(s/it)": 38.477486} +{"loss": 0.64815658, "grad_norm": 0.5388242, "learning_rate": 0.0001, "load_balancing_loss": 1.16430938, "iteration": "155/2862", "elapsed_time": "1h 39m 22s", "remaining_time": "1d 4h 55m 30s", "memory(GiB)": 66.14, "train_speed(s/it)": 38.466875} +{"loss": 0.63810241, "grad_norm": 0.70325947, "learning_rate": 9.999e-05, "load_balancing_loss": 1.16486537, "iteration": "160/2862", "elapsed_time": "1h 42m 34s", "remaining_time": "1d 4h 52m 4s", "memory(GiB)": 66.14, "train_speed(s/it)": 38.461779} +{"loss": 0.64302671, "grad_norm": 0.60749114, "learning_rate": 9.999e-05, "load_balancing_loss": 1.1615839, "iteration": "165/2862", "elapsed_time": "1h 45m 43s", "remaining_time": "1d 4h 47m 53s", "memory(GiB)": 66.14, "train_speed(s/it)": 38.440248} +{"loss": 0.64652723, "grad_norm": 0.55311632, "learning_rate": 9.998e-05, "load_balancing_loss": 1.15939605, "iteration": "170/2862", "elapsed_time": "1h 48m 53s", "remaining_time": "1d 4h 44m 20s", "memory(GiB)": 66.14, "train_speed(s/it)": 38.432291} +{"loss": 0.64580935, "grad_norm": 0.5569815, "learning_rate": 9.997e-05, "load_balancing_loss": 1.15446293, "iteration": "175/2862", "elapsed_time": "1h 52m 6s", "remaining_time": "1d 4h 41m 13s", "memory(GiB)": 66.14, "train_speed(s/it)": 38.434485} +{"loss": 0.65217584, "grad_norm": 0.71926051, "learning_rate": 9.996e-05, "load_balancing_loss": 1.14957988, "iteration": "180/2862", "elapsed_time": "1h 55m 15s", "remaining_time": "1d 4h 37m 19s", "memory(GiB)": 66.14, "train_speed(s/it)": 38.418577} +{"loss": 0.63798219, "grad_norm": 0.62374276, "learning_rate": 9.995e-05, "load_balancing_loss": 1.14977586, "iteration": "185/2862", "elapsed_time": "1h 58m 27s", "remaining_time": "1d 4h 33m 55s", "memory(GiB)": 66.14, "train_speed(s/it)": 38.414139} +{"loss": 0.63894302, "grad_norm": 0.97691458, "learning_rate": 9.993e-05, "load_balancing_loss": 1.14849663, "iteration": "190/2862", "elapsed_time": "2h 1m 37s", "remaining_time": "1d 4h 30m 20s", "memory(GiB)": 66.14, "train_speed(s/it)": 38.405551} +{"loss": 0.62880147, "grad_norm": 0.63020515, "learning_rate": 9.992e-05, "load_balancing_loss": 1.14871883, "iteration": "195/2862", "elapsed_time": "2h 4m 49s", "remaining_time": "1d 4h 27m 8s", "memory(GiB)": 66.14, "train_speed(s/it)": 38.405859} +{"loss": 0.62966287, "grad_norm": 0.54109401, "learning_rate": 9.99e-05, "load_balancing_loss": 1.1471951, "iteration": "200/2862", "elapsed_time": "2h 8m 0s", "remaining_time": "1d 4h 23m 41s", "memory(GiB)": 66.14, "train_speed(s/it)": 38.400052} +{"loss": 0.64291453, "grad_norm": 1.66968703, "learning_rate": 9.988e-05, "load_balancing_loss": 1.14421451, "iteration": "205/2862", "elapsed_time": "2h 11m 10s", "remaining_time": "1d 4h 19m 59s", "memory(GiB)": 66.14, "train_speed(s/it)": 38.38865} +{"loss": 0.64189249, "grad_norm": 0.58357579, "learning_rate": 9.987e-05, "load_balancing_loss": 1.14542663, "iteration": "210/2862", "elapsed_time": "2h 14m 20s", "remaining_time": "1d 4h 16m 28s", "memory(GiB)": 66.14, "train_speed(s/it)": 38.381513} +{"loss": 0.65598613, "grad_norm": 0.8772555, "learning_rate": 9.984e-05, "load_balancing_loss": 1.14091396, "iteration": "215/2862", "elapsed_time": "2h 17m 32s", "remaining_time": "1d 4h 13m 20s", "memory(GiB)": 66.14, "train_speed(s/it)": 38.383234} +{"loss": 0.63320041, "grad_norm": 0.45656875, "learning_rate": 9.982e-05, "load_balancing_loss": 1.14001894, "iteration": "220/2862", "elapsed_time": "2h 20m 43s", "remaining_time": "1d 4h 9m 54s", "memory(GiB)": 66.14, "train_speed(s/it)": 38.377662} +{"loss": 0.64358205, "grad_norm": 0.69913441, "learning_rate": 9.98e-05, "load_balancing_loss": 1.13563967, "iteration": "225/2862", "elapsed_time": "2h 23m 55s", "remaining_time": "1d 4h 6m 41s", "memory(GiB)": 66.14, "train_speed(s/it)": 38.377444} +{"loss": 0.6257779, "grad_norm": 0.56505394, "learning_rate": 9.977e-05, "load_balancing_loss": 1.13196921, "iteration": "230/2862", "elapsed_time": "2h 27m 4s", "remaining_time": "1d 4h 2m 54s", "memory(GiB)": 66.14, "train_speed(s/it)": 38.363974} +{"loss": 0.62794662, "grad_norm": 0.57249409, "learning_rate": 9.975e-05, "load_balancing_loss": 1.1302017, "iteration": "235/2862", "elapsed_time": "2h 30m 11s", "remaining_time": "1d 3h 58m 54s", "memory(GiB)": 66.14, "train_speed(s/it)": 38.345618} +{"loss": 0.63400096, "grad_norm": 0.52483207, "learning_rate": 9.972e-05, "load_balancing_loss": 1.12623239, "iteration": "240/2862", "elapsed_time": "2h 33m 23s", "remaining_time": "1d 3h 55m 38s", "memory(GiB)": 66.14, "train_speed(s/it)": 38.343822} +{"loss": 0.62760597, "grad_norm": 0.56684887, "learning_rate": 9.969e-05, "load_balancing_loss": 1.1231705, "iteration": "245/2862", "elapsed_time": "2h 36m 34s", "remaining_time": "1d 3h 52m 23s", "memory(GiB)": 66.14, "train_speed(s/it)": 38.342646} +{"loss": 0.6374228, "grad_norm": 0.54187369, "learning_rate": 9.966e-05, "load_balancing_loss": 1.12722445, "iteration": "250/2862", "elapsed_time": "2h 39m 43s", "remaining_time": "1d 3h 48m 42s", "memory(GiB)": 66.14, "train_speed(s/it)": 38.33154} +{"loss": 0.63078982, "grad_norm": 0.43893838, "learning_rate": 9.962e-05, "load_balancing_loss": 1.12569869, "iteration": "255/2862", "elapsed_time": "2h 42m 52s", "remaining_time": "1d 3h 45m 1s", "memory(GiB)": 66.14, "train_speed(s/it)": 38.320262} +{"loss": 0.63128757, "grad_norm": 0.58953357, "learning_rate": 9.959e-05, "load_balancing_loss": 1.13062704, "iteration": "260/2862", "elapsed_time": "2h 46m 3s", "remaining_time": "1d 3h 41m 45s", "memory(GiB)": 66.14, "train_speed(s/it)": 38.318614} +{"loss": 0.62989897, "grad_norm": 0.7132504, "learning_rate": 9.955e-05, "load_balancing_loss": 1.12477863, "iteration": "265/2862", "elapsed_time": "2h 49m 12s", "remaining_time": "1d 3h 38m 14s", "memory(GiB)": 66.14, "train_speed(s/it)": 38.311225} +{"loss": 0.62101841, "grad_norm": 0.50350106, "learning_rate": 9.952e-05, "load_balancing_loss": 1.12305701, "iteration": "270/2862", "elapsed_time": "2h 52m 22s", "remaining_time": "1d 3h 34m 44s", "memory(GiB)": 66.14, "train_speed(s/it)": 38.304145} +{"loss": 0.63756287, "grad_norm": 0.42608505, "learning_rate": 9.948e-05, "load_balancing_loss": 1.1239897, "iteration": "275/2862", "elapsed_time": "2h 55m 31s", "remaining_time": "1d 3h 31m 7s", "memory(GiB)": 66.14, "train_speed(s/it)": 38.294331} +{"loss": 0.62812734, "grad_norm": 5.23070574, "learning_rate": 9.944e-05, "load_balancing_loss": 1.12649858, "iteration": "280/2862", "elapsed_time": "2h 58m 41s", "remaining_time": "1d 3h 27m 42s", "memory(GiB)": 66.14, "train_speed(s/it)": 38.289} +{"loss": 0.63140905, "grad_norm": 0.56993622, "learning_rate": 9.94e-05, "load_balancing_loss": 1.12380362, "iteration": "285/2862", "elapsed_time": "3h 1m 50s", "remaining_time": "1d 3h 24m 7s", "memory(GiB)": 66.14, "train_speed(s/it)": 38.279836} +{"loss": 0.62745976, "grad_norm": 0.47628641, "learning_rate": 9.935e-05, "load_balancing_loss": 1.1190877, "iteration": "290/2862", "elapsed_time": "3h 4m 59s", "remaining_time": "1d 3h 20m 32s", "memory(GiB)": 66.14, "train_speed(s/it)": 38.270797} +{"loss": 0.63735753, "grad_norm": 0.69548577, "learning_rate": 9.931e-05, "load_balancing_loss": 1.11576426, "iteration": "295/2862", "elapsed_time": "3h 8m 7s", "remaining_time": "1d 3h 16m 57s", "memory(GiB)": 66.14, "train_speed(s/it)": 38.261246} +{"loss": 0.62280661, "grad_norm": 0.54165751, "learning_rate": 9.926e-05, "load_balancing_loss": 1.11304939, "iteration": "300/2862", "elapsed_time": "3h 11m 15s", "remaining_time": "1d 3h 13m 19s", "memory(GiB)": 66.14, "train_speed(s/it)": 38.250884} +{"loss": 0.64245385, "grad_norm": 0.47031942, "learning_rate": 9.921e-05, "load_balancing_loss": 1.11179042, "iteration": "305/2862", "elapsed_time": "3h 14m 23s", "remaining_time": "1d 3h 9m 41s", "memory(GiB)": 66.14, "train_speed(s/it)": 38.240354} +{"loss": 0.63021505, "grad_norm": 0.45306107, "learning_rate": 9.917e-05, "load_balancing_loss": 1.11493337, "iteration": "310/2862", "elapsed_time": "3h 17m 33s", "remaining_time": "1d 3h 6m 13s", "memory(GiB)": 66.14, "train_speed(s/it)": 38.234047} +{"loss": 0.63003892, "grad_norm": 0.4410415, "learning_rate": 9.912e-05, "load_balancing_loss": 1.11081576, "iteration": "315/2862", "elapsed_time": "3h 20m 41s", "remaining_time": "1d 3h 2m 38s", "memory(GiB)": 66.14, "train_speed(s/it)": 38.224526} +{"loss": 0.62903762, "grad_norm": 0.55305427, "learning_rate": 9.906e-05, "load_balancing_loss": 1.11124086, "iteration": "320/2862", "elapsed_time": "3h 23m 50s", "remaining_time": "1d 2h 59m 14s", "memory(GiB)": 66.14, "train_speed(s/it)": 38.219596} +{"loss": 0.62018895, "grad_norm": 0.56717724, "learning_rate": 9.901e-05, "load_balancing_loss": 1.10812747, "iteration": "325/2862", "elapsed_time": "3h 26m 60s", "remaining_time": "1d 2h 55m 50s", "memory(GiB)": 66.14, "train_speed(s/it)": 38.214279} +{"loss": 0.62456602, "grad_norm": 0.45949227, "learning_rate": 9.895e-05, "load_balancing_loss": 1.1091491, "iteration": "330/2862", "elapsed_time": "3h 30m 8s", "remaining_time": "1d 2h 52m 14s", "memory(GiB)": 66.14, "train_speed(s/it)": 38.204766} +{"loss": 0.62812871, "grad_norm": 0.5530684, "learning_rate": 9.89e-05, "load_balancing_loss": 1.10863829, "iteration": "335/2862", "elapsed_time": "3h 33m 18s", "remaining_time": "1d 2h 48m 58s", "memory(GiB)": 66.14, "train_speed(s/it)": 38.202573} +{"loss": 0.61745751, "grad_norm": 0.74653268, "learning_rate": 9.884e-05, "load_balancing_loss": 1.10720432, "iteration": "340/2862", "elapsed_time": "3h 36m 27s", "remaining_time": "1d 2h 45m 33s", "memory(GiB)": 66.14, "train_speed(s/it)": 38.19691} +{"loss": 0.62382489, "grad_norm": 0.54599899, "learning_rate": 9.878e-05, "load_balancing_loss": 1.10834885, "iteration": "345/2862", "elapsed_time": "3h 39m 36s", "remaining_time": "1d 2h 42m 9s", "memory(GiB)": 66.14, "train_speed(s/it)": 38.191924} +{"loss": 0.63694388, "grad_norm": 0.46733961, "learning_rate": 9.872e-05, "load_balancing_loss": 1.10659921, "iteration": "350/2862", "elapsed_time": "3h 42m 45s", "remaining_time": "1d 2h 38m 45s", "memory(GiB)": 66.14, "train_speed(s/it)": 38.186637} +{"loss": 0.63377011, "grad_norm": 0.66747177, "learning_rate": 9.866e-05, "load_balancing_loss": 1.10813034, "iteration": "355/2862", "elapsed_time": "3h 45m 55s", "remaining_time": "1d 2h 35m 27s", "memory(GiB)": 66.14, "train_speed(s/it)": 38.183927} +{"loss": 0.64274806, "grad_norm": 0.51251173, "learning_rate": 9.859e-05, "load_balancing_loss": 1.10999036, "iteration": "360/2862", "elapsed_time": "3h 49m 6s", "remaining_time": "1d 2h 32m 14s", "memory(GiB)": 66.14, "train_speed(s/it)": 38.182871} +{"loss": 0.6166712, "grad_norm": 0.4310596, "learning_rate": 9.853e-05, "load_balancing_loss": 1.10914075, "iteration": "365/2862", "elapsed_time": "3h 52m 14s", "remaining_time": "1d 2h 28m 46s", "memory(GiB)": 66.14, "train_speed(s/it)": 38.176083} +{"loss": 0.62647635, "grad_norm": 0.55853957, "learning_rate": 9.846e-05, "load_balancing_loss": 1.11089194, "iteration": "370/2862", "elapsed_time": "3h 55m 26s", "remaining_time": "1d 2h 25m 38s", "memory(GiB)": 66.14, "train_speed(s/it)": 38.177409} +{"loss": 0.62540811, "grad_norm": 0.56479514, "learning_rate": 9.839e-05, "load_balancing_loss": 1.11118066, "iteration": "375/2862", "elapsed_time": "3h 58m 32s", "remaining_time": "1d 2h 21m 57s", "memory(GiB)": 66.14, "train_speed(s/it)": 38.165093} +{"loss": 0.61382568, "grad_norm": 0.48117992, "learning_rate": 9.832e-05, "load_balancing_loss": 1.1118468, "iteration": "380/2862", "elapsed_time": "4h 1m 41s", "remaining_time": "1d 2h 18m 32s", "memory(GiB)": 66.14, "train_speed(s/it)": 38.159455} +{"loss": 0.62455583, "grad_norm": 0.56162024, "learning_rate": 9.825e-05, "load_balancing_loss": 1.11472404, "iteration": "385/2862", "elapsed_time": "4h 4m 51s", "remaining_time": "1d 2h 15m 18s", "memory(GiB)": 66.14, "train_speed(s/it)": 38.15821} +{"loss": 0.60557789, "grad_norm": 0.65362388, "learning_rate": 9.818e-05, "load_balancing_loss": 1.11230946, "iteration": "390/2862", "elapsed_time": "4h 7m 57s", "remaining_time": "1d 2h 11m 38s", "memory(GiB)": 66.14, "train_speed(s/it)": 38.146306} +{"loss": 0.63788533, "grad_norm": 0.50842345, "learning_rate": 9.811e-05, "load_balancing_loss": 1.1120522, "iteration": "395/2862", "elapsed_time": "4h 11m 23s", "remaining_time": "1d 2h 9m 59s", "memory(GiB)": 66.14, "train_speed(s/it)": 38.183545} +{"loss": 0.60493052, "grad_norm": 0.50123942, "learning_rate": 9.803e-05, "load_balancing_loss": 1.11187518, "iteration": "400/2862", "elapsed_time": "4h 14m 32s", "remaining_time": "1d 2h 6m 39s", "memory(GiB)": 66.14, "train_speed(s/it)": 38.179915} +{"loss": 0.62107176, "grad_norm": 0.4262934, "learning_rate": 9.796e-05, "load_balancing_loss": 1.11276782, "iteration": "405/2862", "elapsed_time": "4h 17m 42s", "remaining_time": "1d 2h 3m 20s", "memory(GiB)": 66.14, "train_speed(s/it)": 38.176612} +{"loss": 0.63371581, "grad_norm": 0.44057843, "learning_rate": 9.788e-05, "load_balancing_loss": 1.11099696, "iteration": "410/2862", "elapsed_time": "4h 20m 49s", "remaining_time": "1d 1h 59m 49s", "memory(GiB)": 66.14, "train_speed(s/it)": 38.16846} +{"loss": 0.62448335, "grad_norm": 0.50390935, "learning_rate": 9.78e-05, "load_balancing_loss": 1.1089884, "iteration": "415/2862", "elapsed_time": "4h 23m 59s", "remaining_time": "1d 1h 56m 31s", "memory(GiB)": 66.14, "train_speed(s/it)": 38.165351} +{"loss": 0.61878234, "grad_norm": 0.58024359, "learning_rate": 9.772e-05, "load_balancing_loss": 1.11626625, "iteration": "420/2862", "elapsed_time": "4h 27m 7s", "remaining_time": "1d 1h 53m 8s", "memory(GiB)": 66.14, "train_speed(s/it)": 38.160589} +{"loss": 0.61536109, "grad_norm": 0.60605824, "learning_rate": 9.763e-05, "load_balancing_loss": 1.1126219, "iteration": "425/2862", "elapsed_time": "4h 30m 15s", "remaining_time": "1d 1h 49m 39s", "memory(GiB)": 66.14, "train_speed(s/it)": 38.152992} +{"loss": 0.61361283, "grad_norm": 0.57402575, "learning_rate": 9.755e-05, "load_balancing_loss": 1.1153096, "iteration": "430/2862", "elapsed_time": "4h 33m 24s", "remaining_time": "1d 1h 46m 20s", "memory(GiB)": 66.14, "train_speed(s/it)": 38.149804} +{"loss": 0.63549751, "grad_norm": 0.60329789, "learning_rate": 9.746e-05, "load_balancing_loss": 1.11068058, "iteration": "435/2862", "elapsed_time": "4h 36m 33s", "remaining_time": "1d 1h 43m 0s", "memory(GiB)": 66.14, "train_speed(s/it)": 38.145976} +{"loss": 0.60670143, "grad_norm": 0.45085856, "learning_rate": 9.738e-05, "load_balancing_loss": 1.1081804, "iteration": "440/2862", "elapsed_time": "4h 39m 43s", "remaining_time": "1d 1h 39m 41s", "memory(GiB)": 66.14, "train_speed(s/it)": 38.142646} +{"loss": 0.62934071, "grad_norm": 0.45412561, "learning_rate": 9.729e-05, "load_balancing_loss": 1.10521889, "iteration": "445/2862", "elapsed_time": "4h 42m 51s", "remaining_time": "1d 1h 36m 19s", "memory(GiB)": 66.14, "train_speed(s/it)": 38.137793} +{"loss": 0.62091857, "grad_norm": 0.39507988, "learning_rate": 9.72e-05, "load_balancing_loss": 1.1039499, "iteration": "450/2862", "elapsed_time": "4h 45m 59s", "remaining_time": "1d 1h 32m 50s", "memory(GiB)": 66.14, "train_speed(s/it)": 38.130369} +{"loss": 0.62034297, "grad_norm": 0.47943643, "learning_rate": 9.711e-05, "load_balancing_loss": 1.10544491, "iteration": "455/2862", "elapsed_time": "4h 49m 7s", "remaining_time": "1d 1h 29m 30s", "memory(GiB)": 66.14, "train_speed(s/it)": 38.126198} +{"loss": 0.6214838, "grad_norm": 0.53829604, "learning_rate": 9.702e-05, "load_balancing_loss": 1.10437191, "iteration": "460/2862", "elapsed_time": "4h 52m 17s", "remaining_time": "1d 1h 26m 12s", "memory(GiB)": 66.14, "train_speed(s/it)": 38.123242} +{"loss": 0.62548578, "grad_norm": 0.49207643, "learning_rate": 9.692e-05, "load_balancing_loss": 1.10775459, "iteration": "465/2862", "elapsed_time": "4h 55m 26s", "remaining_time": "1d 1h 22m 56s", "memory(GiB)": 66.14, "train_speed(s/it)": 38.121075} +{"loss": 0.61654133, "grad_norm": 0.55357796, "learning_rate": 9.683e-05, "load_balancing_loss": 1.10461164, "iteration": "470/2862", "elapsed_time": "4h 58m 35s", "remaining_time": "1d 1h 19m 36s", "memory(GiB)": 66.14, "train_speed(s/it)": 38.116857} +{"loss": 0.62880951, "grad_norm": 0.51903784, "learning_rate": 9.673e-05, "load_balancing_loss": 1.10801184, "iteration": "475/2862", "elapsed_time": "5h 1m 45s", "remaining_time": "1d 1h 16m 20s", "memory(GiB)": 66.51, "train_speed(s/it)": 38.114759} +{"loss": 0.61341393, "grad_norm": 0.57933998, "learning_rate": 9.663e-05, "load_balancing_loss": 1.10901463, "iteration": "480/2862", "elapsed_time": "5h 4m 53s", "remaining_time": "1d 1h 12m 60s", "memory(GiB)": 66.51, "train_speed(s/it)": 38.110652} +{"loss": 0.62180322, "grad_norm": 0.45913151, "learning_rate": 9.653e-05, "load_balancing_loss": 1.11091006, "iteration": "485/2862", "elapsed_time": "5h 8m 1s", "remaining_time": "1d 1h 9m 35s", "memory(GiB)": 66.51, "train_speed(s/it)": 38.104802} +{"loss": 0.61664373, "grad_norm": 0.43735054, "learning_rate": 9.643e-05, "load_balancing_loss": 1.11004245, "iteration": "490/2862", "elapsed_time": "5h 11m 10s", "remaining_time": "1d 1h 6m 18s", "memory(GiB)": 66.51, "train_speed(s/it)": 38.102135} +{"loss": 0.61429667, "grad_norm": 0.44937652, "learning_rate": 9.633e-05, "load_balancing_loss": 1.10925841, "iteration": "495/2862", "elapsed_time": "5h 14m 21s", "remaining_time": "1d 1h 3m 8s", "memory(GiB)": 66.51, "train_speed(s/it)": 38.102422} +{"loss": 0.60474533, "grad_norm": 0.55225778, "learning_rate": 9.623e-05, "load_balancing_loss": 1.10956681, "iteration": "500/2862", "elapsed_time": "5h 17m 29s", "remaining_time": "1d 0h 59m 50s", "memory(GiB)": 66.51, "train_speed(s/it)": 38.098934} +{"loss": 0.61786431, "grad_norm": 0.49360433, "learning_rate": 9.612e-05, "load_balancing_loss": 1.10594082, "iteration": "505/2862", "elapsed_time": "5h 22m 53s", "remaining_time": "1d 1h 6m 60s", "memory(GiB)": 66.51, "train_speed(s/it)": 38.362276} +{"loss": 0.62161982, "grad_norm": 0.41663298, "learning_rate": 9.602e-05, "load_balancing_loss": 1.10280955, "iteration": "510/2862", "elapsed_time": "5h 26m 2s", "remaining_time": "1d 1h 3m 36s", "memory(GiB)": 66.51, "train_speed(s/it)": 38.357278} +{"loss": 0.62588227, "grad_norm": 0.48927632, "learning_rate": 9.591e-05, "load_balancing_loss": 1.10588932, "iteration": "515/2862", "elapsed_time": "5h 29m 11s", "remaining_time": "1d 1h 0m 10s", "memory(GiB)": 66.69, "train_speed(s/it)": 38.351251} +{"loss": 0.63331157, "grad_norm": 0.46103033, "learning_rate": 9.58e-05, "load_balancing_loss": 1.10367811, "iteration": "520/2862", "elapsed_time": "5h 32m 19s", "remaining_time": "1d 0h 56m 43s", "memory(GiB)": 66.69, "train_speed(s/it)": 38.344571} +{"loss": 0.61706018, "grad_norm": 0.55381107, "learning_rate": 9.569e-05, "load_balancing_loss": 1.10751557, "iteration": "525/2862", "elapsed_time": "5h 35m 26s", "remaining_time": "1d 0h 53m 11s", "memory(GiB)": 66.69, "train_speed(s/it)": 38.336002} +{"loss": 0.62931854, "grad_norm": 0.69997007, "learning_rate": 9.558e-05, "load_balancing_loss": 1.10051215, "iteration": "530/2862", "elapsed_time": "5h 38m 35s", "remaining_time": "1d 0h 49m 47s", "memory(GiB)": 66.69, "train_speed(s/it)": 38.330559} +{"loss": 0.62566501, "grad_norm": 0.56975275, "learning_rate": 9.546e-05, "load_balancing_loss": 1.1003226, "iteration": "535/2862", "elapsed_time": "5h 41m 45s", "remaining_time": "1d 0h 46m 25s", "memory(GiB)": 66.69, "train_speed(s/it)": 38.326363} +{"loss": 0.62632817, "grad_norm": 0.51145422, "learning_rate": 9.535e-05, "load_balancing_loss": 1.09922636, "iteration": "540/2862", "elapsed_time": "5h 44m 51s", "remaining_time": "1d 0h 42m 50s", "memory(GiB)": 66.69, "train_speed(s/it)": 38.31608} +{"loss": 0.61794418, "grad_norm": 0.54916579, "learning_rate": 9.523e-05, "load_balancing_loss": 1.10125315, "iteration": "545/2862", "elapsed_time": "5h 48m 1s", "remaining_time": "1d 0h 39m 31s", "memory(GiB)": 66.69, "train_speed(s/it)": 38.31306} +{"loss": 0.61482143, "grad_norm": 0.52467328, "learning_rate": 9.512e-05, "load_balancing_loss": 1.09990656, "iteration": "550/2862", "elapsed_time": "5h 51m 8s", "remaining_time": "1d 0h 36m 0s", "memory(GiB)": 66.69, "train_speed(s/it)": 38.304561} +{"loss": 0.60380793, "grad_norm": 0.4690879, "learning_rate": 9.5e-05, "load_balancing_loss": 1.09865963, "iteration": "555/2862", "elapsed_time": "5h 54m 15s", "remaining_time": "1d 0h 32m 30s", "memory(GiB)": 66.69, "train_speed(s/it)": 38.296691} +{"loss": 0.63728446, "grad_norm": 0.4267042, "learning_rate": 9.488e-05, "load_balancing_loss": 1.09985709, "iteration": "560/2862", "elapsed_time": "5h 57m 24s", "remaining_time": "1d 0h 29m 10s", "memory(GiB)": 66.69, "train_speed(s/it)": 38.29276} +{"loss": 0.61143982, "grad_norm": 0.46127915, "learning_rate": 9.476e-05, "load_balancing_loss": 1.09968555, "iteration": "565/2862", "elapsed_time": "6h 0m 33s", "remaining_time": "1d 0h 25m 47s", "memory(GiB)": 66.69, "train_speed(s/it)": 38.287658} +{"loss": 0.6227755, "grad_norm": 0.43780816, "learning_rate": 9.464e-05, "load_balancing_loss": 1.09542775, "iteration": "570/2862", "elapsed_time": "6h 3m 41s", "remaining_time": "1d 0h 22m 25s", "memory(GiB)": 66.69, "train_speed(s/it)": 38.283221} +{"loss": 0.61494809, "grad_norm": 0.67821753, "learning_rate": 9.451e-05, "load_balancing_loss": 1.09795272, "iteration": "575/2862", "elapsed_time": "6h 6m 50s", "remaining_time": "1d 0h 19m 4s", "memory(GiB)": 66.69, "train_speed(s/it)": 38.278983} +{"loss": 0.62194055, "grad_norm": 0.97575301, "learning_rate": 9.439e-05, "load_balancing_loss": 1.09378421, "iteration": "580/2862", "elapsed_time": "6h 9m 59s", "remaining_time": "1d 0h 15m 42s", "memory(GiB)": 66.69, "train_speed(s/it)": 38.274188} +{"loss": 0.61773264, "grad_norm": 0.52133185, "learning_rate": 9.426e-05, "load_balancing_loss": 1.0922581, "iteration": "585/2862", "elapsed_time": "6h 13m 9s", "remaining_time": "1d 0h 12m 24s", "memory(GiB)": 66.69, "train_speed(s/it)": 38.271443} +{"loss": 0.59941339, "grad_norm": 0.46672308, "learning_rate": 9.413e-05, "load_balancing_loss": 1.09342921, "iteration": "590/2862", "elapsed_time": "6h 16m 17s", "remaining_time": "1d 0h 8m 60s", "memory(GiB)": 66.69, "train_speed(s/it)": 38.265781} +{"loss": 0.62213844, "grad_norm": 0.50064909, "learning_rate": 9.4e-05, "load_balancing_loss": 1.09240866, "iteration": "595/2862", "elapsed_time": "6h 19m 26s", "remaining_time": "1d 0h 5m 42s", "memory(GiB)": 66.69, "train_speed(s/it)": 38.262714} +{"loss": 0.6103707, "grad_norm": 0.48353022, "learning_rate": 9.387e-05, "load_balancing_loss": 1.09424818, "iteration": "600/2862", "elapsed_time": "6h 22m 36s", "remaining_time": "1d 0h 2m 24s", "memory(GiB)": 66.69, "train_speed(s/it)": 38.259931} +{"loss": 0.61921561, "grad_norm": 0.48387679, "learning_rate": 9.374e-05, "load_balancing_loss": 1.09651244, "iteration": "605/2862", "elapsed_time": "6h 25m 45s", "remaining_time": "23h 59m 3s", "memory(GiB)": 66.69, "train_speed(s/it)": 38.25555} +{"loss": 0.61678958, "grad_norm": 0.40583089, "learning_rate": 9.361e-05, "load_balancing_loss": 1.09563959, "iteration": "610/2862", "elapsed_time": "6h 28m 55s", "remaining_time": "23h 55m 48s", "memory(GiB)": 66.69, "train_speed(s/it)": 38.254011} +{"loss": 0.60773176, "grad_norm": 0.4661153, "learning_rate": 9.347e-05, "load_balancing_loss": 1.09151256, "iteration": "615/2862", "elapsed_time": "6h 32m 2s", "remaining_time": "23h 52m 23s", "memory(GiB)": 66.69, "train_speed(s/it)": 38.247909} +{"loss": 0.61516082, "grad_norm": 0.47261903, "learning_rate": 9.334e-05, "load_balancing_loss": 1.09182119, "iteration": "620/2862", "elapsed_time": "6h 35m 9s", "remaining_time": "23h 48m 56s", "memory(GiB)": 66.69, "train_speed(s/it)": 38.240991} +{"loss": 0.6241101, "grad_norm": 0.49406043, "learning_rate": 9.32e-05, "load_balancing_loss": 1.09400773, "iteration": "625/2862", "elapsed_time": "6h 38m 18s", "remaining_time": "23h 45m 34s", "memory(GiB)": 66.69, "train_speed(s/it)": 38.23615} +{"loss": 0.60991442, "grad_norm": 0.45455068, "learning_rate": 9.306e-05, "load_balancing_loss": 1.0912596, "iteration": "630/2862", "elapsed_time": "6h 41m 26s", "remaining_time": "23h 42m 14s", "memory(GiB)": 66.69, "train_speed(s/it)": 38.231904} +{"loss": 0.62641579, "grad_norm": 0.51071727, "learning_rate": 9.293e-05, "load_balancing_loss": 1.09183466, "iteration": "635/2862", "elapsed_time": "6h 44m 37s", "remaining_time": "23h 39m 0s", "memory(GiB)": 66.69, "train_speed(s/it)": 38.230884} +{"loss": 0.61692482, "grad_norm": 0.47617045, "learning_rate": 9.278e-05, "load_balancing_loss": 1.09407926, "iteration": "640/2862", "elapsed_time": "6h 47m 46s", "remaining_time": "23h 35m 43s", "memory(GiB)": 66.69, "train_speed(s/it)": 38.22795} +{"loss": 0.61825269, "grad_norm": 0.45578802, "learning_rate": 9.264e-05, "load_balancing_loss": 1.09521532, "iteration": "645/2862", "elapsed_time": "6h 50m 56s", "remaining_time": "23h 32m 28s", "memory(GiB)": 66.69, "train_speed(s/it)": 38.22643} +{"loss": 0.61100364, "grad_norm": 0.47920147, "learning_rate": 9.25e-05, "load_balancing_loss": 1.09440076, "iteration": "650/2862", "elapsed_time": "6h 54m 6s", "remaining_time": "23h 29m 12s", "memory(GiB)": 66.69, "train_speed(s/it)": 38.224154} +{"loss": 0.61045498, "grad_norm": 0.56103843, "learning_rate": 9.236e-05, "load_balancing_loss": 1.09659028, "iteration": "655/2862", "elapsed_time": "6h 57m 12s", "remaining_time": "23h 25m 44s", "memory(GiB)": 66.69, "train_speed(s/it)": 38.216537} +{"loss": 0.60285217, "grad_norm": 0.60822356, "learning_rate": 9.221e-05, "load_balancing_loss": 1.09662235, "iteration": "660/2862", "elapsed_time": "7h 0m 21s", "remaining_time": "23h 22m 26s", "memory(GiB)": 66.69, "train_speed(s/it)": 38.213526} +{"loss": 0.6222558, "grad_norm": 0.51277494, "learning_rate": 9.206e-05, "load_balancing_loss": 1.09425962, "iteration": "665/2862", "elapsed_time": "7h 3m 31s", "remaining_time": "23h 19m 10s", "memory(GiB)": 66.69, "train_speed(s/it)": 38.211293} +{"loss": 0.61223519, "grad_norm": 0.41533831, "learning_rate": 9.191e-05, "load_balancing_loss": 1.09455919, "iteration": "670/2862", "elapsed_time": "7h 6m 39s", "remaining_time": "23h 15m 49s", "memory(GiB)": 66.69, "train_speed(s/it)": 38.206737} +{"loss": 0.6140607, "grad_norm": 0.51246196, "learning_rate": 9.177e-05, "load_balancing_loss": 1.09441686, "iteration": "675/2862", "elapsed_time": "7h 9m 47s", "remaining_time": "23h 12m 31s", "memory(GiB)": 66.69, "train_speed(s/it)": 38.203651} +{"loss": 0.6209026, "grad_norm": 0.44568685, "learning_rate": 9.161e-05, "load_balancing_loss": 1.09530187, "iteration": "680/2862", "elapsed_time": "7h 12m 56s", "remaining_time": "23h 9m 14s", "memory(GiB)": 66.69, "train_speed(s/it)": 38.200545} +{"loss": 0.60661626, "grad_norm": 0.6122523, "learning_rate": 9.146e-05, "load_balancing_loss": 1.09260941, "iteration": "685/2862", "elapsed_time": "7h 16m 6s", "remaining_time": "23h 5m 59s", "memory(GiB)": 66.69, "train_speed(s/it)": 38.198722} +{"loss": 0.61764133, "grad_norm": 0.41955352, "learning_rate": 9.131e-05, "load_balancing_loss": 1.09476244, "iteration": "690/2862", "elapsed_time": "7h 19m 15s", "remaining_time": "23h 2m 42s", "memory(GiB)": 67.0, "train_speed(s/it)": 38.196051} +{"loss": 0.61748898, "grad_norm": 0.4751471, "learning_rate": 9.116e-05, "load_balancing_loss": 1.09018314, "iteration": "695/2862", "elapsed_time": "7h 22m 25s", "remaining_time": "22h 59m 26s", "memory(GiB)": 67.0, "train_speed(s/it)": 38.193923} +{"loss": 0.62546003, "grad_norm": 0.55454111, "learning_rate": 9.1e-05, "load_balancing_loss": 1.0918771, "iteration": "700/2862", "elapsed_time": "7h 25m 36s", "remaining_time": "22h 56m 15s", "memory(GiB)": 67.0, "train_speed(s/it)": 38.193606} +{"loss": 0.60928512, "grad_norm": 0.68215096, "learning_rate": 9.084e-05, "load_balancing_loss": 1.08995652, "iteration": "705/2862", "elapsed_time": "7h 28m 45s", "remaining_time": "22h 52m 59s", "memory(GiB)": 67.0, "train_speed(s/it)": 38.191371} +{"loss": 0.60971534, "grad_norm": 0.88597864, "learning_rate": 9.069e-05, "load_balancing_loss": 1.08828866, "iteration": "710/2862", "elapsed_time": "7h 31m 55s", "remaining_time": "22h 49m 44s", "memory(GiB)": 67.0, "train_speed(s/it)": 38.189745} +{"loss": 0.61257267, "grad_norm": 0.43633267, "learning_rate": 9.053e-05, "load_balancing_loss": 1.08914077, "iteration": "715/2862", "elapsed_time": "7h 35m 4s", "remaining_time": "22h 46m 28s", "memory(GiB)": 67.0, "train_speed(s/it)": 38.187221} +{"loss": 0.56610012, "grad_norm": 0.67794299, "learning_rate": 9.037e-05, "load_balancing_loss": 1.0939585, "iteration": "720/2862", "elapsed_time": "7h 38m 16s", "remaining_time": "22h 43m 20s", "memory(GiB)": 67.0, "train_speed(s/it)": 38.188576} +{"loss": 0.56286073, "grad_norm": 1.34115672, "learning_rate": 9.021e-05, "load_balancing_loss": 1.09433687, "iteration": "725/2862", "elapsed_time": "7h 41m 23s", "remaining_time": "22h 39m 59s", "memory(GiB)": 67.0, "train_speed(s/it)": 38.183765} +{"loss": 0.56881213, "grad_norm": 0.86707765, "learning_rate": 9.004e-05, "load_balancing_loss": 1.09263206, "iteration": "730/2862", "elapsed_time": "7h 44m 31s", "remaining_time": "22h 36m 40s", "memory(GiB)": 67.0, "train_speed(s/it)": 38.180085} +{"loss": 0.55686283, "grad_norm": 0.52773482, "learning_rate": 8.988e-05, "load_balancing_loss": 1.09100497, "iteration": "735/2862", "elapsed_time": "7h 47m 39s", "remaining_time": "22h 33m 20s", "memory(GiB)": 67.0, "train_speed(s/it)": 38.175756} +{"loss": 0.5569939, "grad_norm": 0.74331194, "learning_rate": 8.971e-05, "load_balancing_loss": 1.08828092, "iteration": "740/2862", "elapsed_time": "7h 50m 48s", "remaining_time": "22h 30m 3s", "memory(GiB)": 67.53, "train_speed(s/it)": 38.172822} +{"loss": 0.54966849, "grad_norm": 0.69733918, "learning_rate": 8.955e-05, "load_balancing_loss": 1.0915978, "iteration": "745/2862", "elapsed_time": "7h 53m 56s", "remaining_time": "22h 26m 43s", "memory(GiB)": 67.53, "train_speed(s/it)": 38.168698} +{"loss": 0.57043695, "grad_norm": 0.64530599, "learning_rate": 8.938e-05, "load_balancing_loss": 1.09204781, "iteration": "750/2862", "elapsed_time": "7h 57m 5s", "remaining_time": "22h 23m 28s", "memory(GiB)": 67.53, "train_speed(s/it)": 38.166818} +{"loss": 0.5661484, "grad_norm": 0.55951768, "learning_rate": 8.921e-05, "load_balancing_loss": 1.08981049, "iteration": "755/2862", "elapsed_time": "8h 0m 15s", "remaining_time": "22h 20m 14s", "memory(GiB)": 67.53, "train_speed(s/it)": 38.165139} +{"loss": 0.55038577, "grad_norm": 0.60401291, "learning_rate": 8.904e-05, "load_balancing_loss": 1.08848715, "iteration": "760/2862", "elapsed_time": "8h 3m 22s", "remaining_time": "22h 16m 53s", "memory(GiB)": 67.53, "train_speed(s/it)": 38.160505} +{"loss": 0.56808567, "grad_norm": 0.62824565, "learning_rate": 8.887e-05, "load_balancing_loss": 1.08854091, "iteration": "765/2862", "elapsed_time": "8h 6m 31s", "remaining_time": "22h 13m 38s", "memory(GiB)": 67.53, "train_speed(s/it)": 38.158347} +{"loss": 0.5674147, "grad_norm": 0.57974267, "learning_rate": 8.87e-05, "load_balancing_loss": 1.08917105, "iteration": "770/2862", "elapsed_time": "8h 9m 40s", "remaining_time": "22h 10m 23s", "memory(GiB)": 67.53, "train_speed(s/it)": 38.156099} +{"loss": 0.55621862, "grad_norm": 0.61723882, "learning_rate": 8.853e-05, "load_balancing_loss": 1.09054363, "iteration": "775/2862", "elapsed_time": "8h 12m 48s", "remaining_time": "22h 7m 5s", "memory(GiB)": 67.53, "train_speed(s/it)": 38.152832} +{"loss": 0.54619557, "grad_norm": 0.61396116, "learning_rate": 8.835e-05, "load_balancing_loss": 1.08656049, "iteration": "780/2862", "elapsed_time": "8h 15m 57s", "remaining_time": "22h 3m 49s", "memory(GiB)": 67.53, "train_speed(s/it)": 38.150209} +{"loss": 0.53422391, "grad_norm": 0.70218021, "learning_rate": 8.818e-05, "load_balancing_loss": 1.08494425, "iteration": "785/2862", "elapsed_time": "8h 19m 7s", "remaining_time": "22h 0m 34s", "memory(GiB)": 67.53, "train_speed(s/it)": 38.148486} +{"loss": 0.54643208, "grad_norm": 1.1446867, "learning_rate": 8.8e-05, "load_balancing_loss": 1.08644235, "iteration": "790/2862", "elapsed_time": "8h 22m 15s", "remaining_time": "21h 57m 17s", "memory(GiB)": 67.53, "train_speed(s/it)": 38.145418} +{"loss": 0.54062915, "grad_norm": 0.60353804, "learning_rate": 8.783e-05, "load_balancing_loss": 1.08783233, "iteration": "795/2862", "elapsed_time": "8h 25m 24s", "remaining_time": "21h 54m 4s", "memory(GiB)": 67.53, "train_speed(s/it)": 38.143955} +{"loss": 0.5544619, "grad_norm": 0.70179939, "learning_rate": 8.765e-05, "load_balancing_loss": 1.09127593, "iteration": "800/2862", "elapsed_time": "8h 28m 33s", "remaining_time": "21h 50m 48s", "memory(GiB)": 67.53, "train_speed(s/it)": 38.141657} +{"loss": 0.55495203, "grad_norm": 0.60985577, "learning_rate": 8.747e-05, "load_balancing_loss": 1.08704877, "iteration": "805/2862", "elapsed_time": "8h 31m 43s", "remaining_time": "21h 47m 33s", "memory(GiB)": 67.53, "train_speed(s/it)": 38.139755} +{"loss": 0.55005658, "grad_norm": 0.6083231, "learning_rate": 8.729e-05, "load_balancing_loss": 1.08541548, "iteration": "810/2862", "elapsed_time": "8h 34m 53s", "remaining_time": "21h 44m 21s", "memory(GiB)": 67.53, "train_speed(s/it)": 38.139126} +{"loss": 0.55781388, "grad_norm": 0.70168513, "learning_rate": 8.711e-05, "load_balancing_loss": 1.08669555, "iteration": "815/2862", "elapsed_time": "8h 38m 2s", "remaining_time": "21h 41m 7s", "memory(GiB)": 67.53, "train_speed(s/it)": 38.137171} +{"loss": 0.56126457, "grad_norm": 0.63184154, "learning_rate": 8.692e-05, "load_balancing_loss": 1.08715463, "iteration": "820/2862", "elapsed_time": "8h 41m 11s", "remaining_time": "21h 37m 52s", "memory(GiB)": 67.53, "train_speed(s/it)": 38.135267} +{"loss": 0.55942273, "grad_norm": 0.57726228, "learning_rate": 8.674e-05, "load_balancing_loss": 1.08620155, "iteration": "825/2862", "elapsed_time": "8h 44m 19s", "remaining_time": "21h 34m 34s", "memory(GiB)": 67.53, "train_speed(s/it)": 38.131672} +{"loss": 0.57168686, "grad_norm": 0.64649934, "learning_rate": 8.655e-05, "load_balancing_loss": 1.08772087, "iteration": "830/2862", "elapsed_time": "8h 47m 27s", "remaining_time": "21h 31m 18s", "memory(GiB)": 67.53, "train_speed(s/it)": 38.128806} +{"loss": 0.55041707, "grad_norm": 0.55917478, "learning_rate": 8.637e-05, "load_balancing_loss": 1.08700299, "iteration": "835/2862", "elapsed_time": "8h 50m 36s", "remaining_time": "21h 28m 4s", "memory(GiB)": 67.53, "train_speed(s/it)": 38.127329} +{"loss": 0.57390851, "grad_norm": 0.59261793, "learning_rate": 8.618e-05, "load_balancing_loss": 1.08633029, "iteration": "840/2862", "elapsed_time": "8h 53m 45s", "remaining_time": "21h 24m 50s", "memory(GiB)": 67.53, "train_speed(s/it)": 38.125563} +{"loss": 0.55943674, "grad_norm": 1.94022644, "learning_rate": 8.599e-05, "load_balancing_loss": 1.08493412, "iteration": "845/2862", "elapsed_time": "8h 56m 54s", "remaining_time": "21h 21m 35s", "memory(GiB)": 67.53, "train_speed(s/it)": 38.123401} +{"loss": 0.56220865, "grad_norm": 0.65782171, "learning_rate": 8.581e-05, "load_balancing_loss": 1.08774173, "iteration": "850/2862", "elapsed_time": "9h 0m 1s", "remaining_time": "21h 18m 15s", "memory(GiB)": 67.53, "train_speed(s/it)": 38.119025} +{"loss": 0.54640895, "grad_norm": 0.78225279, "learning_rate": 8.562e-05, "load_balancing_loss": 1.09048915, "iteration": "855/2862", "elapsed_time": "9h 3m 10s", "remaining_time": "21h 15m 0s", "memory(GiB)": 67.53, "train_speed(s/it)": 38.11665} +{"loss": 0.56247294, "grad_norm": 0.57628191, "learning_rate": 8.542e-05, "load_balancing_loss": 1.09088147, "iteration": "860/2862", "elapsed_time": "9h 6m 16s", "remaining_time": "21h 11m 39s", "memory(GiB)": 67.53, "train_speed(s/it)": 38.111346} +{"loss": 0.53927451, "grad_norm": 0.58888072, "learning_rate": 8.523e-05, "load_balancing_loss": 1.08895993, "iteration": "865/2862", "elapsed_time": "9h 9m 26s", "remaining_time": "21h 8m 27s", "memory(GiB)": 67.53, "train_speed(s/it)": 38.110821} +{"loss": 0.54937232, "grad_norm": 0.73469859, "learning_rate": 8.504e-05, "load_balancing_loss": 1.08896255, "iteration": "870/2862", "elapsed_time": "9h 12m 35s", "remaining_time": "21h 5m 13s", "memory(GiB)": 67.53, "train_speed(s/it)": 38.108989} +{"loss": 0.54537582, "grad_norm": 0.89345193, "learning_rate": 8.484e-05, "load_balancing_loss": 1.08917069, "iteration": "875/2862", "elapsed_time": "9h 15m 46s", "remaining_time": "21h 2m 4s", "memory(GiB)": 67.53, "train_speed(s/it)": 38.109768} +{"loss": 0.56176287, "grad_norm": 0.56477433, "learning_rate": 8.465e-05, "load_balancing_loss": 1.09066474, "iteration": "880/2862", "elapsed_time": "9h 18m 56s", "remaining_time": "20h 58m 53s", "memory(GiB)": 67.53, "train_speed(s/it)": 38.109365} +{"loss": 0.56008428, "grad_norm": 0.55900043, "learning_rate": 8.445e-05, "load_balancing_loss": 1.08942437, "iteration": "885/2862", "elapsed_time": "9h 22m 4s", "remaining_time": "20h 55m 37s", "memory(GiB)": 67.53, "train_speed(s/it)": 38.106523} +{"loss": 0.55954242, "grad_norm": 0.56023186, "learning_rate": 8.426e-05, "load_balancing_loss": 1.08886302, "iteration": "890/2862", "elapsed_time": "9h 25m 14s", "remaining_time": "20h 52m 25s", "memory(GiB)": 67.53, "train_speed(s/it)": 38.106069} +{"loss": 0.54371321, "grad_norm": 0.73811853, "learning_rate": 8.406e-05, "load_balancing_loss": 1.08929622, "iteration": "895/2862", "elapsed_time": "9h 28m 24s", "remaining_time": "20h 49m 13s", "memory(GiB)": 67.53, "train_speed(s/it)": 38.105405} +{"loss": 0.5504477, "grad_norm": 0.73701507, "learning_rate": 8.386e-05, "load_balancing_loss": 1.08965325, "iteration": "900/2862", "elapsed_time": "9h 31m 33s", "remaining_time": "20h 45m 59s", "memory(GiB)": 67.53, "train_speed(s/it)": 38.103278} +{"loss": 0.56343973, "grad_norm": 0.59839272, "learning_rate": 8.366e-05, "load_balancing_loss": 1.09212387, "iteration": "905/2862", "elapsed_time": "9h 34m 43s", "remaining_time": "20h 42m 46s", "memory(GiB)": 67.53, "train_speed(s/it)": 38.102397} +{"loss": 0.55947858, "grad_norm": 0.63805252, "learning_rate": 8.346e-05, "load_balancing_loss": 1.09114063, "iteration": "910/2862", "elapsed_time": "9h 37m 52s", "remaining_time": "20h 39m 33s", "memory(GiB)": 67.53, "train_speed(s/it)": 38.101071} +{"loss": 0.5516091, "grad_norm": 0.59774798, "learning_rate": 8.326e-05, "load_balancing_loss": 1.09408689, "iteration": "915/2862", "elapsed_time": "9h 41m 1s", "remaining_time": "20h 36m 19s", "memory(GiB)": 67.53, "train_speed(s/it)": 38.099171} +{"loss": 0.54860693, "grad_norm": 0.62712991, "learning_rate": 8.305e-05, "load_balancing_loss": 1.09161675, "iteration": "920/2862", "elapsed_time": "9h 44m 9s", "remaining_time": "20h 33m 5s", "memory(GiB)": 67.53, "train_speed(s/it)": 38.097074} +{"loss": 0.5493558, "grad_norm": 0.65770131, "learning_rate": 8.285e-05, "load_balancing_loss": 1.08785129, "iteration": "925/2862", "elapsed_time": "9h 47m 21s", "remaining_time": "20h 29m 56s", "memory(GiB)": 67.53, "train_speed(s/it)": 38.098314} +{"loss": 0.54495651, "grad_norm": 0.61883271, "learning_rate": 8.265e-05, "load_balancing_loss": 1.09137845, "iteration": "930/2862", "elapsed_time": "9h 50m 30s", "remaining_time": "20h 26m 44s", "memory(GiB)": 67.53, "train_speed(s/it)": 38.097069} +{"loss": 0.56252426, "grad_norm": 2.28242278, "learning_rate": 8.244e-05, "load_balancing_loss": 1.08922672, "iteration": "935/2862", "elapsed_time": "9h 53m 39s", "remaining_time": "20h 23m 29s", "memory(GiB)": 67.53, "train_speed(s/it)": 38.095087} +{"loss": 0.55590671, "grad_norm": 0.60972226, "learning_rate": 8.223e-05, "load_balancing_loss": 1.08996475, "iteration": "940/2862", "elapsed_time": "9h 56m 48s", "remaining_time": "20h 20m 16s", "memory(GiB)": 67.53, "train_speed(s/it)": 38.09377} +{"loss": 0.55473304, "grad_norm": 0.55821127, "learning_rate": 8.203e-05, "load_balancing_loss": 1.08583534, "iteration": "945/2862", "elapsed_time": "9h 59m 56s", "remaining_time": "20h 17m 0s", "memory(GiB)": 67.53, "train_speed(s/it)": 38.090923} +{"loss": 0.56097096, "grad_norm": 0.60691035, "learning_rate": 8.182e-05, "load_balancing_loss": 1.08733737, "iteration": "950/2862", "elapsed_time": "10h 3m 6s", "remaining_time": "20h 13m 48s", "memory(GiB)": 67.53, "train_speed(s/it)": 38.090211} +{"loss": 0.55798054, "grad_norm": 0.67430896, "learning_rate": 8.161e-05, "load_balancing_loss": 1.08754003, "iteration": "955/2862", "elapsed_time": "10h 6m 14s", "remaining_time": "20h 10m 34s", "memory(GiB)": 67.53, "train_speed(s/it)": 38.088182} +{"loss": 0.55036408, "grad_norm": 0.73201156, "learning_rate": 8.14e-05, "load_balancing_loss": 1.08642542, "iteration": "960/2862", "elapsed_time": "10h 9m 23s", "remaining_time": "20h 7m 19s", "memory(GiB)": 67.53, "train_speed(s/it)": 38.085958} +{"loss": 0.55322212, "grad_norm": 0.61244482, "learning_rate": 8.119e-05, "load_balancing_loss": 1.08805561, "iteration": "965/2862", "elapsed_time": "10h 12m 32s", "remaining_time": "20h 4m 8s", "memory(GiB)": 67.53, "train_speed(s/it)": 38.085179} +{"loss": 0.56525868, "grad_norm": 0.64593828, "learning_rate": 8.098e-05, "load_balancing_loss": 1.08753252, "iteration": "970/2862", "elapsed_time": "10h 15m 41s", "remaining_time": "20h 0m 53s", "memory(GiB)": 67.53, "train_speed(s/it)": 38.083137} +{"loss": 0.57126975, "grad_norm": 0.64906681, "learning_rate": 8.076e-05, "load_balancing_loss": 1.0862757, "iteration": "975/2862", "elapsed_time": "10h 18m 49s", "remaining_time": "19h 57m 39s", "memory(GiB)": 67.53, "train_speed(s/it)": 38.081337} +{"loss": 0.55711508, "grad_norm": 0.63430762, "learning_rate": 8.055e-05, "load_balancing_loss": 1.08604527, "iteration": "980/2862", "elapsed_time": "10h 21m 56s", "remaining_time": "19h 54m 21s", "memory(GiB)": 67.53, "train_speed(s/it)": 38.077245} +{"loss": 0.56355762, "grad_norm": 0.68297267, "learning_rate": 8.033e-05, "load_balancing_loss": 1.08249032, "iteration": "985/2862", "elapsed_time": "10h 25m 5s", "remaining_time": "19h 51m 8s", "memory(GiB)": 67.53, "train_speed(s/it)": 38.075741} +{"loss": 0.54146695, "grad_norm": 0.74158061, "learning_rate": 8.012e-05, "load_balancing_loss": 1.08276415, "iteration": "990/2862", "elapsed_time": "10h 28m 14s", "remaining_time": "19h 47m 56s", "memory(GiB)": 67.53, "train_speed(s/it)": 38.074868} +{"loss": 0.56217384, "grad_norm": 0.69572246, "learning_rate": 7.99e-05, "load_balancing_loss": 1.08435428, "iteration": "995/2862", "elapsed_time": "10h 31m 21s", "remaining_time": "19h 44m 40s", "memory(GiB)": 67.53, "train_speed(s/it)": 38.071597} +{"loss": 0.57520211, "grad_norm": 0.63465536, "learning_rate": 7.969e-05, "load_balancing_loss": 1.08254361, "iteration": "1000/2862", "elapsed_time": "10h 34m 32s", "remaining_time": "19h 41m 30s", "memory(GiB)": 67.53, "train_speed(s/it)": 38.072069} +{"loss": 0.55618376, "grad_norm": 0.64119309, "learning_rate": 7.947e-05, "load_balancing_loss": 1.08322978, "iteration": "1005/2862", "elapsed_time": "10h 39m 43s", "remaining_time": "19h 42m 2s", "memory(GiB)": 67.53, "train_speed(s/it)": 38.191764} +{"loss": 0.54508299, "grad_norm": 0.66805166, "learning_rate": 7.925e-05, "load_balancing_loss": 1.08850062, "iteration": "1010/2862", "elapsed_time": "10h 42m 49s", "remaining_time": "19h 38m 43s", "memory(GiB)": 67.53, "train_speed(s/it)": 38.187142} +{"loss": 0.55244488, "grad_norm": 0.57880491, "learning_rate": 7.903e-05, "load_balancing_loss": 1.08512199, "iteration": "1015/2862", "elapsed_time": "10h 45m 56s", "remaining_time": "19h 35m 24s", "memory(GiB)": 67.53, "train_speed(s/it)": 38.183226} +{"loss": 0.54111612, "grad_norm": 0.6488682, "learning_rate": 7.881e-05, "load_balancing_loss": 1.08585584, "iteration": "1020/2862", "elapsed_time": "10h 49m 6s", "remaining_time": "19h 32m 11s", "memory(GiB)": 67.53, "train_speed(s/it)": 38.181987} +{"loss": 0.55530131, "grad_norm": 0.69410485, "learning_rate": 7.859e-05, "load_balancing_loss": 1.0873208, "iteration": "1025/2862", "elapsed_time": "10h 52m 17s", "remaining_time": "19h 29m 1s", "memory(GiB)": 67.53, "train_speed(s/it)": 38.182164} +{"loss": 0.56470412, "grad_norm": 0.61024302, "learning_rate": 7.837e-05, "load_balancing_loss": 1.08720207, "iteration": "1030/2862", "elapsed_time": "10h 55m 26s", "remaining_time": "19h 25m 46s", "memory(GiB)": 67.53, "train_speed(s/it)": 38.180219} +{"loss": 0.5338484, "grad_norm": 0.74117881, "learning_rate": 7.815e-05, "load_balancing_loss": 1.08910406, "iteration": "1035/2862", "elapsed_time": "10h 58m 35s", "remaining_time": "19h 22m 32s", "memory(GiB)": 67.53, "train_speed(s/it)": 38.178572} +{"loss": 0.5446108, "grad_norm": 0.66530323, "learning_rate": 7.792e-05, "load_balancing_loss": 1.08721197, "iteration": "1040/2862", "elapsed_time": "11h 1m 44s", "remaining_time": "19h 19m 19s", "memory(GiB)": 67.53, "train_speed(s/it)": 38.177354} +{"loss": 0.55841708, "grad_norm": 0.66925359, "learning_rate": 7.77e-05, "load_balancing_loss": 1.08597851, "iteration": "1045/2862", "elapsed_time": "11h 4m 54s", "remaining_time": "19h 16m 5s", "memory(GiB)": 67.53, "train_speed(s/it)": 38.175765} +{"loss": 0.55333823, "grad_norm": 0.70094508, "learning_rate": 7.747e-05, "load_balancing_loss": 1.08611906, "iteration": "1050/2862", "elapsed_time": "11h 8m 2s", "remaining_time": "19h 12m 50s", "memory(GiB)": 67.53, "train_speed(s/it)": 38.173114} +{"loss": 0.56483948, "grad_norm": 0.6202541, "learning_rate": 7.725e-05, "load_balancing_loss": 1.08436763, "iteration": "1055/2862", "elapsed_time": "11h 11m 10s", "remaining_time": "19h 9m 34s", "memory(GiB)": 67.53, "train_speed(s/it)": 38.17035} +{"loss": 0.55547988, "grad_norm": 0.64375174, "learning_rate": 7.702e-05, "load_balancing_loss": 1.08325446, "iteration": "1060/2862", "elapsed_time": "11h 14m 19s", "remaining_time": "19h 6m 21s", "memory(GiB)": 67.53, "train_speed(s/it)": 38.16906} +{"loss": 0.55242997, "grad_norm": 0.69477016, "learning_rate": 7.679e-05, "load_balancing_loss": 1.08163869, "iteration": "1065/2862", "elapsed_time": "11h 17m 29s", "remaining_time": "19h 3m 8s", "memory(GiB)": 67.53, "train_speed(s/it)": 38.167893} +{"loss": 0.57185405, "grad_norm": 0.67087454, "learning_rate": 7.657e-05, "load_balancing_loss": 1.08079278, "iteration": "1070/2862", "elapsed_time": "11h 20m 38s", "remaining_time": "18h 59m 54s", "memory(GiB)": 67.53, "train_speed(s/it)": 38.166275} +{"loss": 0.55970436, "grad_norm": 0.66601574, "learning_rate": 7.634e-05, "load_balancing_loss": 1.08292413, "iteration": "1075/2862", "elapsed_time": "11h 23m 48s", "remaining_time": "18h 56m 43s", "memory(GiB)": 67.53, "train_speed(s/it)": 38.166002} +{"loss": 0.55202663, "grad_norm": 0.66544086, "learning_rate": 7.611e-05, "load_balancing_loss": 1.08243275, "iteration": "1080/2862", "elapsed_time": "11h 26m 58s", "remaining_time": "18h 53m 29s", "memory(GiB)": 67.53, "train_speed(s/it)": 38.16448} +{"loss": 0.55325949, "grad_norm": 0.68064898, "learning_rate": 7.588e-05, "load_balancing_loss": 1.08441198, "iteration": "1085/2862", "elapsed_time": "11h 30m 6s", "remaining_time": "18h 50m 14s", "memory(GiB)": 67.53, "train_speed(s/it)": 38.161979} +{"loss": 0.55606556, "grad_norm": 0.74814337, "learning_rate": 7.565e-05, "load_balancing_loss": 1.08251846, "iteration": "1090/2862", "elapsed_time": "11h 33m 14s", "remaining_time": "18h 46m 59s", "memory(GiB)": 67.53, "train_speed(s/it)": 38.159675} +{"loss": 0.54666966, "grad_norm": 0.7193293, "learning_rate": 7.542e-05, "load_balancing_loss": 1.08493316, "iteration": "1095/2862", "elapsed_time": "11h 36m 23s", "remaining_time": "18h 43m 44s", "memory(GiB)": 67.53, "train_speed(s/it)": 38.157598} +{"loss": 0.53757429, "grad_norm": 0.72873604, "learning_rate": 7.518e-05, "load_balancing_loss": 1.08233976, "iteration": "1100/2862", "elapsed_time": "11h 39m 33s", "remaining_time": "18h 40m 33s", "memory(GiB)": 67.53, "train_speed(s/it)": 38.157091} +{"loss": 0.54763567, "grad_norm": 0.61487377, "learning_rate": 7.495e-05, "load_balancing_loss": 1.08240712, "iteration": "1105/2862", "elapsed_time": "11h 42m 43s", "remaining_time": "18h 37m 21s", "memory(GiB)": 67.53, "train_speed(s/it)": 38.156779} +{"loss": 0.55112004, "grad_norm": 0.63726437, "learning_rate": 7.472e-05, "load_balancing_loss": 1.08102095, "iteration": "1110/2862", "elapsed_time": "11h 45m 53s", "remaining_time": "18h 34m 9s", "memory(GiB)": 67.53, "train_speed(s/it)": 38.15574} +{"loss": 0.54378033, "grad_norm": 0.70303839, "learning_rate": 7.448e-05, "load_balancing_loss": 1.0820322, "iteration": "1115/2862", "elapsed_time": "11h 49m 2s", "remaining_time": "18h 30m 56s", "memory(GiB)": 67.53, "train_speed(s/it)": 38.15437} +{"loss": 0.55004179, "grad_norm": 0.73778427, "learning_rate": 7.425e-05, "load_balancing_loss": 1.08022952, "iteration": "1120/2862", "elapsed_time": "11h 52m 9s", "remaining_time": "18h 27m 39s", "memory(GiB)": 67.53, "train_speed(s/it)": 38.150786} +{"loss": 0.54048258, "grad_norm": 0.62197763, "learning_rate": 7.401e-05, "load_balancing_loss": 1.08034766, "iteration": "1125/2862", "elapsed_time": "11h 55m 19s", "remaining_time": "18h 24m 26s", "memory(GiB)": 67.53, "train_speed(s/it)": 38.149867} +{"loss": 0.54224694, "grad_norm": 0.67432618, "learning_rate": 7.378e-05, "load_balancing_loss": 1.08121955, "iteration": "1130/2862", "elapsed_time": "11h 58m 28s", "remaining_time": "18h 21m 13s", "memory(GiB)": 67.53, "train_speed(s/it)": 38.148459} +{"loss": 0.53154922, "grad_norm": 0.64351606, "learning_rate": 7.354e-05, "load_balancing_loss": 1.08184195, "iteration": "1135/2862", "elapsed_time": "12h 1m 36s", "remaining_time": "18h 17m 58s", "memory(GiB)": 67.53, "train_speed(s/it)": 38.145864} +{"loss": 0.5455848, "grad_norm": 0.61994052, "learning_rate": 7.33e-05, "load_balancing_loss": 1.08013761, "iteration": "1140/2862", "elapsed_time": "12h 4m 45s", "remaining_time": "18h 14m 46s", "memory(GiB)": 67.53, "train_speed(s/it)": 38.14514} +{"loss": 0.5601849, "grad_norm": 0.66945899, "learning_rate": 7.307e-05, "load_balancing_loss": 1.0820688, "iteration": "1145/2862", "elapsed_time": "12h 7m 56s", "remaining_time": "18h 11m 35s", "memory(GiB)": 67.53, "train_speed(s/it)": 38.144876} +{"loss": 0.54888517, "grad_norm": 0.60673213, "learning_rate": 7.283e-05, "load_balancing_loss": 1.08092916, "iteration": "1150/2862", "elapsed_time": "12h 11m 4s", "remaining_time": "18h 8m 20s", "memory(GiB)": 67.53, "train_speed(s/it)": 38.142252} +{"loss": 0.54246366, "grad_norm": 0.6282807, "learning_rate": 7.259e-05, "load_balancing_loss": 1.0811305, "iteration": "1155/2862", "elapsed_time": "12h 14m 14s", "remaining_time": "18h 5m 8s", "memory(GiB)": 67.53, "train_speed(s/it)": 38.141835} +{"loss": 0.54538184, "grad_norm": 0.66932315, "learning_rate": 7.235e-05, "load_balancing_loss": 1.08301437, "iteration": "1160/2862", "elapsed_time": "12h 17m 24s", "remaining_time": "18h 1m 57s", "memory(GiB)": 67.53, "train_speed(s/it)": 38.14176} +{"loss": 0.56102228, "grad_norm": 0.6288597, "learning_rate": 7.211e-05, "load_balancing_loss": 1.08173883, "iteration": "1165/2862", "elapsed_time": "12h 20m 34s", "remaining_time": "17h 58m 45s", "memory(GiB)": 67.53, "train_speed(s/it)": 38.140582} +{"loss": 0.542225, "grad_norm": 0.65861619, "learning_rate": 7.187e-05, "load_balancing_loss": 1.07976782, "iteration": "1170/2862", "elapsed_time": "12h 23m 44s", "remaining_time": "17h 55m 33s", "memory(GiB)": 67.53, "train_speed(s/it)": 38.140254} +{"loss": 0.54977101, "grad_norm": 0.74045908, "learning_rate": 7.163e-05, "load_balancing_loss": 1.0816102, "iteration": "1175/2862", "elapsed_time": "12h 26m 54s", "remaining_time": "17h 52m 21s", "memory(GiB)": 67.53, "train_speed(s/it)": 38.139167} +{"loss": 0.55459881, "grad_norm": 0.64708287, "learning_rate": 7.139e-05, "load_balancing_loss": 1.08136892, "iteration": "1180/2862", "elapsed_time": "12h 30m 2s", "remaining_time": "17h 49m 7s", "memory(GiB)": 67.53, "train_speed(s/it)": 38.137536} +{"loss": 0.55771548, "grad_norm": 0.60259342, "learning_rate": 7.114e-05, "load_balancing_loss": 1.08378923, "iteration": "1185/2862", "elapsed_time": "12h 33m 11s", "remaining_time": "17h 45m 54s", "memory(GiB)": 67.53, "train_speed(s/it)": 38.136207} +{"loss": 0.55618614, "grad_norm": 0.71436983, "learning_rate": 7.09e-05, "load_balancing_loss": 1.08073699, "iteration": "1190/2862", "elapsed_time": "12h 36m 20s", "remaining_time": "17h 42m 41s", "memory(GiB)": 67.53, "train_speed(s/it)": 38.134774} +{"loss": 0.55107367, "grad_norm": 0.64207298, "learning_rate": 7.066e-05, "load_balancing_loss": 1.0816201, "iteration": "1195/2862", "elapsed_time": "12h 39m 27s", "remaining_time": "17h 39m 25s", "memory(GiB)": 67.53, "train_speed(s/it)": 38.131534} +{"loss": 0.53021216, "grad_norm": 0.68777603, "learning_rate": 7.041e-05, "load_balancing_loss": 1.07883584, "iteration": "1200/2862", "elapsed_time": "12h 42m 38s", "remaining_time": "17h 36m 15s", "memory(GiB)": 67.53, "train_speed(s/it)": 38.131583} +{"loss": 0.55587763, "grad_norm": 0.68113577, "learning_rate": 7.017e-05, "load_balancing_loss": 1.0764426, "iteration": "1205/2862", "elapsed_time": "12h 45m 48s", "remaining_time": "17h 33m 3s", "memory(GiB)": 67.53, "train_speed(s/it)": 38.130879} +{"loss": 0.55708146, "grad_norm": 0.89596212, "learning_rate": 6.992e-05, "load_balancing_loss": 1.07681048, "iteration": "1210/2862", "elapsed_time": "12h 48m 55s", "remaining_time": "17h 29m 48s", "memory(GiB)": 67.53, "train_speed(s/it)": 38.128321} +{"loss": 0.56438607, "grad_norm": 0.59827387, "learning_rate": 6.968e-05, "load_balancing_loss": 1.07946277, "iteration": "1215/2862", "elapsed_time": "12h 52m 6s", "remaining_time": "17h 26m 38s", "memory(GiB)": 67.53, "train_speed(s/it)": 38.128637} +{"loss": 0.54758316, "grad_norm": 0.55119944, "learning_rate": 6.943e-05, "load_balancing_loss": 1.07743919, "iteration": "1220/2862", "elapsed_time": "12h 55m 16s", "remaining_time": "17h 23m 26s", "memory(GiB)": 67.53, "train_speed(s/it)": 38.127773} +{"loss": 0.55901766, "grad_norm": 0.61358607, "learning_rate": 6.919e-05, "load_balancing_loss": 1.0801419, "iteration": "1225/2862", "elapsed_time": "12h 58m 25s", "remaining_time": "17h 20m 13s", "memory(GiB)": 67.53, "train_speed(s/it)": 38.126347} +{"loss": 0.54055035, "grad_norm": 0.78626603, "learning_rate": 6.894e-05, "load_balancing_loss": 1.07871151, "iteration": "1230/2862", "elapsed_time": "13h 1m 35s", "remaining_time": "17h 17m 2s", "memory(GiB)": 67.53, "train_speed(s/it)": 38.126298} +{"loss": 0.55836159, "grad_norm": 0.65750086, "learning_rate": 6.869e-05, "load_balancing_loss": 1.07651067, "iteration": "1235/2862", "elapsed_time": "13h 4m 44s", "remaining_time": "17h 13m 49s", "memory(GiB)": 67.53, "train_speed(s/it)": 38.124619} +{"loss": 0.54876667, "grad_norm": 0.65784609, "learning_rate": 6.844e-05, "load_balancing_loss": 1.07899225, "iteration": "1240/2862", "elapsed_time": "13h 7m 55s", "remaining_time": "17h 10m 39s", "memory(GiB)": 67.53, "train_speed(s/it)": 38.124877} +{"loss": 0.53257984, "grad_norm": 0.81002712, "learning_rate": 6.819e-05, "load_balancing_loss": 1.07774353, "iteration": "1245/2862", "elapsed_time": "13h 11m 5s", "remaining_time": "17h 7m 27s", "memory(GiB)": 67.53, "train_speed(s/it)": 38.124281} +{"loss": 0.55222422, "grad_norm": 0.63848662, "learning_rate": 6.795e-05, "load_balancing_loss": 1.07948554, "iteration": "1250/2862", "elapsed_time": "13h 14m 14s", "remaining_time": "17h 4m 15s", "memory(GiB)": 67.53, "train_speed(s/it)": 38.123596} +{"loss": 0.56295878, "grad_norm": 0.66445631, "learning_rate": 6.77e-05, "load_balancing_loss": 1.07947981, "iteration": "1255/2862", "elapsed_time": "13h 17m 23s", "remaining_time": "17h 1m 2s", "memory(GiB)": 67.53, "train_speed(s/it)": 38.12191} +{"loss": 0.55031455, "grad_norm": 0.77561492, "learning_rate": 6.745e-05, "load_balancing_loss": 1.08009374, "iteration": "1260/2862", "elapsed_time": "13h 20m 31s", "remaining_time": "16h 57m 48s", "memory(GiB)": 67.53, "train_speed(s/it)": 38.119601} +{"loss": 0.5541783, "grad_norm": 0.6432085, "learning_rate": 6.72e-05, "load_balancing_loss": 1.08068812, "iteration": "1265/2862", "elapsed_time": "13h 23m 38s", "remaining_time": "16h 54m 32s", "memory(GiB)": 67.53, "train_speed(s/it)": 38.116609} +{"loss": 0.54973757, "grad_norm": 0.65922654, "learning_rate": 6.695e-05, "load_balancing_loss": 1.08074498, "iteration": "1270/2862", "elapsed_time": "13h 26m 48s", "remaining_time": "16h 51m 22s", "memory(GiB)": 67.53, "train_speed(s/it)": 38.116784} +{"loss": 0.57179546, "grad_norm": 0.64786094, "learning_rate": 6.67e-05, "load_balancing_loss": 1.08256614, "iteration": "1275/2862", "elapsed_time": "13h 29m 58s", "remaining_time": "16h 48m 10s", "memory(GiB)": 67.53, "train_speed(s/it)": 38.116046} +{"loss": 0.55156744, "grad_norm": 0.66733807, "learning_rate": 6.644e-05, "load_balancing_loss": 1.08049607, "iteration": "1280/2862", "elapsed_time": "13h 33m 8s", "remaining_time": "16h 44m 58s", "memory(GiB)": 67.53, "train_speed(s/it)": 38.115236} +{"loss": 0.55194998, "grad_norm": 0.68065089, "learning_rate": 6.619e-05, "load_balancing_loss": 1.07925868, "iteration": "1285/2862", "elapsed_time": "13h 36m 17s", "remaining_time": "16h 41m 46s", "memory(GiB)": 67.53, "train_speed(s/it)": 38.114268} +{"loss": 0.56066471, "grad_norm": 0.81417298, "learning_rate": 6.594e-05, "load_balancing_loss": 1.07884312, "iteration": "1290/2862", "elapsed_time": "13h 39m 26s", "remaining_time": "16h 38m 34s", "memory(GiB)": 67.53, "train_speed(s/it)": 38.113176} +{"loss": 0.53961945, "grad_norm": 0.71303469, "learning_rate": 6.569e-05, "load_balancing_loss": 1.08002079, "iteration": "1295/2862", "elapsed_time": "13h 42m 35s", "remaining_time": "16h 35m 22s", "memory(GiB)": 67.53, "train_speed(s/it)": 38.112197} +{"loss": 0.54497254, "grad_norm": 0.688568, "learning_rate": 6.544e-05, "load_balancing_loss": 1.08180118, "iteration": "1300/2862", "elapsed_time": "13h 45m 45s", "remaining_time": "16h 32m 10s", "memory(GiB)": 67.53, "train_speed(s/it)": 38.111552} +{"loss": 0.55895621, "grad_norm": 0.670964, "learning_rate": 6.518e-05, "load_balancing_loss": 1.08207417, "iteration": "1305/2862", "elapsed_time": "13h 48m 56s", "remaining_time": "16h 28m 60s", "memory(GiB)": 67.53, "train_speed(s/it)": 38.111747} +{"loss": 0.54409254, "grad_norm": 0.67297357, "learning_rate": 6.493e-05, "load_balancing_loss": 1.07953358, "iteration": "1310/2862", "elapsed_time": "13h 52m 5s", "remaining_time": "16h 25m 48s", "memory(GiB)": 67.53, "train_speed(s/it)": 38.110858} +{"loss": 0.55949742, "grad_norm": 0.63255167, "learning_rate": 6.468e-05, "load_balancing_loss": 1.08172166, "iteration": "1315/2862", "elapsed_time": "13h 55m 14s", "remaining_time": "16h 22m 36s", "memory(GiB)": 67.53, "train_speed(s/it)": 38.109729} +{"loss": 0.53641349, "grad_norm": 0.76252788, "learning_rate": 6.442e-05, "load_balancing_loss": 1.07945192, "iteration": "1320/2862", "elapsed_time": "13h 58m 24s", "remaining_time": "16h 19m 24s", "memory(GiB)": 67.53, "train_speed(s/it)": 38.108907} +{"loss": 0.53488719, "grad_norm": 0.58177418, "learning_rate": 6.417e-05, "load_balancing_loss": 1.07812536, "iteration": "1325/2862", "elapsed_time": "14h 1m 32s", "remaining_time": "16h 16m 10s", "memory(GiB)": 67.53, "train_speed(s/it)": 38.106943} +{"loss": 0.56106138, "grad_norm": 0.71372241, "learning_rate": 6.391e-05, "load_balancing_loss": 1.07791102, "iteration": "1330/2862", "elapsed_time": "14h 4m 43s", "remaining_time": "16h 13m 1s", "memory(GiB)": 67.53, "train_speed(s/it)": 38.107714} +{"loss": 0.55573601, "grad_norm": 0.58337224, "learning_rate": 6.366e-05, "load_balancing_loss": 1.07754493, "iteration": "1335/2862", "elapsed_time": "14h 7m 53s", "remaining_time": "16h 9m 50s", "memory(GiB)": 67.53, "train_speed(s/it)": 38.107101} +{"loss": 0.55872256, "grad_norm": 0.61058879, "learning_rate": 6.34e-05, "load_balancing_loss": 1.07716775, "iteration": "1340/2862", "elapsed_time": "14h 11m 2s", "remaining_time": "16h 6m 37s", "memory(GiB)": 67.53, "train_speed(s/it)": 38.105639} +{"loss": 0.56009823, "grad_norm": 0.70337713, "learning_rate": 6.315e-05, "load_balancing_loss": 1.07735848, "iteration": "1345/2862", "elapsed_time": "14h 14m 12s", "remaining_time": "16h 3m 26s", "memory(GiB)": 67.53, "train_speed(s/it)": 38.105291} +{"loss": 0.56858546, "grad_norm": 0.65816087, "learning_rate": 6.289e-05, "load_balancing_loss": 1.07683945, "iteration": "1350/2862", "elapsed_time": "14h 17m 21s", "remaining_time": "16h 0m 14s", "memory(GiB)": 67.53, "train_speed(s/it)": 38.104308} +{"loss": 0.54676068, "grad_norm": 0.66038114, "learning_rate": 6.263e-05, "load_balancing_loss": 1.07892156, "iteration": "1355/2862", "elapsed_time": "14h 20m 31s", "remaining_time": "15h 57m 2s", "memory(GiB)": 68.72, "train_speed(s/it)": 38.10369} +{"loss": 0.56926614, "grad_norm": 0.66045612, "learning_rate": 6.238e-05, "load_balancing_loss": 1.07764161, "iteration": "1360/2862", "elapsed_time": "14h 23m 40s", "remaining_time": "15h 53m 51s", "memory(GiB)": 68.72, "train_speed(s/it)": 38.103263} +{"loss": 0.54989368, "grad_norm": 0.62565821, "learning_rate": 6.212e-05, "load_balancing_loss": 1.07672238, "iteration": "1365/2862", "elapsed_time": "14h 26m 50s", "remaining_time": "15h 50m 40s", "memory(GiB)": 68.72, "train_speed(s/it)": 38.102587} +{"loss": 0.55878097, "grad_norm": 0.61371517, "learning_rate": 6.187e-05, "load_balancing_loss": 1.0758549, "iteration": "1370/2862", "elapsed_time": "14h 30m 1s", "remaining_time": "15h 47m 29s", "memory(GiB)": 68.72, "train_speed(s/it)": 38.102797} +{"loss": 0.54991931, "grad_norm": 0.59482467, "learning_rate": 6.161e-05, "load_balancing_loss": 1.07566774, "iteration": "1375/2862", "elapsed_time": "14h 33m 11s", "remaining_time": "15h 44m 19s", "memory(GiB)": 68.72, "train_speed(s/it)": 38.102657} +{"loss": 0.5467822, "grad_norm": 0.67765981, "learning_rate": 6.135e-05, "load_balancing_loss": 1.07525313, "iteration": "1380/2862", "elapsed_time": "14h 36m 22s", "remaining_time": "15h 41m 8s", "memory(GiB)": 68.72, "train_speed(s/it)": 38.102747} +{"loss": 0.53518164, "grad_norm": 0.82944214, "learning_rate": 6.109e-05, "load_balancing_loss": 1.07500219, "iteration": "1385/2862", "elapsed_time": "14h 39m 33s", "remaining_time": "15h 37m 58s", "memory(GiB)": 68.72, "train_speed(s/it)": 38.103007} +{"loss": 0.54412353, "grad_norm": 0.69932801, "learning_rate": 6.084e-05, "load_balancing_loss": 1.07691145, "iteration": "1390/2862", "elapsed_time": "14h 42m 42s", "remaining_time": "15h 34m 47s", "memory(GiB)": 68.72, "train_speed(s/it)": 38.102265} +{"loss": 0.53603476, "grad_norm": 0.76441324, "learning_rate": 6.058e-05, "load_balancing_loss": 1.07618272, "iteration": "1395/2862", "elapsed_time": "14h 45m 51s", "remaining_time": "15h 31m 35s", "memory(GiB)": 68.72, "train_speed(s/it)": 38.101268} +{"loss": 0.55156279, "grad_norm": 0.71108174, "learning_rate": 6.032e-05, "load_balancing_loss": 1.07463014, "iteration": "1400/2862", "elapsed_time": "14h 49m 2s", "remaining_time": "15h 28m 24s", "memory(GiB)": 68.72, "train_speed(s/it)": 38.101463} +{"loss": 0.5555802, "grad_norm": 0.80907899, "learning_rate": 6.006e-05, "load_balancing_loss": 1.07739961, "iteration": "1405/2862", "elapsed_time": "14h 52m 10s", "remaining_time": "15h 25m 12s", "memory(GiB)": 68.72, "train_speed(s/it)": 38.099918} +{"loss": 0.56066918, "grad_norm": 0.61714429, "learning_rate": 5.98e-05, "load_balancing_loss": 1.07675648, "iteration": "1410/2862", "elapsed_time": "14h 55m 21s", "remaining_time": "15h 22m 2s", "memory(GiB)": 68.72, "train_speed(s/it)": 38.100333} +{"loss": 0.55744624, "grad_norm": 0.60201305, "learning_rate": 5.954e-05, "load_balancing_loss": 1.07595336, "iteration": "1415/2862", "elapsed_time": "14h 58m 31s", "remaining_time": "15h 18m 50s", "memory(GiB)": 68.72, "train_speed(s/it)": 38.0996} +{"loss": 0.55434203, "grad_norm": 0.65977645, "learning_rate": 5.929e-05, "load_balancing_loss": 1.07509995, "iteration": "1420/2862", "elapsed_time": "15h 1m 41s", "remaining_time": "15h 15m 39s", "memory(GiB)": 68.72, "train_speed(s/it)": 38.099325} +{"loss": 0.55891269, "grad_norm": 0.62629151, "learning_rate": 5.903e-05, "load_balancing_loss": 1.07790494, "iteration": "1425/2862", "elapsed_time": "15h 4m 51s", "remaining_time": "15h 12m 28s", "memory(GiB)": 68.72, "train_speed(s/it)": 38.099127} +{"loss": 0.54857761, "grad_norm": 0.68229699, "learning_rate": 5.877e-05, "load_balancing_loss": 1.07561624, "iteration": "1430/2862", "elapsed_time": "15h 8m 3s", "remaining_time": "15h 9m 19s", "memory(GiB)": 68.72, "train_speed(s/it)": 38.099876} +{"loss": 0.45696479, "grad_norm": 0.60413152, "learning_rate": 5.851e-05, "load_balancing_loss": 1.07471848, "iteration": "1435/2862", "elapsed_time": "15h 11m 27s", "remaining_time": "15h 6m 22s", "memory(GiB)": 68.72, "train_speed(s/it)": 38.109513} +{"loss": 0.44133404, "grad_norm": 0.75081611, "learning_rate": 5.825e-05, "load_balancing_loss": 1.07610011, "iteration": "1440/2862", "elapsed_time": "15h 14m 34s", "remaining_time": "15h 3m 8s", "memory(GiB)": 68.72, "train_speed(s/it)": 38.107155} +{"loss": 0.43955019, "grad_norm": 0.88437718, "learning_rate": 5.799e-05, "load_balancing_loss": 1.0734669, "iteration": "1445/2862", "elapsed_time": "15h 17m 41s", "remaining_time": "14h 59m 54s", "memory(GiB)": 68.72, "train_speed(s/it)": 38.104685} +{"loss": 0.42943901, "grad_norm": 1.27408171, "learning_rate": 5.773e-05, "load_balancing_loss": 1.07396185, "iteration": "1450/2862", "elapsed_time": "15h 20m 51s", "remaining_time": "14h 56m 43s", "memory(GiB)": 68.72, "train_speed(s/it)": 38.104108} +{"loss": 0.40675062, "grad_norm": 1.13589334, "learning_rate": 5.747e-05, "load_balancing_loss": 1.07532442, "iteration": "1455/2862", "elapsed_time": "15h 24m 0s", "remaining_time": "14h 53m 31s", "memory(GiB)": 68.72, "train_speed(s/it)": 38.103128} +{"loss": 0.418603, "grad_norm": 1.29270017, "learning_rate": 5.721e-05, "load_balancing_loss": 1.07616651, "iteration": "1460/2862", "elapsed_time": "15h 27m 11s", "remaining_time": "14h 50m 21s", "memory(GiB)": 68.72, "train_speed(s/it)": 38.103601} +{"loss": 0.41475528, "grad_norm": 1.42923403, "learning_rate": 5.695e-05, "load_balancing_loss": 1.07576978, "iteration": "1465/2862", "elapsed_time": "15h 30m 20s", "remaining_time": "14h 47m 9s", "memory(GiB)": 68.72, "train_speed(s/it)": 38.10214} +{"loss": 0.40098548, "grad_norm": 1.30535972, "learning_rate": 5.669e-05, "load_balancing_loss": 1.07613051, "iteration": "1470/2862", "elapsed_time": "15h 33m 30s", "remaining_time": "14h 43m 58s", "memory(GiB)": 68.72, "train_speed(s/it)": 38.101722} +{"loss": 0.40852982, "grad_norm": 1.16670787, "learning_rate": 5.643e-05, "load_balancing_loss": 1.07564759, "iteration": "1475/2862", "elapsed_time": "15h 36m 37s", "remaining_time": "14h 40m 45s", "memory(GiB)": 68.72, "train_speed(s/it)": 38.099867} +{"loss": 0.44568461, "grad_norm": 1.59923315, "learning_rate": 5.617e-05, "load_balancing_loss": 1.07584882, "iteration": "1480/2862", "elapsed_time": "15h 39m 47s", "remaining_time": "14h 37m 33s", "memory(GiB)": 68.72, "train_speed(s/it)": 38.099145} +{"loss": 0.44157019, "grad_norm": 1.34043467, "learning_rate": 5.591e-05, "load_balancing_loss": 1.07551801, "iteration": "1485/2862", "elapsed_time": "15h 42m 56s", "remaining_time": "14h 34m 22s", "memory(GiB)": 68.72, "train_speed(s/it)": 38.098412} +{"loss": 0.40065175, "grad_norm": 1.19279623, "learning_rate": 5.565e-05, "load_balancing_loss": 1.07647884, "iteration": "1490/2862", "elapsed_time": "15h 46m 5s", "remaining_time": "14h 31m 10s", "memory(GiB)": 68.72, "train_speed(s/it)": 38.097635} +{"loss": 0.42277771, "grad_norm": 1.0835824, "learning_rate": 5.539e-05, "load_balancing_loss": 1.07547271, "iteration": "1495/2862", "elapsed_time": "15h 49m 15s", "remaining_time": "14h 27m 58s", "memory(GiB)": 68.72, "train_speed(s/it)": 38.096784} +{"loss": 0.40937811, "grad_norm": 1.34055424, "learning_rate": 5.513e-05, "load_balancing_loss": 1.07864439, "iteration": "1500/2862", "elapsed_time": "15h 52m 23s", "remaining_time": "14h 24m 46s", "memory(GiB)": 68.72, "train_speed(s/it)": 38.095312} +{"loss": 0.41720241, "grad_norm": 1.15145171, "learning_rate": 5.487e-05, "load_balancing_loss": 1.07685304, "iteration": "1505/2862", "elapsed_time": "15h 57m 32s", "remaining_time": "14h 23m 22s", "memory(GiB)": 68.72, "train_speed(s/it)": 38.174154} +{"loss": 0.4176122, "grad_norm": 1.3456248, "learning_rate": 5.461e-05, "load_balancing_loss": 1.07718396, "iteration": "1510/2862", "elapsed_time": "16h 0m 41s", "remaining_time": "14h 20m 9s", "memory(GiB)": 68.72, "train_speed(s/it)": 38.172707} +{"loss": 0.41413936, "grad_norm": 1.25498283, "learning_rate": 5.435e-05, "load_balancing_loss": 1.07738924, "iteration": "1515/2862", "elapsed_time": "16h 3m 51s", "remaining_time": "14h 16m 58s", "memory(GiB)": 68.72, "train_speed(s/it)": 38.172224} +{"loss": 0.41527992, "grad_norm": 1.31816196, "learning_rate": 5.409e-05, "load_balancing_loss": 1.07909548, "iteration": "1520/2862", "elapsed_time": "16h 6m 60s", "remaining_time": "14h 13m 45s", "memory(GiB)": 68.72, "train_speed(s/it)": 38.17095} +{"loss": 0.40920442, "grad_norm": 1.13207102, "learning_rate": 5.383e-05, "load_balancing_loss": 1.07682538, "iteration": "1525/2862", "elapsed_time": "16h 10m 8s", "remaining_time": "14h 10m 32s", "memory(GiB)": 68.72, "train_speed(s/it)": 38.169285} +{"loss": 0.41990259, "grad_norm": 1.19388056, "learning_rate": 5.357e-05, "load_balancing_loss": 1.07829487, "iteration": "1530/2862", "elapsed_time": "16h 13m 16s", "remaining_time": "14h 7m 19s", "memory(GiB)": 68.72, "train_speed(s/it)": 38.167564} +{"loss": 0.38315356, "grad_norm": 1.30554199, "learning_rate": 5.331e-05, "load_balancing_loss": 1.07828367, "iteration": "1535/2862", "elapsed_time": "16h 16m 25s", "remaining_time": "14h 4m 7s", "memory(GiB)": 68.72, "train_speed(s/it)": 38.166421} +{"loss": 0.39358822, "grad_norm": 1.39771116, "learning_rate": 5.305e-05, "load_balancing_loss": 1.07796884, "iteration": "1540/2862", "elapsed_time": "16h 19m 33s", "remaining_time": "14h 0m 53s", "memory(GiB)": 68.72, "train_speed(s/it)": 38.164438} +{"loss": 0.4089838, "grad_norm": 1.28683293, "learning_rate": 5.279e-05, "load_balancing_loss": 1.07990992, "iteration": "1545/2862", "elapsed_time": "16h 22m 42s", "remaining_time": "13h 57m 40s", "memory(GiB)": 68.72, "train_speed(s/it)": 38.162856} +{"loss": 0.41808048, "grad_norm": 1.13452351, "learning_rate": 5.253e-05, "load_balancing_loss": 1.07897842, "iteration": "1550/2862", "elapsed_time": "16h 25m 51s", "remaining_time": "13h 54m 28s", "memory(GiB)": 68.72, "train_speed(s/it)": 38.161776} +{"loss": 0.38737512, "grad_norm": 1.35664725, "learning_rate": 5.227e-05, "load_balancing_loss": 1.07806242, "iteration": "1555/2862", "elapsed_time": "16h 29m 1s", "remaining_time": "13h 51m 17s", "memory(GiB)": 68.72, "train_speed(s/it)": 38.161613} +{"loss": 0.42602834, "grad_norm": 1.27850652, "learning_rate": 5.202e-05, "load_balancing_loss": 1.07760608, "iteration": "1560/2862", "elapsed_time": "16h 32m 11s", "remaining_time": "13h 48m 5s", "memory(GiB)": 68.72, "train_speed(s/it)": 38.160605} +{"loss": 0.40014791, "grad_norm": 17.11551857, "learning_rate": 5.176e-05, "load_balancing_loss": 1.07601583, "iteration": "1565/2862", "elapsed_time": "16h 35m 20s", "remaining_time": "13h 44m 53s", "memory(GiB)": 68.72, "train_speed(s/it)": 38.159568} +{"loss": 0.41822794, "grad_norm": 4.19012785, "learning_rate": 5.15e-05, "load_balancing_loss": 1.07497537, "iteration": "1570/2862", "elapsed_time": "16h 38m 27s", "remaining_time": "13h 41m 39s", "memory(GiB)": 68.72, "train_speed(s/it)": 38.157362} +{"loss": 0.41106659, "grad_norm": 1.0804106, "learning_rate": 5.124e-05, "load_balancing_loss": 1.07667828, "iteration": "1575/2862", "elapsed_time": "16h 41m 37s", "remaining_time": "13h 38m 28s", "memory(GiB)": 68.72, "train_speed(s/it)": 38.156706} +{"loss": 0.40553227, "grad_norm": 1.17316055, "learning_rate": 5.098e-05, "load_balancing_loss": 1.07645154, "iteration": "1580/2862", "elapsed_time": "16h 44m 47s", "remaining_time": "13h 35m 17s", "memory(GiB)": 68.72, "train_speed(s/it)": 38.156639} +{"loss": 0.41958511, "grad_norm": 1.17240822, "learning_rate": 5.072e-05, "load_balancing_loss": 1.07669163, "iteration": "1585/2862", "elapsed_time": "16h 47m 58s", "remaining_time": "13h 32m 5s", "memory(GiB)": 68.72, "train_speed(s/it)": 38.156219} +{"loss": 0.41465643, "grad_norm": 1.30270851, "learning_rate": 5.046e-05, "load_balancing_loss": 1.07677615, "iteration": "1590/2862", "elapsed_time": "16h 51m 7s", "remaining_time": "13h 28m 54s", "memory(GiB)": 68.72, "train_speed(s/it)": 38.155648} +{"loss": 0.42326277, "grad_norm": 1.36475778, "learning_rate": 5.02e-05, "load_balancing_loss": 1.07896066, "iteration": "1595/2862", "elapsed_time": "16h 54m 18s", "remaining_time": "13h 25m 43s", "memory(GiB)": 68.72, "train_speed(s/it)": 38.155487} +{"loss": 0.41997334, "grad_norm": 1.33872366, "learning_rate": 4.994e-05, "load_balancing_loss": 1.07629943, "iteration": "1600/2862", "elapsed_time": "16h 57m 27s", "remaining_time": "13h 22m 31s", "memory(GiB)": 68.72, "train_speed(s/it)": 38.154212} +{"loss": 0.40688366, "grad_norm": 1.33461893, "learning_rate": 4.969e-05, "load_balancing_loss": 1.07784545, "iteration": "1605/2862", "elapsed_time": "17h 0m 35s", "remaining_time": "13h 19m 18s", "memory(GiB)": 68.72, "train_speed(s/it)": 38.152914} +{"loss": 0.39940009, "grad_norm": 1.29273486, "learning_rate": 4.943e-05, "load_balancing_loss": 1.0769068, "iteration": "1610/2862", "elapsed_time": "17h 3m 45s", "remaining_time": "13h 16m 7s", "memory(GiB)": 68.72, "train_speed(s/it)": 38.152434} +{"loss": 0.41208065, "grad_norm": 1.41675973, "learning_rate": 4.917e-05, "load_balancing_loss": 1.07698524, "iteration": "1615/2862", "elapsed_time": "17h 6m 55s", "remaining_time": "13h 12m 55s", "memory(GiB)": 69.62, "train_speed(s/it)": 38.151749} +{"loss": 0.38745382, "grad_norm": 1.47157884, "learning_rate": 4.891e-05, "load_balancing_loss": 1.07756364, "iteration": "1620/2862", "elapsed_time": "17h 10m 5s", "remaining_time": "13h 9m 44s", "memory(GiB)": 69.62, "train_speed(s/it)": 38.151357} +{"loss": 0.40774322, "grad_norm": 1.23573446, "learning_rate": 4.865e-05, "load_balancing_loss": 1.07825243, "iteration": "1625/2862", "elapsed_time": "17h 13m 15s", "remaining_time": "13h 6m 32s", "memory(GiB)": 69.62, "train_speed(s/it)": 38.150675} +{"loss": 0.40965503, "grad_norm": 1.23221076, "learning_rate": 4.84e-05, "load_balancing_loss": 1.07816017, "iteration": "1630/2862", "elapsed_time": "17h 16m 26s", "remaining_time": "13h 3m 22s", "memory(GiB)": 69.62, "train_speed(s/it)": 38.15107} +{"loss": 0.41262472, "grad_norm": 1.09288919, "learning_rate": 4.814e-05, "load_balancing_loss": 1.07797801, "iteration": "1635/2862", "elapsed_time": "17h 19m 37s", "remaining_time": "13h 0m 11s", "memory(GiB)": 69.62, "train_speed(s/it)": 38.150773} +{"loss": 0.39520696, "grad_norm": 1.15474653, "learning_rate": 4.788e-05, "load_balancing_loss": 1.07876325, "iteration": "1640/2862", "elapsed_time": "17h 22m 44s", "remaining_time": "12h 56m 58s", "memory(GiB)": 69.62, "train_speed(s/it)": 38.148773} +{"loss": 0.40069899, "grad_norm": 1.27235293, "learning_rate": 4.763e-05, "load_balancing_loss": 1.07699752, "iteration": "1645/2862", "elapsed_time": "17h 25m 54s", "remaining_time": "12h 53m 47s", "memory(GiB)": 69.62, "train_speed(s/it)": 38.148561} +{"loss": 0.3964611, "grad_norm": 1.27500522, "learning_rate": 4.737e-05, "load_balancing_loss": 1.08000886, "iteration": "1650/2862", "elapsed_time": "17h 29m 4s", "remaining_time": "12h 50m 35s", "memory(GiB)": 69.62, "train_speed(s/it)": 38.147942} +{"loss": 0.39658529, "grad_norm": 1.41877866, "learning_rate": 4.711e-05, "load_balancing_loss": 1.07916093, "iteration": "1655/2862", "elapsed_time": "17h 32m 16s", "remaining_time": "12h 47m 25s", "memory(GiB)": 69.62, "train_speed(s/it)": 38.148479} +{"loss": 0.3926442, "grad_norm": 1.27852523, "learning_rate": 4.686e-05, "load_balancing_loss": 1.07697058, "iteration": "1660/2862", "elapsed_time": "17h 35m 25s", "remaining_time": "12h 44m 13s", "memory(GiB)": 69.62, "train_speed(s/it)": 38.147361} +{"loss": 0.41000053, "grad_norm": 1.30263424, "learning_rate": 4.66e-05, "load_balancing_loss": 1.07684541, "iteration": "1665/2862", "elapsed_time": "17h 38m 35s", "remaining_time": "12h 41m 2s", "memory(GiB)": 69.62, "train_speed(s/it)": 38.147106} +{"loss": 0.38025159, "grad_norm": 1.34511733, "learning_rate": 4.635e-05, "load_balancing_loss": 1.0784055, "iteration": "1670/2862", "elapsed_time": "17h 41m 44s", "remaining_time": "12h 37m 50s", "memory(GiB)": 69.62, "train_speed(s/it)": 38.146319} +{"loss": 0.40591535, "grad_norm": 1.33152974, "learning_rate": 4.609e-05, "load_balancing_loss": 1.07594764, "iteration": "1675/2862", "elapsed_time": "17h 44m 55s", "remaining_time": "12h 34m 40s", "memory(GiB)": 69.62, "train_speed(s/it)": 38.146467} +{"loss": 0.41429439, "grad_norm": 1.36492574, "learning_rate": 4.584e-05, "load_balancing_loss": 1.07651484, "iteration": "1680/2862", "elapsed_time": "17h 48m 6s", "remaining_time": "12h 31m 29s", "memory(GiB)": 69.62, "train_speed(s/it)": 38.146522} +{"loss": 0.40817145, "grad_norm": 1.20762706, "learning_rate": 4.558e-05, "load_balancing_loss": 1.07747769, "iteration": "1685/2862", "elapsed_time": "17h 51m 16s", "remaining_time": "12h 28m 18s", "memory(GiB)": 69.62, "train_speed(s/it)": 38.146109} +{"loss": 0.41476479, "grad_norm": 1.26555371, "learning_rate": 4.533e-05, "load_balancing_loss": 1.07957542, "iteration": "1690/2862", "elapsed_time": "17h 54m 27s", "remaining_time": "12h 25m 7s", "memory(GiB)": 69.62, "train_speed(s/it)": 38.1461} +{"loss": 0.39947104, "grad_norm": 1.07675481, "learning_rate": 4.508e-05, "load_balancing_loss": 1.07901168, "iteration": "1695/2862", "elapsed_time": "17h 57m 35s", "remaining_time": "12h 21m 55s", "memory(GiB)": 69.62, "train_speed(s/it)": 38.144527} +{"loss": 0.41212744, "grad_norm": 1.12824798, "learning_rate": 4.482e-05, "load_balancing_loss": 1.07793963, "iteration": "1700/2862", "elapsed_time": "18h 0m 44s", "remaining_time": "12h 18m 43s", "memory(GiB)": 69.62, "train_speed(s/it)": 38.143667} +{"loss": 0.42306268, "grad_norm": 1.26784813, "learning_rate": 4.457e-05, "load_balancing_loss": 1.07891202, "iteration": "1705/2862", "elapsed_time": "18h 3m 53s", "remaining_time": "12h 15m 31s", "memory(GiB)": 69.62, "train_speed(s/it)": 38.142358} +{"loss": 0.40270397, "grad_norm": 1.12200141, "learning_rate": 4.432e-05, "load_balancing_loss": 1.07858562, "iteration": "1710/2862", "elapsed_time": "18h 7m 3s", "remaining_time": "12h 12m 19s", "memory(GiB)": 69.62, "train_speed(s/it)": 38.141905} +{"loss": 0.4167673, "grad_norm": 1.15487909, "learning_rate": 4.406e-05, "load_balancing_loss": 1.07522893, "iteration": "1715/2862", "elapsed_time": "18h 10m 12s", "remaining_time": "12h 9m 8s", "memory(GiB)": 69.62, "train_speed(s/it)": 38.141209} +{"loss": 0.40931374, "grad_norm": 1.49670839, "learning_rate": 4.381e-05, "load_balancing_loss": 1.07743585, "iteration": "1720/2862", "elapsed_time": "18h 13m 38s", "remaining_time": "12h 6m 8s", "memory(GiB)": 69.62, "train_speed(s/it)": 38.150211} +{"loss": 0.40457547, "grad_norm": 1.31249368, "learning_rate": 4.356e-05, "load_balancing_loss": 1.07575309, "iteration": "1725/2862", "elapsed_time": "18h 16m 50s", "remaining_time": "12h 2m 57s", "memory(GiB)": 69.62, "train_speed(s/it)": 38.150638} +{"loss": 0.41840816, "grad_norm": 1.09081507, "learning_rate": 4.331e-05, "load_balancing_loss": 1.07824767, "iteration": "1730/2862", "elapsed_time": "18h 19m 59s", "remaining_time": "11h 59m 45s", "memory(GiB)": 69.62, "train_speed(s/it)": 38.149729} +{"loss": 0.41328797, "grad_norm": 1.33691013, "learning_rate": 4.306e-05, "load_balancing_loss": 1.07579732, "iteration": "1735/2862", "elapsed_time": "18h 23m 9s", "remaining_time": "11h 56m 34s", "memory(GiB)": 69.62, "train_speed(s/it)": 38.149229} +{"loss": 0.43215361, "grad_norm": 1.323668, "learning_rate": 4.281e-05, "load_balancing_loss": 1.07660961, "iteration": "1740/2862", "elapsed_time": "18h 26m 20s", "remaining_time": "11h 53m 24s", "memory(GiB)": 69.62, "train_speed(s/it)": 38.149512} +{"loss": 0.39295974, "grad_norm": 1.13952887, "learning_rate": 4.256e-05, "load_balancing_loss": 1.0788399, "iteration": "1745/2862", "elapsed_time": "18h 29m 30s", "remaining_time": "11h 50m 12s", "memory(GiB)": 69.62, "train_speed(s/it)": 38.148892} +{"loss": 0.39473334, "grad_norm": 1.36217129, "learning_rate": 4.231e-05, "load_balancing_loss": 1.07525229, "iteration": "1750/2862", "elapsed_time": "18h 32m 40s", "remaining_time": "11h 47m 1s", "memory(GiB)": 69.62, "train_speed(s/it)": 38.148665} +{"loss": 0.42040306, "grad_norm": 1.29026055, "learning_rate": 4.206e-05, "load_balancing_loss": 1.07491016, "iteration": "1755/2862", "elapsed_time": "18h 35m 48s", "remaining_time": "11h 43m 48s", "memory(GiB)": 69.62, "train_speed(s/it)": 38.146754} +{"loss": 0.38711345, "grad_norm": 1.31231856, "learning_rate": 4.181e-05, "load_balancing_loss": 1.07730973, "iteration": "1760/2862", "elapsed_time": "18h 38m 55s", "remaining_time": "11h 40m 36s", "memory(GiB)": 69.62, "train_speed(s/it)": 38.145164} +{"loss": 0.40632877, "grad_norm": 1.49296224, "learning_rate": 4.156e-05, "load_balancing_loss": 1.0776087, "iteration": "1765/2862", "elapsed_time": "18h 42m 5s", "remaining_time": "11h 37m 25s", "memory(GiB)": 69.62, "train_speed(s/it)": 38.14452} +{"loss": 0.37042087, "grad_norm": 1.28877962, "learning_rate": 4.131e-05, "load_balancing_loss": 1.07773411, "iteration": "1770/2862", "elapsed_time": "18h 45m 14s", "remaining_time": "11h 34m 13s", "memory(GiB)": 69.62, "train_speed(s/it)": 38.143473} +{"loss": 0.37057585, "grad_norm": 1.24698484, "learning_rate": 4.107e-05, "load_balancing_loss": 1.07747555, "iteration": "1775/2862", "elapsed_time": "18h 48m 24s", "remaining_time": "11h 31m 2s", "memory(GiB)": 69.62, "train_speed(s/it)": 38.143127} +{"loss": 0.40481123, "grad_norm": 1.5079143, "learning_rate": 4.082e-05, "load_balancing_loss": 1.07609427, "iteration": "1780/2862", "elapsed_time": "18h 51m 31s", "remaining_time": "11h 27m 49s", "memory(GiB)": 69.62, "train_speed(s/it)": 38.141065} +{"loss": 0.38553971, "grad_norm": 1.33573747, "learning_rate": 4.057e-05, "load_balancing_loss": 1.07640684, "iteration": "1785/2862", "elapsed_time": "18h 54m 42s", "remaining_time": "11h 24m 38s", "memory(GiB)": 69.62, "train_speed(s/it)": 38.140909} +{"loss": 0.40410051, "grad_norm": 1.27927732, "learning_rate": 4.033e-05, "load_balancing_loss": 1.07663369, "iteration": "1790/2862", "elapsed_time": "18h 57m 50s", "remaining_time": "11h 21m 26s", "memory(GiB)": 69.62, "train_speed(s/it)": 38.139787} +{"loss": 0.39568406, "grad_norm": 1.32764673, "learning_rate": 4.008e-05, "load_balancing_loss": 1.07535779, "iteration": "1795/2862", "elapsed_time": "19h 0m 60s", "remaining_time": "11h 18m 14s", "memory(GiB)": 69.62, "train_speed(s/it)": 38.13917} +{"loss": 0.40477052, "grad_norm": 1.31602228, "learning_rate": 3.984e-05, "load_balancing_loss": 1.07779527, "iteration": "1800/2862", "elapsed_time": "19h 4m 9s", "remaining_time": "11h 15m 3s", "memory(GiB)": 69.62, "train_speed(s/it)": 38.138092} +{"loss": 0.40625006, "grad_norm": 1.18200755, "learning_rate": 3.959e-05, "load_balancing_loss": 1.07565486, "iteration": "1805/2862", "elapsed_time": "19h 7m 18s", "remaining_time": "11h 11m 51s", "memory(GiB)": 69.62, "train_speed(s/it)": 38.137372} +{"loss": 0.40294883, "grad_norm": 1.21032751, "learning_rate": 3.935e-05, "load_balancing_loss": 1.0762217, "iteration": "1810/2862", "elapsed_time": "19h 10m 28s", "remaining_time": "11h 8m 40s", "memory(GiB)": 69.62, "train_speed(s/it)": 38.137273} +{"loss": 0.41039038, "grad_norm": 1.27322602, "learning_rate": 3.91e-05, "load_balancing_loss": 1.07930517, "iteration": "1815/2862", "elapsed_time": "19h 13m 38s", "remaining_time": "11h 5m 29s", "memory(GiB)": 69.62, "train_speed(s/it)": 38.136804} +{"loss": 0.37322232, "grad_norm": 1.33370066, "learning_rate": 3.886e-05, "load_balancing_loss": 1.07640243, "iteration": "1820/2862", "elapsed_time": "19h 16m 47s", "remaining_time": "11h 2m 18s", "memory(GiB)": 69.62, "train_speed(s/it)": 38.135917} +{"loss": 0.37738261, "grad_norm": 1.1604377, "learning_rate": 3.862e-05, "load_balancing_loss": 1.07851446, "iteration": "1825/2862", "elapsed_time": "19h 19m 57s", "remaining_time": "10h 59m 6s", "memory(GiB)": 69.62, "train_speed(s/it)": 38.135258} +{"loss": 0.40027362, "grad_norm": 1.29978764, "learning_rate": 3.838e-05, "load_balancing_loss": 1.0755862, "iteration": "1830/2862", "elapsed_time": "19h 23m 8s", "remaining_time": "10h 55m 56s", "memory(GiB)": 69.62, "train_speed(s/it)": 38.135726} +{"loss": 0.38828564, "grad_norm": 1.33128703, "learning_rate": 3.814e-05, "load_balancing_loss": 1.07730436, "iteration": "1835/2862", "elapsed_time": "19h 26m 17s", "remaining_time": "10h 52m 44s", "memory(GiB)": 69.62, "train_speed(s/it)": 38.134679} +{"loss": 0.42200398, "grad_norm": 1.29906583, "learning_rate": 3.79e-05, "load_balancing_loss": 1.07673287, "iteration": "1840/2862", "elapsed_time": "19h 29m 27s", "remaining_time": "10h 49m 33s", "memory(GiB)": 69.62, "train_speed(s/it)": 38.134421} +{"loss": 0.42261603, "grad_norm": 1.25892949, "learning_rate": 3.766e-05, "load_balancing_loss": 1.07418239, "iteration": "1845/2862", "elapsed_time": "19h 32m 37s", "remaining_time": "10h 46m 22s", "memory(GiB)": 69.62, "train_speed(s/it)": 38.134138} +{"loss": 0.4007374, "grad_norm": 1.12754357, "learning_rate": 3.742e-05, "load_balancing_loss": 1.07483149, "iteration": "1850/2862", "elapsed_time": "19h 35m 48s", "remaining_time": "10h 43m 11s", "memory(GiB)": 69.62, "train_speed(s/it)": 38.133812} +{"loss": 0.39793184, "grad_norm": 1.38392186, "learning_rate": 3.718e-05, "load_balancing_loss": 1.07335401, "iteration": "1855/2862", "elapsed_time": "19h 38m 57s", "remaining_time": "10h 40m 0s", "memory(GiB)": 69.62, "train_speed(s/it)": 38.133245} +{"loss": 0.40689924, "grad_norm": 1.31824148, "learning_rate": 3.694e-05, "load_balancing_loss": 1.07678926, "iteration": "1860/2862", "elapsed_time": "19h 42m 6s", "remaining_time": "10h 36m 48s", "memory(GiB)": 69.62, "train_speed(s/it)": 38.132089} +{"loss": 0.39728957, "grad_norm": 1.25131011, "learning_rate": 3.67e-05, "load_balancing_loss": 1.07413137, "iteration": "1865/2862", "elapsed_time": "19h 45m 16s", "remaining_time": "10h 33m 37s", "memory(GiB)": 69.62, "train_speed(s/it)": 38.13171} +{"loss": 0.39670247, "grad_norm": 1.46913719, "learning_rate": 3.646e-05, "load_balancing_loss": 1.07384098, "iteration": "1870/2862", "elapsed_time": "19h 48m 27s", "remaining_time": "10h 30m 27s", "memory(GiB)": 69.62, "train_speed(s/it)": 38.132057} +{"loss": 0.416664, "grad_norm": 1.33213174, "learning_rate": 3.623e-05, "load_balancing_loss": 1.07486427, "iteration": "1875/2862", "elapsed_time": "19h 51m 37s", "remaining_time": "10h 27m 16s", "memory(GiB)": 69.62, "train_speed(s/it)": 38.131672} +{"loss": 0.40672705, "grad_norm": 1.21658182, "learning_rate": 3.599e-05, "load_balancing_loss": 1.07419908, "iteration": "1880/2862", "elapsed_time": "19h 54m 46s", "remaining_time": "10h 24m 4s", "memory(GiB)": 69.62, "train_speed(s/it)": 38.130769} +{"loss": 0.39137027, "grad_norm": 1.2061168, "learning_rate": 3.576e-05, "load_balancing_loss": 1.07522011, "iteration": "1885/2862", "elapsed_time": "19h 57m 57s", "remaining_time": "10h 20m 54s", "memory(GiB)": 69.62, "train_speed(s/it)": 38.130823} +{"loss": 0.43088195, "grad_norm": 1.34364033, "learning_rate": 3.552e-05, "load_balancing_loss": 1.07424128, "iteration": "1890/2862", "elapsed_time": "20h 1m 6s", "remaining_time": "10h 17m 42s", "memory(GiB)": 69.62, "train_speed(s/it)": 38.13011} +{"loss": 0.39871714, "grad_norm": 1.40503168, "learning_rate": 3.529e-05, "load_balancing_loss": 1.07663, "iteration": "1895/2862", "elapsed_time": "20h 4m 17s", "remaining_time": "10h 14m 32s", "memory(GiB)": 69.62, "train_speed(s/it)": 38.130227} +{"loss": 0.39093986, "grad_norm": 1.18492281, "learning_rate": 3.505e-05, "load_balancing_loss": 1.07392788, "iteration": "1900/2862", "elapsed_time": "20h 7m 27s", "remaining_time": "10h 11m 21s", "memory(GiB)": 69.62, "train_speed(s/it)": 38.129834} +{"loss": 0.40388423, "grad_norm": 1.67440736, "learning_rate": 3.482e-05, "load_balancing_loss": 1.07575071, "iteration": "1905/2862", "elapsed_time": "20h 10m 37s", "remaining_time": "10h 8m 10s", "memory(GiB)": 69.62, "train_speed(s/it)": 38.129833} +{"loss": 0.38643983, "grad_norm": 1.34545505, "learning_rate": 3.459e-05, "load_balancing_loss": 1.07562208, "iteration": "1910/2862", "elapsed_time": "20h 13m 49s", "remaining_time": "10h 4m 60s", "memory(GiB)": 69.62, "train_speed(s/it)": 38.130113} +{"loss": 0.39369383, "grad_norm": 1.24357045, "learning_rate": 3.436e-05, "load_balancing_loss": 1.07529008, "iteration": "1915/2862", "elapsed_time": "20h 16m 57s", "remaining_time": "10h 1m 48s", "memory(GiB)": 69.62, "train_speed(s/it)": 38.128886} +{"loss": 0.40963486, "grad_norm": 1.33095837, "learning_rate": 3.413e-05, "load_balancing_loss": 1.07599413, "iteration": "1920/2862", "elapsed_time": "20h 20m 7s", "remaining_time": "9h 58m 37s", "memory(GiB)": 69.62, "train_speed(s/it)": 38.128879} +{"loss": 0.41208026, "grad_norm": 1.26424229, "learning_rate": 3.39e-05, "load_balancing_loss": 1.07607687, "iteration": "1925/2862", "elapsed_time": "20h 23m 17s", "remaining_time": "9h 55m 26s", "memory(GiB)": 69.62, "train_speed(s/it)": 38.128523} +{"loss": 0.39765057, "grad_norm": 1.24484694, "learning_rate": 3.367e-05, "load_balancing_loss": 1.07865644, "iteration": "1930/2862", "elapsed_time": "20h 26m 29s", "remaining_time": "9h 52m 16s", "memory(GiB)": 69.62, "train_speed(s/it)": 38.129185} +{"loss": 0.42688668, "grad_norm": 1.36284781, "learning_rate": 3.344e-05, "load_balancing_loss": 1.07640171, "iteration": "1935/2862", "elapsed_time": "20h 29m 38s", "remaining_time": "9h 49m 5s", "memory(GiB)": 69.62, "train_speed(s/it)": 38.127992} +{"loss": 0.38730043, "grad_norm": 1.2848289, "learning_rate": 3.321e-05, "load_balancing_loss": 1.07557142, "iteration": "1940/2862", "elapsed_time": "20h 32m 50s", "remaining_time": "9h 45m 55s", "memory(GiB)": 69.62, "train_speed(s/it)": 38.128627} +{"loss": 0.37271279, "grad_norm": 3.41408849, "learning_rate": 3.298e-05, "load_balancing_loss": 1.07636762, "iteration": "1945/2862", "elapsed_time": "20h 35m 60s", "remaining_time": "9h 42m 44s", "memory(GiB)": 69.62, "train_speed(s/it)": 38.128437} +{"loss": 0.40567532, "grad_norm": 1.39933527, "learning_rate": 3.276e-05, "load_balancing_loss": 1.07483613, "iteration": "1950/2862", "elapsed_time": "20h 39m 10s", "remaining_time": "9h 39m 33s", "memory(GiB)": 69.62, "train_speed(s/it)": 38.128335} +{"loss": 0.40819916, "grad_norm": 1.49925756, "learning_rate": 3.253e-05, "load_balancing_loss": 1.07671034, "iteration": "1955/2862", "elapsed_time": "20h 42m 19s", "remaining_time": "9h 36m 22s", "memory(GiB)": 69.62, "train_speed(s/it)": 38.127483} +{"loss": 0.40435398, "grad_norm": 1.38659716, "learning_rate": 3.231e-05, "load_balancing_loss": 1.07623374, "iteration": "1960/2862", "elapsed_time": "20h 45m 30s", "remaining_time": "9h 33m 11s", "memory(GiB)": 69.62, "train_speed(s/it)": 38.12759} +{"loss": 0.39452034, "grad_norm": 1.2609843, "learning_rate": 3.208e-05, "load_balancing_loss": 1.07772529, "iteration": "1965/2862", "elapsed_time": "20h 48m 41s", "remaining_time": "9h 30m 0s", "memory(GiB)": 69.62, "train_speed(s/it)": 38.12761} +{"loss": 0.40235746, "grad_norm": 1.2466799, "learning_rate": 3.186e-05, "load_balancing_loss": 1.07749748, "iteration": "1970/2862", "elapsed_time": "20h 51m 50s", "remaining_time": "9h 26m 49s", "memory(GiB)": 69.62, "train_speed(s/it)": 38.126816} +{"loss": 0.3911593, "grad_norm": 1.19487619, "learning_rate": 3.164e-05, "load_balancing_loss": 1.07608473, "iteration": "1975/2862", "elapsed_time": "20h 55m 0s", "remaining_time": "9h 23m 38s", "memory(GiB)": 69.62, "train_speed(s/it)": 38.126768} +{"loss": 0.38464063, "grad_norm": 1.11874855, "learning_rate": 3.141e-05, "load_balancing_loss": 1.07486951, "iteration": "1980/2862", "elapsed_time": "20h 58m 11s", "remaining_time": "9h 20m 28s", "memory(GiB)": 69.62, "train_speed(s/it)": 38.126755} +{"loss": 0.3955746, "grad_norm": 1.35684288, "learning_rate": 3.119e-05, "load_balancing_loss": 1.07700765, "iteration": "1985/2862", "elapsed_time": "21h 1m 20s", "remaining_time": "9h 17m 17s", "memory(GiB)": 69.62, "train_speed(s/it)": 38.126085} +{"loss": 0.41300017, "grad_norm": 1.22297752, "learning_rate": 3.097e-05, "load_balancing_loss": 1.07775331, "iteration": "1990/2862", "elapsed_time": "21h 4m 32s", "remaining_time": "9h 14m 7s", "memory(GiB)": 69.62, "train_speed(s/it)": 38.126843} +{"loss": 0.41676751, "grad_norm": 1.26178074, "learning_rate": 3.075e-05, "load_balancing_loss": 1.07660341, "iteration": "1995/2862", "elapsed_time": "21h 7m 43s", "remaining_time": "9h 10m 56s", "memory(GiB)": 69.62, "train_speed(s/it)": 38.126759} +{"loss": 0.3941898, "grad_norm": 1.92824042, "learning_rate": 3.054e-05, "load_balancing_loss": 1.07540822, "iteration": "2000/2862", "elapsed_time": "21h 10m 53s", "remaining_time": "9h 7m 45s", "memory(GiB)": 69.62, "train_speed(s/it)": 38.126729} +{"loss": 0.39678612, "grad_norm": 1.53153729, "learning_rate": 3.032e-05, "load_balancing_loss": 1.07617557, "iteration": "2005/2862", "elapsed_time": "21h 16m 8s", "remaining_time": "9h 5m 28s", "memory(GiB)": 69.62, "train_speed(s/it)": 38.188686} +{"loss": 0.38377148, "grad_norm": 1.2726872, "learning_rate": 3.01e-05, "load_balancing_loss": 1.07558274, "iteration": "2010/2862", "elapsed_time": "21h 19m 18s", "remaining_time": "9h 2m 16s", "memory(GiB)": 69.62, "train_speed(s/it)": 38.188023} +{"loss": 0.41332173, "grad_norm": 1.47350705, "learning_rate": 2.988e-05, "load_balancing_loss": 1.07692361, "iteration": "2015/2862", "elapsed_time": "21h 22m 29s", "remaining_time": "8h 59m 5s", "memory(GiB)": 69.62, "train_speed(s/it)": 38.188193} +{"loss": 0.39664418, "grad_norm": 1.37847507, "learning_rate": 2.967e-05, "load_balancing_loss": 1.07493055, "iteration": "2020/2862", "elapsed_time": "21h 25m 40s", "remaining_time": "8h 55m 54s", "memory(GiB)": 69.62, "train_speed(s/it)": 38.188063} +{"loss": 0.38876277, "grad_norm": 1.27825463, "learning_rate": 2.945e-05, "load_balancing_loss": 1.07705557, "iteration": "2025/2862", "elapsed_time": "21h 28m 50s", "remaining_time": "8h 52m 43s", "memory(GiB)": 69.62, "train_speed(s/it)": 38.187803} +{"loss": 0.40936607, "grad_norm": 1.32116699, "learning_rate": 2.924e-05, "load_balancing_loss": 1.07454216, "iteration": "2030/2862", "elapsed_time": "21h 32m 2s", "remaining_time": "8h 49m 32s", "memory(GiB)": 69.62, "train_speed(s/it)": 38.187957} +{"loss": 0.40169325, "grad_norm": 1.2685504, "learning_rate": 2.903e-05, "load_balancing_loss": 1.07825971, "iteration": "2035/2862", "elapsed_time": "21h 35m 12s", "remaining_time": "8h 46m 21s", "memory(GiB)": 69.62, "train_speed(s/it)": 38.187609} +{"loss": 0.42359427, "grad_norm": 1.23530579, "learning_rate": 2.882e-05, "load_balancing_loss": 1.07698953, "iteration": "2040/2862", "elapsed_time": "21h 38m 22s", "remaining_time": "8h 43m 10s", "memory(GiB)": 69.62, "train_speed(s/it)": 38.187127} +{"loss": 0.40730575, "grad_norm": 1.28822803, "learning_rate": 2.861e-05, "load_balancing_loss": 1.07671535, "iteration": "2045/2862", "elapsed_time": "21h 41m 32s", "remaining_time": "8h 39m 58s", "memory(GiB)": 69.62, "train_speed(s/it)": 38.186608} +{"loss": 0.39505425, "grad_norm": 1.4858197, "learning_rate": 2.84e-05, "load_balancing_loss": 1.07757378, "iteration": "2050/2862", "elapsed_time": "21h 44m 42s", "remaining_time": "8h 36m 47s", "memory(GiB)": 69.62, "train_speed(s/it)": 38.186262} +{"loss": 0.39479688, "grad_norm": 1.28625596, "learning_rate": 2.819e-05, "load_balancing_loss": 1.07514894, "iteration": "2055/2862", "elapsed_time": "21h 47m 53s", "remaining_time": "8h 33m 36s", "memory(GiB)": 69.62, "train_speed(s/it)": 38.186145} +{"loss": 0.41882974, "grad_norm": 1.19115365, "learning_rate": 2.798e-05, "load_balancing_loss": 1.07720697, "iteration": "2060/2862", "elapsed_time": "21h 51m 1s", "remaining_time": "8h 30m 24s", "memory(GiB)": 69.62, "train_speed(s/it)": 38.184797} +{"loss": 0.386475, "grad_norm": 1.21019292, "learning_rate": 2.777e-05, "load_balancing_loss": 1.07692575, "iteration": "2065/2862", "elapsed_time": "21h 54m 11s", "remaining_time": "8h 27m 13s", "memory(GiB)": 69.62, "train_speed(s/it)": 38.184618} +{"loss": 0.39381614, "grad_norm": 1.44543242, "learning_rate": 2.756e-05, "load_balancing_loss": 1.07542837, "iteration": "2070/2862", "elapsed_time": "21h 57m 22s", "remaining_time": "8h 24m 2s", "memory(GiB)": 69.62, "train_speed(s/it)": 38.184707} +{"loss": 0.42844743, "grad_norm": 1.31893742, "learning_rate": 2.736e-05, "load_balancing_loss": 1.07535994, "iteration": "2075/2862", "elapsed_time": "22h 0m 34s", "remaining_time": "8h 20m 52s", "memory(GiB)": 69.62, "train_speed(s/it)": 38.184999} +{"loss": 0.38819072, "grad_norm": 1.56823194, "learning_rate": 2.715e-05, "load_balancing_loss": 1.07889593, "iteration": "2080/2862", "elapsed_time": "22h 3m 45s", "remaining_time": "8h 17m 41s", "memory(GiB)": 69.62, "train_speed(s/it)": 38.185214} +{"loss": 0.40794894, "grad_norm": 1.39805484, "learning_rate": 2.695e-05, "load_balancing_loss": 1.07414305, "iteration": "2085/2862", "elapsed_time": "22h 6m 56s", "remaining_time": "8h 14m 30s", "memory(GiB)": 69.62, "train_speed(s/it)": 38.184926} +{"loss": 0.38256103, "grad_norm": 1.40689111, "learning_rate": 2.675e-05, "load_balancing_loss": 1.07533646, "iteration": "2090/2862", "elapsed_time": "22h 10m 6s", "remaining_time": "8h 11m 19s", "memory(GiB)": 69.62, "train_speed(s/it)": 38.184667} +{"loss": 0.39294577, "grad_norm": 1.25617206, "learning_rate": 2.655e-05, "load_balancing_loss": 1.07666934, "iteration": "2095/2862", "elapsed_time": "22h 13m 18s", "remaining_time": "8h 8m 8s", "memory(GiB)": 69.62, "train_speed(s/it)": 38.185308} +{"loss": 0.40239733, "grad_norm": 1.24251235, "learning_rate": 2.634e-05, "load_balancing_loss": 1.07623804, "iteration": "2100/2862", "elapsed_time": "22h 16m 28s", "remaining_time": "8h 4m 57s", "memory(GiB)": 69.62, "train_speed(s/it)": 38.184872} +{"loss": 0.41008019, "grad_norm": 1.45632064, "learning_rate": 2.614e-05, "load_balancing_loss": 1.07651174, "iteration": "2105/2862", "elapsed_time": "22h 19m 39s", "remaining_time": "8h 1m 46s", "memory(GiB)": 69.62, "train_speed(s/it)": 38.18458} +{"loss": 0.41292438, "grad_norm": 1.67750049, "learning_rate": 2.595e-05, "load_balancing_loss": 1.07283914, "iteration": "2110/2862", "elapsed_time": "22h 22m 48s", "remaining_time": "7h 58m 34s", "memory(GiB)": 69.62, "train_speed(s/it)": 38.184054} +{"loss": 0.38625473, "grad_norm": 1.36231828, "learning_rate": 2.575e-05, "load_balancing_loss": 1.07894695, "iteration": "2115/2862", "elapsed_time": "22h 25m 59s", "remaining_time": "7h 55m 23s", "memory(GiB)": 69.62, "train_speed(s/it)": 38.183836} +{"loss": 0.40290892, "grad_norm": 1.23041809, "learning_rate": 2.555e-05, "load_balancing_loss": 1.07563365, "iteration": "2120/2862", "elapsed_time": "22h 29m 11s", "remaining_time": "7h 52m 13s", "memory(GiB)": 69.62, "train_speed(s/it)": 38.184283} +{"loss": 0.39321321, "grad_norm": 1.47897172, "learning_rate": 2.535e-05, "load_balancing_loss": 1.0752722, "iteration": "2125/2862", "elapsed_time": "22h 32m 22s", "remaining_time": "7h 49m 2s", "memory(GiB)": 69.62, "train_speed(s/it)": 38.184254} +{"loss": 0.40059534, "grad_norm": 1.2806834, "learning_rate": 2.516e-05, "load_balancing_loss": 1.07578647, "iteration": "2130/2862", "elapsed_time": "22h 35m 31s", "remaining_time": "7h 45m 51s", "memory(GiB)": 69.62, "train_speed(s/it)": 38.183762} +{"loss": 0.39118201, "grad_norm": 1.40522981, "learning_rate": 2.496e-05, "load_balancing_loss": 1.0753473, "iteration": "2135/2862", "elapsed_time": "22h 38m 41s", "remaining_time": "7h 42m 39s", "memory(GiB)": 69.62, "train_speed(s/it)": 38.183347} +{"loss": 0.38725308, "grad_norm": 1.35260403, "learning_rate": 2.477e-05, "load_balancing_loss": 1.07848084, "iteration": "2140/2862", "elapsed_time": "22h 41m 53s", "remaining_time": "7h 39m 29s", "memory(GiB)": 69.62, "train_speed(s/it)": 38.18359} +{"loss": 0.38149917, "grad_norm": 1.3049866, "learning_rate": 2.458e-05, "load_balancing_loss": 1.07550621, "iteration": "2145/2862", "elapsed_time": "22h 45m 1s", "remaining_time": "7h 36m 17s", "memory(GiB)": 69.62, "train_speed(s/it)": 38.182385} +{"loss": 0.31704658, "grad_norm": 1.10877955, "learning_rate": 2.439e-05, "load_balancing_loss": 1.07597578, "iteration": "2150/2862", "elapsed_time": "22h 48m 13s", "remaining_time": "7h 33m 6s", "memory(GiB)": 69.62, "train_speed(s/it)": 38.182985} +{"loss": 0.27159619, "grad_norm": 1.24515581, "learning_rate": 2.42e-05, "load_balancing_loss": 1.07653773, "iteration": "2155/2862", "elapsed_time": "22h 51m 22s", "remaining_time": "7h 29m 55s", "memory(GiB)": 69.62, "train_speed(s/it)": 38.182079} +{"loss": 0.25537905, "grad_norm": 1.62462378, "learning_rate": 2.401e-05, "load_balancing_loss": 1.07372737, "iteration": "2160/2862", "elapsed_time": "22h 54m 34s", "remaining_time": "7h 26m 44s", "memory(GiB)": 69.62, "train_speed(s/it)": 38.182244} +{"loss": 0.24783887, "grad_norm": 2.20151258, "learning_rate": 2.382e-05, "load_balancing_loss": 1.07651186, "iteration": "2165/2862", "elapsed_time": "22h 57m 43s", "remaining_time": "7h 23m 32s", "memory(GiB)": 69.62, "train_speed(s/it)": 38.181355} +{"loss": 0.25563082, "grad_norm": 2.53956771, "learning_rate": 2.363e-05, "load_balancing_loss": 1.07396233, "iteration": "2170/2862", "elapsed_time": "23h 0m 52s", "remaining_time": "7h 20m 21s", "memory(GiB)": 69.62, "train_speed(s/it)": 38.180677} +{"loss": 0.2568289, "grad_norm": 2.19141865, "learning_rate": 2.345e-05, "load_balancing_loss": 1.07468307, "iteration": "2175/2862", "elapsed_time": "23h 4m 3s", "remaining_time": "7h 17m 10s", "memory(GiB)": 69.62, "train_speed(s/it)": 38.180634} +{"loss": 0.25062686, "grad_norm": 1.88980818, "learning_rate": 2.326e-05, "load_balancing_loss": 1.07632828, "iteration": "2180/2862", "elapsed_time": "23h 7m 14s", "remaining_time": "7h 13m 59s", "memory(GiB)": 69.62, "train_speed(s/it)": 38.18055} +{"loss": 0.22871409, "grad_norm": 2.13975334, "learning_rate": 2.308e-05, "load_balancing_loss": 1.07679975, "iteration": "2185/2862", "elapsed_time": "23h 10m 25s", "remaining_time": "7h 10m 48s", "memory(GiB)": 69.62, "train_speed(s/it)": 38.180761} +{"loss": 0.24144404, "grad_norm": 1.78770292, "learning_rate": 2.29e-05, "load_balancing_loss": 1.07685113, "iteration": "2190/2862", "elapsed_time": "23h 13m 36s", "remaining_time": "7h 7m 37s", "memory(GiB)": 69.62, "train_speed(s/it)": 38.180625} +{"loss": 0.23383963, "grad_norm": 1.99427307, "learning_rate": 2.272e-05, "load_balancing_loss": 1.07551849, "iteration": "2195/2862", "elapsed_time": "23h 16m 43s", "remaining_time": "7h 4m 25s", "memory(GiB)": 69.62, "train_speed(s/it)": 38.179025} +{"loss": 0.26286021, "grad_norm": 1.98031163, "learning_rate": 2.254e-05, "load_balancing_loss": 1.07732201, "iteration": "2200/2862", "elapsed_time": "23h 19m 53s", "remaining_time": "7h 1m 14s", "memory(GiB)": 69.62, "train_speed(s/it)": 38.178862} +{"loss": 0.24434468, "grad_norm": 1.87746155, "learning_rate": 2.236e-05, "load_balancing_loss": 1.07685876, "iteration": "2205/2862", "elapsed_time": "23h 23m 4s", "remaining_time": "6h 58m 3s", "memory(GiB)": 69.62, "train_speed(s/it)": 38.178783} +{"loss": 0.24062723, "grad_norm": 1.93359864, "learning_rate": 2.218e-05, "load_balancing_loss": 1.0781635, "iteration": "2210/2862", "elapsed_time": "23h 26m 14s", "remaining_time": "6h 54m 52s", "memory(GiB)": 69.62, "train_speed(s/it)": 38.178432} +{"loss": 0.25753176, "grad_norm": 1.92045856, "learning_rate": 2.2e-05, "load_balancing_loss": 1.07663536, "iteration": "2215/2862", "elapsed_time": "23h 29m 25s", "remaining_time": "6h 51m 41s", "memory(GiB)": 69.62, "train_speed(s/it)": 38.178171} +{"loss": 0.2487563, "grad_norm": 2.06681323, "learning_rate": 2.182e-05, "load_balancing_loss": 1.07543433, "iteration": "2220/2862", "elapsed_time": "23h 32m 35s", "remaining_time": "6h 48m 30s", "memory(GiB)": 69.62, "train_speed(s/it)": 38.17779} +{"loss": 0.23227087, "grad_norm": 1.79434061, "learning_rate": 2.165e-05, "load_balancing_loss": 1.07742143, "iteration": "2225/2862", "elapsed_time": "23h 35m 44s", "remaining_time": "6h 45m 19s", "memory(GiB)": 69.62, "train_speed(s/it)": 38.17722} +{"loss": 0.26160753, "grad_norm": 1.92999661, "learning_rate": 2.147e-05, "load_balancing_loss": 1.07615685, "iteration": "2230/2862", "elapsed_time": "23h 38m 54s", "remaining_time": "6h 42m 8s", "memory(GiB)": 69.62, "train_speed(s/it)": 38.176791} +{"loss": 0.23504007, "grad_norm": 1.86837518, "learning_rate": 2.13e-05, "load_balancing_loss": 1.07834613, "iteration": "2235/2862", "elapsed_time": "23h 42m 5s", "remaining_time": "6h 38m 57s", "memory(GiB)": 69.62, "train_speed(s/it)": 38.176515} +{"loss": 0.2606563, "grad_norm": 2.13133931, "learning_rate": 2.113e-05, "load_balancing_loss": 1.07715857, "iteration": "2240/2862", "elapsed_time": "23h 45m 14s", "remaining_time": "6h 35m 45s", "memory(GiB)": 69.62, "train_speed(s/it)": 38.175912} +{"loss": 0.25501648, "grad_norm": 1.75524414, "learning_rate": 2.096e-05, "load_balancing_loss": 1.07492971, "iteration": "2245/2862", "elapsed_time": "23h 48m 23s", "remaining_time": "6h 32m 34s", "memory(GiB)": 69.62, "train_speed(s/it)": 38.175138} +{"loss": 0.24323751, "grad_norm": 2.24620485, "learning_rate": 2.079e-05, "load_balancing_loss": 1.0769217, "iteration": "2250/2862", "elapsed_time": "23h 51m 34s", "remaining_time": "6h 29m 23s", "memory(GiB)": 69.62, "train_speed(s/it)": 38.175333} +{"loss": 0.27834314, "grad_norm": 2.19255948, "learning_rate": 2.062e-05, "load_balancing_loss": 1.07703376, "iteration": "2255/2862", "elapsed_time": "23h 54m 42s", "remaining_time": "6h 26m 12s", "memory(GiB)": 69.62, "train_speed(s/it)": 38.173901} +{"loss": 0.26227489, "grad_norm": 2.09251356, "learning_rate": 2.045e-05, "load_balancing_loss": 1.07655966, "iteration": "2260/2862", "elapsed_time": "23h 57m 52s", "remaining_time": "6h 23m 0s", "memory(GiB)": 69.62, "train_speed(s/it)": 38.173541} +{"loss": 0.27059948, "grad_norm": 2.10103059, "learning_rate": 2.029e-05, "load_balancing_loss": 1.07700348, "iteration": "2265/2862", "elapsed_time": "1d 0h 1m 4s", "remaining_time": "6h 19m 50s", "memory(GiB)": 69.62, "train_speed(s/it)": 38.173849} +{"loss": 0.2639291, "grad_norm": 1.99291313, "learning_rate": 2.012e-05, "load_balancing_loss": 1.07791507, "iteration": "2270/2862", "elapsed_time": "1d 0h 4m 14s", "remaining_time": "6h 16m 39s", "memory(GiB)": 69.62, "train_speed(s/it)": 38.173654} +{"loss": 0.2378394, "grad_norm": 1.64761662, "learning_rate": 1.996e-05, "load_balancing_loss": 1.07593906, "iteration": "2275/2862", "elapsed_time": "1d 0h 7m 26s", "remaining_time": "6h 13m 28s", "memory(GiB)": 69.62, "train_speed(s/it)": 38.174039} +{"loss": 0.25157797, "grad_norm": 1.87453842, "learning_rate": 1.98e-05, "load_balancing_loss": 1.07713032, "iteration": "2280/2862", "elapsed_time": "1d 0h 10m 37s", "remaining_time": "6h 10m 17s", "memory(GiB)": 69.62, "train_speed(s/it)": 38.174112} +{"loss": 0.2578823, "grad_norm": 1.89719772, "learning_rate": 1.964e-05, "load_balancing_loss": 1.07818067, "iteration": "2285/2862", "elapsed_time": "1d 0h 13m 48s", "remaining_time": "6h 7m 6s", "memory(GiB)": 69.62, "train_speed(s/it)": 38.173996} +{"loss": 0.27494493, "grad_norm": 2.05802631, "learning_rate": 1.948e-05, "load_balancing_loss": 1.07673478, "iteration": "2290/2862", "elapsed_time": "1d 0h 16m 57s", "remaining_time": "6h 3m 55s", "memory(GiB)": 69.62, "train_speed(s/it)": 38.173375} +{"loss": 0.22883876, "grad_norm": 1.82335436, "learning_rate": 1.932e-05, "load_balancing_loss": 1.07805097, "iteration": "2295/2862", "elapsed_time": "1d 0h 20m 5s", "remaining_time": "6h 0m 44s", "memory(GiB)": 69.62, "train_speed(s/it)": 38.172236} +{"loss": 0.24646652, "grad_norm": 2.09691787, "learning_rate": 1.916e-05, "load_balancing_loss": 1.07691991, "iteration": "2300/2862", "elapsed_time": "1d 0h 23m 16s", "remaining_time": "5h 57m 33s", "memory(GiB)": 69.62, "train_speed(s/it)": 38.172158} +{"loss": 0.23308581, "grad_norm": 1.70228934, "learning_rate": 1.9e-05, "load_balancing_loss": 1.07627511, "iteration": "2305/2862", "elapsed_time": "1d 0h 26m 26s", "remaining_time": "5h 54m 22s", "memory(GiB)": 69.62, "train_speed(s/it)": 38.171874} +{"loss": 0.24850851, "grad_norm": 2.04678583, "learning_rate": 1.885e-05, "load_balancing_loss": 1.07668483, "iteration": "2310/2862", "elapsed_time": "1d 0h 29m 37s", "remaining_time": "5h 51m 11s", "memory(GiB)": 69.62, "train_speed(s/it)": 38.171659} +{"loss": 0.24128793, "grad_norm": 2.15102863, "learning_rate": 1.869e-05, "load_balancing_loss": 1.07784021, "iteration": "2315/2862", "elapsed_time": "1d 0h 32m 45s", "remaining_time": "5h 47m 59s", "memory(GiB)": 69.62, "train_speed(s/it)": 38.17073} +{"loss": 0.23435465, "grad_norm": 1.94617677, "learning_rate": 1.854e-05, "load_balancing_loss": 1.07492769, "iteration": "2320/2862", "elapsed_time": "1d 0h 35m 58s", "remaining_time": "5h 44m 49s", "memory(GiB)": 69.62, "train_speed(s/it)": 38.171371} +{"loss": 0.24862853, "grad_norm": 2.16996026, "learning_rate": 1.839e-05, "load_balancing_loss": 1.07604444, "iteration": "2325/2862", "elapsed_time": "1d 0h 39m 8s", "remaining_time": "5h 41m 38s", "memory(GiB)": 69.62, "train_speed(s/it)": 38.17129} +{"loss": 0.24973522, "grad_norm": 1.90018344, "learning_rate": 1.824e-05, "load_balancing_loss": 1.07607198, "iteration": "2330/2862", "elapsed_time": "1d 0h 42m 18s", "remaining_time": "5h 38m 27s", "memory(GiB)": 69.62, "train_speed(s/it)": 38.171007} +{"loss": 0.25158736, "grad_norm": 1.89434373, "learning_rate": 1.809e-05, "load_balancing_loss": 1.07538307, "iteration": "2335/2862", "elapsed_time": "1d 0h 45m 29s", "remaining_time": "5h 35m 16s", "memory(GiB)": 69.62, "train_speed(s/it)": 38.170975} +{"loss": 0.24240969, "grad_norm": 1.63735712, "learning_rate": 1.794e-05, "load_balancing_loss": 1.07635057, "iteration": "2340/2862", "elapsed_time": "1d 0h 48m 40s", "remaining_time": "5h 32m 5s", "memory(GiB)": 69.62, "train_speed(s/it)": 38.170826} +{"loss": 0.25174236, "grad_norm": 2.00542426, "learning_rate": 1.779e-05, "load_balancing_loss": 1.07493675, "iteration": "2345/2862", "elapsed_time": "1d 0h 51m 50s", "remaining_time": "5h 28m 54s", "memory(GiB)": 69.62, "train_speed(s/it)": 38.170755} +{"loss": 0.25669378, "grad_norm": 1.83340836, "learning_rate": 1.765e-05, "load_balancing_loss": 1.07424068, "iteration": "2350/2862", "elapsed_time": "1d 0h 55m 1s", "remaining_time": "5h 25m 43s", "memory(GiB)": 69.62, "train_speed(s/it)": 38.17069} +{"loss": 0.24817356, "grad_norm": 2.05535412, "learning_rate": 1.75e-05, "load_balancing_loss": 1.07660282, "iteration": "2355/2862", "elapsed_time": "1d 0h 58m 11s", "remaining_time": "5h 22m 32s", "memory(GiB)": 69.62, "train_speed(s/it)": 38.170355} +{"loss": 0.25382987, "grad_norm": 2.12521243, "learning_rate": 1.736e-05, "load_balancing_loss": 1.07671225, "iteration": "2360/2862", "elapsed_time": "1d 1h 1m 21s", "remaining_time": "5h 19m 21s", "memory(GiB)": 69.62, "train_speed(s/it)": 38.170074} +{"loss": 0.23888896, "grad_norm": 2.14198327, "learning_rate": 1.722e-05, "load_balancing_loss": 1.07631421, "iteration": "2365/2862", "elapsed_time": "1d 1h 4m 32s", "remaining_time": "5h 16m 11s", "memory(GiB)": 69.62, "train_speed(s/it)": 38.170027} +{"loss": 0.24642362, "grad_norm": 2.19904256, "learning_rate": 1.708e-05, "load_balancing_loss": 1.07679701, "iteration": "2370/2862", "elapsed_time": "1d 1h 7m 42s", "remaining_time": "5h 12m 59s", "memory(GiB)": 69.62, "train_speed(s/it)": 38.169453} +{"loss": 0.23307455, "grad_norm": 1.80679512, "learning_rate": 1.694e-05, "load_balancing_loss": 1.07736897, "iteration": "2375/2862", "elapsed_time": "1d 1h 10m 52s", "remaining_time": "5h 9m 49s", "memory(GiB)": 69.62, "train_speed(s/it)": 38.169471} +{"loss": 0.24716491, "grad_norm": 1.9765172, "learning_rate": 1.68e-05, "load_balancing_loss": 1.07579172, "iteration": "2380/2862", "elapsed_time": "1d 1h 14m 3s", "remaining_time": "5h 6m 38s", "memory(GiB)": 69.62, "train_speed(s/it)": 38.169265} +{"loss": 0.23406748, "grad_norm": 1.83591294, "learning_rate": 1.666e-05, "load_balancing_loss": 1.07666504, "iteration": "2385/2862", "elapsed_time": "1d 1h 17m 16s", "remaining_time": "5h 3m 27s", "memory(GiB)": 69.62, "train_speed(s/it)": 38.170078} +{"loss": 0.23403023, "grad_norm": 1.76172531, "learning_rate": 1.653e-05, "load_balancing_loss": 1.07610989, "iteration": "2390/2862", "elapsed_time": "1d 1h 20m 26s", "remaining_time": "5h 0m 16s", "memory(GiB)": 69.62, "train_speed(s/it)": 38.16984} +{"loss": 0.24741255, "grad_norm": 2.15423489, "learning_rate": 1.639e-05, "load_balancing_loss": 1.07659531, "iteration": "2395/2862", "elapsed_time": "1d 1h 23m 37s", "remaining_time": "4h 57m 5s", "memory(GiB)": 69.62, "train_speed(s/it)": 38.169888} +{"loss": 0.25686955, "grad_norm": 2.02065039, "learning_rate": 1.626e-05, "load_balancing_loss": 1.07649052, "iteration": "2400/2862", "elapsed_time": "1d 1h 26m 45s", "remaining_time": "4h 53m 54s", "memory(GiB)": 69.62, "train_speed(s/it)": 38.168833} +{"loss": 0.23835492, "grad_norm": 2.31986737, "learning_rate": 1.613e-05, "load_balancing_loss": 1.07785654, "iteration": "2405/2862", "elapsed_time": "1d 1h 29m 55s", "remaining_time": "4h 50m 43s", "memory(GiB)": 69.62, "train_speed(s/it)": 38.168516} +{"loss": 0.24495278, "grad_norm": 1.91966975, "learning_rate": 1.6e-05, "load_balancing_loss": 1.07768941, "iteration": "2410/2862", "elapsed_time": "1d 1h 33m 6s", "remaining_time": "4h 47m 32s", "memory(GiB)": 69.62, "train_speed(s/it)": 38.168421} +{"loss": 0.25917763, "grad_norm": 2.2123301, "learning_rate": 1.587e-05, "load_balancing_loss": 1.0773716, "iteration": "2415/2862", "elapsed_time": "1d 1h 36m 17s", "remaining_time": "4h 44m 21s", "memory(GiB)": 69.62, "train_speed(s/it)": 38.168416} +{"loss": 0.24510404, "grad_norm": 1.94068754, "learning_rate": 1.574e-05, "load_balancing_loss": 1.07826483, "iteration": "2420/2862", "elapsed_time": "1d 1h 39m 27s", "remaining_time": "4h 41m 10s", "memory(GiB)": 69.62, "train_speed(s/it)": 38.168238} +{"loss": 0.25003198, "grad_norm": 1.6975534, "learning_rate": 1.562e-05, "load_balancing_loss": 1.0773772, "iteration": "2425/2862", "elapsed_time": "1d 1h 42m 36s", "remaining_time": "4h 37m 59s", "memory(GiB)": 69.62, "train_speed(s/it)": 38.167286} +{"loss": 0.24646218, "grad_norm": 1.63354075, "learning_rate": 1.549e-05, "load_balancing_loss": 1.07632875, "iteration": "2430/2862", "elapsed_time": "1d 1h 45m 45s", "remaining_time": "4h 34m 48s", "memory(GiB)": 69.62, "train_speed(s/it)": 38.166658} +{"loss": 0.2473511, "grad_norm": 1.98558104, "learning_rate": 1.537e-05, "load_balancing_loss": 1.07918215, "iteration": "2435/2862", "elapsed_time": "1d 1h 48m 57s", "remaining_time": "4h 31m 37s", "memory(GiB)": 69.62, "train_speed(s/it)": 38.16706} +{"loss": 0.24150389, "grad_norm": 1.79632902, "learning_rate": 1.524e-05, "load_balancing_loss": 1.07524335, "iteration": "2440/2862", "elapsed_time": "1d 1h 52m 7s", "remaining_time": "4h 28m 26s", "memory(GiB)": 69.62, "train_speed(s/it)": 38.166943} +{"loss": 0.23357612, "grad_norm": 2.15159178, "learning_rate": 1.512e-05, "load_balancing_loss": 1.07641196, "iteration": "2445/2862", "elapsed_time": "1d 1h 55m 19s", "remaining_time": "4h 25m 16s", "memory(GiB)": 69.62, "train_speed(s/it)": 38.167186} +{"loss": 0.23759334, "grad_norm": 2.36352944, "learning_rate": 1.5e-05, "load_balancing_loss": 1.07424915, "iteration": "2450/2862", "elapsed_time": "1d 1h 58m 29s", "remaining_time": "4h 22m 5s", "memory(GiB)": 69.62, "train_speed(s/it)": 38.167029} +{"loss": 0.23622857, "grad_norm": 1.84488189, "learning_rate": 1.489e-05, "load_balancing_loss": 1.07608581, "iteration": "2455/2862", "elapsed_time": "1d 2h 1m 40s", "remaining_time": "4h 18m 54s", "memory(GiB)": 69.62, "train_speed(s/it)": 38.166961} +{"loss": 0.21528246, "grad_norm": 1.86196065, "learning_rate": 1.477e-05, "load_balancing_loss": 1.07581747, "iteration": "2460/2862", "elapsed_time": "1d 2h 4m 48s", "remaining_time": "4h 15m 43s", "memory(GiB)": 69.62, "train_speed(s/it)": 38.165906} +{"loss": 0.24650715, "grad_norm": 2.292593, "learning_rate": 1.465e-05, "load_balancing_loss": 1.07806492, "iteration": "2465/2862", "elapsed_time": "1d 2h 7m 58s", "remaining_time": "4h 12m 32s", "memory(GiB)": 69.62, "train_speed(s/it)": 38.165652} +{"loss": 0.23244515, "grad_norm": 2.3189218, "learning_rate": 1.454e-05, "load_balancing_loss": 1.07748318, "iteration": "2470/2862", "elapsed_time": "1d 2h 11m 9s", "remaining_time": "4h 9m 21s", "memory(GiB)": 69.62, "train_speed(s/it)": 38.165486} +{"loss": 0.22744483, "grad_norm": 2.42864537, "learning_rate": 1.442e-05, "load_balancing_loss": 1.07516623, "iteration": "2475/2862", "elapsed_time": "1d 2h 14m 20s", "remaining_time": "4h 6m 10s", "memory(GiB)": 69.62, "train_speed(s/it)": 38.165601} +{"loss": 0.24447019, "grad_norm": 2.0542326, "learning_rate": 1.431e-05, "load_balancing_loss": 1.07570922, "iteration": "2480/2862", "elapsed_time": "1d 2h 17m 30s", "remaining_time": "4h 2m 59s", "memory(GiB)": 69.62, "train_speed(s/it)": 38.165414} +{"loss": 0.2477521, "grad_norm": 2.3125627, "learning_rate": 1.42e-05, "load_balancing_loss": 1.0758127, "iteration": "2485/2862", "elapsed_time": "1d 2h 20m 41s", "remaining_time": "3h 59m 48s", "memory(GiB)": 69.62, "train_speed(s/it)": 38.165473} +{"loss": 0.22284386, "grad_norm": 1.95567572, "learning_rate": 1.409e-05, "load_balancing_loss": 1.07649696, "iteration": "2490/2862", "elapsed_time": "1d 2h 23m 51s", "remaining_time": "3h 56m 37s", "memory(GiB)": 69.62, "train_speed(s/it)": 38.165103} +{"loss": 0.25561458, "grad_norm": 2.16582513, "learning_rate": 1.399e-05, "load_balancing_loss": 1.07546341, "iteration": "2495/2862", "elapsed_time": "1d 2h 27m 18s", "remaining_time": "3h 53m 29s", "memory(GiB)": 69.62, "train_speed(s/it)": 38.171631} +{"loss": 0.24062988, "grad_norm": 2.20939803, "learning_rate": 1.388e-05, "load_balancing_loss": 1.07545233, "iteration": "2500/2862", "elapsed_time": "1d 2h 30m 29s", "remaining_time": "3h 50m 18s", "memory(GiB)": 69.62, "train_speed(s/it)": 38.171645} +{"loss": 0.23224129, "grad_norm": 1.85384095, "learning_rate": 1.377e-05, "load_balancing_loss": 1.07506502, "iteration": "2505/2862", "elapsed_time": "1d 2h 35m 39s", "remaining_time": "3h 47m 24s", "memory(GiB)": 69.62, "train_speed(s/it)": 38.219235} +{"loss": 0.25839823, "grad_norm": 1.99642611, "learning_rate": 1.367e-05, "load_balancing_loss": 1.07672751, "iteration": "2510/2862", "elapsed_time": "1d 2h 38m 48s", "remaining_time": "3h 44m 13s", "memory(GiB)": 69.62, "train_speed(s/it)": 38.218414} +{"loss": 0.24491943, "grad_norm": 1.68987596, "learning_rate": 1.357e-05, "load_balancing_loss": 1.07731521, "iteration": "2515/2862", "elapsed_time": "1d 2h 41m 60s", "remaining_time": "3h 41m 2s", "memory(GiB)": 69.62, "train_speed(s/it)": 38.218614} +{"loss": 0.24161403, "grad_norm": 2.14477444, "learning_rate": 1.347e-05, "load_balancing_loss": 1.07560217, "iteration": "2520/2862", "elapsed_time": "1d 2h 45m 9s", "remaining_time": "3h 37m 51s", "memory(GiB)": 69.62, "train_speed(s/it)": 38.217998} +{"loss": 0.25069517, "grad_norm": 2.0329721, "learning_rate": 1.337e-05, "load_balancing_loss": 1.07564831, "iteration": "2525/2862", "elapsed_time": "1d 2h 48m 19s", "remaining_time": "3h 34m 39s", "memory(GiB)": 69.62, "train_speed(s/it)": 38.217542} +{"loss": 0.23525266, "grad_norm": 2.05593657, "learning_rate": 1.327e-05, "load_balancing_loss": 1.07761371, "iteration": "2530/2862", "elapsed_time": "1d 2h 51m 30s", "remaining_time": "3h 31m 28s", "memory(GiB)": 69.62, "train_speed(s/it)": 38.217302} +{"loss": 0.24815804, "grad_norm": 1.90253901, "learning_rate": 1.317e-05, "load_balancing_loss": 1.07683122, "iteration": "2535/2862", "elapsed_time": "1d 2h 54m 39s", "remaining_time": "3h 28m 17s", "memory(GiB)": 69.62, "train_speed(s/it)": 38.216591} +{"loss": 0.24685684, "grad_norm": 2.10818124, "learning_rate": 1.308e-05, "load_balancing_loss": 1.07664204, "iteration": "2540/2862", "elapsed_time": "1d 2h 57m 48s", "remaining_time": "3h 25m 5s", "memory(GiB)": 69.62, "train_speed(s/it)": 38.215817} +{"loss": 0.21932231, "grad_norm": 1.7691586, "learning_rate": 1.299e-05, "load_balancing_loss": 1.07802391, "iteration": "2545/2862", "elapsed_time": "1d 3h 0m 57s", "remaining_time": "3h 21m 54s", "memory(GiB)": 69.62, "train_speed(s/it)": 38.215096} +{"loss": 0.25816786, "grad_norm": 2.02979779, "learning_rate": 1.289e-05, "load_balancing_loss": 1.07825553, "iteration": "2550/2862", "elapsed_time": "1d 3h 4m 8s", "remaining_time": "3h 18m 43s", "memory(GiB)": 69.62, "train_speed(s/it)": 38.214877} +{"loss": 0.24592812, "grad_norm": 1.8890183, "learning_rate": 1.28e-05, "load_balancing_loss": 1.07718456, "iteration": "2555/2862", "elapsed_time": "1d 3h 7m 17s", "remaining_time": "3h 15m 32s", "memory(GiB)": 69.62, "train_speed(s/it)": 38.214278} +{"loss": 0.24711178, "grad_norm": 1.96876395, "learning_rate": 1.271e-05, "load_balancing_loss": 1.0783062, "iteration": "2560/2862", "elapsed_time": "1d 3h 10m 29s", "remaining_time": "3h 12m 21s", "memory(GiB)": 69.62, "train_speed(s/it)": 38.214343} +{"loss": 0.22357586, "grad_norm": 2.10441232, "learning_rate": 1.262e-05, "load_balancing_loss": 1.0776875, "iteration": "2565/2862", "elapsed_time": "1d 3h 13m 39s", "remaining_time": "3h 9m 10s", "memory(GiB)": 69.62, "train_speed(s/it)": 38.214197} +{"loss": 0.23675205, "grad_norm": 1.98007762, "learning_rate": 1.254e-05, "load_balancing_loss": 1.07648122, "iteration": "2570/2862", "elapsed_time": "1d 3h 16m 50s", "remaining_time": "3h 5m 58s", "memory(GiB)": 69.62, "train_speed(s/it)": 38.214015} +{"loss": 0.2299343, "grad_norm": 1.7986064, "learning_rate": 1.245e-05, "load_balancing_loss": 1.07663524, "iteration": "2575/2862", "elapsed_time": "1d 3h 19m 58s", "remaining_time": "3h 2m 47s", "memory(GiB)": 69.62, "train_speed(s/it)": 38.212716} +{"loss": 0.23036364, "grad_norm": 2.26003718, "learning_rate": 1.237e-05, "load_balancing_loss": 1.07764935, "iteration": "2580/2862", "elapsed_time": "1d 3h 23m 7s", "remaining_time": "2h 59m 36s", "memory(GiB)": 69.62, "train_speed(s/it)": 38.212139} +{"loss": 0.23611346, "grad_norm": 1.92959249, "learning_rate": 1.229e-05, "load_balancing_loss": 1.07769322, "iteration": "2585/2862", "elapsed_time": "1d 3h 26m 19s", "remaining_time": "2h 56m 25s", "memory(GiB)": 69.62, "train_speed(s/it)": 38.212282} +{"loss": 0.2366786, "grad_norm": 2.24362445, "learning_rate": 1.22e-05, "load_balancing_loss": 1.07776475, "iteration": "2590/2862", "elapsed_time": "1d 3h 29m 28s", "remaining_time": "2h 53m 14s", "memory(GiB)": 69.62, "train_speed(s/it)": 38.211657} +{"loss": 0.25702986, "grad_norm": 2.38170314, "learning_rate": 1.212e-05, "load_balancing_loss": 1.07659972, "iteration": "2595/2862", "elapsed_time": "1d 3h 32m 37s", "remaining_time": "2h 50m 2s", "memory(GiB)": 69.62, "train_speed(s/it)": 38.210633} +{"loss": 0.23981823, "grad_norm": 1.95769882, "learning_rate": 1.205e-05, "load_balancing_loss": 1.07615173, "iteration": "2600/2862", "elapsed_time": "1d 3h 35m 46s", "remaining_time": "2h 46m 51s", "memory(GiB)": 69.62, "train_speed(s/it)": 38.209995} +{"loss": 0.22980945, "grad_norm": 1.97156465, "learning_rate": 1.197e-05, "load_balancing_loss": 1.07588935, "iteration": "2605/2862", "elapsed_time": "1d 3h 38m 56s", "remaining_time": "2h 43m 40s", "memory(GiB)": 69.62, "train_speed(s/it)": 38.20945} +{"loss": 0.22756384, "grad_norm": 2.32482696, "learning_rate": 1.189e-05, "load_balancing_loss": 1.07394159, "iteration": "2610/2862", "elapsed_time": "1d 3h 42m 5s", "remaining_time": "2h 40m 29s", "memory(GiB)": 69.62, "train_speed(s/it)": 38.208954} +{"loss": 0.22550716, "grad_norm": 2.09347916, "learning_rate": 1.182e-05, "load_balancing_loss": 1.07550657, "iteration": "2615/2862", "elapsed_time": "1d 3h 45m 16s", "remaining_time": "2h 37m 18s", "memory(GiB)": 69.62, "train_speed(s/it)": 38.208912} +{"loss": 0.21947721, "grad_norm": 1.74916565, "learning_rate": 1.175e-05, "load_balancing_loss": 1.07660067, "iteration": "2620/2862", "elapsed_time": "1d 3h 48m 24s", "remaining_time": "2h 34m 6s", "memory(GiB)": 69.62, "train_speed(s/it)": 38.207614} +{"loss": 0.24473053, "grad_norm": 2.01635218, "learning_rate": 1.168e-05, "load_balancing_loss": 1.07551575, "iteration": "2625/2862", "elapsed_time": "1d 3h 51m 35s", "remaining_time": "2h 30m 55s", "memory(GiB)": 69.62, "train_speed(s/it)": 38.207622} +{"loss": 0.24197848, "grad_norm": 2.17233658, "learning_rate": 1.161e-05, "load_balancing_loss": 1.07542801, "iteration": "2630/2862", "elapsed_time": "1d 3h 54m 47s", "remaining_time": "2h 27m 44s", "memory(GiB)": 69.62, "train_speed(s/it)": 38.208045} +{"loss": 0.22985883, "grad_norm": 1.73905134, "learning_rate": 1.154e-05, "load_balancing_loss": 1.07572472, "iteration": "2635/2862", "elapsed_time": "1d 3h 57m 55s", "remaining_time": "2h 24m 33s", "memory(GiB)": 69.62, "train_speed(s/it)": 38.20702} +{"loss": 0.26645705, "grad_norm": 2.51843572, "learning_rate": 1.147e-05, "load_balancing_loss": 1.07830846, "iteration": "2640/2862", "elapsed_time": "1d 4h 1m 6s", "remaining_time": "2h 21m 22s", "memory(GiB)": 69.62, "train_speed(s/it)": 38.206856} +{"loss": 0.25177529, "grad_norm": 2.09744191, "learning_rate": 1.141e-05, "load_balancing_loss": 1.07556581, "iteration": "2645/2862", "elapsed_time": "1d 4h 4m 14s", "remaining_time": "2h 18m 11s", "memory(GiB)": 69.62, "train_speed(s/it)": 38.205506} +{"loss": 0.25447246, "grad_norm": 2.38717437, "learning_rate": 1.134e-05, "load_balancing_loss": 1.0770961, "iteration": "2650/2862", "elapsed_time": "1d 4h 7m 23s", "remaining_time": "2h 14m 59s", "memory(GiB)": 69.62, "train_speed(s/it)": 38.205079} +{"loss": 0.23710753, "grad_norm": 1.9754262, "learning_rate": 1.128e-05, "load_balancing_loss": 1.07520258, "iteration": "2655/2862", "elapsed_time": "1d 4h 10m 34s", "remaining_time": "2h 11m 48s", "memory(GiB)": 69.62, "train_speed(s/it)": 38.20486} +{"loss": 0.26270607, "grad_norm": 2.13188839, "learning_rate": 1.122e-05, "load_balancing_loss": 1.07762015, "iteration": "2660/2862", "elapsed_time": "1d 4h 13m 44s", "remaining_time": "2h 8m 37s", "memory(GiB)": 69.62, "train_speed(s/it)": 38.204577} +{"loss": 0.23736344, "grad_norm": 1.83732474, "learning_rate": 1.116e-05, "load_balancing_loss": 1.07627606, "iteration": "2665/2862", "elapsed_time": "1d 4h 16m 50s", "remaining_time": "2h 5m 26s", "memory(GiB)": 69.62, "train_speed(s/it)": 38.202805} +{"loss": 0.22313723, "grad_norm": 2.07002735, "learning_rate": 1.11e-05, "load_balancing_loss": 1.07799435, "iteration": "2670/2862", "elapsed_time": "1d 4h 19m 58s", "remaining_time": "2h 2m 15s", "memory(GiB)": 69.62, "train_speed(s/it)": 38.201553} +{"loss": 0.23402132, "grad_norm": 2.15382504, "learning_rate": 1.105e-05, "load_balancing_loss": 1.07654631, "iteration": "2675/2862", "elapsed_time": "1d 4h 23m 8s", "remaining_time": "1h 59m 4s", "memory(GiB)": 69.62, "train_speed(s/it)": 38.20108} +{"loss": 0.25064856, "grad_norm": 2.4063139, "learning_rate": 1.099e-05, "load_balancing_loss": 1.07543647, "iteration": "2680/2862", "elapsed_time": "1d 4h 26m 19s", "remaining_time": "1h 55m 53s", "memory(GiB)": 69.62, "train_speed(s/it)": 38.200999} +{"loss": 0.2337781, "grad_norm": 2.24290943, "learning_rate": 1.094e-05, "load_balancing_loss": 1.07614195, "iteration": "2685/2862", "elapsed_time": "1d 4h 29m 30s", "remaining_time": "1h 52m 42s", "memory(GiB)": 69.62, "train_speed(s/it)": 38.200958} +{"loss": 0.23094799, "grad_norm": 1.96353018, "learning_rate": 1.089e-05, "load_balancing_loss": 1.07679451, "iteration": "2690/2862", "elapsed_time": "1d 4h 32m 40s", "remaining_time": "1h 49m 31s", "memory(GiB)": 69.62, "train_speed(s/it)": 38.200827} +{"loss": 0.21384194, "grad_norm": 1.80831134, "learning_rate": 1.084e-05, "load_balancing_loss": 1.07458174, "iteration": "2695/2862", "elapsed_time": "1d 4h 35m 52s", "remaining_time": "1h 46m 20s", "memory(GiB)": 69.62, "train_speed(s/it)": 38.20097} +{"loss": 0.23367876, "grad_norm": 2.18348002, "learning_rate": 1.079e-05, "load_balancing_loss": 1.0779177, "iteration": "2700/2862", "elapsed_time": "1d 4h 39m 2s", "remaining_time": "1h 43m 8s", "memory(GiB)": 69.62, "train_speed(s/it)": 38.200594} +{"loss": 0.2547079, "grad_norm": 2.16844249, "learning_rate": 1.074e-05, "load_balancing_loss": 1.07714272, "iteration": "2705/2862", "elapsed_time": "1d 4h 42m 11s", "remaining_time": "1h 39m 57s", "memory(GiB)": 69.62, "train_speed(s/it)": 38.200104} +{"loss": 0.23823532, "grad_norm": 2.06825519, "learning_rate": 1.069e-05, "load_balancing_loss": 1.07521498, "iteration": "2710/2862", "elapsed_time": "1d 4h 45m 20s", "remaining_time": "1h 36m 46s", "memory(GiB)": 69.62, "train_speed(s/it)": 38.199313} +{"loss": 0.22014561, "grad_norm": 1.92247796, "learning_rate": 1.065e-05, "load_balancing_loss": 1.07659853, "iteration": "2715/2862", "elapsed_time": "1d 4h 48m 31s", "remaining_time": "1h 33m 35s", "memory(GiB)": 69.62, "train_speed(s/it)": 38.199377} +{"loss": 0.25063953, "grad_norm": 2.39449382, "learning_rate": 1.06e-05, "load_balancing_loss": 1.07465672, "iteration": "2720/2862", "elapsed_time": "1d 4h 51m 42s", "remaining_time": "1h 30m 24s", "memory(GiB)": 69.62, "train_speed(s/it)": 38.199152} +{"loss": 0.26086858, "grad_norm": 2.34903717, "learning_rate": 1.056e-05, "load_balancing_loss": 1.07580495, "iteration": "2725/2862", "elapsed_time": "1d 4h 54m 51s", "remaining_time": "1h 27m 13s", "memory(GiB)": 69.62, "train_speed(s/it)": 38.198454} +{"loss": 0.23481846, "grad_norm": 2.19462228, "learning_rate": 1.052e-05, "load_balancing_loss": 1.07591891, "iteration": "2730/2862", "elapsed_time": "1d 4h 58m 1s", "remaining_time": "1h 24m 2s", "memory(GiB)": 69.62, "train_speed(s/it)": 38.198001} +{"loss": 0.26091802, "grad_norm": 2.17088294, "learning_rate": 1.048e-05, "load_balancing_loss": 1.07660282, "iteration": "2735/2862", "elapsed_time": "1d 5h 1m 12s", "remaining_time": "1h 20m 51s", "memory(GiB)": 69.62, "train_speed(s/it)": 38.198013} +{"loss": 0.24738051, "grad_norm": 2.08004379, "learning_rate": 1.045e-05, "load_balancing_loss": 1.07492459, "iteration": "2740/2862", "elapsed_time": "1d 5h 4m 22s", "remaining_time": "1h 17m 40s", "memory(GiB)": 69.62, "train_speed(s/it)": 38.197911} +{"loss": 0.24355645, "grad_norm": 2.0581038, "learning_rate": 1.041e-05, "load_balancing_loss": 1.07735491, "iteration": "2745/2862", "elapsed_time": "1d 5h 7m 33s", "remaining_time": "1h 14m 29s", "memory(GiB)": 69.62, "train_speed(s/it)": 38.197798} +{"loss": 0.23633197, "grad_norm": 1.72183287, "learning_rate": 1.038e-05, "load_balancing_loss": 1.07656908, "iteration": "2750/2862", "elapsed_time": "1d 5h 10m 45s", "remaining_time": "1h 11m 18s", "memory(GiB)": 69.62, "train_speed(s/it)": 38.198027} +{"loss": 0.24461527, "grad_norm": 1.90212452, "learning_rate": 1.034e-05, "load_balancing_loss": 1.07708704, "iteration": "2755/2862", "elapsed_time": "1d 5h 13m 56s", "remaining_time": "1h 8m 7s", "memory(GiB)": 69.62, "train_speed(s/it)": 38.198092} +{"loss": 0.25234252, "grad_norm": 2.09068942, "learning_rate": 1.031e-05, "load_balancing_loss": 1.07467842, "iteration": "2760/2862", "elapsed_time": "1d 5h 17m 7s", "remaining_time": "1h 4m 56s", "memory(GiB)": 69.62, "train_speed(s/it)": 38.19802} +{"loss": 0.22896312, "grad_norm": 2.14173865, "learning_rate": 1.028e-05, "load_balancing_loss": 1.07592034, "iteration": "2765/2862", "elapsed_time": "1d 5h 20m 17s", "remaining_time": "1h 1m 45s", "memory(GiB)": 69.62, "train_speed(s/it)": 38.197861} +{"loss": 0.2337994, "grad_norm": 2.36540556, "learning_rate": 1.025e-05, "load_balancing_loss": 1.07510281, "iteration": "2770/2862", "elapsed_time": "1d 5h 23m 28s", "remaining_time": "58m 34s", "memory(GiB)": 69.62, "train_speed(s/it)": 38.197771} +{"loss": 0.22995818, "grad_norm": 2.12258172, "learning_rate": 1.023e-05, "load_balancing_loss": 1.07617807, "iteration": "2775/2862", "elapsed_time": "1d 5h 26m 38s", "remaining_time": "55m 23s", "memory(GiB)": 69.62, "train_speed(s/it)": 38.197636} +{"loss": 0.25972062, "grad_norm": 2.02623916, "learning_rate": 1.02e-05, "load_balancing_loss": 1.07552409, "iteration": "2780/2862", "elapsed_time": "1d 5h 29m 47s", "remaining_time": "52m 12s", "memory(GiB)": 69.62, "train_speed(s/it)": 38.196833} +{"loss": 0.23115718, "grad_norm": 1.78747034, "learning_rate": 1.018e-05, "load_balancing_loss": 1.07693636, "iteration": "2785/2862", "elapsed_time": "1d 5h 32m 57s", "remaining_time": "49m 1s", "memory(GiB)": 69.62, "train_speed(s/it)": 38.196576} +{"loss": 0.23560898, "grad_norm": 1.83378339, "learning_rate": 1.016e-05, "load_balancing_loss": 1.07707691, "iteration": "2790/2862", "elapsed_time": "1d 5h 36m 7s", "remaining_time": "45m 50s", "memory(GiB)": 69.62, "train_speed(s/it)": 38.196189} +{"loss": 0.23471752, "grad_norm": 2.29836273, "learning_rate": 1.013e-05, "load_balancing_loss": 1.0778439, "iteration": "2795/2862", "elapsed_time": "1d 5h 39m 18s", "remaining_time": "42m 39s", "memory(GiB)": 69.62, "train_speed(s/it)": 38.195999} +{"loss": 0.24216737, "grad_norm": 2.07265258, "learning_rate": 1.012e-05, "load_balancing_loss": 1.07613587, "iteration": "2800/2862", "elapsed_time": "1d 5h 42m 26s", "remaining_time": "39m 28s", "memory(GiB)": 69.62, "train_speed(s/it)": 38.195162} +{"loss": 0.25125995, "grad_norm": 1.98510396, "learning_rate": 1.01e-05, "load_balancing_loss": 1.078251, "iteration": "2805/2862", "elapsed_time": "1d 5h 45m 38s", "remaining_time": "36m 17s", "memory(GiB)": 69.62, "train_speed(s/it)": 38.195218} +{"loss": 0.23285887, "grad_norm": 2.03534722, "learning_rate": 1.008e-05, "load_balancing_loss": 1.07632148, "iteration": "2810/2862", "elapsed_time": "1d 5h 48m 48s", "remaining_time": "33m 6s", "memory(GiB)": 69.62, "train_speed(s/it)": 38.195143} +{"loss": 0.24195926, "grad_norm": 2.15544653, "learning_rate": 1.007e-05, "load_balancing_loss": 1.07645893, "iteration": "2815/2862", "elapsed_time": "1d 5h 51m 59s", "remaining_time": "29m 55s", "memory(GiB)": 69.62, "train_speed(s/it)": 38.195073} +{"loss": 0.2487554, "grad_norm": 2.0529604, "learning_rate": 1.005e-05, "load_balancing_loss": 1.07726753, "iteration": "2820/2862", "elapsed_time": "1d 5h 55m 10s", "remaining_time": "26m 44s", "memory(GiB)": 69.62, "train_speed(s/it)": 38.195099} +{"loss": 0.23911181, "grad_norm": 1.91662288, "learning_rate": 1.004e-05, "load_balancing_loss": 1.07614493, "iteration": "2825/2862", "elapsed_time": "1d 5h 58m 19s", "remaining_time": "23m 33s", "memory(GiB)": 69.62, "train_speed(s/it)": 38.19449} +{"loss": 0.23388137, "grad_norm": 1.66680622, "learning_rate": 1.003e-05, "load_balancing_loss": 1.07505226, "iteration": "2830/2862", "elapsed_time": "1d 6h 1m 31s", "remaining_time": "20m 22s", "memory(GiB)": 69.62, "train_speed(s/it)": 38.194849} +{"loss": 0.22151612, "grad_norm": 1.65329564, "learning_rate": 1.002e-05, "load_balancing_loss": 1.07675171, "iteration": "2835/2862", "elapsed_time": "1d 6h 4m 42s", "remaining_time": "17m 11s", "memory(GiB)": 69.62, "train_speed(s/it)": 38.194595} +{"loss": 0.23805271, "grad_norm": 2.06483507, "learning_rate": 1.001e-05, "load_balancing_loss": 1.07739794, "iteration": "2840/2862", "elapsed_time": "1d 6h 7m 53s", "remaining_time": "14m 0s", "memory(GiB)": 69.62, "train_speed(s/it)": 38.194684} +{"loss": 0.2528069, "grad_norm": 1.92453957, "learning_rate": 1.001e-05, "load_balancing_loss": 1.07649803, "iteration": "2845/2862", "elapsed_time": "1d 6h 11m 4s", "remaining_time": "10m 49s", "memory(GiB)": 69.62, "train_speed(s/it)": 38.194854} +{"loss": 0.23866968, "grad_norm": 1.99075317, "learning_rate": 1e-05, "load_balancing_loss": 1.07723415, "iteration": "2850/2862", "elapsed_time": "1d 6h 14m 13s", "remaining_time": "7m 38s", "memory(GiB)": 69.62, "train_speed(s/it)": 38.194159} +{"loss": 0.25391915, "grad_norm": 2.05858397, "learning_rate": 1e-05, "load_balancing_loss": 1.07540548, "iteration": "2855/2862", "elapsed_time": "1d 6h 17m 24s", "remaining_time": "4m 27s", "memory(GiB)": 69.62, "train_speed(s/it)": 38.194006} +{"loss": 0.22982714, "grad_norm": 1.87048066, "learning_rate": 1e-05, "load_balancing_loss": 1.0755924, "iteration": "2860/2862", "elapsed_time": "1d 6h 20m 34s", "remaining_time": "1m 16s", "memory(GiB)": 69.62, "train_speed(s/it)": 38.193595} +{"last_model_checkpoint": "/mnt/bn/strategy-mllm-train/user/tanyue/models/posttrain_models/qwen3_5_35b_a3b_base_highlight_batch1245_epoch4/checkpoint-2862", "best_model_checkpoint": null, "best_metric": null} diff --git a/posttrain_models/qwen3_5_4b_base_highlight_batch1245_epoch4/args.json b/posttrain_models/qwen3_5_4b_base_highlight_batch1245_epoch4/args.json new file mode 100644 index 0000000000000000000000000000000000000000..66fa96327404653aa85875ca21471633385c775c --- /dev/null +++ b/posttrain_models/qwen3_5_4b_base_highlight_batch1245_epoch4/args.json @@ -0,0 +1,398 @@ +{ + "use_ray": false, + "ray_exp_name": null, + "device_groups": null, + "model": "/mnt/bn/strategy-mllm-train/user/tanyue/models/pretrained_models/Qwen3.5-4B", + "model_type": "qwen3_5", + "model_revision": null, + "task_type": "causal_lm", + "torch_dtype": "bfloat16", + "attn_impl": null, + "experts_impl": null, + "new_special_tokens": [], + "num_labels": null, + "problem_type": null, + "rope_scaling": null, + "device_map": null, + "max_memory": {}, + "max_model_len": null, + "local_repo_path": null, + "init_strategy": null, + "template": "qwen3_5", + "system": null, + "max_length": 8192, + "truncation_strategy": "delete", + "max_pixels": null, + "agent_template": null, + "norm_bbox": null, + "use_chat_template": true, + "padding_side": "right", + "padding_free": true, + "loss_scale": "ignore_empty_think", + "sequence_parallel_size": 1, + "template_backend": "swift", + "response_prefix": "", + "enable_thinking": false, + "add_non_thinking_prefix": false, + "dataset": [ + "batch1", + "batch2", + "batch4_part1", + "batch4_part2", + "batch4_part3", + "batch5_part1", + "batch5_part2", + "batch5_part3" + ], + "val_dataset": [], + "cached_dataset": [], + "cached_val_dataset": [], + "split_dataset_ratio": 0.0, + "data_seed": 42, + "dataset_num_proc": 8, + "load_from_cache_file": false, + "dataset_shuffle": true, + "val_dataset_shuffle": false, + "streaming": false, + "interleave_prob": null, + "stopping_strategy": "first_exhausted", + "shuffle_buffer_size": 1000, + "download_mode": "reuse_dataset_if_exists", + "columns": {}, + "strict": false, + "remove_unused_columns": true, + "model_name": null, + "model_author": null, + "custom_dataset_info": [], + "quant_method": null, + "quant_bits": null, + "hqq_axis": null, + "bnb_4bit_compute_dtype": "bfloat16", + "bnb_4bit_quant_type": "nf4", + "bnb_4bit_use_double_quant": true, + "bnb_4bit_quant_storage": null, + "max_new_tokens": null, + "temperature": 0.9, + "top_k": 50, + "top_p": 0.9, + "repetition_penalty": 1.0, + "num_beams": 1, + "stream": false, + "stop_words": [], + "logprobs": false, + "top_logprobs": null, + "structured_outputs_regex": null, + "tuner_backend": "peft", + "tuner_type": "lora", + "train_type": "lora", + "adapters": [], + "external_plugins": [ + "ms-swift/base_highlight_video_dataset.py" + ], + "custom_register_path": [ + "ms-swift/base_highlight_video_dataset.py" + ], + "seed": 42, + "model_kwargs": {}, + "load_args": false, + "load_data_args": false, + "packing": false, + "packing_length": null, + "packing_num_proc": 1, + "lazy_tokenize": true, + "use_hf": false, + "hub_token": null, + "ddp_timeout": 18000000, + "ddp_backend": "nccl", + "ignore_args_error": false, + "use_swift_lora": false, + "freeze_llm": false, + "freeze_vit": true, + "freeze_aligner": true, + "freeze_parameters": [], + "freeze_parameters_regex": null, + "freeze_parameters_ratio": 0.0, + "trainable_parameters": [], + "trainable_parameters_regex": null, + "target_modules": [ + "all-linear" + ], + "target_regex": null, + "modules_to_save": [], + "lora_rank": 8, + "lora_alpha": 32, + "lora_dropout": 0.05, + "lora_bias": "none", + "lora_dtype": null, + "use_rslora": false, + "rlhf_type": null, + "loss_type": null, + "mcore_ref_model": null, + "mcore_ref_adapter": null, + "beta": null, + "rpo_alpha": null, + "reference_free": false, + "label_smoothing": 0.0, + "f_divergence_type": "reverse_kl", + "desirable_weight": 1.0, + "undesirable_weight": 1.0, + "calculate_KL": null, + "center_rewards_coefficient": null, + "teacher_model": null, + "teacher_model_type": null, + "teacher_model_revision": null, + "teacher_model_server": null, + "gkd_logits_topk": null, + "lmbda": 0.5, + "seq_kd": false, + "offload_teacher_model": false, + "sft_alpha": 0.0, + "generation_batch_size": null, + "steps_per_generation": null, + "num_generations": 8, + "num_generations_eval": null, + "max_completion_length": 512, + "importance_sampling_level": "token", + "tau_pos": 1.0, + "tau_neg": 1.05, + "epsilon": 0.2, + "epsilon_high": null, + "delta": null, + "use_vllm": true, + "vllm_mode": null, + "vllm_enable_prefix_caching": true, + "vllm_gpu_memory_utilization": 0.9, + "vllm_tensor_parallel_size": 1, + "vllm_max_model_len": null, + "vllm_enforce_eager": false, + "vllm_limit_mm_per_prompt": null, + "vllm_disable_cascade_attn": false, + "vllm_max_num_seqs": null, + "vllm_mm_processor_cache_gb": null, + "vllm_engine_kwargs": null, + "sleep_level": 0, + "offload_optimizer": false, + "offload_model": false, + "offload_bridge": false, + "vllm_server_base_url": null, + "vllm_server_host": null, + "vllm_server_port": [ + 8000 + ], + "vllm_server_timeout": 240.0, + "vllm_server_group_port": null, + "reward_funcs": [], + "reward_weights": null, + "cosine_min_len_value_wrong": -0.5, + "cosine_max_len_value_wrong": 0.0, + "cosine_min_len_value_correct": 1.0, + "cosine_max_len_value_correct": 0.5, + "cosine_max_len": null, + "repetition_n_grams": 3, + "repetition_max_penalty": -1.0, + "soft_max_length": null, + "soft_cache_length": null, + "dynamic_sample": false, + "max_resample_times": 3, + "overlong_filter": false, + "scale_rewards": "group", + "advantage_estimator": "grpo", + "kl_in_reward": false, + "wandb_log_unique_prompts": null, + "log_completions": false, + "rollout_importance_sampling_mode": null, + "rollout_importance_sampling_threshold": 2.0, + "log_rollout_offpolicy_metrics": false, + "off_policy_sequence_mask_delta": null, + "log_entropy": false, + "top_entropy_quantile": 1.0, + "reward_model": null, + "reward_model_plugin": null, + "sync_ref_model": false, + "ref_model_sync_steps": 512, + "ref_model_mixup_alpha": 0.6, + "async_generate": false, + "move_model_batches": null, + "multi_turn_scheduler": null, + "max_turns": null, + "completion_length_limit_scope": "per_round", + "vllm_server_pass_dataset": false, + "num_iterations": 1, + "micro_batch_size": 8, + "global_batch_size": 64, + "recompute_granularity": "full", + "recompute_method": "uniform", + "recompute_num_layers": 1, + "recompute_modules": [ + "core_attn" + ], + "train_iters": null, + "num_train_epochs": 4, + "masked_softmax_fusion": true, + "bias_dropout_fusion": true, + "bias_activation_fusion": true, + "apply_rope_fusion": false, + "gradient_accumulation_fusion": false, + "cross_entropy_loss_fusion": true, + "cross_entropy_fusion_impl": "te", + "calculate_per_token_loss": true, + "attention_backend": "", + "optimizer": "adam", + "optimizer_cpu_offload": false, + "optimizer_offload_fraction": 1.0, + "use_precision_aware_optimizer": false, + "main_grads_dtype": "float32", + "main_params_dtype": "float32", + "exp_avg_dtype": "float32", + "exp_avg_sq_dtype": "float32", + "manual_gc": false, + "manual_gc_steps": 0, + "manual_gc_eval": true, + "train_dataloader_shuffle": true, + "dataloader_num_workers": 8, + "dataloader_pin_memory": true, + "dataloader_persistent_workers": true, + "dataloader_prefetch_factor": 2, + "data_sharding": false, + "group_by_length": false, + "te_rng_tracker": false, + "data_parallel_random_init": false, + "mlp_padding_free": false, + "lr_warmup_init": 0.0, + "lr": 0.0001, + "lr_decay_style": "cosine", + "lr_decay_iters": null, + "lr_warmup_iters": 0, + "lr_warmup_fraction": 0.05, + "min_lr": 1e-05, + "lr_wsd_decay_style": "exponential", + "lr_wsd_decay_iters": null, + "weight_decay": 0.1, + "weight_decay_incr_style": "constant", + "start_weight_decay": 0.1, + "end_weight_decay": 0.1, + "clip_grad": 1.0, + "adam_beta1": 0.9, + "adam_beta2": 0.95, + "adam_eps": 1e-08, + "sgd_momentum": 0.9, + "output_dir": "/mnt/bn/strategy-mllm-train/user/tanyue/models/posttrain_models/qwen3_5_4b_base_highlight_batch1245_epoch4", + "save_steps": 500, + "no_save_optim": true, + "no_save_rng": true, + "mcore_model": null, + "mcore_adapter": null, + "no_load_optim": false, + "no_load_rng": false, + "finetune": true, + "perform_initialization": false, + "use_cpu_initialization": false, + "async_save": false, + "save_total_limit": null, + "metric_for_best_model": "loss", + "greater_is_better": false, + "use_persistent_ckpt_worker": false, + "dist_ckpt_save_pre_mcore_014": false, + "dist_ckpt_optim_fully_reshardable": false, + "distrib_optim_fully_reshardable_mem_efficient": false, + "local_rank": 0, + "use_distributed_optimizer": true, + "tensor_model_parallel_size": 1, + "pipeline_model_parallel_size": 2, + "decoder_first_pipeline_num_layers": null, + "decoder_last_pipeline_num_layers": null, + "account_for_embedding_in_pipeline_split": false, + "account_for_loss_in_pipeline_split": false, + "overlap_p2p_comm": false, + "align_param_gather": false, + "sequence_parallel": false, + "context_parallel_size": 1, + "tp_comm_overlap": false, + "overlap_grad_reduce": false, + "overlap_param_gather": false, + "overlap_param_gather_with_optimizer_step": false, + "align_grad_reduce": true, + "virtual_pipeline_model_parallel_size": null, + "microbatch_group_size_per_vp_stage": null, + "pipeline_model_parallel_layout": null, + "expert_model_parallel_size": 4, + "expert_tensor_parallel_size": 1, + "report_to": [ + "wandb" + ], + "logging_steps": 5, + "tensorboard_dir": "/mnt/bn/strategy-mllm-train/user/tanyue/models/posttrain_models/qwen3_5_4b_base_highlight_batch1245_epoch4/runs", + "tensorboard_queue_size": 50, + "wandb_project": "highlight_base_highlight_qwen3_5", + "wandb_exp_name": "qwen3_5_4b_base_highlight_batch1245_epoch4", + "swanlab_project": "megatron-swift", + "swanlab_exp_name": null, + "eval_iters": -1, + "eval_steps": 100000, + "fp8_format": null, + "fp8_recipe": "delayed", + "fp8_amax_history_len": 1024, + "fp8_amax_compute_algo": "max", + "fp8_param_gather": false, + "fp16": false, + "bf16": true, + "apply_query_key_layer_scaling": false, + "attention_softmax_in_fp32": true, + "accumulate_allreduce_grads_in_fp32": true, + "moe_router_load_balancing_type": null, + "moe_router_dtype": "fp32", + "moe_token_dispatcher_type": null, + "moe_enable_deepep": false, + "moe_grouped_gemm": true, + "moe_permute_fusion": true, + "moe_aux_loss_coeff": 1e-06, + "moe_z_loss_coeff": null, + "moe_shared_expert_overlap": true, + "moe_layer_recompute": false, + "moe_expert_capacity_factor": null, + "moe_pad_expert_input_to_capacity": false, + "moe_token_drop_policy": null, + "mtp_num_layers": null, + "mtp_loss_scaling_factor": 0.1, + "save_safetensors": true, + "ref_model": null, + "ref_adapters": [], + "merge_lora": true, + "max_shard_size": "5GB", + "vit_gradient_checkpointing": false, + "vit_lr": null, + "aligner_lr": null, + "gradient_checkpointing_kwargs": null, + "check_model": true, + "apply_wd_to_qk_layernorm": false, + "enable_dft_loss": false, + "enable_channel_loss": false, + "save_strategy": "steps", + "callbacks": [ + "print", + "default_flow", + "wandb" + ], + "add_version": false, + "create_checkpoint_symlink": false, + "swift_version": "4.0.1", + "ckpt_dir": null, + "rank": 0, + "global_world_size": 8, + "local_world_size": 8, + "model_suffix": "Qwen3.5-4B", + "model_info": "ModelInfo(model_type='qwen3_5', model_dir='/mnt/bn/strategy-mllm-train/user/tanyue/models/pretrained_models/Qwen3.5-4B', torch_dtype=torch.bfloat16, max_model_len=262144, quant_method=None, quant_bits=None, rope_scaling=None, is_moe_model=False, is_multimodal=True, config=None, task_type='causal_lm', num_labels=None)", + "model_meta": "ModelMeta(model_type='qwen3_5', model_groups=[ModelGroup(models=[Model(ms_model_id='Qwen/Qwen3.5-0.8B', hf_model_id='Qwen/Qwen3.5-0.8B', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen3.5-2B', hf_model_id='Qwen/Qwen3.5-2B', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen3.5-4B', hf_model_id='Qwen/Qwen3.5-4B', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen3.5-9B', hf_model_id='Qwen/Qwen3.5-9B', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen3.5-27B', hf_model_id='Qwen/Qwen3.5-27B', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen3.5-27B-FP8', hf_model_id='Qwen/Qwen3.5-27B-FP8', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen3.5-0.8B-Base', hf_model_id='Qwen/Qwen3.5-0.8B-Base', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen3.5-2B-Base', hf_model_id='Qwen/Qwen3.5-2B-Base', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen3.5-4B-Base', hf_model_id='Qwen/Qwen3.5-4B-Base', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen3.5-9B-Base', hf_model_id='Qwen/Qwen3.5-9B-Base', model_path=None, ms_revision=None, hf_revision=None)], template='qwen3_5', ignore_patterns=None, requires=None, tags=[])], loader=, template='qwen3_5', model_arch=MultiModelKeys(arch_name='qwen2_vl', embedding=None, module_list=None, lm_head=None, q_proj=None, k_proj=None, v_proj=None, o_proj=None, attention=None, mlp=None, down_proj=None, qkv_proj=None, qk_proj=None, qa_proj=None, qb_proj=None, kv_proj=None, kva_proj=None, kvb_proj=None, language_model=['model.language_model', 'lm_head'], aligner=['model.visual.merger'], vision_tower=['model.visual'], generator=[]), architectures=['Qwen3_5ForConditionalGeneration'], additional_saved_files=[], torch_dtype=None, is_multimodal=True, is_reward=False, task_type=None, ignore_patterns=None, requires=['transformers>=5.0.0.dev', 'qwen_vl_utils>=0.0.14', 'decord'], tags=[])", + "model_dir": "/mnt/bn/strategy-mllm-train/user/tanyue/models/pretrained_models/Qwen3.5-4B", + "template_meta": "QwenTemplateMeta(template_type='qwen3_5', prefix=[], prompt=['<|im_start|>user\\n{{QUERY}}<|im_end|>\\n<|im_start|>assistant\\n'], chat_sep=['<|im_end|>\\n'], suffix=['<|im_end|>\\n'], template_cls=, system_prefix=['<|im_start|>system\\n{{SYSTEM}}<|im_end|>\\n'], default_system=None, auto_add_bos=False, stop_words=['<|endoftext|>'], agent_template='qwen3_5', is_thinking=True, thinking_prefix='\\n', non_thinking_prefix='\\n\\n\\n\\n', history_thinking_prefix='')", + "_val_dataset_exists": false, + "hub": "", + "seq_length": 8192, + "is_multimodal": true, + "megatron_model_meta": "MegatronModelMeta(megatron_model_type='qwen3_5', model_types=['qwen3_5', 'qwen3_5_moe'], bridge_cls=, visual_cls=, is_multimodal=True, loader=)", + "fp8": null, + "params_dtype": "bfloat16", + "world_size": 8, + "data_parallel_size": 4, + "num_microbatches": 2 +} \ No newline at end of file diff --git a/posttrain_models/qwen3_5_4b_base_highlight_batch1245_epoch4/checkpoint-1000-merged-vllm-mix-foundation_ft_1000/model.safetensors.index.json b/posttrain_models/qwen3_5_4b_base_highlight_batch1245_epoch4/checkpoint-1000-merged-vllm-mix-foundation_ft_1000/model.safetensors.index.json new file mode 100644 index 0000000000000000000000000000000000000000..eec55fbfbf7a653dc0446bcbc9b9762cda601edb --- /dev/null +++ b/posttrain_models/qwen3_5_4b_base_highlight_batch1245_epoch4/checkpoint-1000-merged-vllm-mix-foundation_ft_1000/model.safetensors.index.json @@ -0,0 +1,730 @@ +{ + "metadata": { + "total_size": 9078531072 + }, + "weight_map": { + "model.language_model.embed_tokens.weight": "model-00001-of-00002.safetensors", + "model.visual.patch_embed.proj.weight": "model-00001-of-00002.safetensors", + "model.visual.patch_embed.proj.bias": "model-00001-of-00002.safetensors", + "model.visual.pos_embed.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.0.norm1.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.0.norm1.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.0.norm2.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.0.norm2.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.0.attn.qkv.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.0.attn.qkv.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.0.attn.proj.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.0.attn.proj.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.0.mlp.linear_fc1.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.0.mlp.linear_fc1.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.0.mlp.linear_fc2.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.0.mlp.linear_fc2.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.1.norm1.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.1.norm1.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.1.norm2.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.1.norm2.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.1.attn.qkv.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.1.attn.qkv.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.1.attn.proj.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.1.attn.proj.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.1.mlp.linear_fc1.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.1.mlp.linear_fc1.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.1.mlp.linear_fc2.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.1.mlp.linear_fc2.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.2.norm1.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.2.norm1.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.2.norm2.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.2.norm2.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.2.attn.qkv.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.2.attn.qkv.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.2.attn.proj.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.2.attn.proj.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.2.mlp.linear_fc1.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.2.mlp.linear_fc1.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.2.mlp.linear_fc2.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.2.mlp.linear_fc2.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.3.norm1.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.3.norm1.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.3.norm2.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.3.norm2.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.3.attn.qkv.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.3.attn.qkv.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.3.attn.proj.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.3.attn.proj.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.3.mlp.linear_fc1.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.3.mlp.linear_fc1.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.3.mlp.linear_fc2.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.3.mlp.linear_fc2.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.4.norm1.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.4.norm1.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.4.norm2.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.4.norm2.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.4.attn.qkv.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.4.attn.qkv.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.4.attn.proj.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.4.attn.proj.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.4.mlp.linear_fc1.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.4.mlp.linear_fc1.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.4.mlp.linear_fc2.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.4.mlp.linear_fc2.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.5.norm1.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.5.norm1.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.5.norm2.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.5.norm2.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.5.attn.qkv.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.5.attn.qkv.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.5.attn.proj.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.5.attn.proj.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.5.mlp.linear_fc1.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.5.mlp.linear_fc1.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.5.mlp.linear_fc2.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.5.mlp.linear_fc2.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.6.norm1.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.6.norm1.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.6.norm2.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.6.norm2.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.6.attn.qkv.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.6.attn.qkv.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.6.attn.proj.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.6.attn.proj.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.6.mlp.linear_fc1.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.6.mlp.linear_fc1.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.6.mlp.linear_fc2.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.6.mlp.linear_fc2.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.7.norm1.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.7.norm1.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.7.norm2.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.7.norm2.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.7.attn.qkv.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.7.attn.qkv.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.7.attn.proj.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.7.attn.proj.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.7.mlp.linear_fc1.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.7.mlp.linear_fc1.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.7.mlp.linear_fc2.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.7.mlp.linear_fc2.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.8.norm1.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.8.norm1.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.8.norm2.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.8.norm2.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.8.attn.qkv.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.8.attn.qkv.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.8.attn.proj.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.8.attn.proj.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.8.mlp.linear_fc1.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.8.mlp.linear_fc1.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.8.mlp.linear_fc2.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.8.mlp.linear_fc2.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.9.norm1.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.9.norm1.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.9.norm2.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.9.norm2.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.9.attn.qkv.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.9.attn.qkv.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.9.attn.proj.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.9.attn.proj.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.9.mlp.linear_fc1.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.9.mlp.linear_fc1.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.9.mlp.linear_fc2.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.9.mlp.linear_fc2.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.10.norm1.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.10.norm1.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.10.norm2.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.10.norm2.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.10.attn.qkv.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.10.attn.qkv.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.10.attn.proj.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.10.attn.proj.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.10.mlp.linear_fc1.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.10.mlp.linear_fc1.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.10.mlp.linear_fc2.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.10.mlp.linear_fc2.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.11.norm1.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.11.norm1.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.11.norm2.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.11.norm2.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.11.attn.qkv.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.11.attn.qkv.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.11.attn.proj.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.11.attn.proj.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.11.mlp.linear_fc1.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.11.mlp.linear_fc1.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.11.mlp.linear_fc2.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.11.mlp.linear_fc2.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.12.norm1.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.12.norm1.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.12.norm2.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.12.norm2.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.12.attn.qkv.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.12.attn.qkv.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.12.attn.proj.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.12.attn.proj.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.12.mlp.linear_fc1.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.12.mlp.linear_fc1.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.12.mlp.linear_fc2.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.12.mlp.linear_fc2.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.13.norm1.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.13.norm1.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.13.norm2.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.13.norm2.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.13.attn.qkv.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.13.attn.qkv.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.13.attn.proj.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.13.attn.proj.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.13.mlp.linear_fc1.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.13.mlp.linear_fc1.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.13.mlp.linear_fc2.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.13.mlp.linear_fc2.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.14.norm1.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.14.norm1.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.14.norm2.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.14.norm2.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.14.attn.qkv.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.14.attn.qkv.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.14.attn.proj.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.14.attn.proj.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.14.mlp.linear_fc1.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.14.mlp.linear_fc1.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.14.mlp.linear_fc2.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.14.mlp.linear_fc2.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.15.norm1.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.15.norm1.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.15.norm2.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.15.norm2.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.15.attn.qkv.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.15.attn.qkv.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.15.attn.proj.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.15.attn.proj.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.15.mlp.linear_fc1.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.15.mlp.linear_fc1.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.15.mlp.linear_fc2.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.15.mlp.linear_fc2.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.16.norm1.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.16.norm1.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.16.norm2.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.16.norm2.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.16.attn.qkv.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.16.attn.qkv.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.16.attn.proj.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.16.attn.proj.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.16.mlp.linear_fc1.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.16.mlp.linear_fc1.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.16.mlp.linear_fc2.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.16.mlp.linear_fc2.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.17.norm1.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.17.norm1.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.17.norm2.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.17.norm2.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.17.attn.qkv.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.17.attn.qkv.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.17.attn.proj.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.17.attn.proj.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.17.mlp.linear_fc1.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.17.mlp.linear_fc1.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.17.mlp.linear_fc2.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.17.mlp.linear_fc2.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.18.norm1.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.18.norm1.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.18.norm2.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.18.norm2.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.18.attn.qkv.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.18.attn.qkv.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.18.attn.proj.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.18.attn.proj.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.18.mlp.linear_fc1.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.18.mlp.linear_fc1.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.18.mlp.linear_fc2.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.18.mlp.linear_fc2.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.19.norm1.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.19.norm1.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.19.norm2.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.19.norm2.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.19.attn.qkv.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.19.attn.qkv.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.19.attn.proj.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.19.attn.proj.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.19.mlp.linear_fc1.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.19.mlp.linear_fc1.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.19.mlp.linear_fc2.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.19.mlp.linear_fc2.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.20.norm1.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.20.norm1.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.20.norm2.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.20.norm2.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.20.attn.qkv.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.20.attn.qkv.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.20.attn.proj.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.20.attn.proj.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.20.mlp.linear_fc1.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.20.mlp.linear_fc1.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.20.mlp.linear_fc2.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.20.mlp.linear_fc2.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.21.norm1.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.21.norm1.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.21.norm2.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.21.norm2.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.21.attn.qkv.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.21.attn.qkv.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.21.attn.proj.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.21.attn.proj.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.21.mlp.linear_fc1.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.21.mlp.linear_fc1.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.21.mlp.linear_fc2.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.21.mlp.linear_fc2.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.22.norm1.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.22.norm1.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.22.norm2.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.22.norm2.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.22.attn.qkv.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.22.attn.qkv.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.22.attn.proj.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.22.attn.proj.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.22.mlp.linear_fc1.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.22.mlp.linear_fc1.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.22.mlp.linear_fc2.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.22.mlp.linear_fc2.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.23.norm1.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.23.norm1.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.23.norm2.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.23.norm2.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.23.attn.qkv.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.23.attn.qkv.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.23.attn.proj.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.23.attn.proj.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.23.mlp.linear_fc1.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.23.mlp.linear_fc1.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.23.mlp.linear_fc2.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.23.mlp.linear_fc2.bias": "model-00001-of-00002.safetensors", + "model.visual.merger.norm.weight": "model-00001-of-00002.safetensors", + "model.visual.merger.norm.bias": "model-00001-of-00002.safetensors", + "model.visual.merger.linear_fc1.weight": "model-00001-of-00002.safetensors", + "model.visual.merger.linear_fc1.bias": "model-00001-of-00002.safetensors", + "model.visual.merger.linear_fc2.weight": "model-00001-of-00002.safetensors", + "model.visual.merger.linear_fc2.bias": "model-00001-of-00002.safetensors", + "model.language_model.layers.0.linear_attn.dt_bias": "model-00001-of-00002.safetensors", + "model.language_model.layers.0.linear_attn.A_log": "model-00001-of-00002.safetensors", + "model.language_model.layers.0.linear_attn.conv1d.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.0.linear_attn.norm.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.0.linear_attn.out_proj.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.0.linear_attn.in_proj_qkv.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.0.linear_attn.in_proj_z.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.0.linear_attn.in_proj_b.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.0.linear_attn.in_proj_a.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.0.input_layernorm.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.0.mlp.gate_proj.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.0.mlp.up_proj.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.0.mlp.down_proj.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.0.post_attention_layernorm.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.1.linear_attn.dt_bias": "model-00001-of-00002.safetensors", + "model.language_model.layers.1.linear_attn.A_log": "model-00001-of-00002.safetensors", + "model.language_model.layers.1.linear_attn.conv1d.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.1.linear_attn.norm.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.1.linear_attn.out_proj.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.1.linear_attn.in_proj_qkv.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.1.linear_attn.in_proj_z.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.1.linear_attn.in_proj_b.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.1.linear_attn.in_proj_a.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.1.input_layernorm.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.1.mlp.gate_proj.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.1.mlp.up_proj.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.1.mlp.down_proj.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.1.post_attention_layernorm.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.2.linear_attn.dt_bias": "model-00001-of-00002.safetensors", + "model.language_model.layers.2.linear_attn.A_log": "model-00001-of-00002.safetensors", + "model.language_model.layers.2.linear_attn.conv1d.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.2.linear_attn.norm.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.2.linear_attn.out_proj.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.2.linear_attn.in_proj_qkv.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.2.linear_attn.in_proj_z.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.2.linear_attn.in_proj_b.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.2.linear_attn.in_proj_a.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.2.input_layernorm.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.2.mlp.gate_proj.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.2.mlp.up_proj.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.2.mlp.down_proj.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.2.post_attention_layernorm.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.3.self_attn.q_proj.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.3.self_attn.k_proj.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.3.self_attn.v_proj.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.3.self_attn.o_proj.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.3.self_attn.q_norm.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.3.self_attn.k_norm.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.3.input_layernorm.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.3.mlp.gate_proj.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.3.mlp.up_proj.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.3.mlp.down_proj.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.3.post_attention_layernorm.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.4.linear_attn.dt_bias": "model-00001-of-00002.safetensors", + "model.language_model.layers.4.linear_attn.A_log": "model-00001-of-00002.safetensors", + "model.language_model.layers.4.linear_attn.conv1d.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.4.linear_attn.norm.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.4.linear_attn.out_proj.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.4.linear_attn.in_proj_qkv.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.4.linear_attn.in_proj_z.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.4.linear_attn.in_proj_b.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.4.linear_attn.in_proj_a.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.4.input_layernorm.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.4.mlp.gate_proj.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.4.mlp.up_proj.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.4.mlp.down_proj.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.4.post_attention_layernorm.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.5.linear_attn.dt_bias": "model-00001-of-00002.safetensors", + "model.language_model.layers.5.linear_attn.A_log": "model-00001-of-00002.safetensors", + "model.language_model.layers.5.linear_attn.conv1d.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.5.linear_attn.norm.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.5.linear_attn.out_proj.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.5.linear_attn.in_proj_qkv.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.5.linear_attn.in_proj_z.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.5.linear_attn.in_proj_b.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.5.linear_attn.in_proj_a.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.5.input_layernorm.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.5.mlp.gate_proj.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.5.mlp.up_proj.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.5.mlp.down_proj.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.5.post_attention_layernorm.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.6.linear_attn.dt_bias": "model-00001-of-00002.safetensors", + "model.language_model.layers.6.linear_attn.A_log": "model-00001-of-00002.safetensors", + "model.language_model.layers.6.linear_attn.conv1d.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.6.linear_attn.norm.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.6.linear_attn.out_proj.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.6.linear_attn.in_proj_qkv.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.6.linear_attn.in_proj_z.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.6.linear_attn.in_proj_b.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.6.linear_attn.in_proj_a.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.6.input_layernorm.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.6.mlp.gate_proj.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.6.mlp.up_proj.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.6.mlp.down_proj.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.6.post_attention_layernorm.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.7.self_attn.q_proj.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.7.self_attn.k_proj.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.7.self_attn.v_proj.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.7.self_attn.o_proj.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.7.self_attn.q_norm.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.7.self_attn.k_norm.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.7.input_layernorm.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.7.mlp.gate_proj.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.7.mlp.up_proj.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.7.mlp.down_proj.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.7.post_attention_layernorm.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.8.linear_attn.dt_bias": "model-00001-of-00002.safetensors", + "model.language_model.layers.8.linear_attn.A_log": "model-00001-of-00002.safetensors", + "model.language_model.layers.8.linear_attn.conv1d.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.8.linear_attn.norm.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.8.linear_attn.out_proj.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.8.linear_attn.in_proj_qkv.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.8.linear_attn.in_proj_z.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.8.linear_attn.in_proj_b.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.8.linear_attn.in_proj_a.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.8.input_layernorm.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.8.mlp.gate_proj.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.8.mlp.up_proj.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.8.mlp.down_proj.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.8.post_attention_layernorm.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.9.linear_attn.dt_bias": "model-00001-of-00002.safetensors", + "model.language_model.layers.9.linear_attn.A_log": "model-00001-of-00002.safetensors", + "model.language_model.layers.9.linear_attn.conv1d.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.9.linear_attn.norm.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.9.linear_attn.out_proj.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.9.linear_attn.in_proj_qkv.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.9.linear_attn.in_proj_z.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.9.linear_attn.in_proj_b.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.9.linear_attn.in_proj_a.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.9.input_layernorm.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.9.mlp.gate_proj.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.9.mlp.up_proj.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.9.mlp.down_proj.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.9.post_attention_layernorm.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.10.linear_attn.dt_bias": "model-00001-of-00002.safetensors", + "model.language_model.layers.10.linear_attn.A_log": "model-00001-of-00002.safetensors", + "model.language_model.layers.10.linear_attn.conv1d.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.10.linear_attn.norm.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.10.linear_attn.out_proj.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.10.linear_attn.in_proj_qkv.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.10.linear_attn.in_proj_z.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.10.linear_attn.in_proj_b.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.10.linear_attn.in_proj_a.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.10.input_layernorm.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.10.mlp.gate_proj.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.10.mlp.up_proj.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.10.mlp.down_proj.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.10.post_attention_layernorm.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.11.self_attn.q_proj.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.11.self_attn.k_proj.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.11.self_attn.v_proj.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.11.self_attn.o_proj.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.11.self_attn.q_norm.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.11.self_attn.k_norm.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.11.input_layernorm.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.11.mlp.gate_proj.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.11.mlp.up_proj.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.11.mlp.down_proj.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.11.post_attention_layernorm.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.12.linear_attn.dt_bias": "model-00001-of-00002.safetensors", + "model.language_model.layers.12.linear_attn.A_log": "model-00001-of-00002.safetensors", + "model.language_model.layers.12.linear_attn.conv1d.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.12.linear_attn.norm.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.12.linear_attn.out_proj.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.12.linear_attn.in_proj_qkv.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.12.linear_attn.in_proj_z.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.12.linear_attn.in_proj_b.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.12.linear_attn.in_proj_a.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.12.input_layernorm.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.12.mlp.gate_proj.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.12.mlp.up_proj.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.12.mlp.down_proj.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.12.post_attention_layernorm.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.13.linear_attn.dt_bias": "model-00001-of-00002.safetensors", + "model.language_model.layers.13.linear_attn.A_log": "model-00001-of-00002.safetensors", + "model.language_model.layers.13.linear_attn.conv1d.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.13.linear_attn.norm.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.13.linear_attn.out_proj.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.13.linear_attn.in_proj_qkv.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.13.linear_attn.in_proj_z.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.13.linear_attn.in_proj_b.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.13.linear_attn.in_proj_a.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.13.input_layernorm.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.13.mlp.gate_proj.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.13.mlp.up_proj.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.13.mlp.down_proj.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.13.post_attention_layernorm.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.14.linear_attn.dt_bias": "model-00002-of-00002.safetensors", + "model.language_model.layers.14.linear_attn.A_log": "model-00002-of-00002.safetensors", + "model.language_model.layers.14.linear_attn.conv1d.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.14.linear_attn.norm.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.14.linear_attn.out_proj.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.14.linear_attn.in_proj_qkv.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.14.linear_attn.in_proj_z.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.14.linear_attn.in_proj_b.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.14.linear_attn.in_proj_a.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.14.input_layernorm.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.14.mlp.gate_proj.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.14.mlp.up_proj.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.14.mlp.down_proj.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.14.post_attention_layernorm.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.15.self_attn.q_proj.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.15.self_attn.k_proj.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.15.self_attn.v_proj.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.15.self_attn.o_proj.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.15.self_attn.q_norm.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.15.self_attn.k_norm.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.15.input_layernorm.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.15.mlp.gate_proj.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.15.mlp.up_proj.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.15.mlp.down_proj.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.15.post_attention_layernorm.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.16.linear_attn.dt_bias": "model-00002-of-00002.safetensors", + "model.language_model.layers.16.linear_attn.A_log": "model-00002-of-00002.safetensors", + "model.language_model.layers.16.linear_attn.conv1d.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.16.linear_attn.norm.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.16.linear_attn.out_proj.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.16.linear_attn.in_proj_qkv.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.16.linear_attn.in_proj_z.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.16.linear_attn.in_proj_b.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.16.linear_attn.in_proj_a.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.16.input_layernorm.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.16.mlp.gate_proj.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.16.mlp.up_proj.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.16.mlp.down_proj.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.16.post_attention_layernorm.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.17.linear_attn.dt_bias": "model-00002-of-00002.safetensors", + "model.language_model.layers.17.linear_attn.A_log": "model-00002-of-00002.safetensors", + "model.language_model.layers.17.linear_attn.conv1d.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.17.linear_attn.norm.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.17.linear_attn.out_proj.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.17.linear_attn.in_proj_qkv.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.17.linear_attn.in_proj_z.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.17.linear_attn.in_proj_b.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.17.linear_attn.in_proj_a.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.17.input_layernorm.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.17.mlp.gate_proj.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.17.mlp.up_proj.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.17.mlp.down_proj.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.17.post_attention_layernorm.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.18.linear_attn.dt_bias": "model-00002-of-00002.safetensors", + "model.language_model.layers.18.linear_attn.A_log": "model-00002-of-00002.safetensors", + "model.language_model.layers.18.linear_attn.conv1d.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.18.linear_attn.norm.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.18.linear_attn.out_proj.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.18.linear_attn.in_proj_qkv.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.18.linear_attn.in_proj_z.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.18.linear_attn.in_proj_b.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.18.linear_attn.in_proj_a.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.18.input_layernorm.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.18.mlp.gate_proj.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.18.mlp.up_proj.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.18.mlp.down_proj.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.18.post_attention_layernorm.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.19.self_attn.q_proj.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.19.self_attn.k_proj.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.19.self_attn.v_proj.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.19.self_attn.o_proj.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.19.self_attn.q_norm.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.19.self_attn.k_norm.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.19.input_layernorm.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.19.mlp.gate_proj.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.19.mlp.up_proj.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.19.mlp.down_proj.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.19.post_attention_layernorm.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.20.linear_attn.dt_bias": "model-00002-of-00002.safetensors", + "model.language_model.layers.20.linear_attn.A_log": "model-00002-of-00002.safetensors", + "model.language_model.layers.20.linear_attn.conv1d.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.20.linear_attn.norm.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.20.linear_attn.out_proj.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.20.linear_attn.in_proj_qkv.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.20.linear_attn.in_proj_z.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.20.linear_attn.in_proj_b.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.20.linear_attn.in_proj_a.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.20.input_layernorm.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.20.mlp.gate_proj.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.20.mlp.up_proj.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.20.mlp.down_proj.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.20.post_attention_layernorm.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.21.linear_attn.dt_bias": "model-00002-of-00002.safetensors", + "model.language_model.layers.21.linear_attn.A_log": "model-00002-of-00002.safetensors", + "model.language_model.layers.21.linear_attn.conv1d.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.21.linear_attn.norm.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.21.linear_attn.out_proj.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.21.linear_attn.in_proj_qkv.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.21.linear_attn.in_proj_z.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.21.linear_attn.in_proj_b.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.21.linear_attn.in_proj_a.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.21.input_layernorm.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.21.mlp.gate_proj.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.21.mlp.up_proj.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.21.mlp.down_proj.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.21.post_attention_layernorm.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.22.linear_attn.dt_bias": "model-00002-of-00002.safetensors", + "model.language_model.layers.22.linear_attn.A_log": "model-00002-of-00002.safetensors", + "model.language_model.layers.22.linear_attn.conv1d.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.22.linear_attn.norm.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.22.linear_attn.out_proj.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.22.linear_attn.in_proj_qkv.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.22.linear_attn.in_proj_z.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.22.linear_attn.in_proj_b.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.22.linear_attn.in_proj_a.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.22.input_layernorm.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.22.mlp.gate_proj.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.22.mlp.up_proj.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.22.mlp.down_proj.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.22.post_attention_layernorm.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.23.self_attn.q_proj.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.23.self_attn.k_proj.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.23.self_attn.v_proj.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.23.self_attn.o_proj.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.23.self_attn.q_norm.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.23.self_attn.k_norm.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.23.input_layernorm.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.23.mlp.gate_proj.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.23.mlp.up_proj.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.23.mlp.down_proj.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.23.post_attention_layernorm.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.24.linear_attn.dt_bias": "model-00002-of-00002.safetensors", + "model.language_model.layers.24.linear_attn.A_log": "model-00002-of-00002.safetensors", + "model.language_model.layers.24.linear_attn.conv1d.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.24.linear_attn.norm.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.24.linear_attn.out_proj.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.24.linear_attn.in_proj_qkv.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.24.linear_attn.in_proj_z.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.24.linear_attn.in_proj_b.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.24.linear_attn.in_proj_a.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.24.input_layernorm.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.24.mlp.gate_proj.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.24.mlp.up_proj.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.24.mlp.down_proj.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.24.post_attention_layernorm.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.25.linear_attn.dt_bias": "model-00002-of-00002.safetensors", + "model.language_model.layers.25.linear_attn.A_log": "model-00002-of-00002.safetensors", + "model.language_model.layers.25.linear_attn.conv1d.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.25.linear_attn.norm.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.25.linear_attn.out_proj.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.25.linear_attn.in_proj_qkv.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.25.linear_attn.in_proj_z.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.25.linear_attn.in_proj_b.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.25.linear_attn.in_proj_a.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.25.input_layernorm.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.25.mlp.gate_proj.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.25.mlp.up_proj.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.25.mlp.down_proj.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.25.post_attention_layernorm.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.26.linear_attn.dt_bias": "model-00002-of-00002.safetensors", + "model.language_model.layers.26.linear_attn.A_log": "model-00002-of-00002.safetensors", + "model.language_model.layers.26.linear_attn.conv1d.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.26.linear_attn.norm.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.26.linear_attn.out_proj.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.26.linear_attn.in_proj_qkv.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.26.linear_attn.in_proj_z.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.26.linear_attn.in_proj_b.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.26.linear_attn.in_proj_a.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.26.input_layernorm.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.26.mlp.gate_proj.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.26.mlp.up_proj.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.26.mlp.down_proj.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.26.post_attention_layernorm.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.27.self_attn.q_proj.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.27.self_attn.k_proj.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.27.self_attn.v_proj.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.27.self_attn.o_proj.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.27.self_attn.q_norm.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.27.self_attn.k_norm.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.27.input_layernorm.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.27.mlp.gate_proj.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.27.mlp.up_proj.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.27.mlp.down_proj.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.27.post_attention_layernorm.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.28.linear_attn.dt_bias": "model-00002-of-00002.safetensors", + "model.language_model.layers.28.linear_attn.A_log": "model-00002-of-00002.safetensors", + "model.language_model.layers.28.linear_attn.conv1d.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.28.linear_attn.norm.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.28.linear_attn.out_proj.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.28.linear_attn.in_proj_qkv.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.28.linear_attn.in_proj_z.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.28.linear_attn.in_proj_b.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.28.linear_attn.in_proj_a.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.28.input_layernorm.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.28.mlp.gate_proj.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.28.mlp.up_proj.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.28.mlp.down_proj.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.28.post_attention_layernorm.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.29.linear_attn.dt_bias": "model-00002-of-00002.safetensors", + "model.language_model.layers.29.linear_attn.A_log": "model-00002-of-00002.safetensors", + "model.language_model.layers.29.linear_attn.conv1d.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.29.linear_attn.norm.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.29.linear_attn.out_proj.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.29.linear_attn.in_proj_qkv.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.29.linear_attn.in_proj_z.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.29.linear_attn.in_proj_b.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.29.linear_attn.in_proj_a.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.29.input_layernorm.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.29.mlp.gate_proj.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.29.mlp.up_proj.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.29.mlp.down_proj.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.29.post_attention_layernorm.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.30.linear_attn.dt_bias": "model-00002-of-00002.safetensors", + "model.language_model.layers.30.linear_attn.A_log": "model-00002-of-00002.safetensors", + "model.language_model.layers.30.linear_attn.conv1d.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.30.linear_attn.norm.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.30.linear_attn.out_proj.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.30.linear_attn.in_proj_qkv.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.30.linear_attn.in_proj_z.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.30.linear_attn.in_proj_b.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.30.linear_attn.in_proj_a.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.30.input_layernorm.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.30.mlp.gate_proj.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.30.mlp.up_proj.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.30.mlp.down_proj.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.30.post_attention_layernorm.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.31.self_attn.q_proj.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.31.self_attn.k_proj.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.31.self_attn.v_proj.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.31.self_attn.o_proj.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.31.self_attn.q_norm.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.31.self_attn.k_norm.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.31.input_layernorm.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.31.mlp.gate_proj.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.31.mlp.up_proj.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.31.mlp.down_proj.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.31.post_attention_layernorm.weight": "model-00002-of-00002.safetensors", + "model.language_model.norm.weight": "model-00002-of-00002.safetensors" + } +} \ No newline at end of file diff --git a/posttrain_models/qwen3_5_4b_base_highlight_batch1245_epoch4/checkpoint-1000-merged/args.json b/posttrain_models/qwen3_5_4b_base_highlight_batch1245_epoch4/checkpoint-1000-merged/args.json new file mode 100644 index 0000000000000000000000000000000000000000..66fa96327404653aa85875ca21471633385c775c --- /dev/null +++ b/posttrain_models/qwen3_5_4b_base_highlight_batch1245_epoch4/checkpoint-1000-merged/args.json @@ -0,0 +1,398 @@ +{ + "use_ray": false, + "ray_exp_name": null, + "device_groups": null, + "model": "/mnt/bn/strategy-mllm-train/user/tanyue/models/pretrained_models/Qwen3.5-4B", + "model_type": "qwen3_5", + "model_revision": null, + "task_type": "causal_lm", + "torch_dtype": "bfloat16", + "attn_impl": null, + "experts_impl": null, + "new_special_tokens": [], + "num_labels": null, + "problem_type": null, + "rope_scaling": null, + "device_map": null, + "max_memory": {}, + "max_model_len": null, + "local_repo_path": null, + "init_strategy": null, + "template": "qwen3_5", + "system": null, + "max_length": 8192, + "truncation_strategy": "delete", + "max_pixels": null, + "agent_template": null, + "norm_bbox": null, + "use_chat_template": true, + "padding_side": "right", + "padding_free": true, + "loss_scale": "ignore_empty_think", + "sequence_parallel_size": 1, + "template_backend": "swift", + "response_prefix": "", + "enable_thinking": false, + "add_non_thinking_prefix": false, + "dataset": [ + "batch1", + "batch2", + "batch4_part1", + "batch4_part2", + "batch4_part3", + "batch5_part1", + "batch5_part2", + "batch5_part3" + ], + "val_dataset": [], + "cached_dataset": [], + "cached_val_dataset": [], + "split_dataset_ratio": 0.0, + "data_seed": 42, + "dataset_num_proc": 8, + "load_from_cache_file": false, + "dataset_shuffle": true, + "val_dataset_shuffle": false, + "streaming": false, + "interleave_prob": null, + "stopping_strategy": "first_exhausted", + "shuffle_buffer_size": 1000, + "download_mode": "reuse_dataset_if_exists", + "columns": {}, + "strict": false, + "remove_unused_columns": true, + "model_name": null, + "model_author": null, + "custom_dataset_info": [], + "quant_method": null, + "quant_bits": null, + "hqq_axis": null, + "bnb_4bit_compute_dtype": "bfloat16", + "bnb_4bit_quant_type": "nf4", + "bnb_4bit_use_double_quant": true, + "bnb_4bit_quant_storage": null, + "max_new_tokens": null, + "temperature": 0.9, + "top_k": 50, + "top_p": 0.9, + "repetition_penalty": 1.0, + "num_beams": 1, + "stream": false, + "stop_words": [], + "logprobs": false, + "top_logprobs": null, + "structured_outputs_regex": null, + "tuner_backend": "peft", + "tuner_type": "lora", + "train_type": "lora", + "adapters": [], + "external_plugins": [ + "ms-swift/base_highlight_video_dataset.py" + ], + "custom_register_path": [ + "ms-swift/base_highlight_video_dataset.py" + ], + "seed": 42, + "model_kwargs": {}, + "load_args": false, + "load_data_args": false, + "packing": false, + "packing_length": null, + "packing_num_proc": 1, + "lazy_tokenize": true, + "use_hf": false, + "hub_token": null, + "ddp_timeout": 18000000, + "ddp_backend": "nccl", + "ignore_args_error": false, + "use_swift_lora": false, + "freeze_llm": false, + "freeze_vit": true, + "freeze_aligner": true, + "freeze_parameters": [], + "freeze_parameters_regex": null, + "freeze_parameters_ratio": 0.0, + "trainable_parameters": [], + "trainable_parameters_regex": null, + "target_modules": [ + "all-linear" + ], + "target_regex": null, + "modules_to_save": [], + "lora_rank": 8, + "lora_alpha": 32, + "lora_dropout": 0.05, + "lora_bias": "none", + "lora_dtype": null, + "use_rslora": false, + "rlhf_type": null, + "loss_type": null, + "mcore_ref_model": null, + "mcore_ref_adapter": null, + "beta": null, + "rpo_alpha": null, + "reference_free": false, + "label_smoothing": 0.0, + "f_divergence_type": "reverse_kl", + "desirable_weight": 1.0, + "undesirable_weight": 1.0, + "calculate_KL": null, + "center_rewards_coefficient": null, + "teacher_model": null, + "teacher_model_type": null, + "teacher_model_revision": null, + "teacher_model_server": null, + "gkd_logits_topk": null, + "lmbda": 0.5, + "seq_kd": false, + "offload_teacher_model": false, + "sft_alpha": 0.0, + "generation_batch_size": null, + "steps_per_generation": null, + "num_generations": 8, + "num_generations_eval": null, + "max_completion_length": 512, + "importance_sampling_level": "token", + "tau_pos": 1.0, + "tau_neg": 1.05, + "epsilon": 0.2, + "epsilon_high": null, + "delta": null, + "use_vllm": true, + "vllm_mode": null, + "vllm_enable_prefix_caching": true, + "vllm_gpu_memory_utilization": 0.9, + "vllm_tensor_parallel_size": 1, + "vllm_max_model_len": null, + "vllm_enforce_eager": false, + "vllm_limit_mm_per_prompt": null, + "vllm_disable_cascade_attn": false, + "vllm_max_num_seqs": null, + "vllm_mm_processor_cache_gb": null, + "vllm_engine_kwargs": null, + "sleep_level": 0, + "offload_optimizer": false, + "offload_model": false, + "offload_bridge": false, + "vllm_server_base_url": null, + "vllm_server_host": null, + "vllm_server_port": [ + 8000 + ], + "vllm_server_timeout": 240.0, + "vllm_server_group_port": null, + "reward_funcs": [], + "reward_weights": null, + "cosine_min_len_value_wrong": -0.5, + "cosine_max_len_value_wrong": 0.0, + "cosine_min_len_value_correct": 1.0, + "cosine_max_len_value_correct": 0.5, + "cosine_max_len": null, + "repetition_n_grams": 3, + "repetition_max_penalty": -1.0, + "soft_max_length": null, + "soft_cache_length": null, + "dynamic_sample": false, + "max_resample_times": 3, + "overlong_filter": false, + "scale_rewards": "group", + "advantage_estimator": "grpo", + "kl_in_reward": false, + "wandb_log_unique_prompts": null, + "log_completions": false, + "rollout_importance_sampling_mode": null, + "rollout_importance_sampling_threshold": 2.0, + "log_rollout_offpolicy_metrics": false, + "off_policy_sequence_mask_delta": null, + "log_entropy": false, + "top_entropy_quantile": 1.0, + "reward_model": null, + "reward_model_plugin": null, + "sync_ref_model": false, + "ref_model_sync_steps": 512, + "ref_model_mixup_alpha": 0.6, + "async_generate": false, + "move_model_batches": null, + "multi_turn_scheduler": null, + "max_turns": null, + "completion_length_limit_scope": "per_round", + "vllm_server_pass_dataset": false, + "num_iterations": 1, + "micro_batch_size": 8, + "global_batch_size": 64, + "recompute_granularity": "full", + "recompute_method": "uniform", + "recompute_num_layers": 1, + "recompute_modules": [ + "core_attn" + ], + "train_iters": null, + "num_train_epochs": 4, + "masked_softmax_fusion": true, + "bias_dropout_fusion": true, + "bias_activation_fusion": true, + "apply_rope_fusion": false, + "gradient_accumulation_fusion": false, + "cross_entropy_loss_fusion": true, + "cross_entropy_fusion_impl": "te", + "calculate_per_token_loss": true, + "attention_backend": "", + "optimizer": "adam", + "optimizer_cpu_offload": false, + "optimizer_offload_fraction": 1.0, + "use_precision_aware_optimizer": false, + "main_grads_dtype": "float32", + "main_params_dtype": "float32", + "exp_avg_dtype": "float32", + "exp_avg_sq_dtype": "float32", + "manual_gc": false, + "manual_gc_steps": 0, + "manual_gc_eval": true, + "train_dataloader_shuffle": true, + "dataloader_num_workers": 8, + "dataloader_pin_memory": true, + "dataloader_persistent_workers": true, + "dataloader_prefetch_factor": 2, + "data_sharding": false, + "group_by_length": false, + "te_rng_tracker": false, + "data_parallel_random_init": false, + "mlp_padding_free": false, + "lr_warmup_init": 0.0, + "lr": 0.0001, + "lr_decay_style": "cosine", + "lr_decay_iters": null, + "lr_warmup_iters": 0, + "lr_warmup_fraction": 0.05, + "min_lr": 1e-05, + "lr_wsd_decay_style": "exponential", + "lr_wsd_decay_iters": null, + "weight_decay": 0.1, + "weight_decay_incr_style": "constant", + "start_weight_decay": 0.1, + "end_weight_decay": 0.1, + "clip_grad": 1.0, + "adam_beta1": 0.9, + "adam_beta2": 0.95, + "adam_eps": 1e-08, + "sgd_momentum": 0.9, + "output_dir": "/mnt/bn/strategy-mllm-train/user/tanyue/models/posttrain_models/qwen3_5_4b_base_highlight_batch1245_epoch4", + "save_steps": 500, + "no_save_optim": true, + "no_save_rng": true, + "mcore_model": null, + "mcore_adapter": null, + "no_load_optim": false, + "no_load_rng": false, + "finetune": true, + "perform_initialization": false, + "use_cpu_initialization": false, + "async_save": false, + "save_total_limit": null, + "metric_for_best_model": "loss", + "greater_is_better": false, + "use_persistent_ckpt_worker": false, + "dist_ckpt_save_pre_mcore_014": false, + "dist_ckpt_optim_fully_reshardable": false, + "distrib_optim_fully_reshardable_mem_efficient": false, + "local_rank": 0, + "use_distributed_optimizer": true, + "tensor_model_parallel_size": 1, + "pipeline_model_parallel_size": 2, + "decoder_first_pipeline_num_layers": null, + "decoder_last_pipeline_num_layers": null, + "account_for_embedding_in_pipeline_split": false, + "account_for_loss_in_pipeline_split": false, + "overlap_p2p_comm": false, + "align_param_gather": false, + "sequence_parallel": false, + "context_parallel_size": 1, + "tp_comm_overlap": false, + "overlap_grad_reduce": false, + "overlap_param_gather": false, + "overlap_param_gather_with_optimizer_step": false, + "align_grad_reduce": true, + "virtual_pipeline_model_parallel_size": null, + "microbatch_group_size_per_vp_stage": null, + "pipeline_model_parallel_layout": null, + "expert_model_parallel_size": 4, + "expert_tensor_parallel_size": 1, + "report_to": [ + "wandb" + ], + "logging_steps": 5, + "tensorboard_dir": "/mnt/bn/strategy-mllm-train/user/tanyue/models/posttrain_models/qwen3_5_4b_base_highlight_batch1245_epoch4/runs", + "tensorboard_queue_size": 50, + "wandb_project": "highlight_base_highlight_qwen3_5", + "wandb_exp_name": "qwen3_5_4b_base_highlight_batch1245_epoch4", + "swanlab_project": "megatron-swift", + "swanlab_exp_name": null, + "eval_iters": -1, + "eval_steps": 100000, + "fp8_format": null, + "fp8_recipe": "delayed", + "fp8_amax_history_len": 1024, + "fp8_amax_compute_algo": "max", + "fp8_param_gather": false, + "fp16": false, + "bf16": true, + "apply_query_key_layer_scaling": false, + "attention_softmax_in_fp32": true, + "accumulate_allreduce_grads_in_fp32": true, + "moe_router_load_balancing_type": null, + "moe_router_dtype": "fp32", + "moe_token_dispatcher_type": null, + "moe_enable_deepep": false, + "moe_grouped_gemm": true, + "moe_permute_fusion": true, + "moe_aux_loss_coeff": 1e-06, + "moe_z_loss_coeff": null, + "moe_shared_expert_overlap": true, + "moe_layer_recompute": false, + "moe_expert_capacity_factor": null, + "moe_pad_expert_input_to_capacity": false, + "moe_token_drop_policy": null, + "mtp_num_layers": null, + "mtp_loss_scaling_factor": 0.1, + "save_safetensors": true, + "ref_model": null, + "ref_adapters": [], + "merge_lora": true, + "max_shard_size": "5GB", + "vit_gradient_checkpointing": false, + "vit_lr": null, + "aligner_lr": null, + "gradient_checkpointing_kwargs": null, + "check_model": true, + "apply_wd_to_qk_layernorm": false, + "enable_dft_loss": false, + "enable_channel_loss": false, + "save_strategy": "steps", + "callbacks": [ + "print", + "default_flow", + "wandb" + ], + "add_version": false, + "create_checkpoint_symlink": false, + "swift_version": "4.0.1", + "ckpt_dir": null, + "rank": 0, + "global_world_size": 8, + "local_world_size": 8, + "model_suffix": "Qwen3.5-4B", + "model_info": "ModelInfo(model_type='qwen3_5', model_dir='/mnt/bn/strategy-mllm-train/user/tanyue/models/pretrained_models/Qwen3.5-4B', torch_dtype=torch.bfloat16, max_model_len=262144, quant_method=None, quant_bits=None, rope_scaling=None, is_moe_model=False, is_multimodal=True, config=None, task_type='causal_lm', num_labels=None)", + "model_meta": "ModelMeta(model_type='qwen3_5', model_groups=[ModelGroup(models=[Model(ms_model_id='Qwen/Qwen3.5-0.8B', hf_model_id='Qwen/Qwen3.5-0.8B', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen3.5-2B', hf_model_id='Qwen/Qwen3.5-2B', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen3.5-4B', hf_model_id='Qwen/Qwen3.5-4B', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen3.5-9B', hf_model_id='Qwen/Qwen3.5-9B', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen3.5-27B', hf_model_id='Qwen/Qwen3.5-27B', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen3.5-27B-FP8', hf_model_id='Qwen/Qwen3.5-27B-FP8', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen3.5-0.8B-Base', hf_model_id='Qwen/Qwen3.5-0.8B-Base', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen3.5-2B-Base', hf_model_id='Qwen/Qwen3.5-2B-Base', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen3.5-4B-Base', hf_model_id='Qwen/Qwen3.5-4B-Base', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen3.5-9B-Base', hf_model_id='Qwen/Qwen3.5-9B-Base', model_path=None, ms_revision=None, hf_revision=None)], template='qwen3_5', ignore_patterns=None, requires=None, tags=[])], loader=, template='qwen3_5', model_arch=MultiModelKeys(arch_name='qwen2_vl', embedding=None, module_list=None, lm_head=None, q_proj=None, k_proj=None, v_proj=None, o_proj=None, attention=None, mlp=None, down_proj=None, qkv_proj=None, qk_proj=None, qa_proj=None, qb_proj=None, kv_proj=None, kva_proj=None, kvb_proj=None, language_model=['model.language_model', 'lm_head'], aligner=['model.visual.merger'], vision_tower=['model.visual'], generator=[]), architectures=['Qwen3_5ForConditionalGeneration'], additional_saved_files=[], torch_dtype=None, is_multimodal=True, is_reward=False, task_type=None, ignore_patterns=None, requires=['transformers>=5.0.0.dev', 'qwen_vl_utils>=0.0.14', 'decord'], tags=[])", + "model_dir": "/mnt/bn/strategy-mllm-train/user/tanyue/models/pretrained_models/Qwen3.5-4B", + "template_meta": "QwenTemplateMeta(template_type='qwen3_5', prefix=[], prompt=['<|im_start|>user\\n{{QUERY}}<|im_end|>\\n<|im_start|>assistant\\n'], chat_sep=['<|im_end|>\\n'], suffix=['<|im_end|>\\n'], template_cls=, system_prefix=['<|im_start|>system\\n{{SYSTEM}}<|im_end|>\\n'], default_system=None, auto_add_bos=False, stop_words=['<|endoftext|>'], agent_template='qwen3_5', is_thinking=True, thinking_prefix='\\n', non_thinking_prefix='\\n\\n\\n\\n', history_thinking_prefix='')", + "_val_dataset_exists": false, + "hub": "", + "seq_length": 8192, + "is_multimodal": true, + "megatron_model_meta": "MegatronModelMeta(megatron_model_type='qwen3_5', model_types=['qwen3_5', 'qwen3_5_moe'], bridge_cls=, visual_cls=, is_multimodal=True, loader=)", + "fp8": null, + "params_dtype": "bfloat16", + "world_size": 8, + "data_parallel_size": 4, + "num_microbatches": 2 +} \ No newline at end of file diff --git a/posttrain_models/qwen3_5_4b_base_highlight_batch1245_epoch4/checkpoint-1000-merged/chat_template.jinja b/posttrain_models/qwen3_5_4b_base_highlight_batch1245_epoch4/checkpoint-1000-merged/chat_template.jinja new file mode 100644 index 0000000000000000000000000000000000000000..a585dec894e63da457d9440ec6aa7caa16d20860 --- /dev/null +++ b/posttrain_models/qwen3_5_4b_base_highlight_batch1245_epoch4/checkpoint-1000-merged/chat_template.jinja @@ -0,0 +1,154 @@ +{%- set image_count = namespace(value=0) %} +{%- set video_count = namespace(value=0) %} +{%- macro render_content(content, do_vision_count, is_system_content=false) %} + {%- if content is string %} + {{- content }} + {%- elif content is iterable and content is not mapping %} + {%- for item in content %} + {%- if 'image' in item or 'image_url' in item or item.type == 'image' %} + {%- if is_system_content %} + {{- raise_exception('System message cannot contain images.') }} + {%- endif %} + {%- if do_vision_count %} + {%- set image_count.value = image_count.value + 1 %} + {%- endif %} + {%- if add_vision_id %} + {{- 'Picture ' ~ image_count.value ~ ': ' }} + {%- endif %} + {{- '<|vision_start|><|image_pad|><|vision_end|>' }} + {%- elif 'video' in item or item.type == 'video' %} + {%- if is_system_content %} + {{- raise_exception('System message cannot contain videos.') }} + {%- endif %} + {%- if do_vision_count %} + {%- set video_count.value = video_count.value + 1 %} + {%- endif %} + {%- if add_vision_id %} + {{- 'Video ' ~ video_count.value ~ ': ' }} + {%- endif %} + {{- '<|vision_start|><|video_pad|><|vision_end|>' }} + {%- elif 'text' in item %} + {{- item.text }} + {%- else %} + {{- raise_exception('Unexpected item type in content.') }} + {%- endif %} + {%- endfor %} + {%- elif content is none or content is undefined %} + {{- '' }} + {%- else %} + {{- raise_exception('Unexpected content type.') }} + {%- endif %} +{%- endmacro %} +{%- if not messages %} + {{- raise_exception('No messages provided.') }} +{%- endif %} +{%- if tools and tools is iterable and tools is not mapping %} + {{- '<|im_start|>system\n' }} + {{- "# Tools\n\nYou have access to the following functions:\n\n" }} + {%- for tool in tools %} + {{- "\n" }} + {{- tool | tojson }} + {%- endfor %} + {{- "\n" }} + {{- '\n\nIf you choose to call a function ONLY reply in the following format with NO suffix:\n\n\n\n\nvalue_1\n\n\nThis is the value for the second parameter\nthat can span\nmultiple lines\n\n\n\n\n\nReminder:\n- Function calls MUST follow the specified format: an inner block must be nested within XML tags\n- Required parameters MUST be specified\n- You may provide optional reasoning for your function call in natural language BEFORE the function call, but NOT after\n- If there is no function call available, answer the question like normal with your current knowledge and do not tell the user about function calls\n' }} + {%- if messages[0].role == 'system' %} + {%- set content = render_content(messages[0].content, false, true)|trim %} + {%- if content %} + {{- '\n\n' + content }} + {%- endif %} + {%- endif %} + {{- '<|im_end|>\n' }} +{%- else %} + {%- if messages[0].role == 'system' %} + {%- set content = render_content(messages[0].content, false, true)|trim %} + {{- '<|im_start|>system\n' + content + '<|im_end|>\n' }} + {%- endif %} +{%- endif %} +{%- set ns = namespace(multi_step_tool=true, last_query_index=messages|length - 1) %} +{%- for message in messages[::-1] %} + {%- set index = (messages|length - 1) - loop.index0 %} + {%- if ns.multi_step_tool and message.role == "user" %} + {%- set content = render_content(message.content, false)|trim %} + {%- if not(content.startswith('') and content.endswith('')) %} + {%- set ns.multi_step_tool = false %} + {%- set ns.last_query_index = index %} + {%- endif %} + {%- endif %} +{%- endfor %} +{%- if ns.multi_step_tool %} + {{- raise_exception('No user query found in messages.') }} +{%- endif %} +{%- for message in messages %} + {%- set content = render_content(message.content, true)|trim %} + {%- if message.role == "system" %} + {%- if not loop.first %} + {{- raise_exception('System message must be at the beginning.') }} + {%- endif %} + {%- elif message.role == "user" %} + {{- '<|im_start|>' + message.role + '\n' + content + '<|im_end|>' + '\n' }} + {%- elif message.role == "assistant" %} + {%- set reasoning_content = '' %} + {%- if message.reasoning_content is string %} + {%- set reasoning_content = message.reasoning_content %} + {%- else %} + {%- if '' in content %} + {%- set reasoning_content = content.split('')[0].rstrip('\n').split('')[-1].lstrip('\n') %} + {%- set content = content.split('')[-1].lstrip('\n') %} + {%- endif %} + {%- endif %} + {%- set reasoning_content = reasoning_content|trim %} + {%- if loop.index0 > ns.last_query_index %} + {{- '<|im_start|>' + message.role + '\n\n' + reasoning_content + '\n\n\n' + content }} + {%- else %} + {{- '<|im_start|>' + message.role + '\n' + content }} + {%- endif %} + {%- if message.tool_calls and message.tool_calls is iterable and message.tool_calls is not mapping %} + {%- for tool_call in message.tool_calls %} + {%- if tool_call.function is defined %} + {%- set tool_call = tool_call.function %} + {%- endif %} + {%- if loop.first %} + {%- if content|trim %} + {{- '\n\n\n\n' }} + {%- else %} + {{- '\n\n' }} + {%- endif %} + {%- else %} + {{- '\n\n\n' }} + {%- endif %} + {%- if tool_call.arguments is defined %} + {%- for args_name, args_value in tool_call.arguments|items %} + {{- '\n' }} + {%- set args_value = args_value | tojson | safe if args_value is mapping or (args_value is sequence and args_value is not string) else args_value | string %} + {{- args_value }} + {{- '\n\n' }} + {%- endfor %} + {%- endif %} + {{- '\n' }} + {%- endfor %} + {%- endif %} + {{- '<|im_end|>\n' }} + {%- elif message.role == "tool" %} + {%- if loop.previtem and loop.previtem.role != "tool" %} + {{- '<|im_start|>user' }} + {%- endif %} + {{- '\n\n' }} + {{- content }} + {{- '\n' }} + {%- if not loop.last and loop.nextitem.role != "tool" %} + {{- '<|im_end|>\n' }} + {%- elif loop.last %} + {{- '<|im_end|>\n' }} + {%- endif %} + {%- else %} + {{- raise_exception('Unexpected message role.') }} + {%- endif %} +{%- endfor %} +{%- if add_generation_prompt %} + {{- '<|im_start|>assistant\n' }} + {%- if enable_thinking is defined and enable_thinking is false %} + {{- '\n\n\n\n' }} + {%- else %} + {{- '\n' }} + {%- endif %} +{%- endif %} \ No newline at end of file diff --git a/posttrain_models/qwen3_5_4b_base_highlight_batch1245_epoch4/checkpoint-1000-merged/config.json b/posttrain_models/qwen3_5_4b_base_highlight_batch1245_epoch4/checkpoint-1000-merged/config.json new file mode 100644 index 0000000000000000000000000000000000000000..80acc45308c2161d827c173628fb655d732cb6be --- /dev/null +++ b/posttrain_models/qwen3_5_4b_base_highlight_batch1245_epoch4/checkpoint-1000-merged/config.json @@ -0,0 +1,110 @@ +{ + "architectures": [ + "Qwen3_5ForConditionalGeneration" + ], + "dtype": "bfloat16", + "hidden_size": 2560, + "image_token_id": 248056, + "model_type": "qwen3_5", + "text_config": { + "attention_bias": false, + "attention_dropout": 0.0, + "attn_output_gate": true, + "bos_token_id": null, + "dtype": "bfloat16", + "eos_token_id": 248044, + "full_attention_interval": 4, + "head_dim": 256, + "hidden_act": "silu", + "hidden_size": 2560, + "initializer_range": 0.02, + "intermediate_size": 9216, + "layer_types": [ + "linear_attention", + "linear_attention", + "linear_attention", + "full_attention", + "linear_attention", + "linear_attention", + "linear_attention", + "full_attention", + "linear_attention", + "linear_attention", + "linear_attention", + "full_attention", + "linear_attention", + "linear_attention", + "linear_attention", + "full_attention", + "linear_attention", + "linear_attention", + "linear_attention", + "full_attention", + "linear_attention", + "linear_attention", + "linear_attention", + "full_attention", + "linear_attention", + "linear_attention", + "linear_attention", + "full_attention", + "linear_attention", + "linear_attention", + "linear_attention", + "full_attention" + ], + "linear_conv_kernel_dim": 4, + "linear_key_head_dim": 128, + "linear_num_key_heads": 16, + "linear_num_value_heads": 32, + "linear_value_head_dim": 128, + "mamba_ssm_dtype": "float32", + "max_position_embeddings": 262144, + "mlp_only_layers": [], + "model_type": "qwen3_5_text", + "mtp_num_hidden_layers": 1, + "mtp_use_dedicated_embeddings": false, + "num_attention_heads": 16, + "num_hidden_layers": 32, + "num_key_value_heads": 4, + "pad_token_id": null, + "partial_rotary_factor": 0.25, + "rms_norm_eps": 1e-06, + "rope_parameters": { + "mrope_interleaved": true, + "mrope_section": [ + 11, + 11, + 10 + ], + "partial_rotary_factor": 0.25, + "rope_theta": 10000000, + "rope_type": "default" + }, + "tie_word_embeddings": true, + "use_cache": true, + "vocab_size": 248320 + }, + "tie_word_embeddings": true, + "transformers_version": "5.2.0", + "video_token_id": 248057, + "vision_config": { + "deepstack_visual_indexes": [], + "depth": 24, + "dtype": "bfloat16", + "hidden_act": "gelu_pytorch_tanh", + "hidden_size": 1024, + "in_channels": 3, + "initializer_range": 0.02, + "intermediate_size": 4096, + "model_type": "qwen3_5", + "num_heads": 16, + "num_position_embeddings": 2304, + "out_hidden_size": 2560, + "patch_size": 16, + "spatial_merge_size": 2, + "temporal_patch_size": 2 + }, + "vision_end_token_id": 248054, + "vision_start_token_id": 248053 +} diff --git a/posttrain_models/qwen3_5_4b_base_highlight_batch1245_epoch4/checkpoint-1000-merged/iter_0001000/common.pt b/posttrain_models/qwen3_5_4b_base_highlight_batch1245_epoch4/checkpoint-1000-merged/iter_0001000/common.pt new file mode 100644 index 0000000000000000000000000000000000000000..fb4ae33a187df8320cbdff9e39d33cd1b1914b96 --- /dev/null +++ b/posttrain_models/qwen3_5_4b_base_highlight_batch1245_epoch4/checkpoint-1000-merged/iter_0001000/common.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:cdd4765a4217aa21a75d85673fc7d045c61f25e688a0015b54a513a738f1cb37 +size 16295 diff --git a/posttrain_models/qwen3_5_4b_base_highlight_batch1245_epoch4/checkpoint-1000-merged/latest_checkpointed_iteration.txt b/posttrain_models/qwen3_5_4b_base_highlight_batch1245_epoch4/checkpoint-1000-merged/latest_checkpointed_iteration.txt new file mode 100644 index 0000000000000000000000000000000000000000..e37d32abba426c06b752a5e53f48c595c84e9270 --- /dev/null +++ b/posttrain_models/qwen3_5_4b_base_highlight_batch1245_epoch4/checkpoint-1000-merged/latest_checkpointed_iteration.txt @@ -0,0 +1 @@ +1000 \ No newline at end of file diff --git a/posttrain_models/qwen3_5_4b_base_highlight_batch1245_epoch4/checkpoint-1000-merged/model.safetensors.index.json b/posttrain_models/qwen3_5_4b_base_highlight_batch1245_epoch4/checkpoint-1000-merged/model.safetensors.index.json new file mode 100644 index 0000000000000000000000000000000000000000..eec55fbfbf7a653dc0446bcbc9b9762cda601edb --- /dev/null +++ b/posttrain_models/qwen3_5_4b_base_highlight_batch1245_epoch4/checkpoint-1000-merged/model.safetensors.index.json @@ -0,0 +1,730 @@ +{ + "metadata": { + "total_size": 9078531072 + }, + "weight_map": { + "model.language_model.embed_tokens.weight": "model-00001-of-00002.safetensors", + "model.visual.patch_embed.proj.weight": "model-00001-of-00002.safetensors", + "model.visual.patch_embed.proj.bias": "model-00001-of-00002.safetensors", + "model.visual.pos_embed.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.0.norm1.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.0.norm1.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.0.norm2.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.0.norm2.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.0.attn.qkv.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.0.attn.qkv.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.0.attn.proj.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.0.attn.proj.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.0.mlp.linear_fc1.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.0.mlp.linear_fc1.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.0.mlp.linear_fc2.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.0.mlp.linear_fc2.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.1.norm1.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.1.norm1.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.1.norm2.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.1.norm2.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.1.attn.qkv.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.1.attn.qkv.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.1.attn.proj.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.1.attn.proj.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.1.mlp.linear_fc1.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.1.mlp.linear_fc1.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.1.mlp.linear_fc2.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.1.mlp.linear_fc2.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.2.norm1.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.2.norm1.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.2.norm2.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.2.norm2.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.2.attn.qkv.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.2.attn.qkv.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.2.attn.proj.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.2.attn.proj.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.2.mlp.linear_fc1.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.2.mlp.linear_fc1.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.2.mlp.linear_fc2.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.2.mlp.linear_fc2.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.3.norm1.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.3.norm1.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.3.norm2.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.3.norm2.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.3.attn.qkv.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.3.attn.qkv.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.3.attn.proj.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.3.attn.proj.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.3.mlp.linear_fc1.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.3.mlp.linear_fc1.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.3.mlp.linear_fc2.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.3.mlp.linear_fc2.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.4.norm1.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.4.norm1.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.4.norm2.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.4.norm2.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.4.attn.qkv.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.4.attn.qkv.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.4.attn.proj.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.4.attn.proj.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.4.mlp.linear_fc1.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.4.mlp.linear_fc1.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.4.mlp.linear_fc2.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.4.mlp.linear_fc2.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.5.norm1.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.5.norm1.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.5.norm2.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.5.norm2.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.5.attn.qkv.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.5.attn.qkv.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.5.attn.proj.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.5.attn.proj.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.5.mlp.linear_fc1.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.5.mlp.linear_fc1.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.5.mlp.linear_fc2.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.5.mlp.linear_fc2.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.6.norm1.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.6.norm1.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.6.norm2.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.6.norm2.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.6.attn.qkv.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.6.attn.qkv.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.6.attn.proj.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.6.attn.proj.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.6.mlp.linear_fc1.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.6.mlp.linear_fc1.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.6.mlp.linear_fc2.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.6.mlp.linear_fc2.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.7.norm1.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.7.norm1.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.7.norm2.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.7.norm2.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.7.attn.qkv.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.7.attn.qkv.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.7.attn.proj.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.7.attn.proj.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.7.mlp.linear_fc1.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.7.mlp.linear_fc1.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.7.mlp.linear_fc2.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.7.mlp.linear_fc2.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.8.norm1.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.8.norm1.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.8.norm2.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.8.norm2.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.8.attn.qkv.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.8.attn.qkv.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.8.attn.proj.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.8.attn.proj.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.8.mlp.linear_fc1.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.8.mlp.linear_fc1.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.8.mlp.linear_fc2.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.8.mlp.linear_fc2.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.9.norm1.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.9.norm1.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.9.norm2.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.9.norm2.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.9.attn.qkv.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.9.attn.qkv.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.9.attn.proj.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.9.attn.proj.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.9.mlp.linear_fc1.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.9.mlp.linear_fc1.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.9.mlp.linear_fc2.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.9.mlp.linear_fc2.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.10.norm1.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.10.norm1.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.10.norm2.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.10.norm2.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.10.attn.qkv.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.10.attn.qkv.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.10.attn.proj.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.10.attn.proj.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.10.mlp.linear_fc1.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.10.mlp.linear_fc1.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.10.mlp.linear_fc2.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.10.mlp.linear_fc2.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.11.norm1.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.11.norm1.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.11.norm2.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.11.norm2.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.11.attn.qkv.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.11.attn.qkv.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.11.attn.proj.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.11.attn.proj.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.11.mlp.linear_fc1.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.11.mlp.linear_fc1.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.11.mlp.linear_fc2.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.11.mlp.linear_fc2.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.12.norm1.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.12.norm1.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.12.norm2.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.12.norm2.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.12.attn.qkv.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.12.attn.qkv.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.12.attn.proj.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.12.attn.proj.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.12.mlp.linear_fc1.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.12.mlp.linear_fc1.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.12.mlp.linear_fc2.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.12.mlp.linear_fc2.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.13.norm1.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.13.norm1.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.13.norm2.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.13.norm2.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.13.attn.qkv.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.13.attn.qkv.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.13.attn.proj.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.13.attn.proj.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.13.mlp.linear_fc1.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.13.mlp.linear_fc1.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.13.mlp.linear_fc2.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.13.mlp.linear_fc2.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.14.norm1.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.14.norm1.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.14.norm2.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.14.norm2.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.14.attn.qkv.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.14.attn.qkv.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.14.attn.proj.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.14.attn.proj.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.14.mlp.linear_fc1.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.14.mlp.linear_fc1.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.14.mlp.linear_fc2.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.14.mlp.linear_fc2.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.15.norm1.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.15.norm1.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.15.norm2.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.15.norm2.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.15.attn.qkv.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.15.attn.qkv.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.15.attn.proj.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.15.attn.proj.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.15.mlp.linear_fc1.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.15.mlp.linear_fc1.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.15.mlp.linear_fc2.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.15.mlp.linear_fc2.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.16.norm1.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.16.norm1.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.16.norm2.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.16.norm2.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.16.attn.qkv.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.16.attn.qkv.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.16.attn.proj.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.16.attn.proj.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.16.mlp.linear_fc1.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.16.mlp.linear_fc1.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.16.mlp.linear_fc2.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.16.mlp.linear_fc2.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.17.norm1.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.17.norm1.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.17.norm2.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.17.norm2.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.17.attn.qkv.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.17.attn.qkv.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.17.attn.proj.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.17.attn.proj.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.17.mlp.linear_fc1.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.17.mlp.linear_fc1.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.17.mlp.linear_fc2.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.17.mlp.linear_fc2.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.18.norm1.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.18.norm1.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.18.norm2.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.18.norm2.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.18.attn.qkv.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.18.attn.qkv.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.18.attn.proj.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.18.attn.proj.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.18.mlp.linear_fc1.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.18.mlp.linear_fc1.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.18.mlp.linear_fc2.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.18.mlp.linear_fc2.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.19.norm1.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.19.norm1.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.19.norm2.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.19.norm2.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.19.attn.qkv.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.19.attn.qkv.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.19.attn.proj.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.19.attn.proj.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.19.mlp.linear_fc1.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.19.mlp.linear_fc1.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.19.mlp.linear_fc2.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.19.mlp.linear_fc2.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.20.norm1.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.20.norm1.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.20.norm2.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.20.norm2.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.20.attn.qkv.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.20.attn.qkv.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.20.attn.proj.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.20.attn.proj.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.20.mlp.linear_fc1.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.20.mlp.linear_fc1.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.20.mlp.linear_fc2.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.20.mlp.linear_fc2.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.21.norm1.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.21.norm1.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.21.norm2.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.21.norm2.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.21.attn.qkv.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.21.attn.qkv.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.21.attn.proj.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.21.attn.proj.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.21.mlp.linear_fc1.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.21.mlp.linear_fc1.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.21.mlp.linear_fc2.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.21.mlp.linear_fc2.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.22.norm1.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.22.norm1.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.22.norm2.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.22.norm2.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.22.attn.qkv.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.22.attn.qkv.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.22.attn.proj.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.22.attn.proj.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.22.mlp.linear_fc1.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.22.mlp.linear_fc1.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.22.mlp.linear_fc2.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.22.mlp.linear_fc2.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.23.norm1.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.23.norm1.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.23.norm2.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.23.norm2.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.23.attn.qkv.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.23.attn.qkv.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.23.attn.proj.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.23.attn.proj.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.23.mlp.linear_fc1.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.23.mlp.linear_fc1.bias": "model-00001-of-00002.safetensors", + "model.visual.blocks.23.mlp.linear_fc2.weight": "model-00001-of-00002.safetensors", + "model.visual.blocks.23.mlp.linear_fc2.bias": "model-00001-of-00002.safetensors", + "model.visual.merger.norm.weight": "model-00001-of-00002.safetensors", + "model.visual.merger.norm.bias": "model-00001-of-00002.safetensors", + "model.visual.merger.linear_fc1.weight": "model-00001-of-00002.safetensors", + "model.visual.merger.linear_fc1.bias": "model-00001-of-00002.safetensors", + "model.visual.merger.linear_fc2.weight": "model-00001-of-00002.safetensors", + "model.visual.merger.linear_fc2.bias": "model-00001-of-00002.safetensors", + "model.language_model.layers.0.linear_attn.dt_bias": "model-00001-of-00002.safetensors", + "model.language_model.layers.0.linear_attn.A_log": "model-00001-of-00002.safetensors", + "model.language_model.layers.0.linear_attn.conv1d.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.0.linear_attn.norm.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.0.linear_attn.out_proj.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.0.linear_attn.in_proj_qkv.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.0.linear_attn.in_proj_z.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.0.linear_attn.in_proj_b.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.0.linear_attn.in_proj_a.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.0.input_layernorm.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.0.mlp.gate_proj.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.0.mlp.up_proj.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.0.mlp.down_proj.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.0.post_attention_layernorm.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.1.linear_attn.dt_bias": "model-00001-of-00002.safetensors", + "model.language_model.layers.1.linear_attn.A_log": "model-00001-of-00002.safetensors", + "model.language_model.layers.1.linear_attn.conv1d.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.1.linear_attn.norm.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.1.linear_attn.out_proj.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.1.linear_attn.in_proj_qkv.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.1.linear_attn.in_proj_z.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.1.linear_attn.in_proj_b.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.1.linear_attn.in_proj_a.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.1.input_layernorm.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.1.mlp.gate_proj.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.1.mlp.up_proj.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.1.mlp.down_proj.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.1.post_attention_layernorm.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.2.linear_attn.dt_bias": "model-00001-of-00002.safetensors", + "model.language_model.layers.2.linear_attn.A_log": "model-00001-of-00002.safetensors", + "model.language_model.layers.2.linear_attn.conv1d.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.2.linear_attn.norm.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.2.linear_attn.out_proj.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.2.linear_attn.in_proj_qkv.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.2.linear_attn.in_proj_z.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.2.linear_attn.in_proj_b.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.2.linear_attn.in_proj_a.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.2.input_layernorm.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.2.mlp.gate_proj.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.2.mlp.up_proj.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.2.mlp.down_proj.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.2.post_attention_layernorm.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.3.self_attn.q_proj.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.3.self_attn.k_proj.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.3.self_attn.v_proj.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.3.self_attn.o_proj.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.3.self_attn.q_norm.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.3.self_attn.k_norm.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.3.input_layernorm.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.3.mlp.gate_proj.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.3.mlp.up_proj.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.3.mlp.down_proj.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.3.post_attention_layernorm.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.4.linear_attn.dt_bias": "model-00001-of-00002.safetensors", + "model.language_model.layers.4.linear_attn.A_log": "model-00001-of-00002.safetensors", + "model.language_model.layers.4.linear_attn.conv1d.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.4.linear_attn.norm.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.4.linear_attn.out_proj.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.4.linear_attn.in_proj_qkv.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.4.linear_attn.in_proj_z.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.4.linear_attn.in_proj_b.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.4.linear_attn.in_proj_a.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.4.input_layernorm.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.4.mlp.gate_proj.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.4.mlp.up_proj.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.4.mlp.down_proj.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.4.post_attention_layernorm.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.5.linear_attn.dt_bias": "model-00001-of-00002.safetensors", + "model.language_model.layers.5.linear_attn.A_log": "model-00001-of-00002.safetensors", + "model.language_model.layers.5.linear_attn.conv1d.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.5.linear_attn.norm.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.5.linear_attn.out_proj.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.5.linear_attn.in_proj_qkv.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.5.linear_attn.in_proj_z.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.5.linear_attn.in_proj_b.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.5.linear_attn.in_proj_a.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.5.input_layernorm.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.5.mlp.gate_proj.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.5.mlp.up_proj.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.5.mlp.down_proj.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.5.post_attention_layernorm.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.6.linear_attn.dt_bias": "model-00001-of-00002.safetensors", + "model.language_model.layers.6.linear_attn.A_log": "model-00001-of-00002.safetensors", + "model.language_model.layers.6.linear_attn.conv1d.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.6.linear_attn.norm.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.6.linear_attn.out_proj.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.6.linear_attn.in_proj_qkv.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.6.linear_attn.in_proj_z.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.6.linear_attn.in_proj_b.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.6.linear_attn.in_proj_a.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.6.input_layernorm.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.6.mlp.gate_proj.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.6.mlp.up_proj.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.6.mlp.down_proj.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.6.post_attention_layernorm.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.7.self_attn.q_proj.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.7.self_attn.k_proj.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.7.self_attn.v_proj.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.7.self_attn.o_proj.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.7.self_attn.q_norm.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.7.self_attn.k_norm.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.7.input_layernorm.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.7.mlp.gate_proj.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.7.mlp.up_proj.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.7.mlp.down_proj.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.7.post_attention_layernorm.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.8.linear_attn.dt_bias": "model-00001-of-00002.safetensors", + "model.language_model.layers.8.linear_attn.A_log": "model-00001-of-00002.safetensors", + "model.language_model.layers.8.linear_attn.conv1d.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.8.linear_attn.norm.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.8.linear_attn.out_proj.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.8.linear_attn.in_proj_qkv.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.8.linear_attn.in_proj_z.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.8.linear_attn.in_proj_b.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.8.linear_attn.in_proj_a.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.8.input_layernorm.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.8.mlp.gate_proj.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.8.mlp.up_proj.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.8.mlp.down_proj.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.8.post_attention_layernorm.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.9.linear_attn.dt_bias": "model-00001-of-00002.safetensors", + "model.language_model.layers.9.linear_attn.A_log": "model-00001-of-00002.safetensors", + "model.language_model.layers.9.linear_attn.conv1d.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.9.linear_attn.norm.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.9.linear_attn.out_proj.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.9.linear_attn.in_proj_qkv.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.9.linear_attn.in_proj_z.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.9.linear_attn.in_proj_b.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.9.linear_attn.in_proj_a.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.9.input_layernorm.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.9.mlp.gate_proj.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.9.mlp.up_proj.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.9.mlp.down_proj.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.9.post_attention_layernorm.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.10.linear_attn.dt_bias": "model-00001-of-00002.safetensors", + "model.language_model.layers.10.linear_attn.A_log": "model-00001-of-00002.safetensors", + "model.language_model.layers.10.linear_attn.conv1d.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.10.linear_attn.norm.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.10.linear_attn.out_proj.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.10.linear_attn.in_proj_qkv.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.10.linear_attn.in_proj_z.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.10.linear_attn.in_proj_b.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.10.linear_attn.in_proj_a.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.10.input_layernorm.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.10.mlp.gate_proj.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.10.mlp.up_proj.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.10.mlp.down_proj.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.10.post_attention_layernorm.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.11.self_attn.q_proj.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.11.self_attn.k_proj.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.11.self_attn.v_proj.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.11.self_attn.o_proj.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.11.self_attn.q_norm.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.11.self_attn.k_norm.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.11.input_layernorm.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.11.mlp.gate_proj.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.11.mlp.up_proj.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.11.mlp.down_proj.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.11.post_attention_layernorm.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.12.linear_attn.dt_bias": "model-00001-of-00002.safetensors", + "model.language_model.layers.12.linear_attn.A_log": "model-00001-of-00002.safetensors", + "model.language_model.layers.12.linear_attn.conv1d.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.12.linear_attn.norm.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.12.linear_attn.out_proj.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.12.linear_attn.in_proj_qkv.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.12.linear_attn.in_proj_z.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.12.linear_attn.in_proj_b.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.12.linear_attn.in_proj_a.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.12.input_layernorm.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.12.mlp.gate_proj.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.12.mlp.up_proj.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.12.mlp.down_proj.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.12.post_attention_layernorm.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.13.linear_attn.dt_bias": "model-00001-of-00002.safetensors", + "model.language_model.layers.13.linear_attn.A_log": "model-00001-of-00002.safetensors", + "model.language_model.layers.13.linear_attn.conv1d.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.13.linear_attn.norm.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.13.linear_attn.out_proj.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.13.linear_attn.in_proj_qkv.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.13.linear_attn.in_proj_z.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.13.linear_attn.in_proj_b.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.13.linear_attn.in_proj_a.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.13.input_layernorm.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.13.mlp.gate_proj.weight": "model-00001-of-00002.safetensors", + "model.language_model.layers.13.mlp.up_proj.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.13.mlp.down_proj.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.13.post_attention_layernorm.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.14.linear_attn.dt_bias": "model-00002-of-00002.safetensors", + "model.language_model.layers.14.linear_attn.A_log": "model-00002-of-00002.safetensors", + "model.language_model.layers.14.linear_attn.conv1d.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.14.linear_attn.norm.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.14.linear_attn.out_proj.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.14.linear_attn.in_proj_qkv.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.14.linear_attn.in_proj_z.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.14.linear_attn.in_proj_b.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.14.linear_attn.in_proj_a.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.14.input_layernorm.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.14.mlp.gate_proj.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.14.mlp.up_proj.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.14.mlp.down_proj.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.14.post_attention_layernorm.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.15.self_attn.q_proj.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.15.self_attn.k_proj.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.15.self_attn.v_proj.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.15.self_attn.o_proj.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.15.self_attn.q_norm.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.15.self_attn.k_norm.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.15.input_layernorm.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.15.mlp.gate_proj.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.15.mlp.up_proj.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.15.mlp.down_proj.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.15.post_attention_layernorm.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.16.linear_attn.dt_bias": "model-00002-of-00002.safetensors", + "model.language_model.layers.16.linear_attn.A_log": "model-00002-of-00002.safetensors", + "model.language_model.layers.16.linear_attn.conv1d.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.16.linear_attn.norm.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.16.linear_attn.out_proj.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.16.linear_attn.in_proj_qkv.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.16.linear_attn.in_proj_z.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.16.linear_attn.in_proj_b.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.16.linear_attn.in_proj_a.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.16.input_layernorm.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.16.mlp.gate_proj.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.16.mlp.up_proj.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.16.mlp.down_proj.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.16.post_attention_layernorm.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.17.linear_attn.dt_bias": "model-00002-of-00002.safetensors", + "model.language_model.layers.17.linear_attn.A_log": "model-00002-of-00002.safetensors", + "model.language_model.layers.17.linear_attn.conv1d.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.17.linear_attn.norm.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.17.linear_attn.out_proj.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.17.linear_attn.in_proj_qkv.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.17.linear_attn.in_proj_z.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.17.linear_attn.in_proj_b.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.17.linear_attn.in_proj_a.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.17.input_layernorm.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.17.mlp.gate_proj.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.17.mlp.up_proj.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.17.mlp.down_proj.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.17.post_attention_layernorm.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.18.linear_attn.dt_bias": "model-00002-of-00002.safetensors", + "model.language_model.layers.18.linear_attn.A_log": "model-00002-of-00002.safetensors", + "model.language_model.layers.18.linear_attn.conv1d.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.18.linear_attn.norm.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.18.linear_attn.out_proj.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.18.linear_attn.in_proj_qkv.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.18.linear_attn.in_proj_z.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.18.linear_attn.in_proj_b.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.18.linear_attn.in_proj_a.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.18.input_layernorm.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.18.mlp.gate_proj.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.18.mlp.up_proj.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.18.mlp.down_proj.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.18.post_attention_layernorm.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.19.self_attn.q_proj.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.19.self_attn.k_proj.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.19.self_attn.v_proj.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.19.self_attn.o_proj.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.19.self_attn.q_norm.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.19.self_attn.k_norm.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.19.input_layernorm.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.19.mlp.gate_proj.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.19.mlp.up_proj.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.19.mlp.down_proj.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.19.post_attention_layernorm.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.20.linear_attn.dt_bias": "model-00002-of-00002.safetensors", + "model.language_model.layers.20.linear_attn.A_log": "model-00002-of-00002.safetensors", + "model.language_model.layers.20.linear_attn.conv1d.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.20.linear_attn.norm.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.20.linear_attn.out_proj.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.20.linear_attn.in_proj_qkv.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.20.linear_attn.in_proj_z.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.20.linear_attn.in_proj_b.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.20.linear_attn.in_proj_a.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.20.input_layernorm.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.20.mlp.gate_proj.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.20.mlp.up_proj.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.20.mlp.down_proj.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.20.post_attention_layernorm.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.21.linear_attn.dt_bias": "model-00002-of-00002.safetensors", + "model.language_model.layers.21.linear_attn.A_log": "model-00002-of-00002.safetensors", + "model.language_model.layers.21.linear_attn.conv1d.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.21.linear_attn.norm.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.21.linear_attn.out_proj.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.21.linear_attn.in_proj_qkv.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.21.linear_attn.in_proj_z.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.21.linear_attn.in_proj_b.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.21.linear_attn.in_proj_a.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.21.input_layernorm.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.21.mlp.gate_proj.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.21.mlp.up_proj.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.21.mlp.down_proj.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.21.post_attention_layernorm.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.22.linear_attn.dt_bias": "model-00002-of-00002.safetensors", + "model.language_model.layers.22.linear_attn.A_log": "model-00002-of-00002.safetensors", + "model.language_model.layers.22.linear_attn.conv1d.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.22.linear_attn.norm.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.22.linear_attn.out_proj.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.22.linear_attn.in_proj_qkv.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.22.linear_attn.in_proj_z.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.22.linear_attn.in_proj_b.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.22.linear_attn.in_proj_a.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.22.input_layernorm.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.22.mlp.gate_proj.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.22.mlp.up_proj.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.22.mlp.down_proj.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.22.post_attention_layernorm.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.23.self_attn.q_proj.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.23.self_attn.k_proj.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.23.self_attn.v_proj.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.23.self_attn.o_proj.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.23.self_attn.q_norm.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.23.self_attn.k_norm.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.23.input_layernorm.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.23.mlp.gate_proj.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.23.mlp.up_proj.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.23.mlp.down_proj.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.23.post_attention_layernorm.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.24.linear_attn.dt_bias": "model-00002-of-00002.safetensors", + "model.language_model.layers.24.linear_attn.A_log": "model-00002-of-00002.safetensors", + "model.language_model.layers.24.linear_attn.conv1d.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.24.linear_attn.norm.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.24.linear_attn.out_proj.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.24.linear_attn.in_proj_qkv.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.24.linear_attn.in_proj_z.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.24.linear_attn.in_proj_b.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.24.linear_attn.in_proj_a.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.24.input_layernorm.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.24.mlp.gate_proj.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.24.mlp.up_proj.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.24.mlp.down_proj.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.24.post_attention_layernorm.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.25.linear_attn.dt_bias": "model-00002-of-00002.safetensors", + "model.language_model.layers.25.linear_attn.A_log": "model-00002-of-00002.safetensors", + "model.language_model.layers.25.linear_attn.conv1d.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.25.linear_attn.norm.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.25.linear_attn.out_proj.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.25.linear_attn.in_proj_qkv.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.25.linear_attn.in_proj_z.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.25.linear_attn.in_proj_b.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.25.linear_attn.in_proj_a.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.25.input_layernorm.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.25.mlp.gate_proj.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.25.mlp.up_proj.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.25.mlp.down_proj.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.25.post_attention_layernorm.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.26.linear_attn.dt_bias": "model-00002-of-00002.safetensors", + "model.language_model.layers.26.linear_attn.A_log": "model-00002-of-00002.safetensors", + "model.language_model.layers.26.linear_attn.conv1d.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.26.linear_attn.norm.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.26.linear_attn.out_proj.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.26.linear_attn.in_proj_qkv.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.26.linear_attn.in_proj_z.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.26.linear_attn.in_proj_b.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.26.linear_attn.in_proj_a.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.26.input_layernorm.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.26.mlp.gate_proj.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.26.mlp.up_proj.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.26.mlp.down_proj.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.26.post_attention_layernorm.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.27.self_attn.q_proj.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.27.self_attn.k_proj.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.27.self_attn.v_proj.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.27.self_attn.o_proj.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.27.self_attn.q_norm.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.27.self_attn.k_norm.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.27.input_layernorm.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.27.mlp.gate_proj.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.27.mlp.up_proj.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.27.mlp.down_proj.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.27.post_attention_layernorm.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.28.linear_attn.dt_bias": "model-00002-of-00002.safetensors", + "model.language_model.layers.28.linear_attn.A_log": "model-00002-of-00002.safetensors", + "model.language_model.layers.28.linear_attn.conv1d.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.28.linear_attn.norm.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.28.linear_attn.out_proj.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.28.linear_attn.in_proj_qkv.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.28.linear_attn.in_proj_z.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.28.linear_attn.in_proj_b.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.28.linear_attn.in_proj_a.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.28.input_layernorm.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.28.mlp.gate_proj.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.28.mlp.up_proj.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.28.mlp.down_proj.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.28.post_attention_layernorm.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.29.linear_attn.dt_bias": "model-00002-of-00002.safetensors", + "model.language_model.layers.29.linear_attn.A_log": "model-00002-of-00002.safetensors", + "model.language_model.layers.29.linear_attn.conv1d.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.29.linear_attn.norm.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.29.linear_attn.out_proj.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.29.linear_attn.in_proj_qkv.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.29.linear_attn.in_proj_z.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.29.linear_attn.in_proj_b.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.29.linear_attn.in_proj_a.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.29.input_layernorm.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.29.mlp.gate_proj.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.29.mlp.up_proj.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.29.mlp.down_proj.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.29.post_attention_layernorm.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.30.linear_attn.dt_bias": "model-00002-of-00002.safetensors", + "model.language_model.layers.30.linear_attn.A_log": "model-00002-of-00002.safetensors", + "model.language_model.layers.30.linear_attn.conv1d.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.30.linear_attn.norm.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.30.linear_attn.out_proj.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.30.linear_attn.in_proj_qkv.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.30.linear_attn.in_proj_z.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.30.linear_attn.in_proj_b.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.30.linear_attn.in_proj_a.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.30.input_layernorm.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.30.mlp.gate_proj.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.30.mlp.up_proj.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.30.mlp.down_proj.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.30.post_attention_layernorm.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.31.self_attn.q_proj.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.31.self_attn.k_proj.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.31.self_attn.v_proj.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.31.self_attn.o_proj.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.31.self_attn.q_norm.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.31.self_attn.k_norm.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.31.input_layernorm.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.31.mlp.gate_proj.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.31.mlp.up_proj.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.31.mlp.down_proj.weight": "model-00002-of-00002.safetensors", + "model.language_model.layers.31.post_attention_layernorm.weight": "model-00002-of-00002.safetensors", + "model.language_model.norm.weight": "model-00002-of-00002.safetensors" + } +} \ No newline at end of file diff --git a/posttrain_models/qwen3_5_4b_base_highlight_batch1245_epoch4/checkpoint-1000-merged/preprocessor_config.json b/posttrain_models/qwen3_5_4b_base_highlight_batch1245_epoch4/checkpoint-1000-merged/preprocessor_config.json new file mode 100644 index 0000000000000000000000000000000000000000..2ea84a437d448ff71b08df68fdd949d5cc4ebb64 --- /dev/null +++ b/posttrain_models/qwen3_5_4b_base_highlight_batch1245_epoch4/checkpoint-1000-merged/preprocessor_config.json @@ -0,0 +1,21 @@ +{ + "size": { + "longest_edge": 16777216, + "shortest_edge": 65536 + }, + "patch_size": 16, + "temporal_patch_size": 2, + "merge_size": 2, + "image_mean": [ + 0.5, + 0.5, + 0.5 + ], + "image_std": [ + 0.5, + 0.5, + 0.5 + ], + "processor_class": "Qwen3VLProcessor", + "image_processor_type": "Qwen2VLImageProcessorFast" +} \ No newline at end of file diff --git a/posttrain_models/qwen3_5_4b_base_highlight_batch1245_epoch4/checkpoint-1000-merged/processor_config.json b/posttrain_models/qwen3_5_4b_base_highlight_batch1245_epoch4/checkpoint-1000-merged/processor_config.json new file mode 100644 index 0000000000000000000000000000000000000000..7ad6acdf4203f22b7b990e36ccc3a1fe38563d5e --- /dev/null +++ b/posttrain_models/qwen3_5_4b_base_highlight_batch1245_epoch4/checkpoint-1000-merged/processor_config.json @@ -0,0 +1,63 @@ +{ + "image_processor": { + "data_format": "channels_first", + "do_convert_rgb": true, + "do_normalize": true, + "do_rescale": true, + "do_resize": true, + "image_mean": [ + 0.5, + 0.5, + 0.5 + ], + "image_processor_type": "Qwen2VLImageProcessorFast", + "image_std": [ + 0.5, + 0.5, + 0.5 + ], + "merge_size": 2, + "patch_size": 16, + "resample": 3, + "rescale_factor": 0.00392156862745098, + "size": { + "longest_edge": 16777216, + "shortest_edge": 65536 + }, + "temporal_patch_size": 2 + }, + "processor_class": "Qwen3VLProcessor", + "video_processor": { + "data_format": "channels_first", + "default_to_square": true, + "do_convert_rgb": true, + "do_normalize": true, + "do_rescale": true, + "do_resize": true, + "do_sample_frames": true, + "fps": 2, + "image_mean": [ + 0.5, + 0.5, + 0.5 + ], + "image_std": [ + 0.5, + 0.5, + 0.5 + ], + "max_frames": 768, + "merge_size": 2, + "min_frames": 4, + "patch_size": 16, + "resample": 3, + "rescale_factor": 0.00392156862745098, + "return_metadata": false, + "size": { + "longest_edge": 25165824, + "shortest_edge": 4096 + }, + "temporal_patch_size": 2, + "video_processor_type": "Qwen3VLVideoProcessor" + } +} diff --git a/posttrain_models/qwen3_5_4b_base_highlight_batch1245_epoch4/checkpoint-1000-merged/tokenizer_config.json b/posttrain_models/qwen3_5_4b_base_highlight_batch1245_epoch4/checkpoint-1000-merged/tokenizer_config.json new file mode 100644 index 0000000000000000000000000000000000000000..a068e2468cff426a9b105006e74e044030a6faf4 --- /dev/null +++ b/posttrain_models/qwen3_5_4b_base_highlight_batch1245_epoch4/checkpoint-1000-merged/tokenizer_config.json @@ -0,0 +1,32 @@ +{ + "add_prefix_space": false, + "audio_bos_token": "<|audio_start|>", + "audio_eos_token": "<|audio_end|>", + "audio_token": "<|audio_pad|>", + "backend": "tokenizers", + "bos_token": null, + "clean_up_tokenization_spaces": false, + "eos_token": "<|im_end|>", + "errors": "replace", + "image_token": "<|image_pad|>", + "is_local": true, + "model_max_length": 262144, + "model_specific_special_tokens": { + "audio_bos_token": "<|audio_start|>", + "audio_eos_token": "<|audio_end|>", + "audio_token": "<|audio_pad|>", + "image_token": "<|image_pad|>", + "video_token": "<|video_pad|>", + "vision_bos_token": "<|vision_start|>", + "vision_eos_token": "<|vision_end|>" + }, + "pad_token": "<|endoftext|>", + "pretokenize_regex": "(?i:'s|'t|'re|'ve|'m|'ll|'d)|[^\\r\\n\\p{L}\\p{N}]?[\\p{L}\\p{M}]+|\\p{N}| ?[^\\s\\p{L}\\p{M}\\p{N}]+[\\r\\n]*|\\s*[\\r\\n]+|\\s+(?!\\S)|\\s+", + "processor_class": "Qwen3VLProcessor", + "split_special_tokens": false, + "tokenizer_class": "TokenizersBackend", + "unk_token": null, + "video_token": "<|video_pad|>", + "vision_bos_token": "<|vision_start|>", + "vision_eos_token": "<|vision_end|>" +} diff --git a/posttrain_models/qwen3_5_4b_base_highlight_batch1245_epoch4/checkpoint-1000/args.json b/posttrain_models/qwen3_5_4b_base_highlight_batch1245_epoch4/checkpoint-1000/args.json new file mode 100644 index 0000000000000000000000000000000000000000..66fa96327404653aa85875ca21471633385c775c --- /dev/null +++ b/posttrain_models/qwen3_5_4b_base_highlight_batch1245_epoch4/checkpoint-1000/args.json @@ -0,0 +1,398 @@ +{ + "use_ray": false, + "ray_exp_name": null, + "device_groups": null, + "model": "/mnt/bn/strategy-mllm-train/user/tanyue/models/pretrained_models/Qwen3.5-4B", + "model_type": "qwen3_5", + "model_revision": null, + "task_type": "causal_lm", + "torch_dtype": "bfloat16", + "attn_impl": null, + "experts_impl": null, + "new_special_tokens": [], + "num_labels": null, + "problem_type": null, + "rope_scaling": null, + "device_map": null, + "max_memory": {}, + "max_model_len": null, + "local_repo_path": null, + "init_strategy": null, + "template": "qwen3_5", + "system": null, + "max_length": 8192, + "truncation_strategy": "delete", + "max_pixels": null, + "agent_template": null, + "norm_bbox": null, + "use_chat_template": true, + "padding_side": "right", + "padding_free": true, + "loss_scale": "ignore_empty_think", + "sequence_parallel_size": 1, + "template_backend": "swift", + "response_prefix": "", + "enable_thinking": false, + "add_non_thinking_prefix": false, + "dataset": [ + "batch1", + "batch2", + "batch4_part1", + "batch4_part2", + "batch4_part3", + "batch5_part1", + "batch5_part2", + "batch5_part3" + ], + "val_dataset": [], + "cached_dataset": [], + "cached_val_dataset": [], + "split_dataset_ratio": 0.0, + "data_seed": 42, + "dataset_num_proc": 8, + "load_from_cache_file": false, + "dataset_shuffle": true, + "val_dataset_shuffle": false, + "streaming": false, + "interleave_prob": null, + "stopping_strategy": "first_exhausted", + "shuffle_buffer_size": 1000, + "download_mode": "reuse_dataset_if_exists", + "columns": {}, + "strict": false, + "remove_unused_columns": true, + "model_name": null, + "model_author": null, + "custom_dataset_info": [], + "quant_method": null, + "quant_bits": null, + "hqq_axis": null, + "bnb_4bit_compute_dtype": "bfloat16", + "bnb_4bit_quant_type": "nf4", + "bnb_4bit_use_double_quant": true, + "bnb_4bit_quant_storage": null, + "max_new_tokens": null, + "temperature": 0.9, + "top_k": 50, + "top_p": 0.9, + "repetition_penalty": 1.0, + "num_beams": 1, + "stream": false, + "stop_words": [], + "logprobs": false, + "top_logprobs": null, + "structured_outputs_regex": null, + "tuner_backend": "peft", + "tuner_type": "lora", + "train_type": "lora", + "adapters": [], + "external_plugins": [ + "ms-swift/base_highlight_video_dataset.py" + ], + "custom_register_path": [ + "ms-swift/base_highlight_video_dataset.py" + ], + "seed": 42, + "model_kwargs": {}, + "load_args": false, + "load_data_args": false, + "packing": false, + "packing_length": null, + "packing_num_proc": 1, + "lazy_tokenize": true, + "use_hf": false, + "hub_token": null, + "ddp_timeout": 18000000, + "ddp_backend": "nccl", + "ignore_args_error": false, + "use_swift_lora": false, + "freeze_llm": false, + "freeze_vit": true, + "freeze_aligner": true, + "freeze_parameters": [], + "freeze_parameters_regex": null, + "freeze_parameters_ratio": 0.0, + "trainable_parameters": [], + "trainable_parameters_regex": null, + "target_modules": [ + "all-linear" + ], + "target_regex": null, + "modules_to_save": [], + "lora_rank": 8, + "lora_alpha": 32, + "lora_dropout": 0.05, + "lora_bias": "none", + "lora_dtype": null, + "use_rslora": false, + "rlhf_type": null, + "loss_type": null, + "mcore_ref_model": null, + "mcore_ref_adapter": null, + "beta": null, + "rpo_alpha": null, + "reference_free": false, + "label_smoothing": 0.0, + "f_divergence_type": "reverse_kl", + "desirable_weight": 1.0, + "undesirable_weight": 1.0, + "calculate_KL": null, + "center_rewards_coefficient": null, + "teacher_model": null, + "teacher_model_type": null, + "teacher_model_revision": null, + "teacher_model_server": null, + "gkd_logits_topk": null, + "lmbda": 0.5, + "seq_kd": false, + "offload_teacher_model": false, + "sft_alpha": 0.0, + "generation_batch_size": null, + "steps_per_generation": null, + "num_generations": 8, + "num_generations_eval": null, + "max_completion_length": 512, + "importance_sampling_level": "token", + "tau_pos": 1.0, + "tau_neg": 1.05, + "epsilon": 0.2, + "epsilon_high": null, + "delta": null, + "use_vllm": true, + "vllm_mode": null, + "vllm_enable_prefix_caching": true, + "vllm_gpu_memory_utilization": 0.9, + "vllm_tensor_parallel_size": 1, + "vllm_max_model_len": null, + "vllm_enforce_eager": false, + "vllm_limit_mm_per_prompt": null, + "vllm_disable_cascade_attn": false, + "vllm_max_num_seqs": null, + "vllm_mm_processor_cache_gb": null, + "vllm_engine_kwargs": null, + "sleep_level": 0, + "offload_optimizer": false, + "offload_model": false, + "offload_bridge": false, + "vllm_server_base_url": null, + "vllm_server_host": null, + "vllm_server_port": [ + 8000 + ], + "vllm_server_timeout": 240.0, + "vllm_server_group_port": null, + "reward_funcs": [], + "reward_weights": null, + "cosine_min_len_value_wrong": -0.5, + "cosine_max_len_value_wrong": 0.0, + "cosine_min_len_value_correct": 1.0, + "cosine_max_len_value_correct": 0.5, + "cosine_max_len": null, + "repetition_n_grams": 3, + "repetition_max_penalty": -1.0, + "soft_max_length": null, + "soft_cache_length": null, + "dynamic_sample": false, + "max_resample_times": 3, + "overlong_filter": false, + "scale_rewards": "group", + "advantage_estimator": "grpo", + "kl_in_reward": false, + "wandb_log_unique_prompts": null, + "log_completions": false, + "rollout_importance_sampling_mode": null, + "rollout_importance_sampling_threshold": 2.0, + "log_rollout_offpolicy_metrics": false, + "off_policy_sequence_mask_delta": null, + "log_entropy": false, + "top_entropy_quantile": 1.0, + "reward_model": null, + "reward_model_plugin": null, + "sync_ref_model": false, + "ref_model_sync_steps": 512, + "ref_model_mixup_alpha": 0.6, + "async_generate": false, + "move_model_batches": null, + "multi_turn_scheduler": null, + "max_turns": null, + "completion_length_limit_scope": "per_round", + "vllm_server_pass_dataset": false, + "num_iterations": 1, + "micro_batch_size": 8, + "global_batch_size": 64, + "recompute_granularity": "full", + "recompute_method": "uniform", + "recompute_num_layers": 1, + "recompute_modules": [ + "core_attn" + ], + "train_iters": null, + "num_train_epochs": 4, + "masked_softmax_fusion": true, + "bias_dropout_fusion": true, + "bias_activation_fusion": true, + "apply_rope_fusion": false, + "gradient_accumulation_fusion": false, + "cross_entropy_loss_fusion": true, + "cross_entropy_fusion_impl": "te", + "calculate_per_token_loss": true, + "attention_backend": "", + "optimizer": "adam", + "optimizer_cpu_offload": false, + "optimizer_offload_fraction": 1.0, + "use_precision_aware_optimizer": false, + "main_grads_dtype": "float32", + "main_params_dtype": "float32", + "exp_avg_dtype": "float32", + "exp_avg_sq_dtype": "float32", + "manual_gc": false, + "manual_gc_steps": 0, + "manual_gc_eval": true, + "train_dataloader_shuffle": true, + "dataloader_num_workers": 8, + "dataloader_pin_memory": true, + "dataloader_persistent_workers": true, + "dataloader_prefetch_factor": 2, + "data_sharding": false, + "group_by_length": false, + "te_rng_tracker": false, + "data_parallel_random_init": false, + "mlp_padding_free": false, + "lr_warmup_init": 0.0, + "lr": 0.0001, + "lr_decay_style": "cosine", + "lr_decay_iters": null, + "lr_warmup_iters": 0, + "lr_warmup_fraction": 0.05, + "min_lr": 1e-05, + "lr_wsd_decay_style": "exponential", + "lr_wsd_decay_iters": null, + "weight_decay": 0.1, + "weight_decay_incr_style": "constant", + "start_weight_decay": 0.1, + "end_weight_decay": 0.1, + "clip_grad": 1.0, + "adam_beta1": 0.9, + "adam_beta2": 0.95, + "adam_eps": 1e-08, + "sgd_momentum": 0.9, + "output_dir": "/mnt/bn/strategy-mllm-train/user/tanyue/models/posttrain_models/qwen3_5_4b_base_highlight_batch1245_epoch4", + "save_steps": 500, + "no_save_optim": true, + "no_save_rng": true, + "mcore_model": null, + "mcore_adapter": null, + "no_load_optim": false, + "no_load_rng": false, + "finetune": true, + "perform_initialization": false, + "use_cpu_initialization": false, + "async_save": false, + "save_total_limit": null, + "metric_for_best_model": "loss", + "greater_is_better": false, + "use_persistent_ckpt_worker": false, + "dist_ckpt_save_pre_mcore_014": false, + "dist_ckpt_optim_fully_reshardable": false, + "distrib_optim_fully_reshardable_mem_efficient": false, + "local_rank": 0, + "use_distributed_optimizer": true, + "tensor_model_parallel_size": 1, + "pipeline_model_parallel_size": 2, + "decoder_first_pipeline_num_layers": null, + "decoder_last_pipeline_num_layers": null, + "account_for_embedding_in_pipeline_split": false, + "account_for_loss_in_pipeline_split": false, + "overlap_p2p_comm": false, + "align_param_gather": false, + "sequence_parallel": false, + "context_parallel_size": 1, + "tp_comm_overlap": false, + "overlap_grad_reduce": false, + "overlap_param_gather": false, + "overlap_param_gather_with_optimizer_step": false, + "align_grad_reduce": true, + "virtual_pipeline_model_parallel_size": null, + "microbatch_group_size_per_vp_stage": null, + "pipeline_model_parallel_layout": null, + "expert_model_parallel_size": 4, + "expert_tensor_parallel_size": 1, + "report_to": [ + "wandb" + ], + "logging_steps": 5, + "tensorboard_dir": "/mnt/bn/strategy-mllm-train/user/tanyue/models/posttrain_models/qwen3_5_4b_base_highlight_batch1245_epoch4/runs", + "tensorboard_queue_size": 50, + "wandb_project": "highlight_base_highlight_qwen3_5", + "wandb_exp_name": "qwen3_5_4b_base_highlight_batch1245_epoch4", + "swanlab_project": "megatron-swift", + "swanlab_exp_name": null, + "eval_iters": -1, + "eval_steps": 100000, + "fp8_format": null, + "fp8_recipe": "delayed", + "fp8_amax_history_len": 1024, + "fp8_amax_compute_algo": "max", + "fp8_param_gather": false, + "fp16": false, + "bf16": true, + "apply_query_key_layer_scaling": false, + "attention_softmax_in_fp32": true, + "accumulate_allreduce_grads_in_fp32": true, + "moe_router_load_balancing_type": null, + "moe_router_dtype": "fp32", + "moe_token_dispatcher_type": null, + "moe_enable_deepep": false, + "moe_grouped_gemm": true, + "moe_permute_fusion": true, + "moe_aux_loss_coeff": 1e-06, + "moe_z_loss_coeff": null, + "moe_shared_expert_overlap": true, + "moe_layer_recompute": false, + "moe_expert_capacity_factor": null, + "moe_pad_expert_input_to_capacity": false, + "moe_token_drop_policy": null, + "mtp_num_layers": null, + "mtp_loss_scaling_factor": 0.1, + "save_safetensors": true, + "ref_model": null, + "ref_adapters": [], + "merge_lora": true, + "max_shard_size": "5GB", + "vit_gradient_checkpointing": false, + "vit_lr": null, + "aligner_lr": null, + "gradient_checkpointing_kwargs": null, + "check_model": true, + "apply_wd_to_qk_layernorm": false, + "enable_dft_loss": false, + "enable_channel_loss": false, + "save_strategy": "steps", + "callbacks": [ + "print", + "default_flow", + "wandb" + ], + "add_version": false, + "create_checkpoint_symlink": false, + "swift_version": "4.0.1", + "ckpt_dir": null, + "rank": 0, + "global_world_size": 8, + "local_world_size": 8, + "model_suffix": "Qwen3.5-4B", + "model_info": "ModelInfo(model_type='qwen3_5', model_dir='/mnt/bn/strategy-mllm-train/user/tanyue/models/pretrained_models/Qwen3.5-4B', torch_dtype=torch.bfloat16, max_model_len=262144, quant_method=None, quant_bits=None, rope_scaling=None, is_moe_model=False, is_multimodal=True, config=None, task_type='causal_lm', num_labels=None)", + "model_meta": "ModelMeta(model_type='qwen3_5', model_groups=[ModelGroup(models=[Model(ms_model_id='Qwen/Qwen3.5-0.8B', hf_model_id='Qwen/Qwen3.5-0.8B', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen3.5-2B', hf_model_id='Qwen/Qwen3.5-2B', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen3.5-4B', hf_model_id='Qwen/Qwen3.5-4B', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen3.5-9B', hf_model_id='Qwen/Qwen3.5-9B', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen3.5-27B', hf_model_id='Qwen/Qwen3.5-27B', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen3.5-27B-FP8', hf_model_id='Qwen/Qwen3.5-27B-FP8', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen3.5-0.8B-Base', hf_model_id='Qwen/Qwen3.5-0.8B-Base', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen3.5-2B-Base', hf_model_id='Qwen/Qwen3.5-2B-Base', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen3.5-4B-Base', hf_model_id='Qwen/Qwen3.5-4B-Base', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen3.5-9B-Base', hf_model_id='Qwen/Qwen3.5-9B-Base', model_path=None, ms_revision=None, hf_revision=None)], template='qwen3_5', ignore_patterns=None, requires=None, tags=[])], loader=, template='qwen3_5', model_arch=MultiModelKeys(arch_name='qwen2_vl', embedding=None, module_list=None, lm_head=None, q_proj=None, k_proj=None, v_proj=None, o_proj=None, attention=None, mlp=None, down_proj=None, qkv_proj=None, qk_proj=None, qa_proj=None, qb_proj=None, kv_proj=None, kva_proj=None, kvb_proj=None, language_model=['model.language_model', 'lm_head'], aligner=['model.visual.merger'], vision_tower=['model.visual'], generator=[]), architectures=['Qwen3_5ForConditionalGeneration'], additional_saved_files=[], torch_dtype=None, is_multimodal=True, is_reward=False, task_type=None, ignore_patterns=None, requires=['transformers>=5.0.0.dev', 'qwen_vl_utils>=0.0.14', 'decord'], tags=[])", + "model_dir": "/mnt/bn/strategy-mllm-train/user/tanyue/models/pretrained_models/Qwen3.5-4B", + "template_meta": "QwenTemplateMeta(template_type='qwen3_5', prefix=[], prompt=['<|im_start|>user\\n{{QUERY}}<|im_end|>\\n<|im_start|>assistant\\n'], chat_sep=['<|im_end|>\\n'], suffix=['<|im_end|>\\n'], template_cls=, system_prefix=['<|im_start|>system\\n{{SYSTEM}}<|im_end|>\\n'], default_system=None, auto_add_bos=False, stop_words=['<|endoftext|>'], agent_template='qwen3_5', is_thinking=True, thinking_prefix='\\n', non_thinking_prefix='\\n\\n\\n\\n', history_thinking_prefix='')", + "_val_dataset_exists": false, + "hub": "", + "seq_length": 8192, + "is_multimodal": true, + "megatron_model_meta": "MegatronModelMeta(megatron_model_type='qwen3_5', model_types=['qwen3_5', 'qwen3_5_moe'], bridge_cls=, visual_cls=, is_multimodal=True, loader=)", + "fp8": null, + "params_dtype": "bfloat16", + "world_size": 8, + "data_parallel_size": 4, + "num_microbatches": 2 +} \ No newline at end of file diff --git a/posttrain_models/qwen3_5_4b_base_highlight_batch1245_epoch4/checkpoint-1000/iter_0001000/.metadata b/posttrain_models/qwen3_5_4b_base_highlight_batch1245_epoch4/checkpoint-1000/iter_0001000/.metadata new file mode 100644 index 0000000000000000000000000000000000000000..71727093f67cb94bcaffc7a5149d753a1571334b Binary files /dev/null and b/posttrain_models/qwen3_5_4b_base_highlight_batch1245_epoch4/checkpoint-1000/iter_0001000/.metadata differ diff --git a/posttrain_models/qwen3_5_4b_base_highlight_batch1245_epoch4/checkpoint-1000/iter_0001000/common.pt b/posttrain_models/qwen3_5_4b_base_highlight_batch1245_epoch4/checkpoint-1000/iter_0001000/common.pt new file mode 100644 index 0000000000000000000000000000000000000000..fb4ae33a187df8320cbdff9e39d33cd1b1914b96 --- /dev/null +++ b/posttrain_models/qwen3_5_4b_base_highlight_batch1245_epoch4/checkpoint-1000/iter_0001000/common.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:cdd4765a4217aa21a75d85673fc7d045c61f25e688a0015b54a513a738f1cb37 +size 16295 diff --git a/posttrain_models/qwen3_5_4b_base_highlight_batch1245_epoch4/checkpoint-1000/iter_0001000/metadata.json b/posttrain_models/qwen3_5_4b_base_highlight_batch1245_epoch4/checkpoint-1000/iter_0001000/metadata.json new file mode 100644 index 0000000000000000000000000000000000000000..c9dbde0bcf4d2d993122dcc7d6bcb1eef8b6fb77 --- /dev/null +++ b/posttrain_models/qwen3_5_4b_base_highlight_batch1245_epoch4/checkpoint-1000/iter_0001000/metadata.json @@ -0,0 +1 @@ +{"sharded_backend": "torch_dist", "sharded_backend_version": 1, "common_backend": "torch", "common_backend_version": 1} \ No newline at end of file diff --git a/posttrain_models/qwen3_5_4b_base_highlight_batch1245_epoch4/checkpoint-1000/latest_checkpointed_iteration.txt b/posttrain_models/qwen3_5_4b_base_highlight_batch1245_epoch4/checkpoint-1000/latest_checkpointed_iteration.txt new file mode 100644 index 0000000000000000000000000000000000000000..e37d32abba426c06b752a5e53f48c595c84e9270 --- /dev/null +++ b/posttrain_models/qwen3_5_4b_base_highlight_batch1245_epoch4/checkpoint-1000/latest_checkpointed_iteration.txt @@ -0,0 +1 @@ +1000 \ No newline at end of file