diff --git a/Conifer/SLS/Qwen3-4B/checkpoint-1100/chat_template.jinja b/Conifer/SLS/Qwen3-4B/checkpoint-1100/chat_template.jinja new file mode 100644 index 0000000000000000000000000000000000000000..01be9b307daa2d425f7c168c9fb145a286e0afb4 --- /dev/null +++ b/Conifer/SLS/Qwen3-4B/checkpoint-1100/chat_template.jinja @@ -0,0 +1,89 @@ +{%- if tools %} + {{- '<|im_start|>system\n' }} + {%- if messages[0].role == 'system' %} + {{- messages[0].content + '\n\n' }} + {%- endif %} + {{- "# Tools\n\nYou may call one or more functions to assist with the user query.\n\nYou are provided with function signatures within XML tags:\n" }} + {%- for tool in tools %} + {{- "\n" }} + {{- tool | tojson }} + {%- endfor %} + {{- "\n\n\nFor each function call, return a json object with function name and arguments within XML tags:\n\n{\"name\": , \"arguments\": }\n<|im_end|>\n" }} +{%- else %} + {%- if messages[0].role == 'system' %} + {{- '<|im_start|>system\n' + messages[0].content + '<|im_end|>\n' }} + {%- endif %} +{%- endif %} +{%- set ns = namespace(multi_step_tool=true, last_query_index=messages|length - 1) %} +{%- for message in messages[::-1] %} + {%- set index = (messages|length - 1) - loop.index0 %} + {%- if ns.multi_step_tool and message.role == "user" and message.content is string and not(message.content.startswith('') and message.content.endswith('')) %} + {%- set ns.multi_step_tool = false %} + {%- set ns.last_query_index = index %} + {%- endif %} +{%- endfor %} +{%- for message in messages %} + {%- if message.content is string %} + {%- set content = message.content %} + {%- else %} + {%- set content = '' %} + {%- endif %} + {%- if (message.role == "user") or (message.role == "system" and not loop.first) %} + {{- '<|im_start|>' + message.role + '\n' + content + '<|im_end|>' + '\n' }} + {%- elif message.role == "assistant" %} + {%- set reasoning_content = '' %} + {%- if message.reasoning_content is string %} + {%- set reasoning_content = message.reasoning_content %} + {%- else %} + {%- if '' in content %} + {%- set reasoning_content = content.split('')[0].rstrip('\n').split('')[-1].lstrip('\n') %} + {%- set content = content.split('')[-1].lstrip('\n') %} + {%- endif %} + {%- endif %} + {%- if loop.index0 > ns.last_query_index %} + {%- if loop.last or (not loop.last and reasoning_content) %} + {{- '<|im_start|>' + message.role + '\n\n' + reasoning_content.strip('\n') + '\n\n\n' + content.lstrip('\n') }} + {%- else %} + {{- '<|im_start|>' + message.role + '\n' + content }} + {%- endif %} + {%- else %} + {{- '<|im_start|>' + message.role + '\n' + content }} + {%- endif %} + {%- if message.tool_calls %} + {%- for tool_call in message.tool_calls %} + {%- if (loop.first and content) or (not loop.first) %} + {{- '\n' }} + {%- endif %} + {%- if tool_call.function %} + {%- set tool_call = tool_call.function %} + {%- endif %} + {{- '\n{"name": "' }} + {{- tool_call.name }} + {{- '", "arguments": ' }} + {%- if tool_call.arguments is string %} + {{- tool_call.arguments }} + {%- else %} + {{- tool_call.arguments | tojson }} + {%- endif %} + {{- '}\n' }} + {%- endfor %} + {%- endif %} + {{- '<|im_end|>\n' }} + {%- elif message.role == "tool" %} + {%- if loop.first or (messages[loop.index0 - 1].role != "tool") %} + {{- '<|im_start|>user' }} + {%- endif %} + {{- '\n\n' }} + {{- content }} + {{- '\n' }} + {%- if loop.last or (messages[loop.index0 + 1].role != "tool") %} + {{- '<|im_end|>\n' }} + {%- endif %} + {%- endif %} +{%- endfor %} +{%- if add_generation_prompt %} + {{- '<|im_start|>assistant\n' }} + {%- if enable_thinking is defined and enable_thinking is false %} + {{- '\n\n\n\n' }} + {%- endif %} +{%- endif %} \ No newline at end of file diff --git a/Conifer/SLS/Qwen3-4B/checkpoint-1100/training_state.json b/Conifer/SLS/Qwen3-4B/checkpoint-1100/training_state.json new file mode 100644 index 0000000000000000000000000000000000000000..e27d11ae3e1bf29bf8118ae7428501e55ec9ecd3 --- /dev/null +++ b/Conifer/SLS/Qwen3-4B/checkpoint-1100/training_state.json @@ -0,0 +1,36 @@ +{ + "version": 1, + "global_step": 1100, + "next_epoch": 2, + "next_micro_idx": 5824, + "contract": { + "agent": "A2", + "rollout": "source_project/conifer_training_hub/13_rl_data/conifer_stratified_sft_full_rl_20260902_rl_from_sft.jsonl", + "rollout_size": 3237712215, + "rollout_sha256": "e448b0ebd5ad4eb57fddb5937e4017fa8dd875009aea106cc654cb6a1d8dd77e", + "reward_field": "reward", + "thinking_enabled": false, + "loss_version": "format_protected_signed_rwr_token_kl_v2", + "negative_mask": "semantic_json_values_and_thinking", + "kl_mode": "token_level_forward_kl", + "model_name_or_path": "Qwen/Qwen3-4B", + "sft_adapter": "source_project/conifer_training_hub/11_runs/sft/conifer_stratified_sft_full_rl_20260902/A2/final", + "kl_coef": 0.05, + "num_epochs": 3.0, + "learning_rate": 5e-06, + "weight_decay": 0.01, + "warmup_ratio": 0.05, + "per_device_batch_size": 1, + "gradient_accumulation_steps": 16, + "max_seq_length": 6144, + "lora_rank": 64, + "lora_alpha": 64, + "lora_dropout": 0.05, + "bf16": true, + "gradient_checkpointing": true, + "seed": 42, + "world_size": 8, + "total_records": 47063, + "total_steps": 1104 + } +} diff --git a/Conifer/SLS/Qwen3-8B/checkpoint-1030/chat_template.jinja b/Conifer/SLS/Qwen3-8B/checkpoint-1030/chat_template.jinja new file mode 100644 index 0000000000000000000000000000000000000000..01be9b307daa2d425f7c168c9fb145a286e0afb4 --- /dev/null +++ b/Conifer/SLS/Qwen3-8B/checkpoint-1030/chat_template.jinja @@ -0,0 +1,89 @@ +{%- if tools %} + {{- '<|im_start|>system\n' }} + {%- if messages[0].role == 'system' %} + {{- messages[0].content + '\n\n' }} + {%- endif %} + {{- "# Tools\n\nYou may call one or more functions to assist with the user query.\n\nYou are provided with function signatures within XML tags:\n" }} + {%- for tool in tools %} + {{- "\n" }} + {{- tool | tojson }} + {%- endfor %} + {{- "\n\n\nFor each function call, return a json object with function name and arguments within XML tags:\n\n{\"name\": , \"arguments\": }\n<|im_end|>\n" }} +{%- else %} + {%- if messages[0].role == 'system' %} + {{- '<|im_start|>system\n' + messages[0].content + '<|im_end|>\n' }} + {%- endif %} +{%- endif %} +{%- set ns = namespace(multi_step_tool=true, last_query_index=messages|length - 1) %} +{%- for message in messages[::-1] %} + {%- set index = (messages|length - 1) - loop.index0 %} + {%- if ns.multi_step_tool and message.role == "user" and message.content is string and not(message.content.startswith('') and message.content.endswith('')) %} + {%- set ns.multi_step_tool = false %} + {%- set ns.last_query_index = index %} + {%- endif %} +{%- endfor %} +{%- for message in messages %} + {%- if message.content is string %} + {%- set content = message.content %} + {%- else %} + {%- set content = '' %} + {%- endif %} + {%- if (message.role == "user") or (message.role == "system" and not loop.first) %} + {{- '<|im_start|>' + message.role + '\n' + content + '<|im_end|>' + '\n' }} + {%- elif message.role == "assistant" %} + {%- set reasoning_content = '' %} + {%- if message.reasoning_content is string %} + {%- set reasoning_content = message.reasoning_content %} + {%- else %} + {%- if '' in content %} + {%- set reasoning_content = content.split('')[0].rstrip('\n').split('')[-1].lstrip('\n') %} + {%- set content = content.split('')[-1].lstrip('\n') %} + {%- endif %} + {%- endif %} + {%- if loop.index0 > ns.last_query_index %} + {%- if loop.last or (not loop.last and reasoning_content) %} + {{- '<|im_start|>' + message.role + '\n\n' + reasoning_content.strip('\n') + '\n\n\n' + content.lstrip('\n') }} + {%- else %} + {{- '<|im_start|>' + message.role + '\n' + content }} + {%- endif %} + {%- else %} + {{- '<|im_start|>' + message.role + '\n' + content }} + {%- endif %} + {%- if message.tool_calls %} + {%- for tool_call in message.tool_calls %} + {%- if (loop.first and content) or (not loop.first) %} + {{- '\n' }} + {%- endif %} + {%- if tool_call.function %} + {%- set tool_call = tool_call.function %} + {%- endif %} + {{- '\n{"name": "' }} + {{- tool_call.name }} + {{- '", "arguments": ' }} + {%- if tool_call.arguments is string %} + {{- tool_call.arguments }} + {%- else %} + {{- tool_call.arguments | tojson }} + {%- endif %} + {{- '}\n' }} + {%- endfor %} + {%- endif %} + {{- '<|im_end|>\n' }} + {%- elif message.role == "tool" %} + {%- if loop.first or (messages[loop.index0 - 1].role != "tool") %} + {{- '<|im_start|>user' }} + {%- endif %} + {{- '\n\n' }} + {{- content }} + {{- '\n' }} + {%- if loop.last or (messages[loop.index0 + 1].role != "tool") %} + {{- '<|im_end|>\n' }} + {%- endif %} + {%- endif %} +{%- endfor %} +{%- if add_generation_prompt %} + {{- '<|im_start|>assistant\n' }} + {%- if enable_thinking is defined and enable_thinking is false %} + {{- '\n\n\n\n' }} + {%- endif %} +{%- endif %} \ No newline at end of file diff --git a/Conifer/SLS/Qwen3-8B/checkpoint-1030/policy_adapter/adapter_config.json b/Conifer/SLS/Qwen3-8B/checkpoint-1030/policy_adapter/adapter_config.json new file mode 100644 index 0000000000000000000000000000000000000000..b3c0282381dfea55351c7175fbad57cbb35a290b --- /dev/null +++ b/Conifer/SLS/Qwen3-8B/checkpoint-1030/policy_adapter/adapter_config.json @@ -0,0 +1,45 @@ +{ + "alora_invocation_tokens": null, + "alpha_pattern": {}, + "arrow_config": null, + "auto_mapping": null, + "base_model_name_or_path": "Qwen/Qwen3-8B", + "bias": "none", + "corda_config": null, + "ensure_weight_tying": false, + "eva_config": null, + "exclude_modules": null, + "fan_in_fan_out": false, + "inference_mode": true, + "init_lora_weights": true, + "layer_replication": null, + "layers_pattern": null, + "layers_to_transform": null, + "loftq_config": {}, + "lora_alpha": 64, + "lora_bias": false, + "lora_dropout": 0.05, + "lora_ga_config": null, + "megatron_config": null, + "megatron_core": "megatron.core", + "modules_to_save": null, + "peft_type": "LORA", + "peft_version": "0.19.1", + "qalora_group_size": 16, + "r": 64, + "rank_pattern": {}, + "revision": null, + "target_modules": [ + "v_proj", + "o_proj", + "q_proj", + "k_proj" + ], + "target_parameters": null, + "task_type": "CAUSAL_LM", + "trainable_token_indices": null, + "use_bdlora": null, + "use_dora": false, + "use_qalora": false, + "use_rslora": false +} diff --git a/Conifer/SLS/Qwen3-8B/checkpoint-1030/tokenizer_config.json b/Conifer/SLS/Qwen3-8B/checkpoint-1030/tokenizer_config.json new file mode 100644 index 0000000000000000000000000000000000000000..5668a4a01433b0648bb972458de616f22f3e728e --- /dev/null +++ b/Conifer/SLS/Qwen3-8B/checkpoint-1030/tokenizer_config.json @@ -0,0 +1,30 @@ +{ + "add_prefix_space": false, + "backend": "tokenizers", + "bos_token": null, + "clean_up_tokenization_spaces": false, + "eos_token": "<|im_end|>", + "errors": "replace", + "extra_special_tokens": [ + "<|im_start|>", + "<|im_end|>", + "<|object_ref_start|>", + "<|object_ref_end|>", + "<|box_start|>", + "<|box_end|>", + "<|quad_start|>", + "<|quad_end|>", + "<|vision_start|>", + "<|vision_end|>", + "<|vision_pad|>", + "<|image_pad|>", + "<|video_pad|>" + ], + "is_local": true, + "local_files_only": false, + "model_max_length": 131072, + "pad_token": "<|endoftext|>", + "split_special_tokens": false, + "tokenizer_class": "Qwen2Tokenizer", + "unk_token": null +} diff --git a/Conifer/SLS/Qwen3-8B/checkpoint-1030/training_state.json b/Conifer/SLS/Qwen3-8B/checkpoint-1030/training_state.json new file mode 100644 index 0000000000000000000000000000000000000000..f6ef55afe21beadfa3a279b847ed09c2f4d2852f --- /dev/null +++ b/Conifer/SLS/Qwen3-8B/checkpoint-1030/training_state.json @@ -0,0 +1,36 @@ +{ + "version": 1, + "global_step": 1030, + "next_epoch": 2, + "next_micro_idx": 5248, + "contract": { + "agent": "A3", + "rollout": "source_project/conifer_training_hub/13_rl_data/conifer_stratified_sft_full_rl_20260902_rl_from_sft.jsonl", + "rollout_size": 3237712215, + "rollout_sha256": "e448b0ebd5ad4eb57fddb5937e4017fa8dd875009aea106cc654cb6a1d8dd77e", + "reward_field": "reward", + "thinking_enabled": false, + "loss_version": "format_protected_signed_rwr_token_kl_v2", + "negative_mask": "semantic_json_values_and_thinking", + "kl_mode": "token_level_forward_kl", + "model_name_or_path": "Qwen/Qwen3-8B", + "sft_adapter": "source_project/conifer_training_hub/11_runs/sft/conifer_stratified_sft_full_rl_20260902/A3/final", + "kl_coef": 0.05, + "num_epochs": 3.0, + "learning_rate": 5e-06, + "weight_decay": 0.01, + "warmup_ratio": 0.05, + "per_device_batch_size": 1, + "gradient_accumulation_steps": 16, + "max_seq_length": 6144, + "lora_rank": 64, + "lora_alpha": 64, + "lora_dropout": 0.05, + "bf16": true, + "gradient_checkpointing": true, + "seed": 42, + "world_size": 8, + "total_records": 44913, + "total_steps": 1053 + } +} diff --git a/Conifer/SLS/Qwen3-8B/checkpoint-1040/chat_template.jinja b/Conifer/SLS/Qwen3-8B/checkpoint-1040/chat_template.jinja new file mode 100644 index 0000000000000000000000000000000000000000..01be9b307daa2d425f7c168c9fb145a286e0afb4 --- /dev/null +++ b/Conifer/SLS/Qwen3-8B/checkpoint-1040/chat_template.jinja @@ -0,0 +1,89 @@ +{%- if tools %} + {{- '<|im_start|>system\n' }} + {%- if messages[0].role == 'system' %} + {{- messages[0].content + '\n\n' }} + {%- endif %} + {{- "# Tools\n\nYou may call one or more functions to assist with the user query.\n\nYou are provided with function signatures within XML tags:\n" }} + {%- for tool in tools %} + {{- "\n" }} + {{- tool | tojson }} + {%- endfor %} + {{- "\n\n\nFor each function call, return a json object with function name and arguments within XML tags:\n\n{\"name\": , \"arguments\": }\n<|im_end|>\n" }} +{%- else %} + {%- if messages[0].role == 'system' %} + {{- '<|im_start|>system\n' + messages[0].content + '<|im_end|>\n' }} + {%- endif %} +{%- endif %} +{%- set ns = namespace(multi_step_tool=true, last_query_index=messages|length - 1) %} +{%- for message in messages[::-1] %} + {%- set index = (messages|length - 1) - loop.index0 %} + {%- if ns.multi_step_tool and message.role == "user" and message.content is string and not(message.content.startswith('') and message.content.endswith('')) %} + {%- set ns.multi_step_tool = false %} + {%- set ns.last_query_index = index %} + {%- endif %} +{%- endfor %} +{%- for message in messages %} + {%- if message.content is string %} + {%- set content = message.content %} + {%- else %} + {%- set content = '' %} + {%- endif %} + {%- if (message.role == "user") or (message.role == "system" and not loop.first) %} + {{- '<|im_start|>' + message.role + '\n' + content + '<|im_end|>' + '\n' }} + {%- elif message.role == "assistant" %} + {%- set reasoning_content = '' %} + {%- if message.reasoning_content is string %} + {%- set reasoning_content = message.reasoning_content %} + {%- else %} + {%- if '' in content %} + {%- set reasoning_content = content.split('')[0].rstrip('\n').split('')[-1].lstrip('\n') %} + {%- set content = content.split('')[-1].lstrip('\n') %} + {%- endif %} + {%- endif %} + {%- if loop.index0 > ns.last_query_index %} + {%- if loop.last or (not loop.last and reasoning_content) %} + {{- '<|im_start|>' + message.role + '\n\n' + reasoning_content.strip('\n') + '\n\n\n' + content.lstrip('\n') }} + {%- else %} + {{- '<|im_start|>' + message.role + '\n' + content }} + {%- endif %} + {%- else %} + {{- '<|im_start|>' + message.role + '\n' + content }} + {%- endif %} + {%- if message.tool_calls %} + {%- for tool_call in message.tool_calls %} + {%- if (loop.first and content) or (not loop.first) %} + {{- '\n' }} + {%- endif %} + {%- if tool_call.function %} + {%- set tool_call = tool_call.function %} + {%- endif %} + {{- '\n{"name": "' }} + {{- tool_call.name }} + {{- '", "arguments": ' }} + {%- if tool_call.arguments is string %} + {{- tool_call.arguments }} + {%- else %} + {{- tool_call.arguments | tojson }} + {%- endif %} + {{- '}\n' }} + {%- endfor %} + {%- endif %} + {{- '<|im_end|>\n' }} + {%- elif message.role == "tool" %} + {%- if loop.first or (messages[loop.index0 - 1].role != "tool") %} + {{- '<|im_start|>user' }} + {%- endif %} + {{- '\n\n' }} + {{- content }} + {{- '\n' }} + {%- if loop.last or (messages[loop.index0 + 1].role != "tool") %} + {{- '<|im_end|>\n' }} + {%- endif %} + {%- endif %} +{%- endfor %} +{%- if add_generation_prompt %} + {{- '<|im_start|>assistant\n' }} + {%- if enable_thinking is defined and enable_thinking is false %} + {{- '\n\n\n\n' }} + {%- endif %} +{%- endif %} \ No newline at end of file diff --git a/Conifer/SLS/Qwen3-8B/checkpoint-1040/policy_adapter/adapter_config.json b/Conifer/SLS/Qwen3-8B/checkpoint-1040/policy_adapter/adapter_config.json new file mode 100644 index 0000000000000000000000000000000000000000..b3c0282381dfea55351c7175fbad57cbb35a290b --- /dev/null +++ b/Conifer/SLS/Qwen3-8B/checkpoint-1040/policy_adapter/adapter_config.json @@ -0,0 +1,45 @@ +{ + "alora_invocation_tokens": null, + "alpha_pattern": {}, + "arrow_config": null, + "auto_mapping": null, + "base_model_name_or_path": "Qwen/Qwen3-8B", + "bias": "none", + "corda_config": null, + "ensure_weight_tying": false, + "eva_config": null, + "exclude_modules": null, + "fan_in_fan_out": false, + "inference_mode": true, + "init_lora_weights": true, + "layer_replication": null, + "layers_pattern": null, + "layers_to_transform": null, + "loftq_config": {}, + "lora_alpha": 64, + "lora_bias": false, + "lora_dropout": 0.05, + "lora_ga_config": null, + "megatron_config": null, + "megatron_core": "megatron.core", + "modules_to_save": null, + "peft_type": "LORA", + "peft_version": "0.19.1", + "qalora_group_size": 16, + "r": 64, + "rank_pattern": {}, + "revision": null, + "target_modules": [ + "v_proj", + "o_proj", + "q_proj", + "k_proj" + ], + "target_parameters": null, + "task_type": "CAUSAL_LM", + "trainable_token_indices": null, + "use_bdlora": null, + "use_dora": false, + "use_qalora": false, + "use_rslora": false +} diff --git a/Conifer/SLS/Qwen3-8B/checkpoint-1040/tokenizer_config.json b/Conifer/SLS/Qwen3-8B/checkpoint-1040/tokenizer_config.json new file mode 100644 index 0000000000000000000000000000000000000000..5668a4a01433b0648bb972458de616f22f3e728e --- /dev/null +++ b/Conifer/SLS/Qwen3-8B/checkpoint-1040/tokenizer_config.json @@ -0,0 +1,30 @@ +{ + "add_prefix_space": false, + "backend": "tokenizers", + "bos_token": null, + "clean_up_tokenization_spaces": false, + "eos_token": "<|im_end|>", + "errors": "replace", + "extra_special_tokens": [ + "<|im_start|>", + "<|im_end|>", + "<|object_ref_start|>", + "<|object_ref_end|>", + "<|box_start|>", + "<|box_end|>", + "<|quad_start|>", + "<|quad_end|>", + "<|vision_start|>", + "<|vision_end|>", + "<|vision_pad|>", + "<|image_pad|>", + "<|video_pad|>" + ], + "is_local": true, + "local_files_only": false, + "model_max_length": 131072, + "pad_token": "<|endoftext|>", + "split_special_tokens": false, + "tokenizer_class": "Qwen2Tokenizer", + "unk_token": null +} diff --git a/Conifer/SLS/Qwen3-8B/checkpoint-1040/training_state.json b/Conifer/SLS/Qwen3-8B/checkpoint-1040/training_state.json new file mode 100644 index 0000000000000000000000000000000000000000..c50db839d4fc64f7937fbf37304bf4e5ec5a18bc --- /dev/null +++ b/Conifer/SLS/Qwen3-8B/checkpoint-1040/training_state.json @@ -0,0 +1,36 @@ +{ + "version": 1, + "global_step": 1040, + "next_epoch": 2, + "next_micro_idx": 5408, + "contract": { + "agent": "A3", + "rollout": "source_project/conifer_training_hub/13_rl_data/conifer_stratified_sft_full_rl_20260902_rl_from_sft.jsonl", + "rollout_size": 3237712215, + "rollout_sha256": "e448b0ebd5ad4eb57fddb5937e4017fa8dd875009aea106cc654cb6a1d8dd77e", + "reward_field": "reward", + "thinking_enabled": false, + "loss_version": "format_protected_signed_rwr_token_kl_v2", + "negative_mask": "semantic_json_values_and_thinking", + "kl_mode": "token_level_forward_kl", + "model_name_or_path": "Qwen/Qwen3-8B", + "sft_adapter": "source_project/conifer_training_hub/11_runs/sft/conifer_stratified_sft_full_rl_20260902/A3/final", + "kl_coef": 0.05, + "num_epochs": 3.0, + "learning_rate": 5e-06, + "weight_decay": 0.01, + "warmup_ratio": 0.05, + "per_device_batch_size": 1, + "gradient_accumulation_steps": 16, + "max_seq_length": 6144, + "lora_rank": 64, + "lora_alpha": 64, + "lora_dropout": 0.05, + "bf16": true, + "gradient_checkpointing": true, + "seed": 42, + "world_size": 8, + "total_records": 44913, + "total_steps": 1053 + } +} diff --git a/Conifer/SLS/Qwen3-8B/final/adapter_config.json b/Conifer/SLS/Qwen3-8B/final/adapter_config.json new file mode 100644 index 0000000000000000000000000000000000000000..b3c0282381dfea55351c7175fbad57cbb35a290b --- /dev/null +++ b/Conifer/SLS/Qwen3-8B/final/adapter_config.json @@ -0,0 +1,45 @@ +{ + "alora_invocation_tokens": null, + "alpha_pattern": {}, + "arrow_config": null, + "auto_mapping": null, + "base_model_name_or_path": "Qwen/Qwen3-8B", + "bias": "none", + "corda_config": null, + "ensure_weight_tying": false, + "eva_config": null, + "exclude_modules": null, + "fan_in_fan_out": false, + "inference_mode": true, + "init_lora_weights": true, + "layer_replication": null, + "layers_pattern": null, + "layers_to_transform": null, + "loftq_config": {}, + "lora_alpha": 64, + "lora_bias": false, + "lora_dropout": 0.05, + "lora_ga_config": null, + "megatron_config": null, + "megatron_core": "megatron.core", + "modules_to_save": null, + "peft_type": "LORA", + "peft_version": "0.19.1", + "qalora_group_size": 16, + "r": 64, + "rank_pattern": {}, + "revision": null, + "target_modules": [ + "v_proj", + "o_proj", + "q_proj", + "k_proj" + ], + "target_parameters": null, + "task_type": "CAUSAL_LM", + "trainable_token_indices": null, + "use_bdlora": null, + "use_dora": false, + "use_qalora": false, + "use_rslora": false +} diff --git a/Conifer/SLS/Qwen3-8B/final/chat_template.jinja b/Conifer/SLS/Qwen3-8B/final/chat_template.jinja new file mode 100644 index 0000000000000000000000000000000000000000..01be9b307daa2d425f7c168c9fb145a286e0afb4 --- /dev/null +++ b/Conifer/SLS/Qwen3-8B/final/chat_template.jinja @@ -0,0 +1,89 @@ +{%- if tools %} + {{- '<|im_start|>system\n' }} + {%- if messages[0].role == 'system' %} + {{- messages[0].content + '\n\n' }} + {%- endif %} + {{- "# Tools\n\nYou may call one or more functions to assist with the user query.\n\nYou are provided with function signatures within XML tags:\n" }} + {%- for tool in tools %} + {{- "\n" }} + {{- tool | tojson }} + {%- endfor %} + {{- "\n\n\nFor each function call, return a json object with function name and arguments within XML tags:\n\n{\"name\": , \"arguments\": }\n<|im_end|>\n" }} +{%- else %} + {%- if messages[0].role == 'system' %} + {{- '<|im_start|>system\n' + messages[0].content + '<|im_end|>\n' }} + {%- endif %} +{%- endif %} +{%- set ns = namespace(multi_step_tool=true, last_query_index=messages|length - 1) %} +{%- for message in messages[::-1] %} + {%- set index = (messages|length - 1) - loop.index0 %} + {%- if ns.multi_step_tool and message.role == "user" and message.content is string and not(message.content.startswith('') and message.content.endswith('')) %} + {%- set ns.multi_step_tool = false %} + {%- set ns.last_query_index = index %} + {%- endif %} +{%- endfor %} +{%- for message in messages %} + {%- if message.content is string %} + {%- set content = message.content %} + {%- else %} + {%- set content = '' %} + {%- endif %} + {%- if (message.role == "user") or (message.role == "system" and not loop.first) %} + {{- '<|im_start|>' + message.role + '\n' + content + '<|im_end|>' + '\n' }} + {%- elif message.role == "assistant" %} + {%- set reasoning_content = '' %} + {%- if message.reasoning_content is string %} + {%- set reasoning_content = message.reasoning_content %} + {%- else %} + {%- if '' in content %} + {%- set reasoning_content = content.split('')[0].rstrip('\n').split('')[-1].lstrip('\n') %} + {%- set content = content.split('')[-1].lstrip('\n') %} + {%- endif %} + {%- endif %} + {%- if loop.index0 > ns.last_query_index %} + {%- if loop.last or (not loop.last and reasoning_content) %} + {{- '<|im_start|>' + message.role + '\n\n' + reasoning_content.strip('\n') + '\n\n\n' + content.lstrip('\n') }} + {%- else %} + {{- '<|im_start|>' + message.role + '\n' + content }} + {%- endif %} + {%- else %} + {{- '<|im_start|>' + message.role + '\n' + content }} + {%- endif %} + {%- if message.tool_calls %} + {%- for tool_call in message.tool_calls %} + {%- if (loop.first and content) or (not loop.first) %} + {{- '\n' }} + {%- endif %} + {%- if tool_call.function %} + {%- set tool_call = tool_call.function %} + {%- endif %} + {{- '\n{"name": "' }} + {{- tool_call.name }} + {{- '", "arguments": ' }} + {%- if tool_call.arguments is string %} + {{- tool_call.arguments }} + {%- else %} + {{- tool_call.arguments | tojson }} + {%- endif %} + {{- '}\n' }} + {%- endfor %} + {%- endif %} + {{- '<|im_end|>\n' }} + {%- elif message.role == "tool" %} + {%- if loop.first or (messages[loop.index0 - 1].role != "tool") %} + {{- '<|im_start|>user' }} + {%- endif %} + {{- '\n\n' }} + {{- content }} + {{- '\n' }} + {%- if loop.last or (messages[loop.index0 + 1].role != "tool") %} + {{- '<|im_end|>\n' }} + {%- endif %} + {%- endif %} +{%- endfor %} +{%- if add_generation_prompt %} + {{- '<|im_start|>assistant\n' }} + {%- if enable_thinking is defined and enable_thinking is false %} + {{- '\n\n\n\n' }} + {%- endif %} +{%- endif %} \ No newline at end of file diff --git a/Conifer/SLS/Qwen3-8B/final/rl_train_summary.json b/Conifer/SLS/Qwen3-8B/final/rl_train_summary.json new file mode 100644 index 0000000000000000000000000000000000000000..d9dd4aa1057af453748737c0ff2582526211ac5f --- /dev/null +++ b/Conifer/SLS/Qwen3-8B/final/rl_train_summary.json @@ -0,0 +1,50 @@ +{ + "summary_version": 2, + "agent": "A3", + "global_step": 1053, + "total_records": 44913, + "algorithm": "rwr", + "kl_coef": 0.05, + "thinking_enabled": false, + "loss_version": "format_protected_signed_rwr_token_kl_v2", + "negative_mask": "semantic_json_values_and_thinking", + "kl_mode": "token_level_forward_kl", + "world_size": 8, + "completed": true, + "rollout": "source_project/conifer_training_hub/13_rl_data/conifer_stratified_sft_full_rl_20260902_rl_from_sft.jsonl", + "rollout_size": 3237712215, + "rollout_sha256": "e448b0ebd5ad4eb57fddb5937e4017fa8dd875009aea106cc654cb6a1d8dd77e", + "adapter_sha256": "686e500b569908330b1a0d9599793e0e6397b2b3599b671c6eb37a2c5af4be67", + "model_name_or_path": "Qwen/Qwen3-8B", + "sft_adapter": "source_project/conifer_training_hub/11_runs/sft/conifer_stratified_sft_full_rl_20260902/A3/final", + "contract": { + "agent": "A3", + "rollout": "source_project/conifer_training_hub/13_rl_data/conifer_stratified_sft_full_rl_20260902_rl_from_sft.jsonl", + "rollout_size": 3237712215, + "rollout_sha256": "e448b0ebd5ad4eb57fddb5937e4017fa8dd875009aea106cc654cb6a1d8dd77e", + "reward_field": "reward", + "thinking_enabled": false, + "loss_version": "format_protected_signed_rwr_token_kl_v2", + "negative_mask": "semantic_json_values_and_thinking", + "kl_mode": "token_level_forward_kl", + "model_name_or_path": "Qwen/Qwen3-8B", + "sft_adapter": "source_project/conifer_training_hub/11_runs/sft/conifer_stratified_sft_full_rl_20260902/A3/final", + "kl_coef": 0.05, + "num_epochs": 3.0, + "learning_rate": 5e-06, + "weight_decay": 0.01, + "warmup_ratio": 0.05, + "per_device_batch_size": 1, + "gradient_accumulation_steps": 16, + "max_seq_length": 6144, + "lora_rank": 64, + "lora_alpha": 64, + "lora_dropout": 0.05, + "bf16": true, + "gradient_checkpointing": true, + "seed": 42, + "world_size": 8, + "total_records": 44913, + "total_steps": 1053 + } +} diff --git a/Conifer/SLS/Qwen3-8B/final/tokenizer_config.json b/Conifer/SLS/Qwen3-8B/final/tokenizer_config.json new file mode 100644 index 0000000000000000000000000000000000000000..5668a4a01433b0648bb972458de616f22f3e728e --- /dev/null +++ b/Conifer/SLS/Qwen3-8B/final/tokenizer_config.json @@ -0,0 +1,30 @@ +{ + "add_prefix_space": false, + "backend": "tokenizers", + "bos_token": null, + "clean_up_tokenization_spaces": false, + "eos_token": "<|im_end|>", + "errors": "replace", + "extra_special_tokens": [ + "<|im_start|>", + "<|im_end|>", + "<|object_ref_start|>", + "<|object_ref_end|>", + "<|box_start|>", + "<|box_end|>", + "<|quad_start|>", + "<|quad_end|>", + "<|vision_start|>", + "<|vision_end|>", + "<|vision_pad|>", + "<|image_pad|>", + "<|video_pad|>" + ], + "is_local": true, + "local_files_only": false, + "model_max_length": 131072, + "pad_token": "<|endoftext|>", + "split_special_tokens": false, + "tokenizer_class": "Qwen2Tokenizer", + "unk_token": null +} diff --git a/MATH/PD/Qwen3-8B/checkpoint-100/adapter_config.json b/MATH/PD/Qwen3-8B/checkpoint-100/adapter_config.json new file mode 100644 index 0000000000000000000000000000000000000000..88f42351f47a6c2cb2e7c315caf403cf833a2600 --- /dev/null +++ b/MATH/PD/Qwen3-8B/checkpoint-100/adapter_config.json @@ -0,0 +1,45 @@ +{ + "alora_invocation_tokens": null, + "alpha_pattern": {}, + "arrow_config": null, + "auto_mapping": null, + "base_model_name_or_path": "Qwen/Qwen3-8B", + "bias": "none", + "corda_config": null, + "ensure_weight_tying": false, + "eva_config": null, + "exclude_modules": null, + "fan_in_fan_out": false, + "inference_mode": true, + "init_lora_weights": true, + "layer_replication": null, + "layers_pattern": null, + "layers_to_transform": null, + "loftq_config": {}, + "lora_alpha": 64, + "lora_bias": false, + "lora_dropout": 0.05, + "lora_ga_config": null, + "megatron_config": null, + "megatron_core": "megatron.core", + "modules_to_save": null, + "peft_type": "LORA", + "peft_version": "0.19.1", + "qalora_group_size": 16, + "r": 64, + "rank_pattern": {}, + "revision": null, + "target_modules": [ + "q_proj", + "v_proj", + "k_proj", + "o_proj" + ], + "target_parameters": null, + "task_type": "CAUSAL_LM", + "trainable_token_indices": null, + "use_bdlora": null, + "use_dora": false, + "use_qalora": false, + "use_rslora": false +} diff --git a/MATH/PD/Qwen3-8B/checkpoint-100/chat_template.jinja b/MATH/PD/Qwen3-8B/checkpoint-100/chat_template.jinja new file mode 100644 index 0000000000000000000000000000000000000000..01be9b307daa2d425f7c168c9fb145a286e0afb4 --- /dev/null +++ b/MATH/PD/Qwen3-8B/checkpoint-100/chat_template.jinja @@ -0,0 +1,89 @@ +{%- if tools %} + {{- '<|im_start|>system\n' }} + {%- if messages[0].role == 'system' %} + {{- messages[0].content + '\n\n' }} + {%- endif %} + {{- "# Tools\n\nYou may call one or more functions to assist with the user query.\n\nYou are provided with function signatures within XML tags:\n" }} + {%- for tool in tools %} + {{- "\n" }} + {{- tool | tojson }} + {%- endfor %} + {{- "\n\n\nFor each function call, return a json object with function name and arguments within XML tags:\n\n{\"name\": , \"arguments\": }\n<|im_end|>\n" }} +{%- else %} + {%- if messages[0].role == 'system' %} + {{- '<|im_start|>system\n' + messages[0].content + '<|im_end|>\n' }} + {%- endif %} +{%- endif %} +{%- set ns = namespace(multi_step_tool=true, last_query_index=messages|length - 1) %} +{%- for message in messages[::-1] %} + {%- set index = (messages|length - 1) - loop.index0 %} + {%- if ns.multi_step_tool and message.role == "user" and message.content is string and not(message.content.startswith('') and message.content.endswith('')) %} + {%- set ns.multi_step_tool = false %} + {%- set ns.last_query_index = index %} + {%- endif %} +{%- endfor %} +{%- for message in messages %} + {%- if message.content is string %} + {%- set content = message.content %} + {%- else %} + {%- set content = '' %} + {%- endif %} + {%- if (message.role == "user") or (message.role == "system" and not loop.first) %} + {{- '<|im_start|>' + message.role + '\n' + content + '<|im_end|>' + '\n' }} + {%- elif message.role == "assistant" %} + {%- set reasoning_content = '' %} + {%- if message.reasoning_content is string %} + {%- set reasoning_content = message.reasoning_content %} + {%- else %} + {%- if '' in content %} + {%- set reasoning_content = content.split('')[0].rstrip('\n').split('')[-1].lstrip('\n') %} + {%- set content = content.split('')[-1].lstrip('\n') %} + {%- endif %} + {%- endif %} + {%- if loop.index0 > ns.last_query_index %} + {%- if loop.last or (not loop.last and reasoning_content) %} + {{- '<|im_start|>' + message.role + '\n\n' + reasoning_content.strip('\n') + '\n\n\n' + content.lstrip('\n') }} + {%- else %} + {{- '<|im_start|>' + message.role + '\n' + content }} + {%- endif %} + {%- else %} + {{- '<|im_start|>' + message.role + '\n' + content }} + {%- endif %} + {%- if message.tool_calls %} + {%- for tool_call in message.tool_calls %} + {%- if (loop.first and content) or (not loop.first) %} + {{- '\n' }} + {%- endif %} + {%- if tool_call.function %} + {%- set tool_call = tool_call.function %} + {%- endif %} + {{- '\n{"name": "' }} + {{- tool_call.name }} + {{- '", "arguments": ' }} + {%- if tool_call.arguments is string %} + {{- tool_call.arguments }} + {%- else %} + {{- tool_call.arguments | tojson }} + {%- endif %} + {{- '}\n' }} + {%- endfor %} + {%- endif %} + {{- '<|im_end|>\n' }} + {%- elif message.role == "tool" %} + {%- if loop.first or (messages[loop.index0 - 1].role != "tool") %} + {{- '<|im_start|>user' }} + {%- endif %} + {{- '\n\n' }} + {{- content }} + {{- '\n' }} + {%- if loop.last or (messages[loop.index0 + 1].role != "tool") %} + {{- '<|im_end|>\n' }} + {%- endif %} + {%- endif %} +{%- endfor %} +{%- if add_generation_prompt %} + {{- '<|im_start|>assistant\n' }} + {%- if enable_thinking is defined and enable_thinking is false %} + {{- '\n\n\n\n' }} + {%- endif %} +{%- endif %} \ No newline at end of file diff --git a/MATH/PD/Qwen3-8B/checkpoint-100/tokenizer_config.json b/MATH/PD/Qwen3-8B/checkpoint-100/tokenizer_config.json new file mode 100644 index 0000000000000000000000000000000000000000..5668a4a01433b0648bb972458de616f22f3e728e --- /dev/null +++ b/MATH/PD/Qwen3-8B/checkpoint-100/tokenizer_config.json @@ -0,0 +1,30 @@ +{ + "add_prefix_space": false, + "backend": "tokenizers", + "bos_token": null, + "clean_up_tokenization_spaces": false, + "eos_token": "<|im_end|>", + "errors": "replace", + "extra_special_tokens": [ + "<|im_start|>", + "<|im_end|>", + "<|object_ref_start|>", + "<|object_ref_end|>", + "<|box_start|>", + "<|box_end|>", + "<|quad_start|>", + "<|quad_end|>", + "<|vision_start|>", + "<|vision_end|>", + "<|vision_pad|>", + "<|image_pad|>", + "<|video_pad|>" + ], + "is_local": true, + "local_files_only": false, + "model_max_length": 131072, + "pad_token": "<|endoftext|>", + "split_special_tokens": false, + "tokenizer_class": "Qwen2Tokenizer", + "unk_token": null +} diff --git a/MATH/PD/Qwen3-8B/checkpoint-100/trainer_state.json b/MATH/PD/Qwen3-8B/checkpoint-100/trainer_state.json new file mode 100644 index 0000000000000000000000000000000000000000..ec44b6e7b74cee09458261a78ac167da741ca0b2 --- /dev/null +++ b/MATH/PD/Qwen3-8B/checkpoint-100/trainer_state.json @@ -0,0 +1,120 @@ +{ + "best_global_step": 100, + "best_metric": 0.07400760799646378, + "best_model_checkpoint": "source_project/experiments/math_specific_sft_rl_v1/artifacts/math_specific_sft_rl_reuse_legacy_v4d_real_rollout_sft_protocol_floor_v2_20260830/01_sft/adapters/A3/checkpoint-100", + "epoch": 1.5190839694656488, + "eval_steps": 50, + "global_step": 100, + "is_hyper_param_search": false, + "is_local_process_zero": true, + "is_world_process_zero": true, + "log_history": [ + { + "epoch": 0.15267175572519084, + "grad_norm": 0.2715332806110382, + "learning_rate": 9e-06, + "loss": 0.19996399879455568, + "step": 10 + }, + { + "epoch": 0.3053435114503817, + "grad_norm": 0.14735907316207886, + "learning_rate": 9.943559569286731e-06, + "loss": 0.18235771656036376, + "step": 20 + }, + { + "epoch": 0.4580152671755725, + "grad_norm": 0.10940974205732346, + "learning_rate": 9.75009217427352e-06, + "loss": 0.13847777843475342, + "step": 30 + }, + { + "epoch": 0.6106870229007634, + "grad_norm": 0.14101381599903107, + "learning_rate": 9.424289624237143e-06, + "loss": 0.13123193979263306, + "step": 40 + }, + { + "epoch": 0.7633587786259542, + "grad_norm": 0.11885557323694229, + "learning_rate": 8.975228612720415e-06, + "loss": 0.12588239908218385, + "step": 50 + }, + { + "epoch": 0.7633587786259542, + "eval_loss": 0.1032027006149292, + "eval_runtime": 18.9159, + "eval_samples_per_second": 5.815, + "eval_steps_per_second": 0.74, + "step": 50 + }, + { + "epoch": 0.916030534351145, + "grad_norm": 0.10043700039386749, + "learning_rate": 8.415419751390155e-06, + "loss": 0.10407333374023438, + "step": 60 + }, + { + "epoch": 1.0610687022900764, + "grad_norm": 0.11581146717071533, + "learning_rate": 7.760459030751285e-06, + "loss": 0.10394111871719361, + "step": 70 + }, + { + "epoch": 1.213740458015267, + "grad_norm": 0.0961180329322815, + "learning_rate": 7.028593323773819e-06, + "loss": 0.09114642143249511, + "step": 80 + }, + { + "epoch": 1.366412213740458, + "grad_norm": 0.10328122973442078, + "learning_rate": 6.240212037280967e-06, + "loss": 0.0868987798690796, + "step": 90 + }, + { + "epoch": 1.5190839694656488, + "grad_norm": 0.09781694412231445, + "learning_rate": 5.41727907343245e-06, + "loss": 0.08267452120780945, + "step": 100 + }, + { + "epoch": 1.5190839694656488, + "eval_loss": 0.07400760799646378, + "eval_runtime": 19.2077, + "eval_samples_per_second": 5.727, + "eval_steps_per_second": 0.729, + "step": 100 + } + ], + "logging_steps": 10, + "max_steps": 198, + "num_input_tokens_seen": 0, + "num_train_epochs": 3, + "save_steps": 50, + "stateful_callbacks": { + "TrainerControl": { + "args": { + "should_epoch_stop": false, + "should_evaluate": false, + "should_log": false, + "should_save": true, + "should_training_stop": false + }, + "attributes": {} + } + }, + "total_flos": 2.0051076510960845e+17, + "train_batch_size": 1, + "trial_name": null, + "trial_params": null +} diff --git a/MATH/PD/Qwen3-8B/checkpoint-150/adapter_config.json b/MATH/PD/Qwen3-8B/checkpoint-150/adapter_config.json new file mode 100644 index 0000000000000000000000000000000000000000..88f42351f47a6c2cb2e7c315caf403cf833a2600 --- /dev/null +++ b/MATH/PD/Qwen3-8B/checkpoint-150/adapter_config.json @@ -0,0 +1,45 @@ +{ + "alora_invocation_tokens": null, + "alpha_pattern": {}, + "arrow_config": null, + "auto_mapping": null, + "base_model_name_or_path": "Qwen/Qwen3-8B", + "bias": "none", + "corda_config": null, + "ensure_weight_tying": false, + "eva_config": null, + "exclude_modules": null, + "fan_in_fan_out": false, + "inference_mode": true, + "init_lora_weights": true, + "layer_replication": null, + "layers_pattern": null, + "layers_to_transform": null, + "loftq_config": {}, + "lora_alpha": 64, + "lora_bias": false, + "lora_dropout": 0.05, + "lora_ga_config": null, + "megatron_config": null, + "megatron_core": "megatron.core", + "modules_to_save": null, + "peft_type": "LORA", + "peft_version": "0.19.1", + "qalora_group_size": 16, + "r": 64, + "rank_pattern": {}, + "revision": null, + "target_modules": [ + "q_proj", + "v_proj", + "k_proj", + "o_proj" + ], + "target_parameters": null, + "task_type": "CAUSAL_LM", + "trainable_token_indices": null, + "use_bdlora": null, + "use_dora": false, + "use_qalora": false, + "use_rslora": false +} diff --git a/MATH/PD/Qwen3-8B/checkpoint-150/chat_template.jinja b/MATH/PD/Qwen3-8B/checkpoint-150/chat_template.jinja new file mode 100644 index 0000000000000000000000000000000000000000..01be9b307daa2d425f7c168c9fb145a286e0afb4 --- /dev/null +++ b/MATH/PD/Qwen3-8B/checkpoint-150/chat_template.jinja @@ -0,0 +1,89 @@ +{%- if tools %} + {{- '<|im_start|>system\n' }} + {%- if messages[0].role == 'system' %} + {{- messages[0].content + '\n\n' }} + {%- endif %} + {{- "# Tools\n\nYou may call one or more functions to assist with the user query.\n\nYou are provided with function signatures within XML tags:\n" }} + {%- for tool in tools %} + {{- "\n" }} + {{- tool | tojson }} + {%- endfor %} + {{- "\n\n\nFor each function call, return a json object with function name and arguments within XML tags:\n\n{\"name\": , \"arguments\": }\n<|im_end|>\n" }} +{%- else %} + {%- if messages[0].role == 'system' %} + {{- '<|im_start|>system\n' + messages[0].content + '<|im_end|>\n' }} + {%- endif %} +{%- endif %} +{%- set ns = namespace(multi_step_tool=true, last_query_index=messages|length - 1) %} +{%- for message in messages[::-1] %} + {%- set index = (messages|length - 1) - loop.index0 %} + {%- if ns.multi_step_tool and message.role == "user" and message.content is string and not(message.content.startswith('') and message.content.endswith('')) %} + {%- set ns.multi_step_tool = false %} + {%- set ns.last_query_index = index %} + {%- endif %} +{%- endfor %} +{%- for message in messages %} + {%- if message.content is string %} + {%- set content = message.content %} + {%- else %} + {%- set content = '' %} + {%- endif %} + {%- if (message.role == "user") or (message.role == "system" and not loop.first) %} + {{- '<|im_start|>' + message.role + '\n' + content + '<|im_end|>' + '\n' }} + {%- elif message.role == "assistant" %} + {%- set reasoning_content = '' %} + {%- if message.reasoning_content is string %} + {%- set reasoning_content = message.reasoning_content %} + {%- else %} + {%- if '' in content %} + {%- set reasoning_content = content.split('')[0].rstrip('\n').split('')[-1].lstrip('\n') %} + {%- set content = content.split('')[-1].lstrip('\n') %} + {%- endif %} + {%- endif %} + {%- if loop.index0 > ns.last_query_index %} + {%- if loop.last or (not loop.last and reasoning_content) %} + {{- '<|im_start|>' + message.role + '\n\n' + reasoning_content.strip('\n') + '\n\n\n' + content.lstrip('\n') }} + {%- else %} + {{- '<|im_start|>' + message.role + '\n' + content }} + {%- endif %} + {%- else %} + {{- '<|im_start|>' + message.role + '\n' + content }} + {%- endif %} + {%- if message.tool_calls %} + {%- for tool_call in message.tool_calls %} + {%- if (loop.first and content) or (not loop.first) %} + {{- '\n' }} + {%- endif %} + {%- if tool_call.function %} + {%- set tool_call = tool_call.function %} + {%- endif %} + {{- '\n{"name": "' }} + {{- tool_call.name }} + {{- '", "arguments": ' }} + {%- if tool_call.arguments is string %} + {{- tool_call.arguments }} + {%- else %} + {{- tool_call.arguments | tojson }} + {%- endif %} + {{- '}\n' }} + {%- endfor %} + {%- endif %} + {{- '<|im_end|>\n' }} + {%- elif message.role == "tool" %} + {%- if loop.first or (messages[loop.index0 - 1].role != "tool") %} + {{- '<|im_start|>user' }} + {%- endif %} + {{- '\n\n' }} + {{- content }} + {{- '\n' }} + {%- if loop.last or (messages[loop.index0 + 1].role != "tool") %} + {{- '<|im_end|>\n' }} + {%- endif %} + {%- endif %} +{%- endfor %} +{%- if add_generation_prompt %} + {{- '<|im_start|>assistant\n' }} + {%- if enable_thinking is defined and enable_thinking is false %} + {{- '\n\n\n\n' }} + {%- endif %} +{%- endif %} \ No newline at end of file diff --git a/MATH/PD/Qwen3-8B/checkpoint-150/tokenizer_config.json b/MATH/PD/Qwen3-8B/checkpoint-150/tokenizer_config.json new file mode 100644 index 0000000000000000000000000000000000000000..5668a4a01433b0648bb972458de616f22f3e728e --- /dev/null +++ b/MATH/PD/Qwen3-8B/checkpoint-150/tokenizer_config.json @@ -0,0 +1,30 @@ +{ + "add_prefix_space": false, + "backend": "tokenizers", + "bos_token": null, + "clean_up_tokenization_spaces": false, + "eos_token": "<|im_end|>", + "errors": "replace", + "extra_special_tokens": [ + "<|im_start|>", + "<|im_end|>", + "<|object_ref_start|>", + "<|object_ref_end|>", + "<|box_start|>", + "<|box_end|>", + "<|quad_start|>", + "<|quad_end|>", + "<|vision_start|>", + "<|vision_end|>", + "<|vision_pad|>", + "<|image_pad|>", + "<|video_pad|>" + ], + "is_local": true, + "local_files_only": false, + "model_max_length": 131072, + "pad_token": "<|endoftext|>", + "split_special_tokens": false, + "tokenizer_class": "Qwen2Tokenizer", + "unk_token": null +} diff --git a/MATH/PD/Qwen3-8B/checkpoint-150/trainer_state.json b/MATH/PD/Qwen3-8B/checkpoint-150/trainer_state.json new file mode 100644 index 0000000000000000000000000000000000000000..6ba3fe250bf830a4d89b3dd5e0112b4b2c797879 --- /dev/null +++ b/MATH/PD/Qwen3-8B/checkpoint-150/trainer_state.json @@ -0,0 +1,163 @@ +{ + "best_global_step": 150, + "best_metric": 0.06524904072284698, + "best_model_checkpoint": "source_project/experiments/math_specific_sft_rl_v1/artifacts/math_specific_sft_rl_reuse_legacy_v4d_real_rollout_sft_protocol_floor_v2_20260830/01_sft/adapters/A3/checkpoint-150", + "epoch": 2.2748091603053435, + "eval_steps": 50, + "global_step": 150, + "is_hyper_param_search": false, + "is_local_process_zero": true, + "is_world_process_zero": true, + "log_history": [ + { + "epoch": 0.15267175572519084, + "grad_norm": 0.2715332806110382, + "learning_rate": 9e-06, + "loss": 0.19996399879455568, + "step": 10 + }, + { + "epoch": 0.3053435114503817, + "grad_norm": 0.14735907316207886, + "learning_rate": 9.943559569286731e-06, + "loss": 0.18235771656036376, + "step": 20 + }, + { + "epoch": 0.4580152671755725, + "grad_norm": 0.10940974205732346, + "learning_rate": 9.75009217427352e-06, + "loss": 0.13847777843475342, + "step": 30 + }, + { + "epoch": 0.6106870229007634, + "grad_norm": 0.14101381599903107, + "learning_rate": 9.424289624237143e-06, + "loss": 0.13123193979263306, + "step": 40 + }, + { + "epoch": 0.7633587786259542, + "grad_norm": 0.11885557323694229, + "learning_rate": 8.975228612720415e-06, + "loss": 0.12588239908218385, + "step": 50 + }, + { + "epoch": 0.7633587786259542, + "eval_loss": 0.1032027006149292, + "eval_runtime": 18.9159, + "eval_samples_per_second": 5.815, + "eval_steps_per_second": 0.74, + "step": 50 + }, + { + "epoch": 0.916030534351145, + "grad_norm": 0.10043700039386749, + "learning_rate": 8.415419751390155e-06, + "loss": 0.10407333374023438, + "step": 60 + }, + { + "epoch": 1.0610687022900764, + "grad_norm": 0.11581146717071533, + "learning_rate": 7.760459030751285e-06, + "loss": 0.10394111871719361, + "step": 70 + }, + { + "epoch": 1.213740458015267, + "grad_norm": 0.0961180329322815, + "learning_rate": 7.028593323773819e-06, + "loss": 0.09114642143249511, + "step": 80 + }, + { + "epoch": 1.366412213740458, + "grad_norm": 0.10328122973442078, + "learning_rate": 6.240212037280967e-06, + "loss": 0.0868987798690796, + "step": 90 + }, + { + "epoch": 1.5190839694656488, + "grad_norm": 0.09781694412231445, + "learning_rate": 5.41727907343245e-06, + "loss": 0.08267452120780945, + "step": 100 + }, + { + "epoch": 1.5190839694656488, + "eval_loss": 0.07400760799646378, + "eval_runtime": 19.2077, + "eval_samples_per_second": 5.727, + "eval_steps_per_second": 0.729, + "step": 100 + }, + { + "epoch": 1.6717557251908397, + "grad_norm": 0.11948485672473907, + "learning_rate": 4.582720926567552e-06, + "loss": 0.0813731849193573, + "step": 110 + }, + { + "epoch": 1.8244274809160306, + "grad_norm": 0.1227552741765976, + "learning_rate": 3.7597879627190337e-06, + "loss": 0.07353355288505554, + "step": 120 + }, + { + "epoch": 1.9770992366412212, + "grad_norm": 0.09954976290464401, + "learning_rate": 2.9714066762261825e-06, + "loss": 0.07269736528396606, + "step": 130 + }, + { + "epoch": 2.122137404580153, + "grad_norm": 0.10711696743965149, + "learning_rate": 2.2395409692487174e-06, + "loss": 0.07261578440666198, + "step": 140 + }, + { + "epoch": 2.2748091603053435, + "grad_norm": 0.10855450481176376, + "learning_rate": 1.5845802486098461e-06, + "loss": 0.07437153458595276, + "step": 150 + }, + { + "epoch": 2.2748091603053435, + "eval_loss": 0.06524904072284698, + "eval_runtime": 18.7372, + "eval_samples_per_second": 5.871, + "eval_steps_per_second": 0.747, + "step": 150 + } + ], + "logging_steps": 10, + "max_steps": 198, + "num_input_tokens_seen": 0, + "num_train_epochs": 3, + "save_steps": 50, + "stateful_callbacks": { + "TrainerControl": { + "args": { + "should_epoch_stop": false, + "should_evaluate": false, + "should_log": false, + "should_save": true, + "should_training_stop": false + }, + "attributes": {} + } + }, + "total_flos": 2.9987447057783194e+17, + "train_batch_size": 1, + "trial_name": null, + "trial_params": null +} diff --git a/MATH/PD/Qwen3-8B/checkpoint-198/adapter_config.json b/MATH/PD/Qwen3-8B/checkpoint-198/adapter_config.json new file mode 100644 index 0000000000000000000000000000000000000000..88f42351f47a6c2cb2e7c315caf403cf833a2600 --- /dev/null +++ b/MATH/PD/Qwen3-8B/checkpoint-198/adapter_config.json @@ -0,0 +1,45 @@ +{ + "alora_invocation_tokens": null, + "alpha_pattern": {}, + "arrow_config": null, + "auto_mapping": null, + "base_model_name_or_path": "Qwen/Qwen3-8B", + "bias": "none", + "corda_config": null, + "ensure_weight_tying": false, + "eva_config": null, + "exclude_modules": null, + "fan_in_fan_out": false, + "inference_mode": true, + "init_lora_weights": true, + "layer_replication": null, + "layers_pattern": null, + "layers_to_transform": null, + "loftq_config": {}, + "lora_alpha": 64, + "lora_bias": false, + "lora_dropout": 0.05, + "lora_ga_config": null, + "megatron_config": null, + "megatron_core": "megatron.core", + "modules_to_save": null, + "peft_type": "LORA", + "peft_version": "0.19.1", + "qalora_group_size": 16, + "r": 64, + "rank_pattern": {}, + "revision": null, + "target_modules": [ + "q_proj", + "v_proj", + "k_proj", + "o_proj" + ], + "target_parameters": null, + "task_type": "CAUSAL_LM", + "trainable_token_indices": null, + "use_bdlora": null, + "use_dora": false, + "use_qalora": false, + "use_rslora": false +} diff --git a/MATH/PD/Qwen3-8B/checkpoint-198/chat_template.jinja b/MATH/PD/Qwen3-8B/checkpoint-198/chat_template.jinja new file mode 100644 index 0000000000000000000000000000000000000000..01be9b307daa2d425f7c168c9fb145a286e0afb4 --- /dev/null +++ b/MATH/PD/Qwen3-8B/checkpoint-198/chat_template.jinja @@ -0,0 +1,89 @@ +{%- if tools %} + {{- '<|im_start|>system\n' }} + {%- if messages[0].role == 'system' %} + {{- messages[0].content + '\n\n' }} + {%- endif %} + {{- "# Tools\n\nYou may call one or more functions to assist with the user query.\n\nYou are provided with function signatures within XML tags:\n" }} + {%- for tool in tools %} + {{- "\n" }} + {{- tool | tojson }} + {%- endfor %} + {{- "\n\n\nFor each function call, return a json object with function name and arguments within XML tags:\n\n{\"name\": , \"arguments\": }\n<|im_end|>\n" }} +{%- else %} + {%- if messages[0].role == 'system' %} + {{- '<|im_start|>system\n' + messages[0].content + '<|im_end|>\n' }} + {%- endif %} +{%- endif %} +{%- set ns = namespace(multi_step_tool=true, last_query_index=messages|length - 1) %} +{%- for message in messages[::-1] %} + {%- set index = (messages|length - 1) - loop.index0 %} + {%- if ns.multi_step_tool and message.role == "user" and message.content is string and not(message.content.startswith('') and message.content.endswith('')) %} + {%- set ns.multi_step_tool = false %} + {%- set ns.last_query_index = index %} + {%- endif %} +{%- endfor %} +{%- for message in messages %} + {%- if message.content is string %} + {%- set content = message.content %} + {%- else %} + {%- set content = '' %} + {%- endif %} + {%- if (message.role == "user") or (message.role == "system" and not loop.first) %} + {{- '<|im_start|>' + message.role + '\n' + content + '<|im_end|>' + '\n' }} + {%- elif message.role == "assistant" %} + {%- set reasoning_content = '' %} + {%- if message.reasoning_content is string %} + {%- set reasoning_content = message.reasoning_content %} + {%- else %} + {%- if '' in content %} + {%- set reasoning_content = content.split('')[0].rstrip('\n').split('')[-1].lstrip('\n') %} + {%- set content = content.split('')[-1].lstrip('\n') %} + {%- endif %} + {%- endif %} + {%- if loop.index0 > ns.last_query_index %} + {%- if loop.last or (not loop.last and reasoning_content) %} + {{- '<|im_start|>' + message.role + '\n\n' + reasoning_content.strip('\n') + '\n\n\n' + content.lstrip('\n') }} + {%- else %} + {{- '<|im_start|>' + message.role + '\n' + content }} + {%- endif %} + {%- else %} + {{- '<|im_start|>' + message.role + '\n' + content }} + {%- endif %} + {%- if message.tool_calls %} + {%- for tool_call in message.tool_calls %} + {%- if (loop.first and content) or (not loop.first) %} + {{- '\n' }} + {%- endif %} + {%- if tool_call.function %} + {%- set tool_call = tool_call.function %} + {%- endif %} + {{- '\n{"name": "' }} + {{- tool_call.name }} + {{- '", "arguments": ' }} + {%- if tool_call.arguments is string %} + {{- tool_call.arguments }} + {%- else %} + {{- tool_call.arguments | tojson }} + {%- endif %} + {{- '}\n' }} + {%- endfor %} + {%- endif %} + {{- '<|im_end|>\n' }} + {%- elif message.role == "tool" %} + {%- if loop.first or (messages[loop.index0 - 1].role != "tool") %} + {{- '<|im_start|>user' }} + {%- endif %} + {{- '\n\n' }} + {{- content }} + {{- '\n' }} + {%- if loop.last or (messages[loop.index0 + 1].role != "tool") %} + {{- '<|im_end|>\n' }} + {%- endif %} + {%- endif %} +{%- endfor %} +{%- if add_generation_prompt %} + {{- '<|im_start|>assistant\n' }} + {%- if enable_thinking is defined and enable_thinking is false %} + {{- '\n\n\n\n' }} + {%- endif %} +{%- endif %} \ No newline at end of file diff --git a/MATH/PD/Qwen3-8B/checkpoint-198/trainer_state.json b/MATH/PD/Qwen3-8B/checkpoint-198/trainer_state.json new file mode 100644 index 0000000000000000000000000000000000000000..8a2cc9a27b40e638fa48f05201c8d99dfa24c610 --- /dev/null +++ b/MATH/PD/Qwen3-8B/checkpoint-198/trainer_state.json @@ -0,0 +1,199 @@ +{ + "best_global_step": 198, + "best_metric": 0.06419320404529572, + "best_model_checkpoint": "source_project/experiments/math_specific_sft_rl_v1/artifacts/math_specific_sft_rl_reuse_legacy_v4d_real_rollout_sft_protocol_floor_v2_20260830/01_sft/adapters/A3/checkpoint-198", + "epoch": 3.0, + "eval_steps": 50, + "global_step": 198, + "is_hyper_param_search": false, + "is_local_process_zero": true, + "is_world_process_zero": true, + "log_history": [ + { + "epoch": 0.15267175572519084, + "grad_norm": 0.2715332806110382, + "learning_rate": 9e-06, + "loss": 0.19996399879455568, + "step": 10 + }, + { + "epoch": 0.3053435114503817, + "grad_norm": 0.14735907316207886, + "learning_rate": 9.943559569286731e-06, + "loss": 0.18235771656036376, + "step": 20 + }, + { + "epoch": 0.4580152671755725, + "grad_norm": 0.10940974205732346, + "learning_rate": 9.75009217427352e-06, + "loss": 0.13847777843475342, + "step": 30 + }, + { + "epoch": 0.6106870229007634, + "grad_norm": 0.14101381599903107, + "learning_rate": 9.424289624237143e-06, + "loss": 0.13123193979263306, + "step": 40 + }, + { + "epoch": 0.7633587786259542, + "grad_norm": 0.11885557323694229, + "learning_rate": 8.975228612720415e-06, + "loss": 0.12588239908218385, + "step": 50 + }, + { + "epoch": 0.7633587786259542, + "eval_loss": 0.1032027006149292, + "eval_runtime": 18.9159, + "eval_samples_per_second": 5.815, + "eval_steps_per_second": 0.74, + "step": 50 + }, + { + "epoch": 0.916030534351145, + "grad_norm": 0.10043700039386749, + "learning_rate": 8.415419751390155e-06, + "loss": 0.10407333374023438, + "step": 60 + }, + { + "epoch": 1.0610687022900764, + "grad_norm": 0.11581146717071533, + "learning_rate": 7.760459030751285e-06, + "loss": 0.10394111871719361, + "step": 70 + }, + { + "epoch": 1.213740458015267, + "grad_norm": 0.0961180329322815, + "learning_rate": 7.028593323773819e-06, + "loss": 0.09114642143249511, + "step": 80 + }, + { + "epoch": 1.366412213740458, + "grad_norm": 0.10328122973442078, + "learning_rate": 6.240212037280967e-06, + "loss": 0.0868987798690796, + "step": 90 + }, + { + "epoch": 1.5190839694656488, + "grad_norm": 0.09781694412231445, + "learning_rate": 5.41727907343245e-06, + "loss": 0.08267452120780945, + "step": 100 + }, + { + "epoch": 1.5190839694656488, + "eval_loss": 0.07400760799646378, + "eval_runtime": 19.2077, + "eval_samples_per_second": 5.727, + "eval_steps_per_second": 0.729, + "step": 100 + }, + { + "epoch": 1.6717557251908397, + "grad_norm": 0.11948485672473907, + "learning_rate": 4.582720926567552e-06, + "loss": 0.0813731849193573, + "step": 110 + }, + { + "epoch": 1.8244274809160306, + "grad_norm": 0.1227552741765976, + "learning_rate": 3.7597879627190337e-06, + "loss": 0.07353355288505554, + "step": 120 + }, + { + "epoch": 1.9770992366412212, + "grad_norm": 0.09954976290464401, + "learning_rate": 2.9714066762261825e-06, + "loss": 0.07269736528396606, + "step": 130 + }, + { + "epoch": 2.122137404580153, + "grad_norm": 0.10711696743965149, + "learning_rate": 2.2395409692487174e-06, + "loss": 0.07261578440666198, + "step": 140 + }, + { + "epoch": 2.2748091603053435, + "grad_norm": 0.10855450481176376, + "learning_rate": 1.5845802486098461e-06, + "loss": 0.07437153458595276, + "step": 150 + }, + { + "epoch": 2.2748091603053435, + "eval_loss": 0.06524904072284698, + "eval_runtime": 18.7372, + "eval_samples_per_second": 5.871, + "eval_steps_per_second": 0.747, + "step": 150 + }, + { + "epoch": 2.427480916030534, + "grad_norm": 0.10708307474851608, + "learning_rate": 1.024771387279585e-06, + "loss": 0.06574302911758423, + "step": 160 + }, + { + "epoch": 2.5801526717557253, + "grad_norm": 0.1650659143924713, + "learning_rate": 5.757103757628573e-07, + "loss": 0.07398568391799927, + "step": 170 + }, + { + "epoch": 2.732824427480916, + "grad_norm": 0.13608039915561676, + "learning_rate": 2.4990782572647977e-07, + "loss": 0.07318060398101807, + "step": 180 + }, + { + "epoch": 2.885496183206107, + "grad_norm": 0.11667497456073761, + "learning_rate": 5.6440430713269325e-08, + "loss": 0.0668993353843689, + "step": 190 + }, + { + "epoch": 3.0, + "eval_loss": 0.06419320404529572, + "eval_runtime": 19.3432, + "eval_samples_per_second": 5.687, + "eval_steps_per_second": 0.724, + "step": 198 + } + ], + "logging_steps": 10, + "max_steps": 198, + "num_input_tokens_seen": 0, + "num_train_epochs": 3, + "save_steps": 50, + "stateful_callbacks": { + "TrainerControl": { + "args": { + "should_epoch_stop": false, + "should_evaluate": false, + "should_log": false, + "should_save": true, + "should_training_stop": true + }, + "attributes": {} + } + }, + "total_flos": 3.956843536753623e+17, + "train_batch_size": 1, + "trial_name": null, + "trial_params": null +} diff --git a/MATH/PD/Qwen3-8B/checkpoint-50/adapter_config.json b/MATH/PD/Qwen3-8B/checkpoint-50/adapter_config.json new file mode 100644 index 0000000000000000000000000000000000000000..88f42351f47a6c2cb2e7c315caf403cf833a2600 --- /dev/null +++ b/MATH/PD/Qwen3-8B/checkpoint-50/adapter_config.json @@ -0,0 +1,45 @@ +{ + "alora_invocation_tokens": null, + "alpha_pattern": {}, + "arrow_config": null, + "auto_mapping": null, + "base_model_name_or_path": "Qwen/Qwen3-8B", + "bias": "none", + "corda_config": null, + "ensure_weight_tying": false, + "eva_config": null, + "exclude_modules": null, + "fan_in_fan_out": false, + "inference_mode": true, + "init_lora_weights": true, + "layer_replication": null, + "layers_pattern": null, + "layers_to_transform": null, + "loftq_config": {}, + "lora_alpha": 64, + "lora_bias": false, + "lora_dropout": 0.05, + "lora_ga_config": null, + "megatron_config": null, + "megatron_core": "megatron.core", + "modules_to_save": null, + "peft_type": "LORA", + "peft_version": "0.19.1", + "qalora_group_size": 16, + "r": 64, + "rank_pattern": {}, + "revision": null, + "target_modules": [ + "q_proj", + "v_proj", + "k_proj", + "o_proj" + ], + "target_parameters": null, + "task_type": "CAUSAL_LM", + "trainable_token_indices": null, + "use_bdlora": null, + "use_dora": false, + "use_qalora": false, + "use_rslora": false +} diff --git a/MATH/PD/Qwen3-8B/checkpoint-50/chat_template.jinja b/MATH/PD/Qwen3-8B/checkpoint-50/chat_template.jinja new file mode 100644 index 0000000000000000000000000000000000000000..01be9b307daa2d425f7c168c9fb145a286e0afb4 --- /dev/null +++ b/MATH/PD/Qwen3-8B/checkpoint-50/chat_template.jinja @@ -0,0 +1,89 @@ +{%- if tools %} + {{- '<|im_start|>system\n' }} + {%- if messages[0].role == 'system' %} + {{- messages[0].content + '\n\n' }} + {%- endif %} + {{- "# Tools\n\nYou may call one or more functions to assist with the user query.\n\nYou are provided with function signatures within XML tags:\n" }} + {%- for tool in tools %} + {{- "\n" }} + {{- tool | tojson }} + {%- endfor %} + {{- "\n\n\nFor each function call, return a json object with function name and arguments within XML tags:\n\n{\"name\": , \"arguments\": }\n<|im_end|>\n" }} +{%- else %} + {%- if messages[0].role == 'system' %} + {{- '<|im_start|>system\n' + messages[0].content + '<|im_end|>\n' }} + {%- endif %} +{%- endif %} +{%- set ns = namespace(multi_step_tool=true, last_query_index=messages|length - 1) %} +{%- for message in messages[::-1] %} + {%- set index = (messages|length - 1) - loop.index0 %} + {%- if ns.multi_step_tool and message.role == "user" and message.content is string and not(message.content.startswith('') and message.content.endswith('')) %} + {%- set ns.multi_step_tool = false %} + {%- set ns.last_query_index = index %} + {%- endif %} +{%- endfor %} +{%- for message in messages %} + {%- if message.content is string %} + {%- set content = message.content %} + {%- else %} + {%- set content = '' %} + {%- endif %} + {%- if (message.role == "user") or (message.role == "system" and not loop.first) %} + {{- '<|im_start|>' + message.role + '\n' + content + '<|im_end|>' + '\n' }} + {%- elif message.role == "assistant" %} + {%- set reasoning_content = '' %} + {%- if message.reasoning_content is string %} + {%- set reasoning_content = message.reasoning_content %} + {%- else %} + {%- if '' in content %} + {%- set reasoning_content = content.split('')[0].rstrip('\n').split('')[-1].lstrip('\n') %} + {%- set content = content.split('')[-1].lstrip('\n') %} + {%- endif %} + {%- endif %} + {%- if loop.index0 > ns.last_query_index %} + {%- if loop.last or (not loop.last and reasoning_content) %} + {{- '<|im_start|>' + message.role + '\n\n' + reasoning_content.strip('\n') + '\n\n\n' + content.lstrip('\n') }} + {%- else %} + {{- '<|im_start|>' + message.role + '\n' + content }} + {%- endif %} + {%- else %} + {{- '<|im_start|>' + message.role + '\n' + content }} + {%- endif %} + {%- if message.tool_calls %} + {%- for tool_call in message.tool_calls %} + {%- if (loop.first and content) or (not loop.first) %} + {{- '\n' }} + {%- endif %} + {%- if tool_call.function %} + {%- set tool_call = tool_call.function %} + {%- endif %} + {{- '\n{"name": "' }} + {{- tool_call.name }} + {{- '", "arguments": ' }} + {%- if tool_call.arguments is string %} + {{- tool_call.arguments }} + {%- else %} + {{- tool_call.arguments | tojson }} + {%- endif %} + {{- '}\n' }} + {%- endfor %} + {%- endif %} + {{- '<|im_end|>\n' }} + {%- elif message.role == "tool" %} + {%- if loop.first or (messages[loop.index0 - 1].role != "tool") %} + {{- '<|im_start|>user' }} + {%- endif %} + {{- '\n\n' }} + {{- content }} + {{- '\n' }} + {%- if loop.last or (messages[loop.index0 + 1].role != "tool") %} + {{- '<|im_end|>\n' }} + {%- endif %} + {%- endif %} +{%- endfor %} +{%- if add_generation_prompt %} + {{- '<|im_start|>assistant\n' }} + {%- if enable_thinking is defined and enable_thinking is false %} + {{- '\n\n\n\n' }} + {%- endif %} +{%- endif %} \ No newline at end of file diff --git a/MATH/PD/Qwen3-8B/checkpoint-50/tokenizer_config.json b/MATH/PD/Qwen3-8B/checkpoint-50/tokenizer_config.json new file mode 100644 index 0000000000000000000000000000000000000000..5668a4a01433b0648bb972458de616f22f3e728e --- /dev/null +++ b/MATH/PD/Qwen3-8B/checkpoint-50/tokenizer_config.json @@ -0,0 +1,30 @@ +{ + "add_prefix_space": false, + "backend": "tokenizers", + "bos_token": null, + "clean_up_tokenization_spaces": false, + "eos_token": "<|im_end|>", + "errors": "replace", + "extra_special_tokens": [ + "<|im_start|>", + "<|im_end|>", + "<|object_ref_start|>", + "<|object_ref_end|>", + "<|box_start|>", + "<|box_end|>", + "<|quad_start|>", + "<|quad_end|>", + "<|vision_start|>", + "<|vision_end|>", + "<|vision_pad|>", + "<|image_pad|>", + "<|video_pad|>" + ], + "is_local": true, + "local_files_only": false, + "model_max_length": 131072, + "pad_token": "<|endoftext|>", + "split_special_tokens": false, + "tokenizer_class": "Qwen2Tokenizer", + "unk_token": null +} diff --git a/MATH/PD/Qwen3-8B/checkpoint-50/trainer_state.json b/MATH/PD/Qwen3-8B/checkpoint-50/trainer_state.json new file mode 100644 index 0000000000000000000000000000000000000000..a926be3c84cd4e9e8c2e0cf6d98401f939089b16 --- /dev/null +++ b/MATH/PD/Qwen3-8B/checkpoint-50/trainer_state.json @@ -0,0 +1,77 @@ +{ + "best_global_step": 50, + "best_metric": 0.1032027006149292, + "best_model_checkpoint": "source_project/experiments/math_specific_sft_rl_v1/artifacts/math_specific_sft_rl_reuse_legacy_v4d_real_rollout_sft_protocol_floor_v2_20260830/01_sft/adapters/A3/checkpoint-50", + "epoch": 0.7633587786259542, + "eval_steps": 50, + "global_step": 50, + "is_hyper_param_search": false, + "is_local_process_zero": true, + "is_world_process_zero": true, + "log_history": [ + { + "epoch": 0.15267175572519084, + "grad_norm": 0.2715332806110382, + "learning_rate": 9e-06, + "loss": 0.19996399879455568, + "step": 10 + }, + { + "epoch": 0.3053435114503817, + "grad_norm": 0.14735907316207886, + "learning_rate": 9.943559569286731e-06, + "loss": 0.18235771656036376, + "step": 20 + }, + { + "epoch": 0.4580152671755725, + "grad_norm": 0.10940974205732346, + "learning_rate": 9.75009217427352e-06, + "loss": 0.13847777843475342, + "step": 30 + }, + { + "epoch": 0.6106870229007634, + "grad_norm": 0.14101381599903107, + "learning_rate": 9.424289624237143e-06, + "loss": 0.13123193979263306, + "step": 40 + }, + { + "epoch": 0.7633587786259542, + "grad_norm": 0.11885557323694229, + "learning_rate": 8.975228612720415e-06, + "loss": 0.12588239908218385, + "step": 50 + }, + { + "epoch": 0.7633587786259542, + "eval_loss": 0.1032027006149292, + "eval_runtime": 18.9159, + "eval_samples_per_second": 5.815, + "eval_steps_per_second": 0.74, + "step": 50 + } + ], + "logging_steps": 10, + "max_steps": 198, + "num_input_tokens_seen": 0, + "num_train_epochs": 3, + "save_steps": 50, + "stateful_callbacks": { + "TrainerControl": { + "args": { + "should_epoch_stop": false, + "should_evaluate": false, + "should_log": false, + "should_save": true, + "should_training_stop": false + }, + "attributes": {} + } + }, + "total_flos": 1.0095803598320435e+17, + "train_batch_size": 1, + "trial_name": null, + "trial_params": null +} diff --git a/MATH/PD/Qwen3-8B/final/adapter_config.json b/MATH/PD/Qwen3-8B/final/adapter_config.json new file mode 100644 index 0000000000000000000000000000000000000000..88f42351f47a6c2cb2e7c315caf403cf833a2600 --- /dev/null +++ b/MATH/PD/Qwen3-8B/final/adapter_config.json @@ -0,0 +1,45 @@ +{ + "alora_invocation_tokens": null, + "alpha_pattern": {}, + "arrow_config": null, + "auto_mapping": null, + "base_model_name_or_path": "Qwen/Qwen3-8B", + "bias": "none", + "corda_config": null, + "ensure_weight_tying": false, + "eva_config": null, + "exclude_modules": null, + "fan_in_fan_out": false, + "inference_mode": true, + "init_lora_weights": true, + "layer_replication": null, + "layers_pattern": null, + "layers_to_transform": null, + "loftq_config": {}, + "lora_alpha": 64, + "lora_bias": false, + "lora_dropout": 0.05, + "lora_ga_config": null, + "megatron_config": null, + "megatron_core": "megatron.core", + "modules_to_save": null, + "peft_type": "LORA", + "peft_version": "0.19.1", + "qalora_group_size": 16, + "r": 64, + "rank_pattern": {}, + "revision": null, + "target_modules": [ + "q_proj", + "v_proj", + "k_proj", + "o_proj" + ], + "target_parameters": null, + "task_type": "CAUSAL_LM", + "trainable_token_indices": null, + "use_bdlora": null, + "use_dora": false, + "use_qalora": false, + "use_rslora": false +} diff --git a/MATH/PD/Qwen3-8B/final/chat_template.jinja b/MATH/PD/Qwen3-8B/final/chat_template.jinja new file mode 100644 index 0000000000000000000000000000000000000000..01be9b307daa2d425f7c168c9fb145a286e0afb4 --- /dev/null +++ b/MATH/PD/Qwen3-8B/final/chat_template.jinja @@ -0,0 +1,89 @@ +{%- if tools %} + {{- '<|im_start|>system\n' }} + {%- if messages[0].role == 'system' %} + {{- messages[0].content + '\n\n' }} + {%- endif %} + {{- "# Tools\n\nYou may call one or more functions to assist with the user query.\n\nYou are provided with function signatures within XML tags:\n" }} + {%- for tool in tools %} + {{- "\n" }} + {{- tool | tojson }} + {%- endfor %} + {{- "\n\n\nFor each function call, return a json object with function name and arguments within XML tags:\n\n{\"name\": , \"arguments\": }\n<|im_end|>\n" }} +{%- else %} + {%- if messages[0].role == 'system' %} + {{- '<|im_start|>system\n' + messages[0].content + '<|im_end|>\n' }} + {%- endif %} +{%- endif %} +{%- set ns = namespace(multi_step_tool=true, last_query_index=messages|length - 1) %} +{%- for message in messages[::-1] %} + {%- set index = (messages|length - 1) - loop.index0 %} + {%- if ns.multi_step_tool and message.role == "user" and message.content is string and not(message.content.startswith('') and message.content.endswith('')) %} + {%- set ns.multi_step_tool = false %} + {%- set ns.last_query_index = index %} + {%- endif %} +{%- endfor %} +{%- for message in messages %} + {%- if message.content is string %} + {%- set content = message.content %} + {%- else %} + {%- set content = '' %} + {%- endif %} + {%- if (message.role == "user") or (message.role == "system" and not loop.first) %} + {{- '<|im_start|>' + message.role + '\n' + content + '<|im_end|>' + '\n' }} + {%- elif message.role == "assistant" %} + {%- set reasoning_content = '' %} + {%- if message.reasoning_content is string %} + {%- set reasoning_content = message.reasoning_content %} + {%- else %} + {%- if '' in content %} + {%- set reasoning_content = content.split('')[0].rstrip('\n').split('')[-1].lstrip('\n') %} + {%- set content = content.split('')[-1].lstrip('\n') %} + {%- endif %} + {%- endif %} + {%- if loop.index0 > ns.last_query_index %} + {%- if loop.last or (not loop.last and reasoning_content) %} + {{- '<|im_start|>' + message.role + '\n\n' + reasoning_content.strip('\n') + '\n\n\n' + content.lstrip('\n') }} + {%- else %} + {{- '<|im_start|>' + message.role + '\n' + content }} + {%- endif %} + {%- else %} + {{- '<|im_start|>' + message.role + '\n' + content }} + {%- endif %} + {%- if message.tool_calls %} + {%- for tool_call in message.tool_calls %} + {%- if (loop.first and content) or (not loop.first) %} + {{- '\n' }} + {%- endif %} + {%- if tool_call.function %} + {%- set tool_call = tool_call.function %} + {%- endif %} + {{- '\n{"name": "' }} + {{- tool_call.name }} + {{- '", "arguments": ' }} + {%- if tool_call.arguments is string %} + {{- tool_call.arguments }} + {%- else %} + {{- tool_call.arguments | tojson }} + {%- endif %} + {{- '}\n' }} + {%- endfor %} + {%- endif %} + {{- '<|im_end|>\n' }} + {%- elif message.role == "tool" %} + {%- if loop.first or (messages[loop.index0 - 1].role != "tool") %} + {{- '<|im_start|>user' }} + {%- endif %} + {{- '\n\n' }} + {{- content }} + {{- '\n' }} + {%- if loop.last or (messages[loop.index0 + 1].role != "tool") %} + {{- '<|im_end|>\n' }} + {%- endif %} + {%- endif %} +{%- endfor %} +{%- if add_generation_prompt %} + {{- '<|im_start|>assistant\n' }} + {%- if enable_thinking is defined and enable_thinking is false %} + {{- '\n\n\n\n' }} + {%- endif %} +{%- endif %} \ No newline at end of file diff --git a/MATH/PD/Qwen3-8B/final/final_eval_metrics.json b/MATH/PD/Qwen3-8B/final/final_eval_metrics.json new file mode 100644 index 0000000000000000000000000000000000000000..5008c7f80f3c03657515141249c2f3527b48d153 --- /dev/null +++ b/MATH/PD/Qwen3-8B/final/final_eval_metrics.json @@ -0,0 +1,7 @@ +{ + "eval_loss": 0.06419320404529572, + "eval_runtime": 18.9529, + "eval_samples_per_second": 5.804, + "eval_steps_per_second": 0.739, + "epoch": 3.0 +} diff --git a/MATH/PD/Qwen3-8B/final/tokenizer_config.json b/MATH/PD/Qwen3-8B/final/tokenizer_config.json new file mode 100644 index 0000000000000000000000000000000000000000..5668a4a01433b0648bb972458de616f22f3e728e --- /dev/null +++ b/MATH/PD/Qwen3-8B/final/tokenizer_config.json @@ -0,0 +1,30 @@ +{ + "add_prefix_space": false, + "backend": "tokenizers", + "bos_token": null, + "clean_up_tokenization_spaces": false, + "eos_token": "<|im_end|>", + "errors": "replace", + "extra_special_tokens": [ + "<|im_start|>", + "<|im_end|>", + "<|object_ref_start|>", + "<|object_ref_end|>", + "<|box_start|>", + "<|box_end|>", + "<|quad_start|>", + "<|quad_end|>", + "<|vision_start|>", + "<|vision_end|>", + "<|vision_pad|>", + "<|image_pad|>", + "<|video_pad|>" + ], + "is_local": true, + "local_files_only": false, + "model_max_length": 131072, + "pad_token": "<|endoftext|>", + "split_special_tokens": false, + "tokenizer_class": "Qwen2Tokenizer", + "unk_token": null +} diff --git a/MATH/PD/Qwen3-8B/final/train_summary.json b/MATH/PD/Qwen3-8B/final/train_summary.json new file mode 100644 index 0000000000000000000000000000000000000000..b834f7923dd48050ae0bc4b232a93b5cca0a9059 --- /dev/null +++ b/MATH/PD/Qwen3-8B/final/train_summary.json @@ -0,0 +1,25 @@ +{ + "agent": "A3", + "dataset": "MATH", + "source_split": "train", + "builder_version": "math_specific_fixed_a1_rollout_sft_v2", + "supervision_source": "controlled_model_rollout", + "initialization": "fresh LoRA on the role base model", + "global_step": 198, + "training_loss": 0.09907432216586488, + "resume_from_checkpoint": null, + "split_stats": { + "train_rows": 2090, + "eval_rows": 110, + "train_groups": 1550, + "eval_groups": 85 + }, + "metrics": { + "train_runtime": 513.1391, + "train_samples_per_second": 12.219, + "train_steps_per_second": 0.386, + "total_flos": 3.956843536753623e+17, + "train_loss": 0.09907432216586488, + "epoch": 3.0 + } +} diff --git a/MuSiQue/PD/Qwen3-1.7B/checkpoint-210/adapter_config.json b/MuSiQue/PD/Qwen3-1.7B/checkpoint-210/adapter_config.json new file mode 100644 index 0000000000000000000000000000000000000000..72e5bf46c94379147b782eafb3b70915beb9e428 --- /dev/null +++ b/MuSiQue/PD/Qwen3-1.7B/checkpoint-210/adapter_config.json @@ -0,0 +1,45 @@ +{ + "alora_invocation_tokens": null, + "alpha_pattern": {}, + "arrow_config": null, + "auto_mapping": null, + "base_model_name_or_path": "Qwen/Qwen3-1.7B", + "bias": "none", + "corda_config": null, + "ensure_weight_tying": false, + "eva_config": null, + "exclude_modules": null, + "fan_in_fan_out": false, + "inference_mode": true, + "init_lora_weights": true, + "layer_replication": null, + "layers_pattern": null, + "layers_to_transform": null, + "loftq_config": {}, + "lora_alpha": 64, + "lora_bias": false, + "lora_dropout": 0.05, + "lora_ga_config": null, + "megatron_config": null, + "megatron_core": "megatron.core", + "modules_to_save": null, + "peft_type": "LORA", + "peft_version": "0.19.1", + "qalora_group_size": 16, + "r": 64, + "rank_pattern": {}, + "revision": null, + "target_modules": [ + "q_proj", + "v_proj", + "k_proj", + "o_proj" + ], + "target_parameters": null, + "task_type": "CAUSAL_LM", + "trainable_token_indices": null, + "use_bdlora": null, + "use_dora": false, + "use_qalora": false, + "use_rslora": false +} diff --git a/MuSiQue/PD/Qwen3-1.7B/checkpoint-210/chat_template.jinja b/MuSiQue/PD/Qwen3-1.7B/checkpoint-210/chat_template.jinja new file mode 100644 index 0000000000000000000000000000000000000000..01be9b307daa2d425f7c168c9fb145a286e0afb4 --- /dev/null +++ b/MuSiQue/PD/Qwen3-1.7B/checkpoint-210/chat_template.jinja @@ -0,0 +1,89 @@ +{%- if tools %} + {{- '<|im_start|>system\n' }} + {%- if messages[0].role == 'system' %} + {{- messages[0].content + '\n\n' }} + {%- endif %} + {{- "# Tools\n\nYou may call one or more functions to assist with the user query.\n\nYou are provided with function signatures within XML tags:\n" }} + {%- for tool in tools %} + {{- "\n" }} + {{- tool | tojson }} + {%- endfor %} + {{- "\n\n\nFor each function call, return a json object with function name and arguments within XML tags:\n\n{\"name\": , \"arguments\": }\n<|im_end|>\n" }} +{%- else %} + {%- if messages[0].role == 'system' %} + {{- '<|im_start|>system\n' + messages[0].content + '<|im_end|>\n' }} + {%- endif %} +{%- endif %} +{%- set ns = namespace(multi_step_tool=true, last_query_index=messages|length - 1) %} +{%- for message in messages[::-1] %} + {%- set index = (messages|length - 1) - loop.index0 %} + {%- if ns.multi_step_tool and message.role == "user" and message.content is string and not(message.content.startswith('') and message.content.endswith('')) %} + {%- set ns.multi_step_tool = false %} + {%- set ns.last_query_index = index %} + {%- endif %} +{%- endfor %} +{%- for message in messages %} + {%- if message.content is string %} + {%- set content = message.content %} + {%- else %} + {%- set content = '' %} + {%- endif %} + {%- if (message.role == "user") or (message.role == "system" and not loop.first) %} + {{- '<|im_start|>' + message.role + '\n' + content + '<|im_end|>' + '\n' }} + {%- elif message.role == "assistant" %} + {%- set reasoning_content = '' %} + {%- if message.reasoning_content is string %} + {%- set reasoning_content = message.reasoning_content %} + {%- else %} + {%- if '' in content %} + {%- set reasoning_content = content.split('')[0].rstrip('\n').split('')[-1].lstrip('\n') %} + {%- set content = content.split('')[-1].lstrip('\n') %} + {%- endif %} + {%- endif %} + {%- if loop.index0 > ns.last_query_index %} + {%- if loop.last or (not loop.last and reasoning_content) %} + {{- '<|im_start|>' + message.role + '\n\n' + reasoning_content.strip('\n') + '\n\n\n' + content.lstrip('\n') }} + {%- else %} + {{- '<|im_start|>' + message.role + '\n' + content }} + {%- endif %} + {%- else %} + {{- '<|im_start|>' + message.role + '\n' + content }} + {%- endif %} + {%- if message.tool_calls %} + {%- for tool_call in message.tool_calls %} + {%- if (loop.first and content) or (not loop.first) %} + {{- '\n' }} + {%- endif %} + {%- if tool_call.function %} + {%- set tool_call = tool_call.function %} + {%- endif %} + {{- '\n{"name": "' }} + {{- tool_call.name }} + {{- '", "arguments": ' }} + {%- if tool_call.arguments is string %} + {{- tool_call.arguments }} + {%- else %} + {{- tool_call.arguments | tojson }} + {%- endif %} + {{- '}\n' }} + {%- endfor %} + {%- endif %} + {{- '<|im_end|>\n' }} + {%- elif message.role == "tool" %} + {%- if loop.first or (messages[loop.index0 - 1].role != "tool") %} + {{- '<|im_start|>user' }} + {%- endif %} + {{- '\n\n' }} + {{- content }} + {{- '\n' }} + {%- if loop.last or (messages[loop.index0 + 1].role != "tool") %} + {{- '<|im_end|>\n' }} + {%- endif %} + {%- endif %} +{%- endfor %} +{%- if add_generation_prompt %} + {{- '<|im_start|>assistant\n' }} + {%- if enable_thinking is defined and enable_thinking is false %} + {{- '\n\n\n\n' }} + {%- endif %} +{%- endif %} \ No newline at end of file diff --git a/MuSiQue/PD/Qwen3-1.7B/checkpoint-210/tokenizer_config.json b/MuSiQue/PD/Qwen3-1.7B/checkpoint-210/tokenizer_config.json new file mode 100644 index 0000000000000000000000000000000000000000..5668a4a01433b0648bb972458de616f22f3e728e --- /dev/null +++ b/MuSiQue/PD/Qwen3-1.7B/checkpoint-210/tokenizer_config.json @@ -0,0 +1,30 @@ +{ + "add_prefix_space": false, + "backend": "tokenizers", + "bos_token": null, + "clean_up_tokenization_spaces": false, + "eos_token": "<|im_end|>", + "errors": "replace", + "extra_special_tokens": [ + "<|im_start|>", + "<|im_end|>", + "<|object_ref_start|>", + "<|object_ref_end|>", + "<|box_start|>", + "<|box_end|>", + "<|quad_start|>", + "<|quad_end|>", + "<|vision_start|>", + "<|vision_end|>", + "<|vision_pad|>", + "<|image_pad|>", + "<|video_pad|>" + ], + "is_local": true, + "local_files_only": false, + "model_max_length": 131072, + "pad_token": "<|endoftext|>", + "split_special_tokens": false, + "tokenizer_class": "Qwen2Tokenizer", + "unk_token": null +} diff --git a/MuSiQue/PD/Qwen3-1.7B/checkpoint-210/trainer_state.json b/MuSiQue/PD/Qwen3-1.7B/checkpoint-210/trainer_state.json new file mode 100644 index 0000000000000000000000000000000000000000..01fd7320486ade757b9f6c57588a77b2410ce3d9 --- /dev/null +++ b/MuSiQue/PD/Qwen3-1.7B/checkpoint-210/trainer_state.json @@ -0,0 +1,214 @@ +{ + "best_global_step": 210, + "best_metric": 0.42751801013946533, + "best_model_checkpoint": "jca/sft_runs/0706v2/A1/checkpoint-210", + "epoch": 5.0, + "eval_steps": 100, + "global_step": 210, + "is_hyper_param_search": false, + "is_local_process_zero": true, + "is_world_process_zero": true, + "log_history": [ + { + "epoch": 0.24242424242424243, + "grad_norm": 4.720747947692871, + "learning_rate": 1.6363636363636366e-05, + "loss": 1.6281307220458985, + "step": 10 + }, + { + "epoch": 0.48484848484848486, + "grad_norm": 1.0954445600509644, + "learning_rate": 1.992035358593258e-05, + "loss": 0.9036008834838867, + "step": 20 + }, + { + "epoch": 0.7272727272727273, + "grad_norm": 0.3447713851928711, + "learning_rate": 1.959896320156857e-05, + "loss": 0.5538296222686767, + "step": 30 + }, + { + "epoch": 0.9696969696969697, + "grad_norm": 0.28223949670791626, + "learning_rate": 1.903883800488824e-05, + "loss": 0.5149138927459717, + "step": 40 + }, + { + "epoch": 1.1939393939393939, + "grad_norm": 0.21286053955554962, + "learning_rate": 1.8253908811219764e-05, + "loss": 0.4824307918548584, + "step": 50 + }, + { + "epoch": 1.4363636363636363, + "grad_norm": 0.1979370266199112, + "learning_rate": 1.7263697512646397e-05, + "loss": 0.45679168701171874, + "step": 60 + }, + { + "epoch": 1.6787878787878787, + "grad_norm": 0.19309963285923004, + "learning_rate": 1.609283155106517e-05, + "loss": 0.45611371994018557, + "step": 70 + }, + { + "epoch": 1.9212121212121214, + "grad_norm": 0.2116900086402893, + "learning_rate": 1.4770431411648898e-05, + "loss": 0.4341451644897461, + "step": 80 + }, + { + "epoch": 2.1454545454545455, + "grad_norm": 0.2067466527223587, + "learning_rate": 1.3329386370297615e-05, + "loss": 0.42741823196411133, + "step": 90 + }, + { + "epoch": 2.3878787878787877, + "grad_norm": 0.2089332640171051, + "learning_rate": 1.1805536507889021e-05, + "loss": 0.4203496932983398, + "step": 100 + }, + { + "epoch": 2.3878787878787877, + "eval_loss": 0.4448707103729248, + "eval_runtime": 27.0017, + "eval_samples_per_second": 10.962, + "eval_steps_per_second": 0.704, + "step": 100 + }, + { + "epoch": 2.6303030303030304, + "grad_norm": 0.189452663064003, + "learning_rate": 1.0236781335366239e-05, + "loss": 0.41803560256958006, + "step": 110 + }, + { + "epoch": 2.8727272727272726, + "grad_norm": 0.217136412858963, + "learning_rate": 8.662137198955211e-06, + "loss": 0.4174006938934326, + "step": 120 + }, + { + "epoch": 3.096969696969697, + "grad_norm": 0.20486938953399658, + "learning_rate": 7.1207669086883366e-06, + "loss": 0.4287893295288086, + "step": 130 + }, + { + "epoch": 3.3393939393939394, + "grad_norm": 0.19223496317863464, + "learning_rate": 5.651005724242072e-06, + "loss": 0.4109820365905762, + "step": 140 + }, + { + "epoch": 3.581818181818182, + "grad_norm": 0.22415931522846222, + "learning_rate": 4.289407922693053e-06, + "loss": 0.40108532905578614, + "step": 150 + }, + { + "epoch": 3.824242424242424, + "grad_norm": 0.23860600590705872, + "learning_rate": 3.0698376609066828e-06, + "loss": 0.39721534252166746, + "step": 160 + }, + { + "epoch": 4.048484848484849, + "grad_norm": 0.21809329092502594, + "learning_rate": 2.022626743624807e-06, + "loss": 0.42003021240234373, + "step": 170 + }, + { + "epoch": 4.290909090909091, + "grad_norm": 0.220177561044693, + "learning_rate": 1.1738202443145307e-06, + "loss": 0.41559562683105467, + "step": 180 + }, + { + "epoch": 4.533333333333333, + "grad_norm": 0.23265336453914642, + "learning_rate": 5.445287408633304e-07, + "loss": 0.41177845001220703, + "step": 190 + }, + { + "epoch": 4.775757575757575, + "grad_norm": 0.2148609608411789, + "learning_rate": 1.504032765988961e-07, + "loss": 0.3848694086074829, + "step": 200 + }, + { + "epoch": 4.775757575757575, + "eval_loss": 0.4275880753993988, + "eval_runtime": 26.7562, + "eval_samples_per_second": 11.063, + "eval_steps_per_second": 0.71, + "step": 200 + }, + { + "epoch": 5.0, + "grad_norm": 0.36726605892181396, + "learning_rate": 1.246104823426908e-09, + "loss": 0.38492980003356936, + "step": 210 + }, + { + "epoch": 5.0, + "eval_loss": 0.42751801013946533, + "eval_runtime": 26.7689, + "eval_samples_per_second": 11.058, + "eval_steps_per_second": 0.71, + "step": 210 + } + ], + "logging_steps": 10, + "max_steps": 210, + "num_input_tokens_seen": 0, + "num_train_epochs": 5, + "save_steps": 100, + "stateful_callbacks": { + "EarlyStoppingCallback": { + "args": { + "early_stopping_patience": 3, + "early_stopping_threshold": 0.0 + }, + "attributes": { + "early_stopping_patience_counter": 0 + } + }, + "TrainerControl": { + "args": { + "should_epoch_stop": false, + "should_evaluate": false, + "should_log": false, + "should_save": true, + "should_training_stop": true + }, + "attributes": {} + } + }, + "total_flos": 4.188032724977582e+17, + "train_batch_size": 2, + "trial_name": null, + "trial_params": null +} diff --git a/MuSiQue/PD/Qwen3-1.7B/final/adapter_config.json b/MuSiQue/PD/Qwen3-1.7B/final/adapter_config.json new file mode 100644 index 0000000000000000000000000000000000000000..72e5bf46c94379147b782eafb3b70915beb9e428 --- /dev/null +++ b/MuSiQue/PD/Qwen3-1.7B/final/adapter_config.json @@ -0,0 +1,45 @@ +{ + "alora_invocation_tokens": null, + "alpha_pattern": {}, + "arrow_config": null, + "auto_mapping": null, + "base_model_name_or_path": "Qwen/Qwen3-1.7B", + "bias": "none", + "corda_config": null, + "ensure_weight_tying": false, + "eva_config": null, + "exclude_modules": null, + "fan_in_fan_out": false, + "inference_mode": true, + "init_lora_weights": true, + "layer_replication": null, + "layers_pattern": null, + "layers_to_transform": null, + "loftq_config": {}, + "lora_alpha": 64, + "lora_bias": false, + "lora_dropout": 0.05, + "lora_ga_config": null, + "megatron_config": null, + "megatron_core": "megatron.core", + "modules_to_save": null, + "peft_type": "LORA", + "peft_version": "0.19.1", + "qalora_group_size": 16, + "r": 64, + "rank_pattern": {}, + "revision": null, + "target_modules": [ + "q_proj", + "v_proj", + "k_proj", + "o_proj" + ], + "target_parameters": null, + "task_type": "CAUSAL_LM", + "trainable_token_indices": null, + "use_bdlora": null, + "use_dora": false, + "use_qalora": false, + "use_rslora": false +} diff --git a/MuSiQue/PD/Qwen3-1.7B/final/chat_template.jinja b/MuSiQue/PD/Qwen3-1.7B/final/chat_template.jinja new file mode 100644 index 0000000000000000000000000000000000000000..01be9b307daa2d425f7c168c9fb145a286e0afb4 --- /dev/null +++ b/MuSiQue/PD/Qwen3-1.7B/final/chat_template.jinja @@ -0,0 +1,89 @@ +{%- if tools %} + {{- '<|im_start|>system\n' }} + {%- if messages[0].role == 'system' %} + {{- messages[0].content + '\n\n' }} + {%- endif %} + {{- "# Tools\n\nYou may call one or more functions to assist with the user query.\n\nYou are provided with function signatures within XML tags:\n" }} + {%- for tool in tools %} + {{- "\n" }} + {{- tool | tojson }} + {%- endfor %} + {{- "\n\n\nFor each function call, return a json object with function name and arguments within XML tags:\n\n{\"name\": , \"arguments\": }\n<|im_end|>\n" }} +{%- else %} + {%- if messages[0].role == 'system' %} + {{- '<|im_start|>system\n' + messages[0].content + '<|im_end|>\n' }} + {%- endif %} +{%- endif %} +{%- set ns = namespace(multi_step_tool=true, last_query_index=messages|length - 1) %} +{%- for message in messages[::-1] %} + {%- set index = (messages|length - 1) - loop.index0 %} + {%- if ns.multi_step_tool and message.role == "user" and message.content is string and not(message.content.startswith('') and message.content.endswith('')) %} + {%- set ns.multi_step_tool = false %} + {%- set ns.last_query_index = index %} + {%- endif %} +{%- endfor %} +{%- for message in messages %} + {%- if message.content is string %} + {%- set content = message.content %} + {%- else %} + {%- set content = '' %} + {%- endif %} + {%- if (message.role == "user") or (message.role == "system" and not loop.first) %} + {{- '<|im_start|>' + message.role + '\n' + content + '<|im_end|>' + '\n' }} + {%- elif message.role == "assistant" %} + {%- set reasoning_content = '' %} + {%- if message.reasoning_content is string %} + {%- set reasoning_content = message.reasoning_content %} + {%- else %} + {%- if '' in content %} + {%- set reasoning_content = content.split('')[0].rstrip('\n').split('')[-1].lstrip('\n') %} + {%- set content = content.split('')[-1].lstrip('\n') %} + {%- endif %} + {%- endif %} + {%- if loop.index0 > ns.last_query_index %} + {%- if loop.last or (not loop.last and reasoning_content) %} + {{- '<|im_start|>' + message.role + '\n\n' + reasoning_content.strip('\n') + '\n\n\n' + content.lstrip('\n') }} + {%- else %} + {{- '<|im_start|>' + message.role + '\n' + content }} + {%- endif %} + {%- else %} + {{- '<|im_start|>' + message.role + '\n' + content }} + {%- endif %} + {%- if message.tool_calls %} + {%- for tool_call in message.tool_calls %} + {%- if (loop.first and content) or (not loop.first) %} + {{- '\n' }} + {%- endif %} + {%- if tool_call.function %} + {%- set tool_call = tool_call.function %} + {%- endif %} + {{- '\n{"name": "' }} + {{- tool_call.name }} + {{- '", "arguments": ' }} + {%- if tool_call.arguments is string %} + {{- tool_call.arguments }} + {%- else %} + {{- tool_call.arguments | tojson }} + {%- endif %} + {{- '}\n' }} + {%- endfor %} + {%- endif %} + {{- '<|im_end|>\n' }} + {%- elif message.role == "tool" %} + {%- if loop.first or (messages[loop.index0 - 1].role != "tool") %} + {{- '<|im_start|>user' }} + {%- endif %} + {{- '\n\n' }} + {{- content }} + {{- '\n' }} + {%- if loop.last or (messages[loop.index0 + 1].role != "tool") %} + {{- '<|im_end|>\n' }} + {%- endif %} + {%- endif %} +{%- endfor %} +{%- if add_generation_prompt %} + {{- '<|im_start|>assistant\n' }} + {%- if enable_thinking is defined and enable_thinking is false %} + {{- '\n\n\n\n' }} + {%- endif %} +{%- endif %} \ No newline at end of file diff --git a/MuSiQue/PD/Qwen3-1.7B/final/tokenizer_config.json b/MuSiQue/PD/Qwen3-1.7B/final/tokenizer_config.json new file mode 100644 index 0000000000000000000000000000000000000000..5668a4a01433b0648bb972458de616f22f3e728e --- /dev/null +++ b/MuSiQue/PD/Qwen3-1.7B/final/tokenizer_config.json @@ -0,0 +1,30 @@ +{ + "add_prefix_space": false, + "backend": "tokenizers", + "bos_token": null, + "clean_up_tokenization_spaces": false, + "eos_token": "<|im_end|>", + "errors": "replace", + "extra_special_tokens": [ + "<|im_start|>", + "<|im_end|>", + "<|object_ref_start|>", + "<|object_ref_end|>", + "<|box_start|>", + "<|box_end|>", + "<|quad_start|>", + "<|quad_end|>", + "<|vision_start|>", + "<|vision_end|>", + "<|vision_pad|>", + "<|image_pad|>", + "<|video_pad|>" + ], + "is_local": true, + "local_files_only": false, + "model_max_length": 131072, + "pad_token": "<|endoftext|>", + "split_special_tokens": false, + "tokenizer_class": "Qwen2Tokenizer", + "unk_token": null +} diff --git a/MuSiQue/SLS/Qwen3-1.7B/final/adapter_config.json b/MuSiQue/SLS/Qwen3-1.7B/final/adapter_config.json new file mode 100644 index 0000000000000000000000000000000000000000..75fcbf770754ef66b1b2e9ea3e9e1cb3f18ffefb --- /dev/null +++ b/MuSiQue/SLS/Qwen3-1.7B/final/adapter_config.json @@ -0,0 +1,45 @@ +{ + "alora_invocation_tokens": null, + "alpha_pattern": {}, + "arrow_config": null, + "auto_mapping": null, + "base_model_name_or_path": "Qwen/Qwen3-1.7B", + "bias": "none", + "corda_config": null, + "ensure_weight_tying": false, + "eva_config": null, + "exclude_modules": null, + "fan_in_fan_out": false, + "inference_mode": true, + "init_lora_weights": true, + "layer_replication": null, + "layers_pattern": null, + "layers_to_transform": null, + "loftq_config": {}, + "lora_alpha": 64, + "lora_bias": false, + "lora_dropout": 0.05, + "lora_ga_config": null, + "megatron_config": null, + "megatron_core": "megatron.core", + "modules_to_save": null, + "peft_type": "LORA", + "peft_version": "0.19.1", + "qalora_group_size": 16, + "r": 64, + "rank_pattern": {}, + "revision": null, + "target_modules": [ + "k_proj", + "v_proj", + "o_proj", + "q_proj" + ], + "target_parameters": null, + "task_type": "CAUSAL_LM", + "trainable_token_indices": null, + "use_bdlora": null, + "use_dora": false, + "use_qalora": false, + "use_rslora": false +} diff --git a/MuSiQue/SLS/Qwen3-1.7B/final/chat_template.jinja b/MuSiQue/SLS/Qwen3-1.7B/final/chat_template.jinja new file mode 100644 index 0000000000000000000000000000000000000000..01be9b307daa2d425f7c168c9fb145a286e0afb4 --- /dev/null +++ b/MuSiQue/SLS/Qwen3-1.7B/final/chat_template.jinja @@ -0,0 +1,89 @@ +{%- if tools %} + {{- '<|im_start|>system\n' }} + {%- if messages[0].role == 'system' %} + {{- messages[0].content + '\n\n' }} + {%- endif %} + {{- "# Tools\n\nYou may call one or more functions to assist with the user query.\n\nYou are provided with function signatures within XML tags:\n" }} + {%- for tool in tools %} + {{- "\n" }} + {{- tool | tojson }} + {%- endfor %} + {{- "\n\n\nFor each function call, return a json object with function name and arguments within XML tags:\n\n{\"name\": , \"arguments\": }\n<|im_end|>\n" }} +{%- else %} + {%- if messages[0].role == 'system' %} + {{- '<|im_start|>system\n' + messages[0].content + '<|im_end|>\n' }} + {%- endif %} +{%- endif %} +{%- set ns = namespace(multi_step_tool=true, last_query_index=messages|length - 1) %} +{%- for message in messages[::-1] %} + {%- set index = (messages|length - 1) - loop.index0 %} + {%- if ns.multi_step_tool and message.role == "user" and message.content is string and not(message.content.startswith('') and message.content.endswith('')) %} + {%- set ns.multi_step_tool = false %} + {%- set ns.last_query_index = index %} + {%- endif %} +{%- endfor %} +{%- for message in messages %} + {%- if message.content is string %} + {%- set content = message.content %} + {%- else %} + {%- set content = '' %} + {%- endif %} + {%- if (message.role == "user") or (message.role == "system" and not loop.first) %} + {{- '<|im_start|>' + message.role + '\n' + content + '<|im_end|>' + '\n' }} + {%- elif message.role == "assistant" %} + {%- set reasoning_content = '' %} + {%- if message.reasoning_content is string %} + {%- set reasoning_content = message.reasoning_content %} + {%- else %} + {%- if '' in content %} + {%- set reasoning_content = content.split('')[0].rstrip('\n').split('')[-1].lstrip('\n') %} + {%- set content = content.split('')[-1].lstrip('\n') %} + {%- endif %} + {%- endif %} + {%- if loop.index0 > ns.last_query_index %} + {%- if loop.last or (not loop.last and reasoning_content) %} + {{- '<|im_start|>' + message.role + '\n\n' + reasoning_content.strip('\n') + '\n\n\n' + content.lstrip('\n') }} + {%- else %} + {{- '<|im_start|>' + message.role + '\n' + content }} + {%- endif %} + {%- else %} + {{- '<|im_start|>' + message.role + '\n' + content }} + {%- endif %} + {%- if message.tool_calls %} + {%- for tool_call in message.tool_calls %} + {%- if (loop.first and content) or (not loop.first) %} + {{- '\n' }} + {%- endif %} + {%- if tool_call.function %} + {%- set tool_call = tool_call.function %} + {%- endif %} + {{- '\n{"name": "' }} + {{- tool_call.name }} + {{- '", "arguments": ' }} + {%- if tool_call.arguments is string %} + {{- tool_call.arguments }} + {%- else %} + {{- tool_call.arguments | tojson }} + {%- endif %} + {{- '}\n' }} + {%- endfor %} + {%- endif %} + {{- '<|im_end|>\n' }} + {%- elif message.role == "tool" %} + {%- if loop.first or (messages[loop.index0 - 1].role != "tool") %} + {{- '<|im_start|>user' }} + {%- endif %} + {{- '\n\n' }} + {{- content }} + {{- '\n' }} + {%- if loop.last or (messages[loop.index0 + 1].role != "tool") %} + {{- '<|im_end|>\n' }} + {%- endif %} + {%- endif %} +{%- endfor %} +{%- if add_generation_prompt %} + {{- '<|im_start|>assistant\n' }} + {%- if enable_thinking is defined and enable_thinking is false %} + {{- '\n\n\n\n' }} + {%- endif %} +{%- endif %} \ No newline at end of file diff --git a/MuSiQue/SLS/Qwen3-1.7B/final/rl_train_summary.json b/MuSiQue/SLS/Qwen3-1.7B/final/rl_train_summary.json new file mode 100644 index 0000000000000000000000000000000000000000..6fe9f74d13413d241d516e32569d4ebfd3beb78d --- /dev/null +++ b/MuSiQue/SLS/Qwen3-1.7B/final/rl_train_summary.json @@ -0,0 +1,8 @@ +{ + "agent": "A1", + "global_step": 302, + "total_records": 2416, + "algorithm": "rwr", + "kl_coef": 0.2, + "world_size": 8 +} diff --git a/MuSiQue/SLS/Qwen3-1.7B/final/tokenizer_config.json b/MuSiQue/SLS/Qwen3-1.7B/final/tokenizer_config.json new file mode 100644 index 0000000000000000000000000000000000000000..5668a4a01433b0648bb972458de616f22f3e728e --- /dev/null +++ b/MuSiQue/SLS/Qwen3-1.7B/final/tokenizer_config.json @@ -0,0 +1,30 @@ +{ + "add_prefix_space": false, + "backend": "tokenizers", + "bos_token": null, + "clean_up_tokenization_spaces": false, + "eos_token": "<|im_end|>", + "errors": "replace", + "extra_special_tokens": [ + "<|im_start|>", + "<|im_end|>", + "<|object_ref_start|>", + "<|object_ref_end|>", + "<|box_start|>", + "<|box_end|>", + "<|quad_start|>", + "<|quad_end|>", + "<|vision_start|>", + "<|vision_end|>", + "<|vision_pad|>", + "<|image_pad|>", + "<|video_pad|>" + ], + "is_local": true, + "local_files_only": false, + "model_max_length": 131072, + "pad_token": "<|endoftext|>", + "split_special_tokens": false, + "tokenizer_class": "Qwen2Tokenizer", + "unk_token": null +} diff --git a/MuSiQue/SLS/Qwen3-4B/final/adapter_config.json b/MuSiQue/SLS/Qwen3-4B/final/adapter_config.json new file mode 100644 index 0000000000000000000000000000000000000000..9e4f92d62739074b61b493d1cac69c11f5a1b5ed --- /dev/null +++ b/MuSiQue/SLS/Qwen3-4B/final/adapter_config.json @@ -0,0 +1,45 @@ +{ + "alora_invocation_tokens": null, + "alpha_pattern": {}, + "arrow_config": null, + "auto_mapping": null, + "base_model_name_or_path": "Qwen/Qwen3-4B", + "bias": "none", + "corda_config": null, + "ensure_weight_tying": false, + "eva_config": null, + "exclude_modules": null, + "fan_in_fan_out": false, + "inference_mode": true, + "init_lora_weights": true, + "layer_replication": null, + "layers_pattern": null, + "layers_to_transform": null, + "loftq_config": {}, + "lora_alpha": 64, + "lora_bias": false, + "lora_dropout": 0.05, + "lora_ga_config": null, + "megatron_config": null, + "megatron_core": "megatron.core", + "modules_to_save": null, + "peft_type": "LORA", + "peft_version": "0.19.1", + "qalora_group_size": 16, + "r": 64, + "rank_pattern": {}, + "revision": null, + "target_modules": [ + "o_proj", + "q_proj", + "k_proj", + "v_proj" + ], + "target_parameters": null, + "task_type": "CAUSAL_LM", + "trainable_token_indices": null, + "use_bdlora": null, + "use_dora": false, + "use_qalora": false, + "use_rslora": false +} diff --git a/MuSiQue/SLS/Qwen3-4B/final/chat_template.jinja b/MuSiQue/SLS/Qwen3-4B/final/chat_template.jinja new file mode 100644 index 0000000000000000000000000000000000000000..01be9b307daa2d425f7c168c9fb145a286e0afb4 --- /dev/null +++ b/MuSiQue/SLS/Qwen3-4B/final/chat_template.jinja @@ -0,0 +1,89 @@ +{%- if tools %} + {{- '<|im_start|>system\n' }} + {%- if messages[0].role == 'system' %} + {{- messages[0].content + '\n\n' }} + {%- endif %} + {{- "# Tools\n\nYou may call one or more functions to assist with the user query.\n\nYou are provided with function signatures within XML tags:\n" }} + {%- for tool in tools %} + {{- "\n" }} + {{- tool | tojson }} + {%- endfor %} + {{- "\n\n\nFor each function call, return a json object with function name and arguments within XML tags:\n\n{\"name\": , \"arguments\": }\n<|im_end|>\n" }} +{%- else %} + {%- if messages[0].role == 'system' %} + {{- '<|im_start|>system\n' + messages[0].content + '<|im_end|>\n' }} + {%- endif %} +{%- endif %} +{%- set ns = namespace(multi_step_tool=true, last_query_index=messages|length - 1) %} +{%- for message in messages[::-1] %} + {%- set index = (messages|length - 1) - loop.index0 %} + {%- if ns.multi_step_tool and message.role == "user" and message.content is string and not(message.content.startswith('') and message.content.endswith('')) %} + {%- set ns.multi_step_tool = false %} + {%- set ns.last_query_index = index %} + {%- endif %} +{%- endfor %} +{%- for message in messages %} + {%- if message.content is string %} + {%- set content = message.content %} + {%- else %} + {%- set content = '' %} + {%- endif %} + {%- if (message.role == "user") or (message.role == "system" and not loop.first) %} + {{- '<|im_start|>' + message.role + '\n' + content + '<|im_end|>' + '\n' }} + {%- elif message.role == "assistant" %} + {%- set reasoning_content = '' %} + {%- if message.reasoning_content is string %} + {%- set reasoning_content = message.reasoning_content %} + {%- else %} + {%- if '' in content %} + {%- set reasoning_content = content.split('')[0].rstrip('\n').split('')[-1].lstrip('\n') %} + {%- set content = content.split('')[-1].lstrip('\n') %} + {%- endif %} + {%- endif %} + {%- if loop.index0 > ns.last_query_index %} + {%- if loop.last or (not loop.last and reasoning_content) %} + {{- '<|im_start|>' + message.role + '\n\n' + reasoning_content.strip('\n') + '\n\n\n' + content.lstrip('\n') }} + {%- else %} + {{- '<|im_start|>' + message.role + '\n' + content }} + {%- endif %} + {%- else %} + {{- '<|im_start|>' + message.role + '\n' + content }} + {%- endif %} + {%- if message.tool_calls %} + {%- for tool_call in message.tool_calls %} + {%- if (loop.first and content) or (not loop.first) %} + {{- '\n' }} + {%- endif %} + {%- if tool_call.function %} + {%- set tool_call = tool_call.function %} + {%- endif %} + {{- '\n{"name": "' }} + {{- tool_call.name }} + {{- '", "arguments": ' }} + {%- if tool_call.arguments is string %} + {{- tool_call.arguments }} + {%- else %} + {{- tool_call.arguments | tojson }} + {%- endif %} + {{- '}\n' }} + {%- endfor %} + {%- endif %} + {{- '<|im_end|>\n' }} + {%- elif message.role == "tool" %} + {%- if loop.first or (messages[loop.index0 - 1].role != "tool") %} + {{- '<|im_start|>user' }} + {%- endif %} + {{- '\n\n' }} + {{- content }} + {{- '\n' }} + {%- if loop.last or (messages[loop.index0 + 1].role != "tool") %} + {{- '<|im_end|>\n' }} + {%- endif %} + {%- endif %} +{%- endfor %} +{%- if add_generation_prompt %} + {{- '<|im_start|>assistant\n' }} + {%- if enable_thinking is defined and enable_thinking is false %} + {{- '\n\n\n\n' }} + {%- endif %} +{%- endif %} \ No newline at end of file diff --git a/MuSiQue/SLS/Qwen3-4B/final/rl_train_summary.json b/MuSiQue/SLS/Qwen3-4B/final/rl_train_summary.json new file mode 100644 index 0000000000000000000000000000000000000000..5c66ae50652aa99f66db7cd7a56042bfc5fdc17f --- /dev/null +++ b/MuSiQue/SLS/Qwen3-4B/final/rl_train_summary.json @@ -0,0 +1,8 @@ +{ + "agent": "A2", + "global_step": 216, + "total_records": 1727, + "algorithm": "rwr", + "kl_coef": 0.2, + "world_size": 8 +} diff --git a/MuSiQue/SLS/Qwen3-4B/final/tokenizer_config.json b/MuSiQue/SLS/Qwen3-4B/final/tokenizer_config.json new file mode 100644 index 0000000000000000000000000000000000000000..5668a4a01433b0648bb972458de616f22f3e728e --- /dev/null +++ b/MuSiQue/SLS/Qwen3-4B/final/tokenizer_config.json @@ -0,0 +1,30 @@ +{ + "add_prefix_space": false, + "backend": "tokenizers", + "bos_token": null, + "clean_up_tokenization_spaces": false, + "eos_token": "<|im_end|>", + "errors": "replace", + "extra_special_tokens": [ + "<|im_start|>", + "<|im_end|>", + "<|object_ref_start|>", + "<|object_ref_end|>", + "<|box_start|>", + "<|box_end|>", + "<|quad_start|>", + "<|quad_end|>", + "<|vision_start|>", + "<|vision_end|>", + "<|vision_pad|>", + "<|image_pad|>", + "<|video_pad|>" + ], + "is_local": true, + "local_files_only": false, + "model_max_length": 131072, + "pad_token": "<|endoftext|>", + "split_special_tokens": false, + "tokenizer_class": "Qwen2Tokenizer", + "unk_token": null +} diff --git a/MuSiQue/SLS/Qwen3-8B/final/adapter_config.json b/MuSiQue/SLS/Qwen3-8B/final/adapter_config.json new file mode 100644 index 0000000000000000000000000000000000000000..be54f9c0c7cbd90d34dc3ef09e5c5cc0414e3e36 --- /dev/null +++ b/MuSiQue/SLS/Qwen3-8B/final/adapter_config.json @@ -0,0 +1,45 @@ +{ + "alora_invocation_tokens": null, + "alpha_pattern": {}, + "arrow_config": null, + "auto_mapping": null, + "base_model_name_or_path": "Qwen/Qwen3-8B", + "bias": "none", + "corda_config": null, + "ensure_weight_tying": false, + "eva_config": null, + "exclude_modules": null, + "fan_in_fan_out": false, + "inference_mode": true, + "init_lora_weights": true, + "layer_replication": null, + "layers_pattern": null, + "layers_to_transform": null, + "loftq_config": {}, + "lora_alpha": 64, + "lora_bias": false, + "lora_dropout": 0.05, + "lora_ga_config": null, + "megatron_config": null, + "megatron_core": "megatron.core", + "modules_to_save": null, + "peft_type": "LORA", + "peft_version": "0.19.1", + "qalora_group_size": 16, + "r": 64, + "rank_pattern": {}, + "revision": null, + "target_modules": [ + "o_proj", + "v_proj", + "q_proj", + "k_proj" + ], + "target_parameters": null, + "task_type": "CAUSAL_LM", + "trainable_token_indices": null, + "use_bdlora": null, + "use_dora": false, + "use_qalora": false, + "use_rslora": false +} diff --git a/MuSiQue/SLS/Qwen3-8B/final/chat_template.jinja b/MuSiQue/SLS/Qwen3-8B/final/chat_template.jinja new file mode 100644 index 0000000000000000000000000000000000000000..01be9b307daa2d425f7c168c9fb145a286e0afb4 --- /dev/null +++ b/MuSiQue/SLS/Qwen3-8B/final/chat_template.jinja @@ -0,0 +1,89 @@ +{%- if tools %} + {{- '<|im_start|>system\n' }} + {%- if messages[0].role == 'system' %} + {{- messages[0].content + '\n\n' }} + {%- endif %} + {{- "# Tools\n\nYou may call one or more functions to assist with the user query.\n\nYou are provided with function signatures within XML tags:\n" }} + {%- for tool in tools %} + {{- "\n" }} + {{- tool | tojson }} + {%- endfor %} + {{- "\n\n\nFor each function call, return a json object with function name and arguments within XML tags:\n\n{\"name\": , \"arguments\": }\n<|im_end|>\n" }} +{%- else %} + {%- if messages[0].role == 'system' %} + {{- '<|im_start|>system\n' + messages[0].content + '<|im_end|>\n' }} + {%- endif %} +{%- endif %} +{%- set ns = namespace(multi_step_tool=true, last_query_index=messages|length - 1) %} +{%- for message in messages[::-1] %} + {%- set index = (messages|length - 1) - loop.index0 %} + {%- if ns.multi_step_tool and message.role == "user" and message.content is string and not(message.content.startswith('') and message.content.endswith('')) %} + {%- set ns.multi_step_tool = false %} + {%- set ns.last_query_index = index %} + {%- endif %} +{%- endfor %} +{%- for message in messages %} + {%- if message.content is string %} + {%- set content = message.content %} + {%- else %} + {%- set content = '' %} + {%- endif %} + {%- if (message.role == "user") or (message.role == "system" and not loop.first) %} + {{- '<|im_start|>' + message.role + '\n' + content + '<|im_end|>' + '\n' }} + {%- elif message.role == "assistant" %} + {%- set reasoning_content = '' %} + {%- if message.reasoning_content is string %} + {%- set reasoning_content = message.reasoning_content %} + {%- else %} + {%- if '' in content %} + {%- set reasoning_content = content.split('')[0].rstrip('\n').split('')[-1].lstrip('\n') %} + {%- set content = content.split('')[-1].lstrip('\n') %} + {%- endif %} + {%- endif %} + {%- if loop.index0 > ns.last_query_index %} + {%- if loop.last or (not loop.last and reasoning_content) %} + {{- '<|im_start|>' + message.role + '\n\n' + reasoning_content.strip('\n') + '\n\n\n' + content.lstrip('\n') }} + {%- else %} + {{- '<|im_start|>' + message.role + '\n' + content }} + {%- endif %} + {%- else %} + {{- '<|im_start|>' + message.role + '\n' + content }} + {%- endif %} + {%- if message.tool_calls %} + {%- for tool_call in message.tool_calls %} + {%- if (loop.first and content) or (not loop.first) %} + {{- '\n' }} + {%- endif %} + {%- if tool_call.function %} + {%- set tool_call = tool_call.function %} + {%- endif %} + {{- '\n{"name": "' }} + {{- tool_call.name }} + {{- '", "arguments": ' }} + {%- if tool_call.arguments is string %} + {{- tool_call.arguments }} + {%- else %} + {{- tool_call.arguments | tojson }} + {%- endif %} + {{- '}\n' }} + {%- endfor %} + {%- endif %} + {{- '<|im_end|>\n' }} + {%- elif message.role == "tool" %} + {%- if loop.first or (messages[loop.index0 - 1].role != "tool") %} + {{- '<|im_start|>user' }} + {%- endif %} + {{- '\n\n' }} + {{- content }} + {{- '\n' }} + {%- if loop.last or (messages[loop.index0 + 1].role != "tool") %} + {{- '<|im_end|>\n' }} + {%- endif %} + {%- endif %} +{%- endfor %} +{%- if add_generation_prompt %} + {{- '<|im_start|>assistant\n' }} + {%- if enable_thinking is defined and enable_thinking is false %} + {{- '\n\n\n\n' }} + {%- endif %} +{%- endif %} \ No newline at end of file diff --git a/MuSiQue/SLS/Qwen3-8B/final/rl_train_summary.json b/MuSiQue/SLS/Qwen3-8B/final/rl_train_summary.json new file mode 100644 index 0000000000000000000000000000000000000000..a66276518046c0f352beb4b860d27a58e55ad7a7 --- /dev/null +++ b/MuSiQue/SLS/Qwen3-8B/final/rl_train_summary.json @@ -0,0 +1,8 @@ +{ + "agent": "A3", + "global_step": 178, + "total_records": 1411, + "algorithm": "rwr", + "kl_coef": 0.2, + "world_size": 8 +} diff --git a/MuSiQue/SLS/Qwen3-8B/final/tokenizer_config.json b/MuSiQue/SLS/Qwen3-8B/final/tokenizer_config.json new file mode 100644 index 0000000000000000000000000000000000000000..5668a4a01433b0648bb972458de616f22f3e728e --- /dev/null +++ b/MuSiQue/SLS/Qwen3-8B/final/tokenizer_config.json @@ -0,0 +1,30 @@ +{ + "add_prefix_space": false, + "backend": "tokenizers", + "bos_token": null, + "clean_up_tokenization_spaces": false, + "eos_token": "<|im_end|>", + "errors": "replace", + "extra_special_tokens": [ + "<|im_start|>", + "<|im_end|>", + "<|object_ref_start|>", + "<|object_ref_end|>", + "<|box_start|>", + "<|box_end|>", + "<|quad_start|>", + "<|quad_end|>", + "<|vision_start|>", + "<|vision_end|>", + "<|vision_pad|>", + "<|image_pad|>", + "<|video_pad|>" + ], + "is_local": true, + "local_files_only": false, + "model_max_length": 131072, + "pad_token": "<|endoftext|>", + "split_special_tokens": false, + "tokenizer_class": "Qwen2Tokenizer", + "unk_token": null +} diff --git a/MultiPL-E/PD/Qwen3-1.7B/checkpoint-270/adapter_config.json b/MultiPL-E/PD/Qwen3-1.7B/checkpoint-270/adapter_config.json new file mode 100644 index 0000000000000000000000000000000000000000..fed0130b211f99ee7a02618d94e65397d75a189f --- /dev/null +++ b/MultiPL-E/PD/Qwen3-1.7B/checkpoint-270/adapter_config.json @@ -0,0 +1,45 @@ +{ + "alora_invocation_tokens": null, + "alpha_pattern": {}, + "arrow_config": null, + "auto_mapping": null, + "base_model_name_or_path": "Qwen/Qwen3-1.7B", + "bias": "none", + "corda_config": null, + "ensure_weight_tying": false, + "eva_config": null, + "exclude_modules": null, + "fan_in_fan_out": false, + "inference_mode": true, + "init_lora_weights": true, + "layer_replication": null, + "layers_pattern": null, + "layers_to_transform": null, + "loftq_config": {}, + "lora_alpha": 64, + "lora_bias": false, + "lora_dropout": 0.05, + "lora_ga_config": null, + "megatron_config": null, + "megatron_core": "megatron.core", + "modules_to_save": null, + "peft_type": "LORA", + "peft_version": "0.19.1", + "qalora_group_size": 16, + "r": 64, + "rank_pattern": {}, + "revision": null, + "target_modules": [ + "o_proj", + "v_proj", + "q_proj", + "k_proj" + ], + "target_parameters": null, + "task_type": "CAUSAL_LM", + "trainable_token_indices": null, + "use_bdlora": null, + "use_dora": false, + "use_qalora": false, + "use_rslora": false +} diff --git a/MultiPL-E/PD/Qwen3-1.7B/checkpoint-270/chat_template.jinja b/MultiPL-E/PD/Qwen3-1.7B/checkpoint-270/chat_template.jinja new file mode 100644 index 0000000000000000000000000000000000000000..01be9b307daa2d425f7c168c9fb145a286e0afb4 --- /dev/null +++ b/MultiPL-E/PD/Qwen3-1.7B/checkpoint-270/chat_template.jinja @@ -0,0 +1,89 @@ +{%- if tools %} + {{- '<|im_start|>system\n' }} + {%- if messages[0].role == 'system' %} + {{- messages[0].content + '\n\n' }} + {%- endif %} + {{- "# Tools\n\nYou may call one or more functions to assist with the user query.\n\nYou are provided with function signatures within XML tags:\n" }} + {%- for tool in tools %} + {{- "\n" }} + {{- tool | tojson }} + {%- endfor %} + {{- "\n\n\nFor each function call, return a json object with function name and arguments within XML tags:\n\n{\"name\": , \"arguments\": }\n<|im_end|>\n" }} +{%- else %} + {%- if messages[0].role == 'system' %} + {{- '<|im_start|>system\n' + messages[0].content + '<|im_end|>\n' }} + {%- endif %} +{%- endif %} +{%- set ns = namespace(multi_step_tool=true, last_query_index=messages|length - 1) %} +{%- for message in messages[::-1] %} + {%- set index = (messages|length - 1) - loop.index0 %} + {%- if ns.multi_step_tool and message.role == "user" and message.content is string and not(message.content.startswith('') and message.content.endswith('')) %} + {%- set ns.multi_step_tool = false %} + {%- set ns.last_query_index = index %} + {%- endif %} +{%- endfor %} +{%- for message in messages %} + {%- if message.content is string %} + {%- set content = message.content %} + {%- else %} + {%- set content = '' %} + {%- endif %} + {%- if (message.role == "user") or (message.role == "system" and not loop.first) %} + {{- '<|im_start|>' + message.role + '\n' + content + '<|im_end|>' + '\n' }} + {%- elif message.role == "assistant" %} + {%- set reasoning_content = '' %} + {%- if message.reasoning_content is string %} + {%- set reasoning_content = message.reasoning_content %} + {%- else %} + {%- if '' in content %} + {%- set reasoning_content = content.split('')[0].rstrip('\n').split('')[-1].lstrip('\n') %} + {%- set content = content.split('')[-1].lstrip('\n') %} + {%- endif %} + {%- endif %} + {%- if loop.index0 > ns.last_query_index %} + {%- if loop.last or (not loop.last and reasoning_content) %} + {{- '<|im_start|>' + message.role + '\n\n' + reasoning_content.strip('\n') + '\n\n\n' + content.lstrip('\n') }} + {%- else %} + {{- '<|im_start|>' + message.role + '\n' + content }} + {%- endif %} + {%- else %} + {{- '<|im_start|>' + message.role + '\n' + content }} + {%- endif %} + {%- if message.tool_calls %} + {%- for tool_call in message.tool_calls %} + {%- if (loop.first and content) or (not loop.first) %} + {{- '\n' }} + {%- endif %} + {%- if tool_call.function %} + {%- set tool_call = tool_call.function %} + {%- endif %} + {{- '\n{"name": "' }} + {{- tool_call.name }} + {{- '", "arguments": ' }} + {%- if tool_call.arguments is string %} + {{- tool_call.arguments }} + {%- else %} + {{- tool_call.arguments | tojson }} + {%- endif %} + {{- '}\n' }} + {%- endfor %} + {%- endif %} + {{- '<|im_end|>\n' }} + {%- elif message.role == "tool" %} + {%- if loop.first or (messages[loop.index0 - 1].role != "tool") %} + {{- '<|im_start|>user' }} + {%- endif %} + {{- '\n\n' }} + {{- content }} + {{- '\n' }} + {%- if loop.last or (messages[loop.index0 + 1].role != "tool") %} + {{- '<|im_end|>\n' }} + {%- endif %} + {%- endif %} +{%- endfor %} +{%- if add_generation_prompt %} + {{- '<|im_start|>assistant\n' }} + {%- if enable_thinking is defined and enable_thinking is false %} + {{- '\n\n\n\n' }} + {%- endif %} +{%- endif %} \ No newline at end of file diff --git a/MultiPL-E/PD/Qwen3-1.7B/checkpoint-270/tokenizer_config.json b/MultiPL-E/PD/Qwen3-1.7B/checkpoint-270/tokenizer_config.json new file mode 100644 index 0000000000000000000000000000000000000000..5668a4a01433b0648bb972458de616f22f3e728e --- /dev/null +++ b/MultiPL-E/PD/Qwen3-1.7B/checkpoint-270/tokenizer_config.json @@ -0,0 +1,30 @@ +{ + "add_prefix_space": false, + "backend": "tokenizers", + "bos_token": null, + "clean_up_tokenization_spaces": false, + "eos_token": "<|im_end|>", + "errors": "replace", + "extra_special_tokens": [ + "<|im_start|>", + "<|im_end|>", + "<|object_ref_start|>", + "<|object_ref_end|>", + "<|box_start|>", + "<|box_end|>", + "<|quad_start|>", + "<|quad_end|>", + "<|vision_start|>", + "<|vision_end|>", + "<|vision_pad|>", + "<|image_pad|>", + "<|video_pad|>" + ], + "is_local": true, + "local_files_only": false, + "model_max_length": 131072, + "pad_token": "<|endoftext|>", + "split_special_tokens": false, + "tokenizer_class": "Qwen2Tokenizer", + "unk_token": null +} diff --git a/MultiPL-E/PD/Qwen3-1.7B/checkpoint-270/trainer_state.json b/MultiPL-E/PD/Qwen3-1.7B/checkpoint-270/trainer_state.json new file mode 100644 index 0000000000000000000000000000000000000000..866723e05b91c6d44f7bac9ae4a1d4831e626448 --- /dev/null +++ b/MultiPL-E/PD/Qwen3-1.7B/checkpoint-270/trainer_state.json @@ -0,0 +1,304 @@ +{ + "best_global_step": 270, + "best_metric": 0.3026478886604309, + "best_model_checkpoint": "jca/sft_runs/multipl_e_8lang_v5_diverse_a1_5ep/A1/checkpoint-270", + "epoch": 4.825112107623318, + "eval_steps": 30, + "global_step": 270, + "is_hyper_param_search": false, + "is_local_process_zero": true, + "is_world_process_zero": true, + "log_history": [ + { + "epoch": 0.17937219730941703, + "grad_norm": 2.7016327381134033, + "learning_rate": 1.2857142857142859e-05, + "loss": 0.9510406494140625, + "step": 10 + }, + { + "epoch": 0.35874439461883406, + "grad_norm": 0.4876834750175476, + "learning_rate": 1.9982569081367844e-05, + "loss": 0.6075239181518555, + "step": 20 + }, + { + "epoch": 0.5381165919282511, + "grad_norm": 0.26797446608543396, + "learning_rate": 1.9843486124773546e-05, + "loss": 0.4491765022277832, + "step": 30 + }, + { + "epoch": 0.5381165919282511, + "eval_loss": 0.42794305086135864, + "eval_runtime": 30.5701, + "eval_samples_per_second": 12.921, + "eval_steps_per_second": 0.818, + "step": 30 + }, + { + "epoch": 0.7174887892376681, + "grad_norm": 0.21848726272583008, + "learning_rate": 1.956725799620305e-05, + "loss": 0.40464367866516116, + "step": 40 + }, + { + "epoch": 0.8968609865470852, + "grad_norm": 0.19735607504844666, + "learning_rate": 1.9157733266550577e-05, + "loss": 0.37493760585784913, + "step": 50 + }, + { + "epoch": 1.0717488789237668, + "grad_norm": 0.19580143690109253, + "learning_rate": 1.8620617672459097e-05, + "loss": 0.3587300777435303, + "step": 60 + }, + { + "epoch": 1.0717488789237668, + "eval_loss": 0.3591115474700928, + "eval_runtime": 29.4512, + "eval_samples_per_second": 13.412, + "eval_steps_per_second": 0.849, + "step": 60 + }, + { + "epoch": 1.251121076233184, + "grad_norm": 0.17649081349372864, + "learning_rate": 1.7963394620679945e-05, + "loss": 0.3459214687347412, + "step": 70 + }, + { + "epoch": 1.4304932735426008, + "grad_norm": 0.18987315893173218, + "learning_rate": 1.7195220924898883e-05, + "loss": 0.32703309059143065, + "step": 80 + }, + { + "epoch": 1.609865470852018, + "grad_norm": 0.17722012102603912, + "learning_rate": 1.6326799227683806e-05, + "loss": 0.3273223876953125, + "step": 90 + }, + { + "epoch": 1.609865470852018, + "eval_loss": 0.3318503797054291, + "eval_runtime": 27.4154, + "eval_samples_per_second": 14.408, + "eval_steps_per_second": 0.912, + "step": 90 + }, + { + "epoch": 1.789237668161435, + "grad_norm": 0.16768372058868408, + "learning_rate": 1.5370228885045662e-05, + "loss": 0.3189870357513428, + "step": 100 + }, + { + "epoch": 1.9686098654708521, + "grad_norm": 0.16524484753608704, + "learning_rate": 1.4338837391175582e-05, + "loss": 0.30961763858795166, + "step": 110 + }, + { + "epoch": 2.1434977578475336, + "grad_norm": 0.18033911287784576, + "learning_rate": 1.3246994692046837e-05, + "loss": 0.30172014236450195, + "step": 120 + }, + { + "epoch": 2.1434977578475336, + "eval_loss": 0.31922078132629395, + "eval_runtime": 27.178, + "eval_samples_per_second": 14.534, + "eval_steps_per_second": 0.92, + "step": 120 + }, + { + "epoch": 2.3228699551569507, + "grad_norm": 0.1823396533727646, + "learning_rate": 1.2109912974972424e-05, + "loss": 0.3013172149658203, + "step": 130 + }, + { + "epoch": 2.502242152466368, + "grad_norm": 0.19819572567939758, + "learning_rate": 1.0943434723566624e-05, + "loss": 0.30026633739471437, + "step": 140 + }, + { + "epoch": 2.681614349775785, + "grad_norm": 0.18170028924942017, + "learning_rate": 9.763811991052021e-06, + "loss": 0.29707751274108884, + "step": 150 + }, + { + "epoch": 2.681614349775785, + "eval_loss": 0.3111979067325592, + "eval_runtime": 27.199, + "eval_samples_per_second": 14.523, + "eval_steps_per_second": 0.919, + "step": 150 + }, + { + "epoch": 2.8609865470852016, + "grad_norm": 0.18932338058948517, + "learning_rate": 8.587479967204584e-06, + "loss": 0.3032960891723633, + "step": 160 + }, + { + "epoch": 3.0358744394618835, + "grad_norm": 0.19508804380893707, + "learning_rate": 7.430827993733809e-06, + "loss": 0.2967501640319824, + "step": 170 + }, + { + "epoch": 3.2152466367713006, + "grad_norm": 0.20429708063602448, + "learning_rate": 6.30997121844481e-06, + "loss": 0.2913133859634399, + "step": 180 + }, + { + "epoch": 3.2152466367713006, + "eval_loss": 0.3067827820777893, + "eval_runtime": 27.6297, + "eval_samples_per_second": 14.296, + "eval_steps_per_second": 0.905, + "step": 180 + }, + { + "epoch": 3.3946188340807173, + "grad_norm": 0.1792595535516739, + "learning_rate": 5.240526069629265e-06, + "loss": 0.29277753829956055, + "step": 190 + }, + { + "epoch": 3.5739910313901344, + "grad_norm": 0.16619998216629028, + "learning_rate": 4.237392678906633e-06, + "loss": 0.28897757530212403, + "step": 200 + }, + { + "epoch": 3.7533632286995515, + "grad_norm": 0.18059593439102173, + "learning_rate": 3.3145472839270575e-06, + "loss": 0.29484267234802247, + "step": 210 + }, + { + "epoch": 3.7533632286995515, + "eval_loss": 0.3041122257709503, + "eval_runtime": 27.4044, + "eval_samples_per_second": 14.414, + "eval_steps_per_second": 0.912, + "step": 210 + }, + { + "epoch": 3.9327354260089686, + "grad_norm": 0.18860650062561035, + "learning_rate": 2.484847503302238e-06, + "loss": 0.2927693843841553, + "step": 220 + }, + { + "epoch": 4.1076233183856505, + "grad_norm": 0.19515369832515717, + "learning_rate": 1.7598531967871468e-06, + "loss": 0.29400765895843506, + "step": 230 + }, + { + "epoch": 4.286995515695067, + "grad_norm": 0.17510154843330383, + "learning_rate": 1.1496654065926927e-06, + "loss": 0.28560829162597656, + "step": 240 + }, + { + "epoch": 4.286995515695067, + "eval_loss": 0.3028334677219391, + "eval_runtime": 26.3261, + "eval_samples_per_second": 15.004, + "eval_steps_per_second": 0.95, + "step": 240 + }, + { + "epoch": 4.466367713004484, + "grad_norm": 0.1877126544713974, + "learning_rate": 6.627856237925812e-07, + "loss": 0.2830939769744873, + "step": 250 + }, + { + "epoch": 4.645739910313901, + "grad_norm": 0.1783999502658844, + "learning_rate": 3.059973406066963e-07, + "loss": 0.28902902603149416, + "step": 260 + }, + { + "epoch": 4.825112107623318, + "grad_norm": 0.18093666434288025, + "learning_rate": 8.42715388433446e-08, + "loss": 0.2830434560775757, + "step": 270 + }, + { + "epoch": 4.825112107623318, + "eval_loss": 0.3026478886604309, + "eval_runtime": 27.5009, + "eval_samples_per_second": 14.363, + "eval_steps_per_second": 0.909, + "step": 270 + } + ], + "logging_steps": 10, + "max_steps": 280, + "num_input_tokens_seen": 0, + "num_train_epochs": 5, + "save_steps": 30, + "stateful_callbacks": { + "EarlyStoppingCallback": { + "args": { + "early_stopping_patience": 3, + "early_stopping_threshold": 0.0 + }, + "attributes": { + "early_stopping_patience_counter": 0 + } + }, + "TrainerControl": { + "args": { + "should_epoch_stop": false, + "should_evaluate": false, + "should_log": false, + "should_save": true, + "should_training_stop": false + }, + "attributes": {} + } + }, + "total_flos": 1.443434129742889e+17, + "train_batch_size": 2, + "trial_name": null, + "trial_params": null +} diff --git a/MultiPL-E/PD/Qwen3-1.7B/checkpoint-280/adapter_config.json b/MultiPL-E/PD/Qwen3-1.7B/checkpoint-280/adapter_config.json new file mode 100644 index 0000000000000000000000000000000000000000..fed0130b211f99ee7a02618d94e65397d75a189f --- /dev/null +++ b/MultiPL-E/PD/Qwen3-1.7B/checkpoint-280/adapter_config.json @@ -0,0 +1,45 @@ +{ + "alora_invocation_tokens": null, + "alpha_pattern": {}, + "arrow_config": null, + "auto_mapping": null, + "base_model_name_or_path": "Qwen/Qwen3-1.7B", + "bias": "none", + "corda_config": null, + "ensure_weight_tying": false, + "eva_config": null, + "exclude_modules": null, + "fan_in_fan_out": false, + "inference_mode": true, + "init_lora_weights": true, + "layer_replication": null, + "layers_pattern": null, + "layers_to_transform": null, + "loftq_config": {}, + "lora_alpha": 64, + "lora_bias": false, + "lora_dropout": 0.05, + "lora_ga_config": null, + "megatron_config": null, + "megatron_core": "megatron.core", + "modules_to_save": null, + "peft_type": "LORA", + "peft_version": "0.19.1", + "qalora_group_size": 16, + "r": 64, + "rank_pattern": {}, + "revision": null, + "target_modules": [ + "o_proj", + "v_proj", + "q_proj", + "k_proj" + ], + "target_parameters": null, + "task_type": "CAUSAL_LM", + "trainable_token_indices": null, + "use_bdlora": null, + "use_dora": false, + "use_qalora": false, + "use_rslora": false +} diff --git a/MultiPL-E/PD/Qwen3-1.7B/checkpoint-280/chat_template.jinja b/MultiPL-E/PD/Qwen3-1.7B/checkpoint-280/chat_template.jinja new file mode 100644 index 0000000000000000000000000000000000000000..01be9b307daa2d425f7c168c9fb145a286e0afb4 --- /dev/null +++ b/MultiPL-E/PD/Qwen3-1.7B/checkpoint-280/chat_template.jinja @@ -0,0 +1,89 @@ +{%- if tools %} + {{- '<|im_start|>system\n' }} + {%- if messages[0].role == 'system' %} + {{- messages[0].content + '\n\n' }} + {%- endif %} + {{- "# Tools\n\nYou may call one or more functions to assist with the user query.\n\nYou are provided with function signatures within XML tags:\n" }} + {%- for tool in tools %} + {{- "\n" }} + {{- tool | tojson }} + {%- endfor %} + {{- "\n\n\nFor each function call, return a json object with function name and arguments within XML tags:\n\n{\"name\": , \"arguments\": }\n<|im_end|>\n" }} +{%- else %} + {%- if messages[0].role == 'system' %} + {{- '<|im_start|>system\n' + messages[0].content + '<|im_end|>\n' }} + {%- endif %} +{%- endif %} +{%- set ns = namespace(multi_step_tool=true, last_query_index=messages|length - 1) %} +{%- for message in messages[::-1] %} + {%- set index = (messages|length - 1) - loop.index0 %} + {%- if ns.multi_step_tool and message.role == "user" and message.content is string and not(message.content.startswith('') and message.content.endswith('')) %} + {%- set ns.multi_step_tool = false %} + {%- set ns.last_query_index = index %} + {%- endif %} +{%- endfor %} +{%- for message in messages %} + {%- if message.content is string %} + {%- set content = message.content %} + {%- else %} + {%- set content = '' %} + {%- endif %} + {%- if (message.role == "user") or (message.role == "system" and not loop.first) %} + {{- '<|im_start|>' + message.role + '\n' + content + '<|im_end|>' + '\n' }} + {%- elif message.role == "assistant" %} + {%- set reasoning_content = '' %} + {%- if message.reasoning_content is string %} + {%- set reasoning_content = message.reasoning_content %} + {%- else %} + {%- if '' in content %} + {%- set reasoning_content = content.split('')[0].rstrip('\n').split('')[-1].lstrip('\n') %} + {%- set content = content.split('')[-1].lstrip('\n') %} + {%- endif %} + {%- endif %} + {%- if loop.index0 > ns.last_query_index %} + {%- if loop.last or (not loop.last and reasoning_content) %} + {{- '<|im_start|>' + message.role + '\n\n' + reasoning_content.strip('\n') + '\n\n\n' + content.lstrip('\n') }} + {%- else %} + {{- '<|im_start|>' + message.role + '\n' + content }} + {%- endif %} + {%- else %} + {{- '<|im_start|>' + message.role + '\n' + content }} + {%- endif %} + {%- if message.tool_calls %} + {%- for tool_call in message.tool_calls %} + {%- if (loop.first and content) or (not loop.first) %} + {{- '\n' }} + {%- endif %} + {%- if tool_call.function %} + {%- set tool_call = tool_call.function %} + {%- endif %} + {{- '\n{"name": "' }} + {{- tool_call.name }} + {{- '", "arguments": ' }} + {%- if tool_call.arguments is string %} + {{- tool_call.arguments }} + {%- else %} + {{- tool_call.arguments | tojson }} + {%- endif %} + {{- '}\n' }} + {%- endfor %} + {%- endif %} + {{- '<|im_end|>\n' }} + {%- elif message.role == "tool" %} + {%- if loop.first or (messages[loop.index0 - 1].role != "tool") %} + {{- '<|im_start|>user' }} + {%- endif %} + {{- '\n\n' }} + {{- content }} + {{- '\n' }} + {%- if loop.last or (messages[loop.index0 + 1].role != "tool") %} + {{- '<|im_end|>\n' }} + {%- endif %} + {%- endif %} +{%- endfor %} +{%- if add_generation_prompt %} + {{- '<|im_start|>assistant\n' }} + {%- if enable_thinking is defined and enable_thinking is false %} + {{- '\n\n\n\n' }} + {%- endif %} +{%- endif %} \ No newline at end of file diff --git a/MultiPL-E/PD/Qwen3-1.7B/checkpoint-280/tokenizer_config.json b/MultiPL-E/PD/Qwen3-1.7B/checkpoint-280/tokenizer_config.json new file mode 100644 index 0000000000000000000000000000000000000000..5668a4a01433b0648bb972458de616f22f3e728e --- /dev/null +++ b/MultiPL-E/PD/Qwen3-1.7B/checkpoint-280/tokenizer_config.json @@ -0,0 +1,30 @@ +{ + "add_prefix_space": false, + "backend": "tokenizers", + "bos_token": null, + "clean_up_tokenization_spaces": false, + "eos_token": "<|im_end|>", + "errors": "replace", + "extra_special_tokens": [ + "<|im_start|>", + "<|im_end|>", + "<|object_ref_start|>", + "<|object_ref_end|>", + "<|box_start|>", + "<|box_end|>", + "<|quad_start|>", + "<|quad_end|>", + "<|vision_start|>", + "<|vision_end|>", + "<|vision_pad|>", + "<|image_pad|>", + "<|video_pad|>" + ], + "is_local": true, + "local_files_only": false, + "model_max_length": 131072, + "pad_token": "<|endoftext|>", + "split_special_tokens": false, + "tokenizer_class": "Qwen2Tokenizer", + "unk_token": null +} diff --git a/MultiPL-E/PD/Qwen3-1.7B/checkpoint-280/trainer_state.json b/MultiPL-E/PD/Qwen3-1.7B/checkpoint-280/trainer_state.json new file mode 100644 index 0000000000000000000000000000000000000000..2441212c1d0ca658c3722570b58fda4f8ea56cfc --- /dev/null +++ b/MultiPL-E/PD/Qwen3-1.7B/checkpoint-280/trainer_state.json @@ -0,0 +1,319 @@ +{ + "best_global_step": 270, + "best_metric": 0.3026478886604309, + "best_model_checkpoint": "jca/sft_runs/multipl_e_8lang_v5_diverse_a1_5ep/A1/checkpoint-270", + "epoch": 5.0, + "eval_steps": 30, + "global_step": 280, + "is_hyper_param_search": false, + "is_local_process_zero": true, + "is_world_process_zero": true, + "log_history": [ + { + "epoch": 0.17937219730941703, + "grad_norm": 2.7016327381134033, + "learning_rate": 1.2857142857142859e-05, + "loss": 0.9510406494140625, + "step": 10 + }, + { + "epoch": 0.35874439461883406, + "grad_norm": 0.4876834750175476, + "learning_rate": 1.9982569081367844e-05, + "loss": 0.6075239181518555, + "step": 20 + }, + { + "epoch": 0.5381165919282511, + "grad_norm": 0.26797446608543396, + "learning_rate": 1.9843486124773546e-05, + "loss": 0.4491765022277832, + "step": 30 + }, + { + "epoch": 0.5381165919282511, + "eval_loss": 0.42794305086135864, + "eval_runtime": 30.5701, + "eval_samples_per_second": 12.921, + "eval_steps_per_second": 0.818, + "step": 30 + }, + { + "epoch": 0.7174887892376681, + "grad_norm": 0.21848726272583008, + "learning_rate": 1.956725799620305e-05, + "loss": 0.40464367866516116, + "step": 40 + }, + { + "epoch": 0.8968609865470852, + "grad_norm": 0.19735607504844666, + "learning_rate": 1.9157733266550577e-05, + "loss": 0.37493760585784913, + "step": 50 + }, + { + "epoch": 1.0717488789237668, + "grad_norm": 0.19580143690109253, + "learning_rate": 1.8620617672459097e-05, + "loss": 0.3587300777435303, + "step": 60 + }, + { + "epoch": 1.0717488789237668, + "eval_loss": 0.3591115474700928, + "eval_runtime": 29.4512, + "eval_samples_per_second": 13.412, + "eval_steps_per_second": 0.849, + "step": 60 + }, + { + "epoch": 1.251121076233184, + "grad_norm": 0.17649081349372864, + "learning_rate": 1.7963394620679945e-05, + "loss": 0.3459214687347412, + "step": 70 + }, + { + "epoch": 1.4304932735426008, + "grad_norm": 0.18987315893173218, + "learning_rate": 1.7195220924898883e-05, + "loss": 0.32703309059143065, + "step": 80 + }, + { + "epoch": 1.609865470852018, + "grad_norm": 0.17722012102603912, + "learning_rate": 1.6326799227683806e-05, + "loss": 0.3273223876953125, + "step": 90 + }, + { + "epoch": 1.609865470852018, + "eval_loss": 0.3318503797054291, + "eval_runtime": 27.4154, + "eval_samples_per_second": 14.408, + "eval_steps_per_second": 0.912, + "step": 90 + }, + { + "epoch": 1.789237668161435, + "grad_norm": 0.16768372058868408, + "learning_rate": 1.5370228885045662e-05, + "loss": 0.3189870357513428, + "step": 100 + }, + { + "epoch": 1.9686098654708521, + "grad_norm": 0.16524484753608704, + "learning_rate": 1.4338837391175582e-05, + "loss": 0.30961763858795166, + "step": 110 + }, + { + "epoch": 2.1434977578475336, + "grad_norm": 0.18033911287784576, + "learning_rate": 1.3246994692046837e-05, + "loss": 0.30172014236450195, + "step": 120 + }, + { + "epoch": 2.1434977578475336, + "eval_loss": 0.31922078132629395, + "eval_runtime": 27.178, + "eval_samples_per_second": 14.534, + "eval_steps_per_second": 0.92, + "step": 120 + }, + { + "epoch": 2.3228699551569507, + "grad_norm": 0.1823396533727646, + "learning_rate": 1.2109912974972424e-05, + "loss": 0.3013172149658203, + "step": 130 + }, + { + "epoch": 2.502242152466368, + "grad_norm": 0.19819572567939758, + "learning_rate": 1.0943434723566624e-05, + "loss": 0.30026633739471437, + "step": 140 + }, + { + "epoch": 2.681614349775785, + "grad_norm": 0.18170028924942017, + "learning_rate": 9.763811991052021e-06, + "loss": 0.29707751274108884, + "step": 150 + }, + { + "epoch": 2.681614349775785, + "eval_loss": 0.3111979067325592, + "eval_runtime": 27.199, + "eval_samples_per_second": 14.523, + "eval_steps_per_second": 0.919, + "step": 150 + }, + { + "epoch": 2.8609865470852016, + "grad_norm": 0.18932338058948517, + "learning_rate": 8.587479967204584e-06, + "loss": 0.3032960891723633, + "step": 160 + }, + { + "epoch": 3.0358744394618835, + "grad_norm": 0.19508804380893707, + "learning_rate": 7.430827993733809e-06, + "loss": 0.2967501640319824, + "step": 170 + }, + { + "epoch": 3.2152466367713006, + "grad_norm": 0.20429708063602448, + "learning_rate": 6.30997121844481e-06, + "loss": 0.2913133859634399, + "step": 180 + }, + { + "epoch": 3.2152466367713006, + "eval_loss": 0.3067827820777893, + "eval_runtime": 27.6297, + "eval_samples_per_second": 14.296, + "eval_steps_per_second": 0.905, + "step": 180 + }, + { + "epoch": 3.3946188340807173, + "grad_norm": 0.1792595535516739, + "learning_rate": 5.240526069629265e-06, + "loss": 0.29277753829956055, + "step": 190 + }, + { + "epoch": 3.5739910313901344, + "grad_norm": 0.16619998216629028, + "learning_rate": 4.237392678906633e-06, + "loss": 0.28897757530212403, + "step": 200 + }, + { + "epoch": 3.7533632286995515, + "grad_norm": 0.18059593439102173, + "learning_rate": 3.3145472839270575e-06, + "loss": 0.29484267234802247, + "step": 210 + }, + { + "epoch": 3.7533632286995515, + "eval_loss": 0.3041122257709503, + "eval_runtime": 27.4044, + "eval_samples_per_second": 14.414, + "eval_steps_per_second": 0.912, + "step": 210 + }, + { + "epoch": 3.9327354260089686, + "grad_norm": 0.18860650062561035, + "learning_rate": 2.484847503302238e-06, + "loss": 0.2927693843841553, + "step": 220 + }, + { + "epoch": 4.1076233183856505, + "grad_norm": 0.19515369832515717, + "learning_rate": 1.7598531967871468e-06, + "loss": 0.29400765895843506, + "step": 230 + }, + { + "epoch": 4.286995515695067, + "grad_norm": 0.17510154843330383, + "learning_rate": 1.1496654065926927e-06, + "loss": 0.28560829162597656, + "step": 240 + }, + { + "epoch": 4.286995515695067, + "eval_loss": 0.3028334677219391, + "eval_runtime": 26.3261, + "eval_samples_per_second": 15.004, + "eval_steps_per_second": 0.95, + "step": 240 + }, + { + "epoch": 4.466367713004484, + "grad_norm": 0.1877126544713974, + "learning_rate": 6.627856237925812e-07, + "loss": 0.2830939769744873, + "step": 250 + }, + { + "epoch": 4.645739910313901, + "grad_norm": 0.1783999502658844, + "learning_rate": 3.059973406066963e-07, + "loss": 0.28902902603149416, + "step": 260 + }, + { + "epoch": 4.825112107623318, + "grad_norm": 0.18093666434288025, + "learning_rate": 8.42715388433446e-08, + "loss": 0.2830434560775757, + "step": 270 + }, + { + "epoch": 4.825112107623318, + "eval_loss": 0.3026478886604309, + "eval_runtime": 27.5009, + "eval_samples_per_second": 14.363, + "eval_steps_per_second": 0.909, + "step": 270 + }, + { + "epoch": 5.0, + "grad_norm": 0.2123940885066986, + "learning_rate": 6.974312904517444e-10, + "loss": 0.29397642612457275, + "step": 280 + }, + { + "epoch": 5.0, + "eval_loss": 0.30267491936683655, + "eval_runtime": 28.1798, + "eval_samples_per_second": 14.017, + "eval_steps_per_second": 0.887, + "step": 280 + } + ], + "logging_steps": 10, + "max_steps": 280, + "num_input_tokens_seen": 0, + "num_train_epochs": 5, + "save_steps": 30, + "stateful_callbacks": { + "EarlyStoppingCallback": { + "args": { + "early_stopping_patience": 3, + "early_stopping_threshold": 0.0 + }, + "attributes": { + "early_stopping_patience_counter": 1 + } + }, + "TrainerControl": { + "args": { + "should_epoch_stop": false, + "should_evaluate": false, + "should_log": false, + "should_save": true, + "should_training_stop": true + }, + "attributes": {} + } + }, + "total_flos": 1.4952225877000192e+17, + "train_batch_size": 2, + "trial_name": null, + "trial_params": null +} diff --git a/MultiPL-E/PD/Qwen3-1.7B/final/adapter_config.json b/MultiPL-E/PD/Qwen3-1.7B/final/adapter_config.json new file mode 100644 index 0000000000000000000000000000000000000000..fed0130b211f99ee7a02618d94e65397d75a189f --- /dev/null +++ b/MultiPL-E/PD/Qwen3-1.7B/final/adapter_config.json @@ -0,0 +1,45 @@ +{ + "alora_invocation_tokens": null, + "alpha_pattern": {}, + "arrow_config": null, + "auto_mapping": null, + "base_model_name_or_path": "Qwen/Qwen3-1.7B", + "bias": "none", + "corda_config": null, + "ensure_weight_tying": false, + "eva_config": null, + "exclude_modules": null, + "fan_in_fan_out": false, + "inference_mode": true, + "init_lora_weights": true, + "layer_replication": null, + "layers_pattern": null, + "layers_to_transform": null, + "loftq_config": {}, + "lora_alpha": 64, + "lora_bias": false, + "lora_dropout": 0.05, + "lora_ga_config": null, + "megatron_config": null, + "megatron_core": "megatron.core", + "modules_to_save": null, + "peft_type": "LORA", + "peft_version": "0.19.1", + "qalora_group_size": 16, + "r": 64, + "rank_pattern": {}, + "revision": null, + "target_modules": [ + "o_proj", + "v_proj", + "q_proj", + "k_proj" + ], + "target_parameters": null, + "task_type": "CAUSAL_LM", + "trainable_token_indices": null, + "use_bdlora": null, + "use_dora": false, + "use_qalora": false, + "use_rslora": false +} diff --git a/MultiPL-E/PD/Qwen3-1.7B/final/chat_template.jinja b/MultiPL-E/PD/Qwen3-1.7B/final/chat_template.jinja new file mode 100644 index 0000000000000000000000000000000000000000..01be9b307daa2d425f7c168c9fb145a286e0afb4 --- /dev/null +++ b/MultiPL-E/PD/Qwen3-1.7B/final/chat_template.jinja @@ -0,0 +1,89 @@ +{%- if tools %} + {{- '<|im_start|>system\n' }} + {%- if messages[0].role == 'system' %} + {{- messages[0].content + '\n\n' }} + {%- endif %} + {{- "# Tools\n\nYou may call one or more functions to assist with the user query.\n\nYou are provided with function signatures within XML tags:\n" }} + {%- for tool in tools %} + {{- "\n" }} + {{- tool | tojson }} + {%- endfor %} + {{- "\n\n\nFor each function call, return a json object with function name and arguments within XML tags:\n\n{\"name\": , \"arguments\": }\n<|im_end|>\n" }} +{%- else %} + {%- if messages[0].role == 'system' %} + {{- '<|im_start|>system\n' + messages[0].content + '<|im_end|>\n' }} + {%- endif %} +{%- endif %} +{%- set ns = namespace(multi_step_tool=true, last_query_index=messages|length - 1) %} +{%- for message in messages[::-1] %} + {%- set index = (messages|length - 1) - loop.index0 %} + {%- if ns.multi_step_tool and message.role == "user" and message.content is string and not(message.content.startswith('') and message.content.endswith('')) %} + {%- set ns.multi_step_tool = false %} + {%- set ns.last_query_index = index %} + {%- endif %} +{%- endfor %} +{%- for message in messages %} + {%- if message.content is string %} + {%- set content = message.content %} + {%- else %} + {%- set content = '' %} + {%- endif %} + {%- if (message.role == "user") or (message.role == "system" and not loop.first) %} + {{- '<|im_start|>' + message.role + '\n' + content + '<|im_end|>' + '\n' }} + {%- elif message.role == "assistant" %} + {%- set reasoning_content = '' %} + {%- if message.reasoning_content is string %} + {%- set reasoning_content = message.reasoning_content %} + {%- else %} + {%- if '' in content %} + {%- set reasoning_content = content.split('')[0].rstrip('\n').split('')[-1].lstrip('\n') %} + {%- set content = content.split('')[-1].lstrip('\n') %} + {%- endif %} + {%- endif %} + {%- if loop.index0 > ns.last_query_index %} + {%- if loop.last or (not loop.last and reasoning_content) %} + {{- '<|im_start|>' + message.role + '\n\n' + reasoning_content.strip('\n') + '\n\n\n' + content.lstrip('\n') }} + {%- else %} + {{- '<|im_start|>' + message.role + '\n' + content }} + {%- endif %} + {%- else %} + {{- '<|im_start|>' + message.role + '\n' + content }} + {%- endif %} + {%- if message.tool_calls %} + {%- for tool_call in message.tool_calls %} + {%- if (loop.first and content) or (not loop.first) %} + {{- '\n' }} + {%- endif %} + {%- if tool_call.function %} + {%- set tool_call = tool_call.function %} + {%- endif %} + {{- '\n{"name": "' }} + {{- tool_call.name }} + {{- '", "arguments": ' }} + {%- if tool_call.arguments is string %} + {{- tool_call.arguments }} + {%- else %} + {{- tool_call.arguments | tojson }} + {%- endif %} + {{- '}\n' }} + {%- endfor %} + {%- endif %} + {{- '<|im_end|>\n' }} + {%- elif message.role == "tool" %} + {%- if loop.first or (messages[loop.index0 - 1].role != "tool") %} + {{- '<|im_start|>user' }} + {%- endif %} + {{- '\n\n' }} + {{- content }} + {{- '\n' }} + {%- if loop.last or (messages[loop.index0 + 1].role != "tool") %} + {{- '<|im_end|>\n' }} + {%- endif %} + {%- endif %} +{%- endfor %} +{%- if add_generation_prompt %} + {{- '<|im_start|>assistant\n' }} + {%- if enable_thinking is defined and enable_thinking is false %} + {{- '\n\n\n\n' }} + {%- endif %} +{%- endif %} \ No newline at end of file diff --git a/MultiPL-E/PD/Qwen3-1.7B/final/final_eval_metrics.json b/MultiPL-E/PD/Qwen3-1.7B/final/final_eval_metrics.json new file mode 100644 index 0000000000000000000000000000000000000000..964194ab46ec1fbbe033794d26526c32b45849a9 --- /dev/null +++ b/MultiPL-E/PD/Qwen3-1.7B/final/final_eval_metrics.json @@ -0,0 +1,7 @@ +{ + "eval_loss": 0.3026478886604309, + "eval_runtime": 28.991, + "eval_samples_per_second": 13.625, + "eval_steps_per_second": 0.862, + "epoch": 5.0 +} diff --git a/MultiPL-E/PD/Qwen3-1.7B/final/tokenizer_config.json b/MultiPL-E/PD/Qwen3-1.7B/final/tokenizer_config.json new file mode 100644 index 0000000000000000000000000000000000000000..5668a4a01433b0648bb972458de616f22f3e728e --- /dev/null +++ b/MultiPL-E/PD/Qwen3-1.7B/final/tokenizer_config.json @@ -0,0 +1,30 @@ +{ + "add_prefix_space": false, + "backend": "tokenizers", + "bos_token": null, + "clean_up_tokenization_spaces": false, + "eos_token": "<|im_end|>", + "errors": "replace", + "extra_special_tokens": [ + "<|im_start|>", + "<|im_end|>", + "<|object_ref_start|>", + "<|object_ref_end|>", + "<|box_start|>", + "<|box_end|>", + "<|quad_start|>", + "<|quad_end|>", + "<|vision_start|>", + "<|vision_end|>", + "<|vision_pad|>", + "<|image_pad|>", + "<|video_pad|>" + ], + "is_local": true, + "local_files_only": false, + "model_max_length": 131072, + "pad_token": "<|endoftext|>", + "split_special_tokens": false, + "tokenizer_class": "Qwen2Tokenizer", + "unk_token": null +} diff --git a/MultiPL-E/PD/Qwen3-1.7B/final/train_summary.json b/MultiPL-E/PD/Qwen3-1.7B/final/train_summary.json new file mode 100644 index 0000000000000000000000000000000000000000..09dd7b67f300f4d06ec3f5880306a50579d99ba9 --- /dev/null +++ b/MultiPL-E/PD/Qwen3-1.7B/final/train_summary.json @@ -0,0 +1,12 @@ +{ + "global_step": 280, + "training_loss": 0.34874288950647625, + "metrics": { + "train_runtime": 633.0604, + "train_samples_per_second": 28.102, + "train_steps_per_second": 0.442, + "total_flos": 1.4952225877000192e+17, + "train_loss": 0.34874288950647625, + "epoch": 5.0 + } +} diff --git a/MultiPL-E/PD/Qwen3-4B/checkpoint-118/adapter_config.json b/MultiPL-E/PD/Qwen3-4B/checkpoint-118/adapter_config.json new file mode 100644 index 0000000000000000000000000000000000000000..c9d8501bf248a66a00095e9e4a1d853eae68dd23 --- /dev/null +++ b/MultiPL-E/PD/Qwen3-4B/checkpoint-118/adapter_config.json @@ -0,0 +1,45 @@ +{ + "alora_invocation_tokens": null, + "alpha_pattern": {}, + "arrow_config": null, + "auto_mapping": null, + "base_model_name_or_path": "Qwen/Qwen3-4B", + "bias": "none", + "corda_config": null, + "ensure_weight_tying": false, + "eva_config": null, + "exclude_modules": null, + "fan_in_fan_out": false, + "inference_mode": true, + "init_lora_weights": true, + "layer_replication": null, + "layers_pattern": null, + "layers_to_transform": null, + "loftq_config": {}, + "lora_alpha": 64, + "lora_bias": false, + "lora_dropout": 0.05, + "lora_ga_config": null, + "megatron_config": null, + "megatron_core": "megatron.core", + "modules_to_save": null, + "peft_type": "LORA", + "peft_version": "0.19.1", + "qalora_group_size": 16, + "r": 64, + "rank_pattern": {}, + "revision": null, + "target_modules": [ + "v_proj", + "k_proj", + "o_proj", + "q_proj" + ], + "target_parameters": null, + "task_type": "CAUSAL_LM", + "trainable_token_indices": null, + "use_bdlora": null, + "use_dora": false, + "use_qalora": false, + "use_rslora": false +} diff --git a/MultiPL-E/PD/Qwen3-4B/checkpoint-118/chat_template.jinja b/MultiPL-E/PD/Qwen3-4B/checkpoint-118/chat_template.jinja new file mode 100644 index 0000000000000000000000000000000000000000..01be9b307daa2d425f7c168c9fb145a286e0afb4 --- /dev/null +++ b/MultiPL-E/PD/Qwen3-4B/checkpoint-118/chat_template.jinja @@ -0,0 +1,89 @@ +{%- if tools %} + {{- '<|im_start|>system\n' }} + {%- if messages[0].role == 'system' %} + {{- messages[0].content + '\n\n' }} + {%- endif %} + {{- "# Tools\n\nYou may call one or more functions to assist with the user query.\n\nYou are provided with function signatures within XML tags:\n" }} + {%- for tool in tools %} + {{- "\n" }} + {{- tool | tojson }} + {%- endfor %} + {{- "\n\n\nFor each function call, return a json object with function name and arguments within XML tags:\n\n{\"name\": , \"arguments\": }\n<|im_end|>\n" }} +{%- else %} + {%- if messages[0].role == 'system' %} + {{- '<|im_start|>system\n' + messages[0].content + '<|im_end|>\n' }} + {%- endif %} +{%- endif %} +{%- set ns = namespace(multi_step_tool=true, last_query_index=messages|length - 1) %} +{%- for message in messages[::-1] %} + {%- set index = (messages|length - 1) - loop.index0 %} + {%- if ns.multi_step_tool and message.role == "user" and message.content is string and not(message.content.startswith('') and message.content.endswith('')) %} + {%- set ns.multi_step_tool = false %} + {%- set ns.last_query_index = index %} + {%- endif %} +{%- endfor %} +{%- for message in messages %} + {%- if message.content is string %} + {%- set content = message.content %} + {%- else %} + {%- set content = '' %} + {%- endif %} + {%- if (message.role == "user") or (message.role == "system" and not loop.first) %} + {{- '<|im_start|>' + message.role + '\n' + content + '<|im_end|>' + '\n' }} + {%- elif message.role == "assistant" %} + {%- set reasoning_content = '' %} + {%- if message.reasoning_content is string %} + {%- set reasoning_content = message.reasoning_content %} + {%- else %} + {%- if '' in content %} + {%- set reasoning_content = content.split('')[0].rstrip('\n').split('')[-1].lstrip('\n') %} + {%- set content = content.split('')[-1].lstrip('\n') %} + {%- endif %} + {%- endif %} + {%- if loop.index0 > ns.last_query_index %} + {%- if loop.last or (not loop.last and reasoning_content) %} + {{- '<|im_start|>' + message.role + '\n\n' + reasoning_content.strip('\n') + '\n\n\n' + content.lstrip('\n') }} + {%- else %} + {{- '<|im_start|>' + message.role + '\n' + content }} + {%- endif %} + {%- else %} + {{- '<|im_start|>' + message.role + '\n' + content }} + {%- endif %} + {%- if message.tool_calls %} + {%- for tool_call in message.tool_calls %} + {%- if (loop.first and content) or (not loop.first) %} + {{- '\n' }} + {%- endif %} + {%- if tool_call.function %} + {%- set tool_call = tool_call.function %} + {%- endif %} + {{- '\n{"name": "' }} + {{- tool_call.name }} + {{- '", "arguments": ' }} + {%- if tool_call.arguments is string %} + {{- tool_call.arguments }} + {%- else %} + {{- tool_call.arguments | tojson }} + {%- endif %} + {{- '}\n' }} + {%- endfor %} + {%- endif %} + {{- '<|im_end|>\n' }} + {%- elif message.role == "tool" %} + {%- if loop.first or (messages[loop.index0 - 1].role != "tool") %} + {{- '<|im_start|>user' }} + {%- endif %} + {{- '\n\n' }} + {{- content }} + {{- '\n' }} + {%- if loop.last or (messages[loop.index0 + 1].role != "tool") %} + {{- '<|im_end|>\n' }} + {%- endif %} + {%- endif %} +{%- endfor %} +{%- if add_generation_prompt %} + {{- '<|im_start|>assistant\n' }} + {%- if enable_thinking is defined and enable_thinking is false %} + {{- '\n\n\n\n' }} + {%- endif %} +{%- endif %} \ No newline at end of file diff --git a/MultiPL-E/PD/Qwen3-4B/checkpoint-118/tokenizer_config.json b/MultiPL-E/PD/Qwen3-4B/checkpoint-118/tokenizer_config.json new file mode 100644 index 0000000000000000000000000000000000000000..5668a4a01433b0648bb972458de616f22f3e728e --- /dev/null +++ b/MultiPL-E/PD/Qwen3-4B/checkpoint-118/tokenizer_config.json @@ -0,0 +1,30 @@ +{ + "add_prefix_space": false, + "backend": "tokenizers", + "bos_token": null, + "clean_up_tokenization_spaces": false, + "eos_token": "<|im_end|>", + "errors": "replace", + "extra_special_tokens": [ + "<|im_start|>", + "<|im_end|>", + "<|object_ref_start|>", + "<|object_ref_end|>", + "<|box_start|>", + "<|box_end|>", + "<|quad_start|>", + "<|quad_end|>", + "<|vision_start|>", + "<|vision_end|>", + "<|vision_pad|>", + "<|image_pad|>", + "<|video_pad|>" + ], + "is_local": true, + "local_files_only": false, + "model_max_length": 131072, + "pad_token": "<|endoftext|>", + "split_special_tokens": false, + "tokenizer_class": "Qwen2Tokenizer", + "unk_token": null +} diff --git a/MultiPL-E/PD/Qwen3-4B/checkpoint-118/trainer_state.json b/MultiPL-E/PD/Qwen3-4B/checkpoint-118/trainer_state.json new file mode 100644 index 0000000000000000000000000000000000000000..78819a2ef1d8eb3b1d5091b738d81244ef71da0e --- /dev/null +++ b/MultiPL-E/PD/Qwen3-4B/checkpoint-118/trainer_state.json @@ -0,0 +1,152 @@ +{ + "best_global_step": 118, + "best_metric": 0.15761800110340118, + "best_model_checkpoint": "jca/sft_runs/multipl_e_8lang_v5_diverse_a2_selfcorr2x_2ep_20260819/A2/checkpoint-118", + "epoch": 2.0, + "eval_steps": 30, + "global_step": 118, + "is_hyper_param_search": false, + "is_local_process_zero": true, + "is_world_process_zero": true, + "log_history": [ + { + "epoch": 0.17094017094017094, + "grad_norm": 0.1693044900894165, + "learning_rate": 7.985845976470477e-06, + "loss": 0.15684274435043336, + "step": 10 + }, + { + "epoch": 0.3418803418803419, + "grad_norm": 0.14446397125720978, + "learning_rate": 7.736995761178398e-06, + "loss": 0.1547669291496277, + "step": 20 + }, + { + "epoch": 0.5128205128205128, + "grad_norm": 0.14958830177783966, + "learning_rate": 7.196041941432996e-06, + "loss": 0.15154476165771485, + "step": 30 + }, + { + "epoch": 0.5128205128205128, + "eval_loss": 0.1592281460762024, + "eval_runtime": 43.1669, + "eval_samples_per_second": 9.614, + "eval_steps_per_second": 0.602, + "step": 30 + }, + { + "epoch": 0.6837606837606838, + "grad_norm": 0.149710014462471, + "learning_rate": 6.405268365193623e-06, + "loss": 0.15802257061004638, + "step": 40 + }, + { + "epoch": 0.8547008547008547, + "grad_norm": 0.15043851733207703, + "learning_rate": 5.4264861306492515e-06, + "loss": 0.15393251180648804, + "step": 50 + }, + { + "epoch": 1.017094017094017, + "grad_norm": 0.13903117179870605, + "learning_rate": 4.33620209971699e-06, + "loss": 0.15015512704849243, + "step": 60 + }, + { + "epoch": 1.017094017094017, + "eval_loss": 0.15827643871307373, + "eval_runtime": 42.5579, + "eval_samples_per_second": 9.751, + "eval_steps_per_second": 0.611, + "step": 60 + }, + { + "epoch": 1.188034188034188, + "grad_norm": 0.16060711443424225, + "learning_rate": 3.219638711935487e-06, + "loss": 0.15185315608978273, + "step": 70 + }, + { + "epoch": 1.358974358974359, + "grad_norm": 0.14025266468524933, + "learning_rate": 2.1640725421298483e-06, + "loss": 0.1490501880645752, + "step": 80 + }, + { + "epoch": 1.5299145299145298, + "grad_norm": 0.1372736692428589, + "learning_rate": 1.252012295860393e-06, + "loss": 0.16009281873703002, + "step": 90 + }, + { + "epoch": 1.5299145299145298, + "eval_loss": 0.1576949506998062, + "eval_runtime": 42.3323, + "eval_samples_per_second": 9.803, + "eval_steps_per_second": 0.614, + "step": 90 + }, + { + "epoch": 1.7008547008547008, + "grad_norm": 0.13545867800712585, + "learning_rate": 5.547494870703642e-07, + "loss": 0.1535496473312378, + "step": 100 + }, + { + "epoch": 1.8717948717948718, + "grad_norm": 0.14962707459926605, + "learning_rate": 1.2678591151095463e-07, + "loss": 0.14652007818222046, + "step": 110 + }, + { + "epoch": 2.0, + "eval_loss": 0.15761800110340118, + "eval_runtime": 42.6333, + "eval_samples_per_second": 9.734, + "eval_steps_per_second": 0.61, + "step": 118 + } + ], + "logging_steps": 10, + "max_steps": 118, + "num_input_tokens_seen": 0, + "num_train_epochs": 2, + "save_steps": 30, + "stateful_callbacks": { + "EarlyStoppingCallback": { + "args": { + "early_stopping_patience": 3, + "early_stopping_threshold": 0.0 + }, + "attributes": { + "early_stopping_patience_counter": 0 + } + }, + "TrainerControl": { + "args": { + "should_epoch_stop": false, + "should_evaluate": false, + "should_log": false, + "should_save": true, + "should_training_stop": true + }, + "attributes": {} + } + }, + "total_flos": 1.8702598943434342e+17, + "train_batch_size": 2, + "trial_name": null, + "trial_params": null +} diff --git a/MultiPL-E/PD/Qwen3-4B/final/adapter_config.json b/MultiPL-E/PD/Qwen3-4B/final/adapter_config.json new file mode 100644 index 0000000000000000000000000000000000000000..c9d8501bf248a66a00095e9e4a1d853eae68dd23 --- /dev/null +++ b/MultiPL-E/PD/Qwen3-4B/final/adapter_config.json @@ -0,0 +1,45 @@ +{ + "alora_invocation_tokens": null, + "alpha_pattern": {}, + "arrow_config": null, + "auto_mapping": null, + "base_model_name_or_path": "Qwen/Qwen3-4B", + "bias": "none", + "corda_config": null, + "ensure_weight_tying": false, + "eva_config": null, + "exclude_modules": null, + "fan_in_fan_out": false, + "inference_mode": true, + "init_lora_weights": true, + "layer_replication": null, + "layers_pattern": null, + "layers_to_transform": null, + "loftq_config": {}, + "lora_alpha": 64, + "lora_bias": false, + "lora_dropout": 0.05, + "lora_ga_config": null, + "megatron_config": null, + "megatron_core": "megatron.core", + "modules_to_save": null, + "peft_type": "LORA", + "peft_version": "0.19.1", + "qalora_group_size": 16, + "r": 64, + "rank_pattern": {}, + "revision": null, + "target_modules": [ + "v_proj", + "k_proj", + "o_proj", + "q_proj" + ], + "target_parameters": null, + "task_type": "CAUSAL_LM", + "trainable_token_indices": null, + "use_bdlora": null, + "use_dora": false, + "use_qalora": false, + "use_rslora": false +} diff --git a/MultiPL-E/PD/Qwen3-4B/final/chat_template.jinja b/MultiPL-E/PD/Qwen3-4B/final/chat_template.jinja new file mode 100644 index 0000000000000000000000000000000000000000..01be9b307daa2d425f7c168c9fb145a286e0afb4 --- /dev/null +++ b/MultiPL-E/PD/Qwen3-4B/final/chat_template.jinja @@ -0,0 +1,89 @@ +{%- if tools %} + {{- '<|im_start|>system\n' }} + {%- if messages[0].role == 'system' %} + {{- messages[0].content + '\n\n' }} + {%- endif %} + {{- "# Tools\n\nYou may call one or more functions to assist with the user query.\n\nYou are provided with function signatures within XML tags:\n" }} + {%- for tool in tools %} + {{- "\n" }} + {{- tool | tojson }} + {%- endfor %} + {{- "\n\n\nFor each function call, return a json object with function name and arguments within XML tags:\n\n{\"name\": , \"arguments\": }\n<|im_end|>\n" }} +{%- else %} + {%- if messages[0].role == 'system' %} + {{- '<|im_start|>system\n' + messages[0].content + '<|im_end|>\n' }} + {%- endif %} +{%- endif %} +{%- set ns = namespace(multi_step_tool=true, last_query_index=messages|length - 1) %} +{%- for message in messages[::-1] %} + {%- set index = (messages|length - 1) - loop.index0 %} + {%- if ns.multi_step_tool and message.role == "user" and message.content is string and not(message.content.startswith('') and message.content.endswith('')) %} + {%- set ns.multi_step_tool = false %} + {%- set ns.last_query_index = index %} + {%- endif %} +{%- endfor %} +{%- for message in messages %} + {%- if message.content is string %} + {%- set content = message.content %} + {%- else %} + {%- set content = '' %} + {%- endif %} + {%- if (message.role == "user") or (message.role == "system" and not loop.first) %} + {{- '<|im_start|>' + message.role + '\n' + content + '<|im_end|>' + '\n' }} + {%- elif message.role == "assistant" %} + {%- set reasoning_content = '' %} + {%- if message.reasoning_content is string %} + {%- set reasoning_content = message.reasoning_content %} + {%- else %} + {%- if '' in content %} + {%- set reasoning_content = content.split('')[0].rstrip('\n').split('')[-1].lstrip('\n') %} + {%- set content = content.split('')[-1].lstrip('\n') %} + {%- endif %} + {%- endif %} + {%- if loop.index0 > ns.last_query_index %} + {%- if loop.last or (not loop.last and reasoning_content) %} + {{- '<|im_start|>' + message.role + '\n\n' + reasoning_content.strip('\n') + '\n\n\n' + content.lstrip('\n') }} + {%- else %} + {{- '<|im_start|>' + message.role + '\n' + content }} + {%- endif %} + {%- else %} + {{- '<|im_start|>' + message.role + '\n' + content }} + {%- endif %} + {%- if message.tool_calls %} + {%- for tool_call in message.tool_calls %} + {%- if (loop.first and content) or (not loop.first) %} + {{- '\n' }} + {%- endif %} + {%- if tool_call.function %} + {%- set tool_call = tool_call.function %} + {%- endif %} + {{- '\n{"name": "' }} + {{- tool_call.name }} + {{- '", "arguments": ' }} + {%- if tool_call.arguments is string %} + {{- tool_call.arguments }} + {%- else %} + {{- tool_call.arguments | tojson }} + {%- endif %} + {{- '}\n' }} + {%- endfor %} + {%- endif %} + {{- '<|im_end|>\n' }} + {%- elif message.role == "tool" %} + {%- if loop.first or (messages[loop.index0 - 1].role != "tool") %} + {{- '<|im_start|>user' }} + {%- endif %} + {{- '\n\n' }} + {{- content }} + {{- '\n' }} + {%- if loop.last or (messages[loop.index0 + 1].role != "tool") %} + {{- '<|im_end|>\n' }} + {%- endif %} + {%- endif %} +{%- endfor %} +{%- if add_generation_prompt %} + {{- '<|im_start|>assistant\n' }} + {%- if enable_thinking is defined and enable_thinking is false %} + {{- '\n\n\n\n' }} + {%- endif %} +{%- endif %} \ No newline at end of file diff --git a/MultiPL-E/PD/Qwen3-4B/final/final_eval_metrics.json b/MultiPL-E/PD/Qwen3-4B/final/final_eval_metrics.json new file mode 100644 index 0000000000000000000000000000000000000000..74e946c0e667d3673fe38bf66aa05cea5d9769c2 --- /dev/null +++ b/MultiPL-E/PD/Qwen3-4B/final/final_eval_metrics.json @@ -0,0 +1,7 @@ +{ + "eval_loss": 0.15761800110340118, + "eval_runtime": 42.3353, + "eval_samples_per_second": 9.803, + "eval_steps_per_second": 0.614, + "epoch": 2.0 +} diff --git a/MultiPL-E/PD/Qwen3-4B/final/tokenizer_config.json b/MultiPL-E/PD/Qwen3-4B/final/tokenizer_config.json new file mode 100644 index 0000000000000000000000000000000000000000..5668a4a01433b0648bb972458de616f22f3e728e --- /dev/null +++ b/MultiPL-E/PD/Qwen3-4B/final/tokenizer_config.json @@ -0,0 +1,30 @@ +{ + "add_prefix_space": false, + "backend": "tokenizers", + "bos_token": null, + "clean_up_tokenization_spaces": false, + "eos_token": "<|im_end|>", + "errors": "replace", + "extra_special_tokens": [ + "<|im_start|>", + "<|im_end|>", + "<|object_ref_start|>", + "<|object_ref_end|>", + "<|box_start|>", + "<|box_end|>", + "<|quad_start|>", + "<|quad_end|>", + "<|vision_start|>", + "<|vision_end|>", + "<|vision_pad|>", + "<|image_pad|>", + "<|video_pad|>" + ], + "is_local": true, + "local_files_only": false, + "model_max_length": 131072, + "pad_token": "<|endoftext|>", + "split_special_tokens": false, + "tokenizer_class": "Qwen2Tokenizer", + "unk_token": null +} diff --git a/MultiPL-E/PD/Qwen3-4B/final/train_summary.json b/MultiPL-E/PD/Qwen3-4B/final/train_summary.json new file mode 100644 index 0000000000000000000000000000000000000000..860c86c626329606d96bf7e6a647d49e2912dcf3 --- /dev/null +++ b/MultiPL-E/PD/Qwen3-4B/final/train_summary.json @@ -0,0 +1,12 @@ +{ + "global_step": 118, + "training_loss": 0.15227285280066022, + "metrics": { + "train_runtime": 496.5453, + "train_samples_per_second": 15.08, + "train_steps_per_second": 0.238, + "total_flos": 1.8702598943434342e+17, + "train_loss": 0.15227285280066022, + "epoch": 2.0 + } +} diff --git a/MultiPL-E/PD/Qwen3-8B/checkpoint-45/adapter_config.json b/MultiPL-E/PD/Qwen3-8B/checkpoint-45/adapter_config.json new file mode 100644 index 0000000000000000000000000000000000000000..99ff45f9ddf750b6b81d94ba4c6ad47d2f66ae00 --- /dev/null +++ b/MultiPL-E/PD/Qwen3-8B/checkpoint-45/adapter_config.json @@ -0,0 +1,45 @@ +{ + "alora_invocation_tokens": null, + "alpha_pattern": {}, + "arrow_config": null, + "auto_mapping": null, + "base_model_name_or_path": "Qwen/Qwen3-8B", + "bias": "none", + "corda_config": null, + "ensure_weight_tying": false, + "eva_config": null, + "exclude_modules": null, + "fan_in_fan_out": false, + "inference_mode": true, + "init_lora_weights": true, + "layer_replication": null, + "layers_pattern": null, + "layers_to_transform": null, + "loftq_config": {}, + "lora_alpha": 64, + "lora_bias": false, + "lora_dropout": 0.05, + "lora_ga_config": null, + "megatron_config": null, + "megatron_core": "megatron.core", + "modules_to_save": null, + "peft_type": "LORA", + "peft_version": "0.19.1", + "qalora_group_size": 16, + "r": 64, + "rank_pattern": {}, + "revision": null, + "target_modules": [ + "q_proj", + "o_proj", + "v_proj", + "k_proj" + ], + "target_parameters": null, + "task_type": "CAUSAL_LM", + "trainable_token_indices": null, + "use_bdlora": null, + "use_dora": false, + "use_qalora": false, + "use_rslora": false +} diff --git a/MultiPL-E/PD/Qwen3-8B/checkpoint-45/chat_template.jinja b/MultiPL-E/PD/Qwen3-8B/checkpoint-45/chat_template.jinja new file mode 100644 index 0000000000000000000000000000000000000000..01be9b307daa2d425f7c168c9fb145a286e0afb4 --- /dev/null +++ b/MultiPL-E/PD/Qwen3-8B/checkpoint-45/chat_template.jinja @@ -0,0 +1,89 @@ +{%- if tools %} + {{- '<|im_start|>system\n' }} + {%- if messages[0].role == 'system' %} + {{- messages[0].content + '\n\n' }} + {%- endif %} + {{- "# Tools\n\nYou may call one or more functions to assist with the user query.\n\nYou are provided with function signatures within XML tags:\n" }} + {%- for tool in tools %} + {{- "\n" }} + {{- tool | tojson }} + {%- endfor %} + {{- "\n\n\nFor each function call, return a json object with function name and arguments within XML tags:\n\n{\"name\": , \"arguments\": }\n<|im_end|>\n" }} +{%- else %} + {%- if messages[0].role == 'system' %} + {{- '<|im_start|>system\n' + messages[0].content + '<|im_end|>\n' }} + {%- endif %} +{%- endif %} +{%- set ns = namespace(multi_step_tool=true, last_query_index=messages|length - 1) %} +{%- for message in messages[::-1] %} + {%- set index = (messages|length - 1) - loop.index0 %} + {%- if ns.multi_step_tool and message.role == "user" and message.content is string and not(message.content.startswith('') and message.content.endswith('')) %} + {%- set ns.multi_step_tool = false %} + {%- set ns.last_query_index = index %} + {%- endif %} +{%- endfor %} +{%- for message in messages %} + {%- if message.content is string %} + {%- set content = message.content %} + {%- else %} + {%- set content = '' %} + {%- endif %} + {%- if (message.role == "user") or (message.role == "system" and not loop.first) %} + {{- '<|im_start|>' + message.role + '\n' + content + '<|im_end|>' + '\n' }} + {%- elif message.role == "assistant" %} + {%- set reasoning_content = '' %} + {%- if message.reasoning_content is string %} + {%- set reasoning_content = message.reasoning_content %} + {%- else %} + {%- if '' in content %} + {%- set reasoning_content = content.split('')[0].rstrip('\n').split('')[-1].lstrip('\n') %} + {%- set content = content.split('')[-1].lstrip('\n') %} + {%- endif %} + {%- endif %} + {%- if loop.index0 > ns.last_query_index %} + {%- if loop.last or (not loop.last and reasoning_content) %} + {{- '<|im_start|>' + message.role + '\n\n' + reasoning_content.strip('\n') + '\n\n\n' + content.lstrip('\n') }} + {%- else %} + {{- '<|im_start|>' + message.role + '\n' + content }} + {%- endif %} + {%- else %} + {{- '<|im_start|>' + message.role + '\n' + content }} + {%- endif %} + {%- if message.tool_calls %} + {%- for tool_call in message.tool_calls %} + {%- if (loop.first and content) or (not loop.first) %} + {{- '\n' }} + {%- endif %} + {%- if tool_call.function %} + {%- set tool_call = tool_call.function %} + {%- endif %} + {{- '\n{"name": "' }} + {{- tool_call.name }} + {{- '", "arguments": ' }} + {%- if tool_call.arguments is string %} + {{- tool_call.arguments }} + {%- else %} + {{- tool_call.arguments | tojson }} + {%- endif %} + {{- '}\n' }} + {%- endfor %} + {%- endif %} + {{- '<|im_end|>\n' }} + {%- elif message.role == "tool" %} + {%- if loop.first or (messages[loop.index0 - 1].role != "tool") %} + {{- '<|im_start|>user' }} + {%- endif %} + {{- '\n\n' }} + {{- content }} + {{- '\n' }} + {%- if loop.last or (messages[loop.index0 + 1].role != "tool") %} + {{- '<|im_end|>\n' }} + {%- endif %} + {%- endif %} +{%- endfor %} +{%- if add_generation_prompt %} + {{- '<|im_start|>assistant\n' }} + {%- if enable_thinking is defined and enable_thinking is false %} + {{- '\n\n\n\n' }} + {%- endif %} +{%- endif %} \ No newline at end of file diff --git a/MultiPL-E/PD/Qwen3-8B/checkpoint-45/tokenizer_config.json b/MultiPL-E/PD/Qwen3-8B/checkpoint-45/tokenizer_config.json new file mode 100644 index 0000000000000000000000000000000000000000..5668a4a01433b0648bb972458de616f22f3e728e --- /dev/null +++ b/MultiPL-E/PD/Qwen3-8B/checkpoint-45/tokenizer_config.json @@ -0,0 +1,30 @@ +{ + "add_prefix_space": false, + "backend": "tokenizers", + "bos_token": null, + "clean_up_tokenization_spaces": false, + "eos_token": "<|im_end|>", + "errors": "replace", + "extra_special_tokens": [ + "<|im_start|>", + "<|im_end|>", + "<|object_ref_start|>", + "<|object_ref_end|>", + "<|box_start|>", + "<|box_end|>", + "<|quad_start|>", + "<|quad_end|>", + "<|vision_start|>", + "<|vision_end|>", + "<|vision_pad|>", + "<|image_pad|>", + "<|video_pad|>" + ], + "is_local": true, + "local_files_only": false, + "model_max_length": 131072, + "pad_token": "<|endoftext|>", + "split_special_tokens": false, + "tokenizer_class": "Qwen2Tokenizer", + "unk_token": null +} diff --git a/MultiPL-E/PD/Qwen3-8B/checkpoint-45/trainer_state.json b/MultiPL-E/PD/Qwen3-8B/checkpoint-45/trainer_state.json new file mode 100644 index 0000000000000000000000000000000000000000..096383a94c0ebdb54916b6667f169bdf37204065 --- /dev/null +++ b/MultiPL-E/PD/Qwen3-8B/checkpoint-45/trainer_state.json @@ -0,0 +1,146 @@ +{ + "best_global_step": 45, + "best_metric": 0.09639278054237366, + "best_model_checkpoint": "jca/sft_runs/multipl_e_8lang_v5_diverse_a3_5ep/A3/checkpoint-45", + "epoch": 5.0, + "eval_steps": 10, + "global_step": 45, + "is_hyper_param_search": false, + "is_local_process_zero": true, + "is_world_process_zero": true, + "log_history": [ + { + "epoch": 0.5633802816901409, + "grad_norm": 0.5089372992515564, + "learning_rate": 1.9972037971811802e-05, + "loss": 0.20812680721282958, + "step": 5 + }, + { + "epoch": 1.1126760563380282, + "grad_norm": 0.19522583484649658, + "learning_rate": 1.900968867902419e-05, + "loss": 0.1552955389022827, + "step": 10 + }, + { + "epoch": 1.1126760563380282, + "eval_loss": 0.12053649872541428, + "eval_runtime": 33.8764, + "eval_samples_per_second": 1.83, + "eval_steps_per_second": 0.236, + "step": 10 + }, + { + "epoch": 1.676056338028169, + "grad_norm": 0.12821316719055176, + "learning_rate": 1.6801727377709195e-05, + "loss": 0.11046926975250244, + "step": 15 + }, + { + "epoch": 2.2253521126760565, + "grad_norm": 0.1584511697292328, + "learning_rate": 1.3653410243663953e-05, + "loss": 0.10451444387435913, + "step": 20 + }, + { + "epoch": 2.2253521126760565, + "eval_loss": 0.10305467247962952, + "eval_runtime": 34.712, + "eval_samples_per_second": 1.786, + "eval_steps_per_second": 0.23, + "step": 20 + }, + { + "epoch": 2.788732394366197, + "grad_norm": 0.1459791213274002, + "learning_rate": 1e-05, + "loss": 0.10007953643798828, + "step": 25 + }, + { + "epoch": 3.3380281690140845, + "grad_norm": 0.11733456701040268, + "learning_rate": 6.34658975633605e-06, + "loss": 0.09617980122566223, + "step": 30 + }, + { + "epoch": 3.3380281690140845, + "eval_loss": 0.09819617122411728, + "eval_runtime": 33.521, + "eval_samples_per_second": 1.85, + "eval_steps_per_second": 0.239, + "step": 30 + }, + { + "epoch": 3.9014084507042255, + "grad_norm": 0.1395888328552246, + "learning_rate": 3.1982726222908046e-06, + "loss": 0.0960239291191101, + "step": 35 + }, + { + "epoch": 4.450704225352113, + "grad_norm": 0.15123504400253296, + "learning_rate": 9.903113209758098e-07, + "loss": 0.09316462874412537, + "step": 40 + }, + { + "epoch": 4.450704225352113, + "eval_loss": 0.09643875807523727, + "eval_runtime": 34.7357, + "eval_samples_per_second": 1.785, + "eval_steps_per_second": 0.23, + "step": 40 + }, + { + "epoch": 5.0, + "grad_norm": 0.10066380351781845, + "learning_rate": 2.796202818819871e-08, + "loss": 0.09446485042572021, + "step": 45 + }, + { + "epoch": 5.0, + "eval_loss": 0.09639278054237366, + "eval_runtime": 34.0738, + "eval_samples_per_second": 1.82, + "eval_steps_per_second": 0.235, + "step": 45 + } + ], + "logging_steps": 5, + "max_steps": 45, + "num_input_tokens_seen": 0, + "num_train_epochs": 5, + "save_steps": 10, + "stateful_callbacks": { + "EarlyStoppingCallback": { + "args": { + "early_stopping_patience": 3, + "early_stopping_threshold": 0.0 + }, + "attributes": { + "early_stopping_patience_counter": 0 + } + }, + "TrainerControl": { + "args": { + "should_epoch_stop": false, + "should_evaluate": false, + "should_log": false, + "should_save": true, + "should_training_stop": true + }, + "attributes": {} + } + }, + "total_flos": 1.5990721803504845e+17, + "train_batch_size": 1, + "trial_name": null, + "trial_params": null +} diff --git a/MultiPL-E/PD/Qwen3-8B/final/adapter_config.json b/MultiPL-E/PD/Qwen3-8B/final/adapter_config.json new file mode 100644 index 0000000000000000000000000000000000000000..99ff45f9ddf750b6b81d94ba4c6ad47d2f66ae00 --- /dev/null +++ b/MultiPL-E/PD/Qwen3-8B/final/adapter_config.json @@ -0,0 +1,45 @@ +{ + "alora_invocation_tokens": null, + "alpha_pattern": {}, + "arrow_config": null, + "auto_mapping": null, + "base_model_name_or_path": "Qwen/Qwen3-8B", + "bias": "none", + "corda_config": null, + "ensure_weight_tying": false, + "eva_config": null, + "exclude_modules": null, + "fan_in_fan_out": false, + "inference_mode": true, + "init_lora_weights": true, + "layer_replication": null, + "layers_pattern": null, + "layers_to_transform": null, + "loftq_config": {}, + "lora_alpha": 64, + "lora_bias": false, + "lora_dropout": 0.05, + "lora_ga_config": null, + "megatron_config": null, + "megatron_core": "megatron.core", + "modules_to_save": null, + "peft_type": "LORA", + "peft_version": "0.19.1", + "qalora_group_size": 16, + "r": 64, + "rank_pattern": {}, + "revision": null, + "target_modules": [ + "q_proj", + "o_proj", + "v_proj", + "k_proj" + ], + "target_parameters": null, + "task_type": "CAUSAL_LM", + "trainable_token_indices": null, + "use_bdlora": null, + "use_dora": false, + "use_qalora": false, + "use_rslora": false +} diff --git a/MultiPL-E/PD/Qwen3-8B/final/chat_template.jinja b/MultiPL-E/PD/Qwen3-8B/final/chat_template.jinja new file mode 100644 index 0000000000000000000000000000000000000000..01be9b307daa2d425f7c168c9fb145a286e0afb4 --- /dev/null +++ b/MultiPL-E/PD/Qwen3-8B/final/chat_template.jinja @@ -0,0 +1,89 @@ +{%- if tools %} + {{- '<|im_start|>system\n' }} + {%- if messages[0].role == 'system' %} + {{- messages[0].content + '\n\n' }} + {%- endif %} + {{- "# Tools\n\nYou may call one or more functions to assist with the user query.\n\nYou are provided with function signatures within XML tags:\n" }} + {%- for tool in tools %} + {{- "\n" }} + {{- tool | tojson }} + {%- endfor %} + {{- "\n\n\nFor each function call, return a json object with function name and arguments within XML tags:\n\n{\"name\": , \"arguments\": }\n<|im_end|>\n" }} +{%- else %} + {%- if messages[0].role == 'system' %} + {{- '<|im_start|>system\n' + messages[0].content + '<|im_end|>\n' }} + {%- endif %} +{%- endif %} +{%- set ns = namespace(multi_step_tool=true, last_query_index=messages|length - 1) %} +{%- for message in messages[::-1] %} + {%- set index = (messages|length - 1) - loop.index0 %} + {%- if ns.multi_step_tool and message.role == "user" and message.content is string and not(message.content.startswith('') and message.content.endswith('')) %} + {%- set ns.multi_step_tool = false %} + {%- set ns.last_query_index = index %} + {%- endif %} +{%- endfor %} +{%- for message in messages %} + {%- if message.content is string %} + {%- set content = message.content %} + {%- else %} + {%- set content = '' %} + {%- endif %} + {%- if (message.role == "user") or (message.role == "system" and not loop.first) %} + {{- '<|im_start|>' + message.role + '\n' + content + '<|im_end|>' + '\n' }} + {%- elif message.role == "assistant" %} + {%- set reasoning_content = '' %} + {%- if message.reasoning_content is string %} + {%- set reasoning_content = message.reasoning_content %} + {%- else %} + {%- if '' in content %} + {%- set reasoning_content = content.split('')[0].rstrip('\n').split('')[-1].lstrip('\n') %} + {%- set content = content.split('')[-1].lstrip('\n') %} + {%- endif %} + {%- endif %} + {%- if loop.index0 > ns.last_query_index %} + {%- if loop.last or (not loop.last and reasoning_content) %} + {{- '<|im_start|>' + message.role + '\n\n' + reasoning_content.strip('\n') + '\n\n\n' + content.lstrip('\n') }} + {%- else %} + {{- '<|im_start|>' + message.role + '\n' + content }} + {%- endif %} + {%- else %} + {{- '<|im_start|>' + message.role + '\n' + content }} + {%- endif %} + {%- if message.tool_calls %} + {%- for tool_call in message.tool_calls %} + {%- if (loop.first and content) or (not loop.first) %} + {{- '\n' }} + {%- endif %} + {%- if tool_call.function %} + {%- set tool_call = tool_call.function %} + {%- endif %} + {{- '\n{"name": "' }} + {{- tool_call.name }} + {{- '", "arguments": ' }} + {%- if tool_call.arguments is string %} + {{- tool_call.arguments }} + {%- else %} + {{- tool_call.arguments | tojson }} + {%- endif %} + {{- '}\n' }} + {%- endfor %} + {%- endif %} + {{- '<|im_end|>\n' }} + {%- elif message.role == "tool" %} + {%- if loop.first or (messages[loop.index0 - 1].role != "tool") %} + {{- '<|im_start|>user' }} + {%- endif %} + {{- '\n\n' }} + {{- content }} + {{- '\n' }} + {%- if loop.last or (messages[loop.index0 + 1].role != "tool") %} + {{- '<|im_end|>\n' }} + {%- endif %} + {%- endif %} +{%- endfor %} +{%- if add_generation_prompt %} + {{- '<|im_start|>assistant\n' }} + {%- if enable_thinking is defined and enable_thinking is false %} + {{- '\n\n\n\n' }} + {%- endif %} +{%- endif %} \ No newline at end of file diff --git a/MultiPL-E/PD/Qwen3-8B/final/final_eval_metrics.json b/MultiPL-E/PD/Qwen3-8B/final/final_eval_metrics.json new file mode 100644 index 0000000000000000000000000000000000000000..d70ae32af7b58b44d77a75bb7582c2b2820cb2f4 --- /dev/null +++ b/MultiPL-E/PD/Qwen3-8B/final/final_eval_metrics.json @@ -0,0 +1,7 @@ +{ + "eval_loss": 0.09639278054237366, + "eval_runtime": 35.5838, + "eval_samples_per_second": 1.742, + "eval_steps_per_second": 0.225, + "epoch": 5.0 +} diff --git a/MultiPL-E/PD/Qwen3-8B/final/tokenizer_config.json b/MultiPL-E/PD/Qwen3-8B/final/tokenizer_config.json new file mode 100644 index 0000000000000000000000000000000000000000..5668a4a01433b0648bb972458de616f22f3e728e --- /dev/null +++ b/MultiPL-E/PD/Qwen3-8B/final/tokenizer_config.json @@ -0,0 +1,30 @@ +{ + "add_prefix_space": false, + "backend": "tokenizers", + "bos_token": null, + "clean_up_tokenization_spaces": false, + "eos_token": "<|im_end|>", + "errors": "replace", + "extra_special_tokens": [ + "<|im_start|>", + "<|im_end|>", + "<|object_ref_start|>", + "<|object_ref_end|>", + "<|box_start|>", + "<|box_end|>", + "<|quad_start|>", + "<|quad_end|>", + "<|vision_start|>", + "<|vision_end|>", + "<|vision_pad|>", + "<|image_pad|>", + "<|video_pad|>" + ], + "is_local": true, + "local_files_only": false, + "model_max_length": 131072, + "pad_token": "<|endoftext|>", + "split_special_tokens": false, + "tokenizer_class": "Qwen2Tokenizer", + "unk_token": null +} diff --git a/MultiPL-E/PD/Qwen3-8B/final/train_summary.json b/MultiPL-E/PD/Qwen3-8B/final/train_summary.json new file mode 100644 index 0000000000000000000000000000000000000000..b1827104622ad10b0562f157dd74c48cfdf0ada3 --- /dev/null +++ b/MultiPL-E/PD/Qwen3-8B/final/train_summary.json @@ -0,0 +1,12 @@ +{ + "global_step": 45, + "training_loss": 0.1175909784105089, + "metrics": { + "train_runtime": 346.098, + "train_samples_per_second": 8.162, + "train_steps_per_second": 0.13, + "total_flos": 1.5990721803504845e+17, + "train_loss": 0.1175909784105089, + "epoch": 5.0 + } +} diff --git a/MultiPL-E/SLS/Qwen3-4B/checkpoint-120/chat_template.jinja b/MultiPL-E/SLS/Qwen3-4B/checkpoint-120/chat_template.jinja new file mode 100644 index 0000000000000000000000000000000000000000..01be9b307daa2d425f7c168c9fb145a286e0afb4 --- /dev/null +++ b/MultiPL-E/SLS/Qwen3-4B/checkpoint-120/chat_template.jinja @@ -0,0 +1,89 @@ +{%- if tools %} + {{- '<|im_start|>system\n' }} + {%- if messages[0].role == 'system' %} + {{- messages[0].content + '\n\n' }} + {%- endif %} + {{- "# Tools\n\nYou may call one or more functions to assist with the user query.\n\nYou are provided with function signatures within XML tags:\n" }} + {%- for tool in tools %} + {{- "\n" }} + {{- tool | tojson }} + {%- endfor %} + {{- "\n\n\nFor each function call, return a json object with function name and arguments within XML tags:\n\n{\"name\": , \"arguments\": }\n<|im_end|>\n" }} +{%- else %} + {%- if messages[0].role == 'system' %} + {{- '<|im_start|>system\n' + messages[0].content + '<|im_end|>\n' }} + {%- endif %} +{%- endif %} +{%- set ns = namespace(multi_step_tool=true, last_query_index=messages|length - 1) %} +{%- for message in messages[::-1] %} + {%- set index = (messages|length - 1) - loop.index0 %} + {%- if ns.multi_step_tool and message.role == "user" and message.content is string and not(message.content.startswith('') and message.content.endswith('')) %} + {%- set ns.multi_step_tool = false %} + {%- set ns.last_query_index = index %} + {%- endif %} +{%- endfor %} +{%- for message in messages %} + {%- if message.content is string %} + {%- set content = message.content %} + {%- else %} + {%- set content = '' %} + {%- endif %} + {%- if (message.role == "user") or (message.role == "system" and not loop.first) %} + {{- '<|im_start|>' + message.role + '\n' + content + '<|im_end|>' + '\n' }} + {%- elif message.role == "assistant" %} + {%- set reasoning_content = '' %} + {%- if message.reasoning_content is string %} + {%- set reasoning_content = message.reasoning_content %} + {%- else %} + {%- if '' in content %} + {%- set reasoning_content = content.split('')[0].rstrip('\n').split('')[-1].lstrip('\n') %} + {%- set content = content.split('')[-1].lstrip('\n') %} + {%- endif %} + {%- endif %} + {%- if loop.index0 > ns.last_query_index %} + {%- if loop.last or (not loop.last and reasoning_content) %} + {{- '<|im_start|>' + message.role + '\n\n' + reasoning_content.strip('\n') + '\n\n\n' + content.lstrip('\n') }} + {%- else %} + {{- '<|im_start|>' + message.role + '\n' + content }} + {%- endif %} + {%- else %} + {{- '<|im_start|>' + message.role + '\n' + content }} + {%- endif %} + {%- if message.tool_calls %} + {%- for tool_call in message.tool_calls %} + {%- if (loop.first and content) or (not loop.first) %} + {{- '\n' }} + {%- endif %} + {%- if tool_call.function %} + {%- set tool_call = tool_call.function %} + {%- endif %} + {{- '\n{"name": "' }} + {{- tool_call.name }} + {{- '", "arguments": ' }} + {%- if tool_call.arguments is string %} + {{- tool_call.arguments }} + {%- else %} + {{- tool_call.arguments | tojson }} + {%- endif %} + {{- '}\n' }} + {%- endfor %} + {%- endif %} + {{- '<|im_end|>\n' }} + {%- elif message.role == "tool" %} + {%- if loop.first or (messages[loop.index0 - 1].role != "tool") %} + {{- '<|im_start|>user' }} + {%- endif %} + {{- '\n\n' }} + {{- content }} + {{- '\n' }} + {%- if loop.last or (messages[loop.index0 + 1].role != "tool") %} + {{- '<|im_end|>\n' }} + {%- endif %} + {%- endif %} +{%- endfor %} +{%- if add_generation_prompt %} + {{- '<|im_start|>assistant\n' }} + {%- if enable_thinking is defined and enable_thinking is false %} + {{- '\n\n\n\n' }} + {%- endif %} +{%- endif %} \ No newline at end of file diff --git a/MultiPL-E/SLS/Qwen3-4B/checkpoint-120/policy_adapter/adapter_config.json b/MultiPL-E/SLS/Qwen3-4B/checkpoint-120/policy_adapter/adapter_config.json new file mode 100644 index 0000000000000000000000000000000000000000..3e043613f8b2fe102d2bdcf2ee77b26fb1d282f4 --- /dev/null +++ b/MultiPL-E/SLS/Qwen3-4B/checkpoint-120/policy_adapter/adapter_config.json @@ -0,0 +1,45 @@ +{ + "alora_invocation_tokens": null, + "alpha_pattern": {}, + "arrow_config": null, + "auto_mapping": null, + "base_model_name_or_path": "Qwen/Qwen3-4B", + "bias": "none", + "corda_config": null, + "ensure_weight_tying": false, + "eva_config": null, + "exclude_modules": null, + "fan_in_fan_out": false, + "inference_mode": true, + "init_lora_weights": true, + "layer_replication": null, + "layers_pattern": null, + "layers_to_transform": null, + "loftq_config": {}, + "lora_alpha": 64, + "lora_bias": false, + "lora_dropout": 0.05, + "lora_ga_config": null, + "megatron_config": null, + "megatron_core": "megatron.core", + "modules_to_save": null, + "peft_type": "LORA", + "peft_version": "0.19.1", + "qalora_group_size": 16, + "r": 64, + "rank_pattern": {}, + "revision": null, + "target_modules": [ + "o_proj", + "k_proj", + "v_proj", + "q_proj" + ], + "target_parameters": null, + "task_type": "CAUSAL_LM", + "trainable_token_indices": null, + "use_bdlora": null, + "use_dora": false, + "use_qalora": false, + "use_rslora": false +} diff --git a/MultiPL-E/SLS/Qwen3-4B/checkpoint-120/tokenizer_config.json b/MultiPL-E/SLS/Qwen3-4B/checkpoint-120/tokenizer_config.json new file mode 100644 index 0000000000000000000000000000000000000000..5668a4a01433b0648bb972458de616f22f3e728e --- /dev/null +++ b/MultiPL-E/SLS/Qwen3-4B/checkpoint-120/tokenizer_config.json @@ -0,0 +1,30 @@ +{ + "add_prefix_space": false, + "backend": "tokenizers", + "bos_token": null, + "clean_up_tokenization_spaces": false, + "eos_token": "<|im_end|>", + "errors": "replace", + "extra_special_tokens": [ + "<|im_start|>", + "<|im_end|>", + "<|object_ref_start|>", + "<|object_ref_end|>", + "<|box_start|>", + "<|box_end|>", + "<|quad_start|>", + "<|quad_end|>", + "<|vision_start|>", + "<|vision_end|>", + "<|vision_pad|>", + "<|image_pad|>", + "<|video_pad|>" + ], + "is_local": true, + "local_files_only": false, + "model_max_length": 131072, + "pad_token": "<|endoftext|>", + "split_special_tokens": false, + "tokenizer_class": "Qwen2Tokenizer", + "unk_token": null +} diff --git a/MultiPL-E/SLS/Qwen3-4B/checkpoint-120/training_state.json b/MultiPL-E/SLS/Qwen3-4B/checkpoint-120/training_state.json new file mode 100644 index 0000000000000000000000000000000000000000..9da1aa79e96851305196f5c1485db53283274412 --- /dev/null +++ b/MultiPL-E/SLS/Qwen3-4B/checkpoint-120/training_state.json @@ -0,0 +1,35 @@ +{ + "version": 1, + "global_step": 120, + "next_epoch": 1, + "next_micro_idx": 0, + "contract": { + "agent": "A2", + "rollout": "source_project/rl_data/clean/multipl_e_8lang_best_sft_gpt5_a2_paired_core_20260827.jsonl", + "rollout_size": 12146806, + "reward_field": "train_weight", + "thinking_enabled": false, + "loss_version": "format_protected_signed_rwr_token_kl_v2", + "negative_mask": "semantic_json_values_and_thinking", + "kl_mode": "token_level_forward_kl", + "model_name_or_path": "Qwen/Qwen3-4B", + "sft_adapter": "source_project/sft_runs/multipl_e_8lang_v5_diverse_a2_selfcorr2x_2ep_20260819/A2/final", + "kl_coef": 0.2, + "num_epochs": 1.0, + "learning_rate": 5e-08, + "weight_decay": 0.01, + "warmup_ratio": 0.05, + "per_device_batch_size": 1, + "gradient_accumulation_steps": 2, + "max_seq_length": 12000, + "lora_rank": 64, + "lora_alpha": 64, + "lora_dropout": 0.05, + "bf16": true, + "gradient_checkpointing": true, + "seed": 42, + "world_size": 8, + "total_records": 1908, + "total_steps": 120 + } +} diff --git a/MultiPL-E/SLS/Qwen3-4B/final/adapter_config.json b/MultiPL-E/SLS/Qwen3-4B/final/adapter_config.json new file mode 100644 index 0000000000000000000000000000000000000000..3e043613f8b2fe102d2bdcf2ee77b26fb1d282f4 --- /dev/null +++ b/MultiPL-E/SLS/Qwen3-4B/final/adapter_config.json @@ -0,0 +1,45 @@ +{ + "alora_invocation_tokens": null, + "alpha_pattern": {}, + "arrow_config": null, + "auto_mapping": null, + "base_model_name_or_path": "Qwen/Qwen3-4B", + "bias": "none", + "corda_config": null, + "ensure_weight_tying": false, + "eva_config": null, + "exclude_modules": null, + "fan_in_fan_out": false, + "inference_mode": true, + "init_lora_weights": true, + "layer_replication": null, + "layers_pattern": null, + "layers_to_transform": null, + "loftq_config": {}, + "lora_alpha": 64, + "lora_bias": false, + "lora_dropout": 0.05, + "lora_ga_config": null, + "megatron_config": null, + "megatron_core": "megatron.core", + "modules_to_save": null, + "peft_type": "LORA", + "peft_version": "0.19.1", + "qalora_group_size": 16, + "r": 64, + "rank_pattern": {}, + "revision": null, + "target_modules": [ + "o_proj", + "k_proj", + "v_proj", + "q_proj" + ], + "target_parameters": null, + "task_type": "CAUSAL_LM", + "trainable_token_indices": null, + "use_bdlora": null, + "use_dora": false, + "use_qalora": false, + "use_rslora": false +} diff --git a/MultiPL-E/SLS/Qwen3-4B/final/chat_template.jinja b/MultiPL-E/SLS/Qwen3-4B/final/chat_template.jinja new file mode 100644 index 0000000000000000000000000000000000000000..01be9b307daa2d425f7c168c9fb145a286e0afb4 --- /dev/null +++ b/MultiPL-E/SLS/Qwen3-4B/final/chat_template.jinja @@ -0,0 +1,89 @@ +{%- if tools %} + {{- '<|im_start|>system\n' }} + {%- if messages[0].role == 'system' %} + {{- messages[0].content + '\n\n' }} + {%- endif %} + {{- "# Tools\n\nYou may call one or more functions to assist with the user query.\n\nYou are provided with function signatures within XML tags:\n" }} + {%- for tool in tools %} + {{- "\n" }} + {{- tool | tojson }} + {%- endfor %} + {{- "\n\n\nFor each function call, return a json object with function name and arguments within XML tags:\n\n{\"name\": , \"arguments\": }\n<|im_end|>\n" }} +{%- else %} + {%- if messages[0].role == 'system' %} + {{- '<|im_start|>system\n' + messages[0].content + '<|im_end|>\n' }} + {%- endif %} +{%- endif %} +{%- set ns = namespace(multi_step_tool=true, last_query_index=messages|length - 1) %} +{%- for message in messages[::-1] %} + {%- set index = (messages|length - 1) - loop.index0 %} + {%- if ns.multi_step_tool and message.role == "user" and message.content is string and not(message.content.startswith('') and message.content.endswith('')) %} + {%- set ns.multi_step_tool = false %} + {%- set ns.last_query_index = index %} + {%- endif %} +{%- endfor %} +{%- for message in messages %} + {%- if message.content is string %} + {%- set content = message.content %} + {%- else %} + {%- set content = '' %} + {%- endif %} + {%- if (message.role == "user") or (message.role == "system" and not loop.first) %} + {{- '<|im_start|>' + message.role + '\n' + content + '<|im_end|>' + '\n' }} + {%- elif message.role == "assistant" %} + {%- set reasoning_content = '' %} + {%- if message.reasoning_content is string %} + {%- set reasoning_content = message.reasoning_content %} + {%- else %} + {%- if '' in content %} + {%- set reasoning_content = content.split('')[0].rstrip('\n').split('')[-1].lstrip('\n') %} + {%- set content = content.split('')[-1].lstrip('\n') %} + {%- endif %} + {%- endif %} + {%- if loop.index0 > ns.last_query_index %} + {%- if loop.last or (not loop.last and reasoning_content) %} + {{- '<|im_start|>' + message.role + '\n\n' + reasoning_content.strip('\n') + '\n\n\n' + content.lstrip('\n') }} + {%- else %} + {{- '<|im_start|>' + message.role + '\n' + content }} + {%- endif %} + {%- else %} + {{- '<|im_start|>' + message.role + '\n' + content }} + {%- endif %} + {%- if message.tool_calls %} + {%- for tool_call in message.tool_calls %} + {%- if (loop.first and content) or (not loop.first) %} + {{- '\n' }} + {%- endif %} + {%- if tool_call.function %} + {%- set tool_call = tool_call.function %} + {%- endif %} + {{- '\n{"name": "' }} + {{- tool_call.name }} + {{- '", "arguments": ' }} + {%- if tool_call.arguments is string %} + {{- tool_call.arguments }} + {%- else %} + {{- tool_call.arguments | tojson }} + {%- endif %} + {{- '}\n' }} + {%- endfor %} + {%- endif %} + {{- '<|im_end|>\n' }} + {%- elif message.role == "tool" %} + {%- if loop.first or (messages[loop.index0 - 1].role != "tool") %} + {{- '<|im_start|>user' }} + {%- endif %} + {{- '\n\n' }} + {{- content }} + {{- '\n' }} + {%- if loop.last or (messages[loop.index0 + 1].role != "tool") %} + {{- '<|im_end|>\n' }} + {%- endif %} + {%- endif %} +{%- endfor %} +{%- if add_generation_prompt %} + {{- '<|im_start|>assistant\n' }} + {%- if enable_thinking is defined and enable_thinking is false %} + {{- '\n\n\n\n' }} + {%- endif %} +{%- endif %} \ No newline at end of file diff --git a/MultiPL-E/SLS/Qwen3-4B/final/rl_train_summary.json b/MultiPL-E/SLS/Qwen3-4B/final/rl_train_summary.json new file mode 100644 index 0000000000000000000000000000000000000000..9f9236b0de712d47c3ed9eefc0fcfd7f5fe79941 --- /dev/null +++ b/MultiPL-E/SLS/Qwen3-4B/final/rl_train_summary.json @@ -0,0 +1,12 @@ +{ + "agent": "A2", + "global_step": 120, + "total_records": 1908, + "algorithm": "rwr", + "kl_coef": 0.2, + "thinking_enabled": false, + "loss_version": "format_protected_signed_rwr_token_kl_v2", + "kl_mode": "token_level_forward_kl", + "world_size": 8, + "completed": true +} diff --git a/MultiPL-E/SLS/Qwen3-4B/final/tokenizer_config.json b/MultiPL-E/SLS/Qwen3-4B/final/tokenizer_config.json new file mode 100644 index 0000000000000000000000000000000000000000..5668a4a01433b0648bb972458de616f22f3e728e --- /dev/null +++ b/MultiPL-E/SLS/Qwen3-4B/final/tokenizer_config.json @@ -0,0 +1,30 @@ +{ + "add_prefix_space": false, + "backend": "tokenizers", + "bos_token": null, + "clean_up_tokenization_spaces": false, + "eos_token": "<|im_end|>", + "errors": "replace", + "extra_special_tokens": [ + "<|im_start|>", + "<|im_end|>", + "<|object_ref_start|>", + "<|object_ref_end|>", + "<|box_start|>", + "<|box_end|>", + "<|quad_start|>", + "<|quad_end|>", + "<|vision_start|>", + "<|vision_end|>", + "<|vision_pad|>", + "<|image_pad|>", + "<|video_pad|>" + ], + "is_local": true, + "local_files_only": false, + "model_max_length": 131072, + "pad_token": "<|endoftext|>", + "split_special_tokens": false, + "tokenizer_class": "Qwen2Tokenizer", + "unk_token": null +} diff --git a/MultiPL-E/SLS/Qwen3-8B/checkpoint-100/chat_template.jinja b/MultiPL-E/SLS/Qwen3-8B/checkpoint-100/chat_template.jinja new file mode 100644 index 0000000000000000000000000000000000000000..01be9b307daa2d425f7c168c9fb145a286e0afb4 --- /dev/null +++ b/MultiPL-E/SLS/Qwen3-8B/checkpoint-100/chat_template.jinja @@ -0,0 +1,89 @@ +{%- if tools %} + {{- '<|im_start|>system\n' }} + {%- if messages[0].role == 'system' %} + {{- messages[0].content + '\n\n' }} + {%- endif %} + {{- "# Tools\n\nYou may call one or more functions to assist with the user query.\n\nYou are provided with function signatures within XML tags:\n" }} + {%- for tool in tools %} + {{- "\n" }} + {{- tool | tojson }} + {%- endfor %} + {{- "\n\n\nFor each function call, return a json object with function name and arguments within XML tags:\n\n{\"name\": , \"arguments\": }\n<|im_end|>\n" }} +{%- else %} + {%- if messages[0].role == 'system' %} + {{- '<|im_start|>system\n' + messages[0].content + '<|im_end|>\n' }} + {%- endif %} +{%- endif %} +{%- set ns = namespace(multi_step_tool=true, last_query_index=messages|length - 1) %} +{%- for message in messages[::-1] %} + {%- set index = (messages|length - 1) - loop.index0 %} + {%- if ns.multi_step_tool and message.role == "user" and message.content is string and not(message.content.startswith('') and message.content.endswith('')) %} + {%- set ns.multi_step_tool = false %} + {%- set ns.last_query_index = index %} + {%- endif %} +{%- endfor %} +{%- for message in messages %} + {%- if message.content is string %} + {%- set content = message.content %} + {%- else %} + {%- set content = '' %} + {%- endif %} + {%- if (message.role == "user") or (message.role == "system" and not loop.first) %} + {{- '<|im_start|>' + message.role + '\n' + content + '<|im_end|>' + '\n' }} + {%- elif message.role == "assistant" %} + {%- set reasoning_content = '' %} + {%- if message.reasoning_content is string %} + {%- set reasoning_content = message.reasoning_content %} + {%- else %} + {%- if '' in content %} + {%- set reasoning_content = content.split('')[0].rstrip('\n').split('')[-1].lstrip('\n') %} + {%- set content = content.split('')[-1].lstrip('\n') %} + {%- endif %} + {%- endif %} + {%- if loop.index0 > ns.last_query_index %} + {%- if loop.last or (not loop.last and reasoning_content) %} + {{- '<|im_start|>' + message.role + '\n\n' + reasoning_content.strip('\n') + '\n\n\n' + content.lstrip('\n') }} + {%- else %} + {{- '<|im_start|>' + message.role + '\n' + content }} + {%- endif %} + {%- else %} + {{- '<|im_start|>' + message.role + '\n' + content }} + {%- endif %} + {%- if message.tool_calls %} + {%- for tool_call in message.tool_calls %} + {%- if (loop.first and content) or (not loop.first) %} + {{- '\n' }} + {%- endif %} + {%- if tool_call.function %} + {%- set tool_call = tool_call.function %} + {%- endif %} + {{- '\n{"name": "' }} + {{- tool_call.name }} + {{- '", "arguments": ' }} + {%- if tool_call.arguments is string %} + {{- tool_call.arguments }} + {%- else %} + {{- tool_call.arguments | tojson }} + {%- endif %} + {{- '}\n' }} + {%- endfor %} + {%- endif %} + {{- '<|im_end|>\n' }} + {%- elif message.role == "tool" %} + {%- if loop.first or (messages[loop.index0 - 1].role != "tool") %} + {{- '<|im_start|>user' }} + {%- endif %} + {{- '\n\n' }} + {{- content }} + {{- '\n' }} + {%- if loop.last or (messages[loop.index0 + 1].role != "tool") %} + {{- '<|im_end|>\n' }} + {%- endif %} + {%- endif %} +{%- endfor %} +{%- if add_generation_prompt %} + {{- '<|im_start|>assistant\n' }} + {%- if enable_thinking is defined and enable_thinking is false %} + {{- '\n\n\n\n' }} + {%- endif %} +{%- endif %} \ No newline at end of file diff --git a/MultiPL-E/SLS/Qwen3-8B/checkpoint-100/policy_adapter/adapter_config.json b/MultiPL-E/SLS/Qwen3-8B/checkpoint-100/policy_adapter/adapter_config.json new file mode 100644 index 0000000000000000000000000000000000000000..aad2d15e6ba8c4ed4916aea16ee1679ef7bce598 --- /dev/null +++ b/MultiPL-E/SLS/Qwen3-8B/checkpoint-100/policy_adapter/adapter_config.json @@ -0,0 +1,45 @@ +{ + "alora_invocation_tokens": null, + "alpha_pattern": {}, + "arrow_config": null, + "auto_mapping": null, + "base_model_name_or_path": "Qwen/Qwen3-8B", + "bias": "none", + "corda_config": null, + "ensure_weight_tying": false, + "eva_config": null, + "exclude_modules": null, + "fan_in_fan_out": false, + "inference_mode": true, + "init_lora_weights": true, + "layer_replication": null, + "layers_pattern": null, + "layers_to_transform": null, + "loftq_config": {}, + "lora_alpha": 64, + "lora_bias": false, + "lora_dropout": 0.05, + "lora_ga_config": null, + "megatron_config": null, + "megatron_core": "megatron.core", + "modules_to_save": null, + "peft_type": "LORA", + "peft_version": "0.19.1", + "qalora_group_size": 16, + "r": 64, + "rank_pattern": {}, + "revision": null, + "target_modules": [ + "k_proj", + "o_proj", + "v_proj", + "q_proj" + ], + "target_parameters": null, + "task_type": "CAUSAL_LM", + "trainable_token_indices": null, + "use_bdlora": null, + "use_dora": false, + "use_qalora": false, + "use_rslora": false +} diff --git a/MultiPL-E/SLS/Qwen3-8B/checkpoint-100/tokenizer_config.json b/MultiPL-E/SLS/Qwen3-8B/checkpoint-100/tokenizer_config.json new file mode 100644 index 0000000000000000000000000000000000000000..5668a4a01433b0648bb972458de616f22f3e728e --- /dev/null +++ b/MultiPL-E/SLS/Qwen3-8B/checkpoint-100/tokenizer_config.json @@ -0,0 +1,30 @@ +{ + "add_prefix_space": false, + "backend": "tokenizers", + "bos_token": null, + "clean_up_tokenization_spaces": false, + "eos_token": "<|im_end|>", + "errors": "replace", + "extra_special_tokens": [ + "<|im_start|>", + "<|im_end|>", + "<|object_ref_start|>", + "<|object_ref_end|>", + "<|box_start|>", + "<|box_end|>", + "<|quad_start|>", + "<|quad_end|>", + "<|vision_start|>", + "<|vision_end|>", + "<|vision_pad|>", + "<|image_pad|>", + "<|video_pad|>" + ], + "is_local": true, + "local_files_only": false, + "model_max_length": 131072, + "pad_token": "<|endoftext|>", + "split_special_tokens": false, + "tokenizer_class": "Qwen2Tokenizer", + "unk_token": null +} diff --git a/MultiPL-E/SLS/Qwen3-8B/checkpoint-100/training_state.json b/MultiPL-E/SLS/Qwen3-8B/checkpoint-100/training_state.json new file mode 100644 index 0000000000000000000000000000000000000000..b7e91be6b9d4ba94de00be58b927d3d9fe20310c --- /dev/null +++ b/MultiPL-E/SLS/Qwen3-8B/checkpoint-100/training_state.json @@ -0,0 +1,36 @@ +{ + "version": 1, + "global_step": 100, + "next_epoch": 0, + "next_micro_idx": 200, + "contract": { + "agent": "A1", + "rollout": "source_project/rl_data/clean/multipl_e_8lang_best_sft_gpt5_a1_paired_core_20260825.jsonl", + "rollout_size": 10472495, + "rollout_sha256": "0763f753edd1881fde70e7ff1a240fbe92df13e50e633f25b8f249beee9fa779", + "reward_field": "train_weight", + "thinking_enabled": false, + "loss_version": "format_protected_signed_rwr_token_kl_v2", + "negative_mask": "semantic_json_values_and_thinking", + "kl_mode": "token_level_forward_kl", + "model_name_or_path": "Qwen/Qwen3-8B", + "sft_adapter": "source_project/sft_runs/multipl_e_8lang_v5_diverse_a3_5ep/A3/final", + "kl_coef": 0.2, + "num_epochs": 1.0, + "learning_rate": 5e-08, + "weight_decay": 0.01, + "warmup_ratio": 0.05, + "per_device_batch_size": 1, + "gradient_accumulation_steps": 2, + "max_seq_length": 12000, + "lora_rank": 64, + "lora_alpha": 64, + "lora_dropout": 0.05, + "bf16": true, + "gradient_checkpointing": true, + "seed": 42, + "world_size": 8, + "total_records": 1860, + "total_steps": 117 + } +} diff --git a/MultiPL-E/SLS/Qwen3-8B/final/adapter_config.json b/MultiPL-E/SLS/Qwen3-8B/final/adapter_config.json new file mode 100644 index 0000000000000000000000000000000000000000..aad2d15e6ba8c4ed4916aea16ee1679ef7bce598 --- /dev/null +++ b/MultiPL-E/SLS/Qwen3-8B/final/adapter_config.json @@ -0,0 +1,45 @@ +{ + "alora_invocation_tokens": null, + "alpha_pattern": {}, + "arrow_config": null, + "auto_mapping": null, + "base_model_name_or_path": "Qwen/Qwen3-8B", + "bias": "none", + "corda_config": null, + "ensure_weight_tying": false, + "eva_config": null, + "exclude_modules": null, + "fan_in_fan_out": false, + "inference_mode": true, + "init_lora_weights": true, + "layer_replication": null, + "layers_pattern": null, + "layers_to_transform": null, + "loftq_config": {}, + "lora_alpha": 64, + "lora_bias": false, + "lora_dropout": 0.05, + "lora_ga_config": null, + "megatron_config": null, + "megatron_core": "megatron.core", + "modules_to_save": null, + "peft_type": "LORA", + "peft_version": "0.19.1", + "qalora_group_size": 16, + "r": 64, + "rank_pattern": {}, + "revision": null, + "target_modules": [ + "k_proj", + "o_proj", + "v_proj", + "q_proj" + ], + "target_parameters": null, + "task_type": "CAUSAL_LM", + "trainable_token_indices": null, + "use_bdlora": null, + "use_dora": false, + "use_qalora": false, + "use_rslora": false +} diff --git a/MultiPL-E/SLS/Qwen3-8B/final/chat_template.jinja b/MultiPL-E/SLS/Qwen3-8B/final/chat_template.jinja new file mode 100644 index 0000000000000000000000000000000000000000..01be9b307daa2d425f7c168c9fb145a286e0afb4 --- /dev/null +++ b/MultiPL-E/SLS/Qwen3-8B/final/chat_template.jinja @@ -0,0 +1,89 @@ +{%- if tools %} + {{- '<|im_start|>system\n' }} + {%- if messages[0].role == 'system' %} + {{- messages[0].content + '\n\n' }} + {%- endif %} + {{- "# Tools\n\nYou may call one or more functions to assist with the user query.\n\nYou are provided with function signatures within XML tags:\n" }} + {%- for tool in tools %} + {{- "\n" }} + {{- tool | tojson }} + {%- endfor %} + {{- "\n\n\nFor each function call, return a json object with function name and arguments within XML tags:\n\n{\"name\": , \"arguments\": }\n<|im_end|>\n" }} +{%- else %} + {%- if messages[0].role == 'system' %} + {{- '<|im_start|>system\n' + messages[0].content + '<|im_end|>\n' }} + {%- endif %} +{%- endif %} +{%- set ns = namespace(multi_step_tool=true, last_query_index=messages|length - 1) %} +{%- for message in messages[::-1] %} + {%- set index = (messages|length - 1) - loop.index0 %} + {%- if ns.multi_step_tool and message.role == "user" and message.content is string and not(message.content.startswith('') and message.content.endswith('')) %} + {%- set ns.multi_step_tool = false %} + {%- set ns.last_query_index = index %} + {%- endif %} +{%- endfor %} +{%- for message in messages %} + {%- if message.content is string %} + {%- set content = message.content %} + {%- else %} + {%- set content = '' %} + {%- endif %} + {%- if (message.role == "user") or (message.role == "system" and not loop.first) %} + {{- '<|im_start|>' + message.role + '\n' + content + '<|im_end|>' + '\n' }} + {%- elif message.role == "assistant" %} + {%- set reasoning_content = '' %} + {%- if message.reasoning_content is string %} + {%- set reasoning_content = message.reasoning_content %} + {%- else %} + {%- if '' in content %} + {%- set reasoning_content = content.split('')[0].rstrip('\n').split('')[-1].lstrip('\n') %} + {%- set content = content.split('')[-1].lstrip('\n') %} + {%- endif %} + {%- endif %} + {%- if loop.index0 > ns.last_query_index %} + {%- if loop.last or (not loop.last and reasoning_content) %} + {{- '<|im_start|>' + message.role + '\n\n' + reasoning_content.strip('\n') + '\n\n\n' + content.lstrip('\n') }} + {%- else %} + {{- '<|im_start|>' + message.role + '\n' + content }} + {%- endif %} + {%- else %} + {{- '<|im_start|>' + message.role + '\n' + content }} + {%- endif %} + {%- if message.tool_calls %} + {%- for tool_call in message.tool_calls %} + {%- if (loop.first and content) or (not loop.first) %} + {{- '\n' }} + {%- endif %} + {%- if tool_call.function %} + {%- set tool_call = tool_call.function %} + {%- endif %} + {{- '\n{"name": "' }} + {{- tool_call.name }} + {{- '", "arguments": ' }} + {%- if tool_call.arguments is string %} + {{- tool_call.arguments }} + {%- else %} + {{- tool_call.arguments | tojson }} + {%- endif %} + {{- '}\n' }} + {%- endfor %} + {%- endif %} + {{- '<|im_end|>\n' }} + {%- elif message.role == "tool" %} + {%- if loop.first or (messages[loop.index0 - 1].role != "tool") %} + {{- '<|im_start|>user' }} + {%- endif %} + {{- '\n\n' }} + {{- content }} + {{- '\n' }} + {%- if loop.last or (messages[loop.index0 + 1].role != "tool") %} + {{- '<|im_end|>\n' }} + {%- endif %} + {%- endif %} +{%- endfor %} +{%- if add_generation_prompt %} + {{- '<|im_start|>assistant\n' }} + {%- if enable_thinking is defined and enable_thinking is false %} + {{- '\n\n\n\n' }} + {%- endif %} +{%- endif %} \ No newline at end of file diff --git a/MultiPL-E/SLS/Qwen3-8B/final/rl_train_summary.json b/MultiPL-E/SLS/Qwen3-8B/final/rl_train_summary.json new file mode 100644 index 0000000000000000000000000000000000000000..ceb620065646d8848e0dbbbd9d0db20f288def6b --- /dev/null +++ b/MultiPL-E/SLS/Qwen3-8B/final/rl_train_summary.json @@ -0,0 +1,50 @@ +{ + "summary_version": 2, + "agent": "A1", + "global_step": 117, + "total_records": 1860, + "algorithm": "rwr", + "kl_coef": 0.2, + "thinking_enabled": false, + "loss_version": "format_protected_signed_rwr_token_kl_v2", + "negative_mask": "semantic_json_values_and_thinking", + "kl_mode": "token_level_forward_kl", + "world_size": 8, + "completed": true, + "rollout": "source_project/rl_data/clean/multipl_e_8lang_best_sft_gpt5_a1_paired_core_20260825.jsonl", + "rollout_size": 10472495, + "rollout_sha256": "0763f753edd1881fde70e7ff1a240fbe92df13e50e633f25b8f249beee9fa779", + "adapter_sha256": "878c81880664b1b272a71195ff97e95c84ea2b5bec9b6b0cf53feb0e7f8b5bfe", + "model_name_or_path": "Qwen/Qwen3-8B", + "sft_adapter": "source_project/sft_runs/multipl_e_8lang_v5_diverse_a3_5ep/A3/final", + "contract": { + "agent": "A1", + "rollout": "source_project/rl_data/clean/multipl_e_8lang_best_sft_gpt5_a1_paired_core_20260825.jsonl", + "rollout_size": 10472495, + "rollout_sha256": "0763f753edd1881fde70e7ff1a240fbe92df13e50e633f25b8f249beee9fa779", + "reward_field": "train_weight", + "thinking_enabled": false, + "loss_version": "format_protected_signed_rwr_token_kl_v2", + "negative_mask": "semantic_json_values_and_thinking", + "kl_mode": "token_level_forward_kl", + "model_name_or_path": "Qwen/Qwen3-8B", + "sft_adapter": "source_project/sft_runs/multipl_e_8lang_v5_diverse_a3_5ep/A3/final", + "kl_coef": 0.2, + "num_epochs": 1.0, + "learning_rate": 5e-08, + "weight_decay": 0.01, + "warmup_ratio": 0.05, + "per_device_batch_size": 1, + "gradient_accumulation_steps": 2, + "max_seq_length": 12000, + "lora_rank": 64, + "lora_alpha": 64, + "lora_dropout": 0.05, + "bf16": true, + "gradient_checkpointing": true, + "seed": 42, + "world_size": 8, + "total_records": 1860, + "total_steps": 117 + } +} diff --git a/MultiPL-E/SLS/Qwen3-8B/final/tokenizer_config.json b/MultiPL-E/SLS/Qwen3-8B/final/tokenizer_config.json new file mode 100644 index 0000000000000000000000000000000000000000..5668a4a01433b0648bb972458de616f22f3e728e --- /dev/null +++ b/MultiPL-E/SLS/Qwen3-8B/final/tokenizer_config.json @@ -0,0 +1,30 @@ +{ + "add_prefix_space": false, + "backend": "tokenizers", + "bos_token": null, + "clean_up_tokenization_spaces": false, + "eos_token": "<|im_end|>", + "errors": "replace", + "extra_special_tokens": [ + "<|im_start|>", + "<|im_end|>", + "<|object_ref_start|>", + "<|object_ref_end|>", + "<|box_start|>", + "<|box_end|>", + "<|quad_start|>", + "<|quad_end|>", + "<|vision_start|>", + "<|vision_end|>", + "<|vision_pad|>", + "<|image_pad|>", + "<|video_pad|>" + ], + "is_local": true, + "local_files_only": false, + "model_max_length": 131072, + "pad_token": "<|endoftext|>", + "split_special_tokens": false, + "tokenizer_class": "Qwen2Tokenizer", + "unk_token": null +}