import os import json import torch import transformers import warnings from transformers import AutoTokenizer, AutoModelForCausalLM, Qwen3Config, Qwen3ForCausalLM, Qwen3MoeConfig, Qwen3MoeForCausalLM from models import LMConfig, LMForCausalLM from models.lm.lora import apply_lora, merge_lora warnings.filterwarnings('ignore', category=UserWarning) def convert_torch2transformers_omni(torch_path, transformers_path, dtype=torch.float16, tokenizer_path='checkpoint/tokenizer'): LMConfig.register_for_auto_class() LMForCausalLM.register_for_auto_class("AutoModelForCausalLM") lm_model = LMForCausalLM(lm_config) device = torch.device('cuda' if torch.cuda.is_available() else 'cpu') state_dict = torch.load(torch_path, map_location=device) lm_model.load_state_dict(state_dict, strict=False) lm_model = lm_model.to(dtype) # 转换模型权重精度 model_params = sum(p.numel() for p in lm_model.parameters() if p.requires_grad) print(f'模型参数: {model_params / 1e6} 百万 = {model_params / 1e9} B (Billion)') lm_model.save_pretrained(transformers_path, safe_serialization=False) tokenizer = AutoTokenizer.from_pretrained(tokenizer_path) tokenizer.save_pretrained(transformers_path) # ======= transformers-5.0的兼容低版本写法 ======= if int(transformers.__version__.split('.')[0]) >= 5: tokenizer_config_path, config_path = os.path.join(transformers_path, "tokenizer_config.json"), os.path.join(transformers_path, "config.json") json.dump({**json.load(open(tokenizer_config_path, 'r', encoding='utf-8')), "tokenizer_class": "PreTrainedTokenizerFast", "extra_special_tokens": {}}, open(tokenizer_config_path, 'w', encoding='utf-8'), indent=2, ensure_ascii=False) config = json.load(open(config_path, 'r', encoding='utf-8')) config['rope_theta'] = lm_config.rope_theta; config['rope_scaling'] = None; del config['rope_parameters'] json.dump(config, open(config_path, 'w', encoding='utf-8'), indent=2, ensure_ascii=False) print(f"模型已保存为 Transformers-Omni 格式: {transformers_path}") # QwenForCausalLM/LlamaForCausalLM结构兼容生态 def convert_torch2transformers(torch_path, transformers_path, dtype=torch.float16, tokenizer_path='checkpoint/tokenizer'): device = torch.device('cuda' if torch.cuda.is_available() else 'cpu') state_dict = torch.load(torch_path, map_location=device) common_config = { "vocab_size": lm_config.vocab_size, "hidden_size": lm_config.hidden_size, "intermediate_size": lm_config.intermediate_size, "num_hidden_layers": lm_config.num_hidden_layers, "num_attention_heads": lm_config.num_attention_heads, "num_key_value_heads": lm_config.num_key_value_heads, "head_dim": lm_config.hidden_size // lm_config.num_attention_heads, "max_position_embeddings": lm_config.max_position_embeddings, "rms_norm_eps": lm_config.rms_norm_eps, "rope_theta": lm_config.rope_theta, "tie_word_embeddings": lm_config.tie_word_embeddings } if not lm_config.use_moe: qwen_config = Qwen3Config( **common_config, use_sliding_window=False, sliding_window=None ) qwen_model = Qwen3ForCausalLM(qwen_config) else: qwen_config = Qwen3MoeConfig( **common_config, num_experts=lm_config.num_experts, num_experts_per_tok=lm_config.num_experts_per_tok, moe_intermediate_size=lm_config.moe_intermediate_size, norm_topk_prob=lm_config.norm_topk_prob ) qwen_model = Qwen3MoeForCausalLM(qwen_config) # ======= transformers-5.0的兼容低版本写法 ======= if int(transformers.__version__.split('.')[0]) >= 5: new_sd = {k: v for k, v in state_dict.items() if 'experts.' not in k or 'gate.weight' in k} for l in range(lm_config.num_hidden_layers): p = f'model.layers.{l}.mlp.experts' new_sd[f'{p}.gate_up_proj'] = torch.cat([torch.stack([state_dict[f'{p}.{e}.gate_proj.weight'] for e in range(lm_config.num_experts)]), torch.stack([state_dict[f'{p}.{e}.up_proj.weight'] for e in range(lm_config.num_experts)])], dim=1) new_sd[f'{p}.down_proj'] = torch.stack([state_dict[f'{p}.{e}.down_proj.weight'] for e in range(lm_config.num_experts)]) state_dict = new_sd qwen_model.load_state_dict(state_dict, strict=True) qwen_model = qwen_model.to(dtype) # 转换模型权重精度 qwen_model.save_pretrained(transformers_path) model_params = sum(p.numel() for p in qwen_model.parameters() if p.requires_grad) print(f'模型参数: {model_params / 1e6} 百万 = {model_params / 1e9} B (Billion)') tokenizer = AutoTokenizer.from_pretrained(tokenizer_path) tokenizer.save_pretrained(transformers_path) # ======= transformers-5.0的兼容低版本写法 ======= if int(transformers.__version__.split('.')[0]) >= 5: tokenizer_config_path, config_path = os.path.join(transformers_path, "tokenizer_config.json"), os.path.join(transformers_path, "config.json") json.dump({**json.load(open(tokenizer_config_path, 'r', encoding='utf-8')), "tokenizer_class": "PreTrainedTokenizerFast", "extra_special_tokens": {}}, open(tokenizer_config_path, 'w', encoding='utf-8'), indent=2, ensure_ascii=False) config = json.load(open(config_path, 'r', encoding='utf-8')) config['rope_theta'] = lm_config.rope_theta; config['rope_scaling'] = None; del config['rope_parameters'] json.dump(config, open(config_path, 'w', encoding='utf-8'), indent=2, ensure_ascii=False) print(f"模型已保存为 Transformers 格式: {transformers_path}") def convert_transformers2torch(transformers_path, torch_path): model = AutoModelForCausalLM.from_pretrained(transformers_path, trust_remote_code=True) torch.save({k: v.cpu().half() for k, v in model.state_dict().items()}, torch_path) print(f"模型已保存为 PyTorch 格式: {torch_path}") def convert_merge_base_lora(base_torch_path, lora_path, merged_torch_path): device = torch.device('cuda' if torch.cuda.is_available() else 'cpu') lm_model = LMForCausalLM(lm_config).to(device) state_dict = torch.load(base_torch_path, map_location=device) lm_model.load_state_dict(state_dict, strict=False) apply_lora(lm_model) merge_lora(lm_model, lora_path, merged_torch_path) print(f"LoRA 已合并并保存为基模结构 PyTorch 格式: {merged_torch_path}") def convert_jinja_to_json(jinja_path): with open(jinja_path, 'r') as f: template = f.read() escaped = json.dumps(template) print(f'"chat_template": {escaped}') def convert_json_to_jinja(json_file_path, output_path): with open(json_file_path, 'r') as f: config = json.load(f) template = config['chat_template'] with open(output_path, 'w') as f: f.write(template) print(f"模板已保存为 jinja 文件: {output_path}") if __name__ == '__main__': import argparse parser = argparse.ArgumentParser(description="转换模型格式") parser.add_argument('torch_path', type=str, help="输入 .pth 权重路径") parser.add_argument('output_dir', type=str, help="输出目录") parser.add_argument('--hidden_size', type=int, default=None, help="hidden_size（未指定则从 checkpoint 推断）") parser.add_argument('--num_hidden_layers', type=int, default=None, help="层数（未指定则从 checkpoint 推断）") parser.add_argument('--use_moe', action='store_true', help="MoE 架构") parser.add_argument('--max_seq_len', type=int, default=8192, help="最大序列长度") parser.add_argument('--mode', choices=['omni', 'qwen'], default='omni', help="输出格式（omni=原生LM格式, qwen=Qwen3兼容格式）") parser.add_argument('--dtype', type=str, default='float16', help="权重精度") parser.add_argument('--tokenizer_path', type=str, default='checkpoint/tokenizer', help="tokenizer 路径") args = parser.parse_args() state = torch.load(args.torch_path, map_location='cpu') hs = args.hidden_size or state['model.embed_tokens.weight'].shape[1] nl = args.num_hidden_layers or (max(int(k.split('.')[2]) for k in state if k.startswith('model.layers.')) + 1) dtype = getattr(torch, args.dtype) lm_config = LMConfig(hidden_size=hs, num_hidden_layers=nl, max_seq_len=args.max_seq_len, use_moe=args.use_moe) if args.mode == 'omni': convert_torch2transformers_omni(args.torch_path, args.output_dir, dtype, args.tokenizer_path) else: convert_torch2transformers(args.torch_path, args.output_dir, dtype, args.tokenizer_path)