""" 批量提取pose embedding的脚本 从2858个视频对应的pose数据中提取embedding,并保存到指定目录 流程: 1. 遍历dance_2858_video目录下的所有视频文件名 2. 从文件名解析出原始pose目录名和segment索引 3. 根据segment索引计算应该读取pose的哪些帧(每个segment 81帧,16fps) 4. 用dwpose_embedding处理这些帧 5. 保存为对应的embedding文件(.pt格式) """ import torch import os import pickle import argparse from PIL import Image from io import BytesIO from tqdm import tqdm import numpy as np import torch.nn as nn from einops import rearrange import torchvision.transforms as T import imageio from pathlib import Path class DwposeEmbedding(nn.Module): """ 将pose视频转为[1, 32760, 5120]维度embedding的模块 时间下采样4倍,空间下采样16倍 输入: [B, 3, 84, 832, 480] (B, C, T+3, H, W) - 需要在时间维度前面补3帧 输出: [B, 5120, 21, 52, 30] -> reshape为 [B, 32760, 5120] """ def __init__(self): super().__init__() concat_dim = 4 self.embedding = nn.Sequential( nn.Conv3d(3, concat_dim * 4, (3,3,3), stride=(1,1,1), padding=(1,1,1)), nn.SiLU(), nn.Conv3d(concat_dim * 4, concat_dim * 4, (3,3,3), stride=(1,1,1), padding=(1,1,1)), nn.SiLU(), nn.Conv3d(concat_dim * 4, concat_dim * 4, (3,3,3), stride=(1,1,1), padding=(1,1,1)), nn.SiLU(), nn.Conv3d(concat_dim * 4, concat_dim * 4, (3,3,3), stride=(1,2,2), padding=(1,1,1)), nn.SiLU(), nn.Conv3d(concat_dim * 4, concat_dim * 4, 3, stride=(2,2,2), padding=1), nn.SiLU(), nn.Conv3d(concat_dim * 4, concat_dim * 4, 3, stride=(2,2,2), padding=1), nn.SiLU(), nn.Conv3d(concat_dim * 4, 5120, (1,2,2), stride=(1,2,2), padding=0) ) def forward(self, x): """ x: [B, 3, T, H, W] - pose视频,T=84(81帧+前面复制的3帧) 输出: [B, 32760, 5120] """ out = self.embedding(x) # [B, 5120, 21, 52, 30] out = rearrange(out, 'b c f h w -> b (f h w) c').contiguous() # [B, 32760, 5120] return out def load_dwpose_embedding(checkpoint_path, device='cuda'): """ 从checkpoint加载dwpose_embedding模块 """ from diffsynth import load_state_dict, load_state_dict_from_folder model = DwposeEmbedding() # 加载checkpoint try: state_dict = load_state_dict(checkpoint_path) except: state_dict = load_state_dict_from_folder(checkpoint_path) # 提取dwpose_embedding的权重 state_dict_new = {} for key in state_dict.keys(): if "dwpose_embedding" in key: new_key = key.split("dwpose_embedding.")[1] new_key = "embedding." + new_key state_dict_new[new_key] = state_dict[key] if len(state_dict_new) == 0: print(f"警告: 在checkpoint中未找到dwpose_embedding的权重,使用随机初始化") else: model.load_state_dict(state_dict_new, strict=True) print(f"成功加载dwpose_embedding权重,共{len(state_dict_new)}个参数") model = model.to(device).eval() return model def parse_video_filename(video_filename): """ 解析视频文件名,提取原始pose目录名和segment索引 例如: "movies_000005_0000_seg0.mp4" -> ("movies_000005_0000", 0) """ # 去掉.mp4后缀 name = video_filename.replace('.mp4', '') # 找到最后一个_seg parts = name.rsplit('_seg', 1) if len(parts) != 2: raise ValueError(f"无法解析文件名: {video_filename}") pose_dir_name = parts[0] seg_index = int(parts[1]) return pose_dir_name, seg_index def get_video_fps(video_path): """ 使用ffprobe获取视频帧率 Args: video_path: 视频文件路径 Returns: float: 视频帧率,获取失败返回None """ import subprocess cmd = [ 'ffprobe', '-v', 'error', '-select_streams', 'v:0', '-show_entries', 'stream=r_frame_rate', '-of', 'default=noprint_wrappers=1:nokey=1', str(video_path) ] try: result = subprocess.run(cmd, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, text=True) fps_str = result.stdout.strip() # fps_str 格式可能是 "30/1" 或 "30000/1001" if '/' in fps_str: num, den = fps_str.split('/') return float(num) / float(den) return float(fps_str) except: return None def load_pose_frames_with_sampling(pose_dir, start_time, duration, original_fps, target_fps, target_frames, height=832, width=480, augmentation='none', augmentation_ratio=1.2): """ 从pose目录加载指定时间范围的帧,并进行下采样 Args: pose_dir: pose数据目录路径 start_time: 起始时间(秒) duration: 时长(秒) original_fps: 原始视频帧率 target_fps: 目标帧率 target_frames: 目标帧数 height: 目标高度 width: 目标宽度 augmentation: 数据增强类型,可选 'none', 'fat', 'thin' - 'fat': 横向拉伸,让人变胖 - 'thin': 纵向拉伸,让人变瘦 augmentation_ratio: 拉伸比例,默认1.2表示拉伸20% Returns: torch.Tensor: shape [T, 3, H, W], 值范围 [0, 255] """ dwpose_path = os.path.join(pose_dir, 'dw_pose_with_foot_wo_face.pkl') if not os.path.exists(dwpose_path): raise FileNotFoundError(f"未找到pose文件: {dwpose_path}") with open(dwpose_path, 'rb') as f: dwpose_all = pickle.load(f) keys = list(dwpose_all.keys()) total_frames = len(keys) # 计算在原始帧率下的起始帧索引 start_frame = int(start_time * original_fps) # 计算采样步长:每隔多少帧取一帧 # 例如:original_fps=30, target_fps=16 -> step=30/16=1.875 sample_step = original_fps / target_fps # 计算需要采样的帧索引 frame_indices = [] for i in range(target_frames): frame_idx = int(start_frame + i * sample_step) # 确保不超出范围 if frame_idx >= total_frames: frame_idx = total_frames - 1 frame_indices.append(frame_idx) # 检查起始帧是否有效 if start_frame >= total_frames: raise ValueError(f"起始帧{start_frame}超出总帧数{total_frames}") # 定义最终resize transform resize_transform = T.Resize((height, width), antialias=True) dwpose_list = [] for frame_idx in frame_indices: key = keys[frame_idx] pose_img = Image.open(BytesIO(dwpose_all[key])) if pose_img.mode != 'RGB': pose_img = pose_img.convert('RGB') # 获取原始图像尺寸 orig_w, orig_h = pose_img.size # PIL的size是(width, height) if augmentation == 'fat': # fat版本:裁剪掉左右两侧,保留中间部分,再resize回原尺寸 # 例如ratio=1.2时,保留中间 1/1.2 = 83.3% 的宽度 crop_w = int(orig_w / augmentation_ratio) left = (orig_w - crop_w) // 2 right = left + crop_w pose_img = pose_img.crop((left, 0, right, orig_h)) # (left, top, right, bottom) elif augmentation == 'thin': # thin版本:裁剪掉上下两侧,保留中间部分,再resize回原尺寸 # 例如ratio=1.2时,保留中间 1/1.2 = 83.3% 的高度 crop_h = int(orig_h / augmentation_ratio) top = (orig_h - crop_h) // 2 bottom = top + crop_h pose_img = pose_img.crop((0, top, orig_w, bottom)) # (left, top, right, bottom) # Resize到目标尺寸 pose_resized = resize_transform(pose_img) # PIL Image 转换为 numpy array,再转为 tensor pose_array = np.array(pose_resized, dtype=np.uint8) # [H, W, C] pose_tensor = torch.from_numpy(pose_array.copy()) # [H, W, C] pose_tensor = pose_tensor.permute(2, 0, 1) # [C, H, W] dwpose_list.append(pose_tensor) # Stack成 [T, C, H, W] dwpose_tensor = torch.stack(dwpose_list, dim=0) return dwpose_tensor def process_single_video(model, video_filename, pose_base_dir, output_dir, target_fps=16, target_frames=81, height=832, width=480, device='cuda', pose_video_dir=None, original_video_dir=None, default_original_fps=30.0, augmentation='none', augmentation_ratio=1.2): """ 处理单个视频对应的pose数据 Args: model: dwpose_embedding模型 video_filename: 视频文件名 (如 "movies_000005_0000_seg0.mp4") pose_base_dir: pose数据根目录 output_dir: 输出目录 target_fps: 目标帧率 target_frames: 每个segment的帧数 height: 目标高度 width: 目标宽度 device: 计算设备 pose_video_dir: pose视频保存目录,如果为None则不保存pose视频 original_video_dir: 原始视频目录,用于获取帧率 default_original_fps: 默认原始视频帧率(当无法获取时使用) augmentation: 数据增强类型,可选 'none', 'fat', 'thin' augmentation_ratio: 拉伸比例 Returns: bool: 是否成功处理 """ try: # 解析文件名 pose_dir_name, seg_index = parse_video_filename(video_filename) # 构建pose目录路径 pose_dir = os.path.join(pose_base_dir, pose_dir_name) if not os.path.exists(pose_dir): print(f"警告: pose目录不存在: {pose_dir}") return False # 获取原始视频帧率 original_fps = default_original_fps if original_video_dir: # 原始视频文件名(去掉_segX后缀) original_video_name = pose_dir_name + '.mp4' original_video_path = os.path.join(original_video_dir, original_video_name) if os.path.exists(original_video_path): detected_fps = get_video_fps(original_video_path) if detected_fps: original_fps = detected_fps # 计算时间范围 # segment_duration = target_frames / target_fps = 81 / 16 = 5.0625秒 segment_duration = target_frames / target_fps start_time = seg_index * segment_duration # 加载pose帧(带下采样和可选的数据增强) dwpose_tensor = load_pose_frames_with_sampling( pose_dir, start_time, segment_duration, original_fps, target_fps, target_frames, height, width, augmentation, augmentation_ratio ) # 检查帧数是否足够 actual_frames = dwpose_tensor.shape[0] if actual_frames < target_frames: print(f"警告: {video_filename} 帧数不足 ({actual_frames} < {target_frames}),跳过") return False # 保存pose视频(如果指定了目录) if pose_video_dir is not None: pose_video_path = os.path.join(pose_video_dir, video_filename) # dwpose_tensor: [T, C, H, W] -> [T, H, W, C] for imageio video_frames = dwpose_tensor.permute(0, 2, 3, 1).numpy() # [T, H, W, C], uint8 imageio.mimwrite(pose_video_path, video_frames, fps=target_fps, codec='libx264') # 准备输入: 需要在时间维度前面复制3帧,变成84帧 # dwpose_tensor: [81, 3, H, W] first_frame = dwpose_tensor[:1] # [1, 3, H, W] first_3_frames = first_frame.repeat(3, 1, 1, 1) # [3, 3, H, W] dwpose_padded = torch.cat([first_3_frames, dwpose_tensor], dim=0) # [84, 3, H, W] # 转换为 [1, 3, 84, H, W] 并归一化到 [0, 1] dwpose_input = dwpose_padded.unsqueeze(0).permute(0, 2, 1, 3, 4).float() / 255.0 dwpose_input = dwpose_input.to(device) # 前向推理 with torch.no_grad(): embedding = model(dwpose_input) # [1, 32760, 5120] # 保存结果 output_filename = video_filename.replace('.mp4', '.pt') output_path = os.path.join(output_dir, output_filename) torch.save(embedding.cpu(), output_path) return True except Exception as e: print(f"处理 {video_filename} 时出错: {e}") return False def main(): parser = argparse.ArgumentParser(description="批量提取pose embedding") parser.add_argument( "--checkpoint_path", type=str, default="/cpfs01/VideoGen/user_data/buwendong/code/UniAnimate-DiT/wan/UniA-20260120-191336/epoch=3-step=820.ckpt/output_dir", help="训练好的checkpoint路径" ) parser.add_argument( "--video_dir", type=str, default="/cpfs01/VideoGen/user_data/buwendong/code/dance_2858_video", help="2858个视频所在目录" ) parser.add_argument( "--pose_base_dir", type=str, default="/cpfs01/VideoGen/user_data/buwendong/code/UniAnimate-DiT/dance_1723_new/training_video_pose", help="1723个pose数据所在目录" ) parser.add_argument( "--output_dir", type=str, default="/cpfs01/VideoGen/user_data/buwendong/code/dance_2858_pose_embed_v2", help="输出目录" ) parser.add_argument( "--pose_video_dir", type=str, default="/cpfs01/VideoGen/user_data/buwendong/code/dance_2858_pose_video_v2", help="pose视频保存目录,设为空字符串则不保存pose视频" ) parser.add_argument( "--original_video_dir", type=str, default="/cpfs01/VideoGen/user_data/buwendong/code/dance_1723_video", help="原始视频目录,用于获取视频帧率" ) parser.add_argument( "--default_original_fps", type=float, default=30.0, help="默认原始视频帧率(当无法从视频获取时使用)" ) parser.add_argument( "--target_fps", type=int, default=16, help="目标帧率" ) parser.add_argument( "--target_frames", type=int, default=81, help="每个segment的帧数" ) parser.add_argument( "--height", type=int, default=832, help="目标高度" ) parser.add_argument( "--width", type=int, default=480, help="目标宽度" ) parser.add_argument( "--device", type=str, default="cuda", help="计算设备" ) parser.add_argument( "--batch_size", type=int, default=1, help="批处理大小(暂未实现多batch)" ) parser.add_argument( "--augmentation", type=str, default="none", choices=["none", "fat", "thin"], help="数据增强类型:none=无增强,fat=横向拉伸(人变胖),thin=纵向拉伸(人变瘦)" ) parser.add_argument( "--augmentation_ratio", type=float, default=1.2, help="拉伸比例,默认1.2表示拉伸20%%。比例越大,fat/thin效果越明显" ) args = parser.parse_args() # 创建输出目录 os.makedirs(args.output_dir, exist_ok=True) # 处理pose_video_dir参数 pose_video_dir = args.pose_video_dir if args.pose_video_dir else None if pose_video_dir: os.makedirs(pose_video_dir, exist_ok=True) print(f"pose视频将保存到: {pose_video_dir}") # 显示数据增强设置 if args.augmentation != 'none': print(f"数据增强模式: {args.augmentation}, 拉伸比例: {args.augmentation_ratio}") else: print("数据增强模式: 无") # 加载模型 print(f"正在加载模型: {args.checkpoint_path}") model = load_dwpose_embedding(args.checkpoint_path, args.device) # 获取所有视频文件名 video_files = [f for f in os.listdir(args.video_dir) if f.endswith('.mp4')] video_files.sort() print(f"找到 {len(video_files)} 个视频文件") # 检查已处理的文件,支持断点续传(同时检查embedding和pose视频) existing_embed_files = set(f.replace('.pt', '.mp4') for f in os.listdir(args.output_dir) if f.endswith('.pt')) if pose_video_dir: existing_pose_videos = set(f for f in os.listdir(pose_video_dir) if f.endswith('.mp4')) # 只有两者都存在才算处理完成 existing_files = existing_embed_files & existing_pose_videos else: existing_files = existing_embed_files remaining_files = [f for f in video_files if f not in existing_files] print(f"已处理 {len(existing_files)} 个,还需处理 {len(remaining_files)} 个") # 处理每个视频 success_count = 0 fail_count = 0 desc = f"提取pose embedding ({args.augmentation})" if args.augmentation != 'none' else "提取pose embedding" for video_file in tqdm(remaining_files, desc=desc): success = process_single_video( model=model, video_filename=video_file, pose_base_dir=args.pose_base_dir, output_dir=args.output_dir, target_fps=args.target_fps, target_frames=args.target_frames, height=args.height, width=args.width, device=args.device, pose_video_dir=pose_video_dir, original_video_dir=args.original_video_dir, default_original_fps=args.default_original_fps, augmentation=args.augmentation, augmentation_ratio=args.augmentation_ratio ) if success: success_count += 1 else: fail_count += 1 print(f"\n处理完成!") print(f"成功: {success_count}") print(f"失败: {fail_count}") print(f"总计: {success_count + fail_count}") if __name__ == "__main__": main()