Download extract_pose_embed_shape.py from Tinghong-Ye22/code-UniAnimate-DiT: direct link, hf CLI and curl.
- Browser
- Download file 18.3 kB
-
https://huggingface.co/Tinghong-Ye22/code-UniAnimate-DiT/resolve/main/extract_pose_embed_shape.py
- Command line
-
hf download hf://Tinghong-Ye22/code-UniAnimate-DiT/extract_pose_embed_shape.py
-
curl -L -o extract_pose_embed_shape.py https://huggingface.co/Tinghong-Ye22/code-UniAnimate-DiT/resolve/main/extract_pose_embed_shape.py
18.3 kB
| """ | |
| 批量提取pose embedding的脚本 | |
| 从2858个视频对应的pose数据中提取embedding,并保存到指定目录 | |
| 流程: | |
| 1. 遍历dance_2858_video目录下的所有视频文件名 | |
| 2. 从文件名解析出原始pose目录名和segment索引 | |
| 3. 根据segment索引计算应该读取pose的哪些帧(每个segment 81帧,16fps) | |
| 4. 用dwpose_embedding处理这些帧 | |
| 5. 保存为对应的embedding文件(.pt格式) | |
| """ | |
| import torch | |
| import os | |
| import pickle | |
| import argparse | |
| from PIL import Image | |
| from io import BytesIO | |
| from tqdm import tqdm | |
| import numpy as np | |
| import torch.nn as nn | |
| from einops import rearrange | |
| import torchvision.transforms as T | |
| import imageio | |
| from pathlib import Path | |
| class DwposeEmbedding(nn.Module): | |
| """ | |
| 将pose视频转为[1, 32760, 5120]维度embedding的模块 | |
| 时间下采样4倍,空间下采样16倍 | |
| 输入: [B, 3, 84, 832, 480] (B, C, T+3, H, W) - 需要在时间维度前面补3帧 | |
| 输出: [B, 5120, 21, 52, 30] -> reshape为 [B, 32760, 5120] | |
| """ | |
| def __init__(self): | |
| super().__init__() | |
| concat_dim = 4 | |
| self.embedding = nn.Sequential( | |
| nn.Conv3d(3, concat_dim * 4, (3,3,3), stride=(1,1,1), padding=(1,1,1)), | |
| nn.SiLU(), | |
| nn.Conv3d(concat_dim * 4, concat_dim * 4, (3,3,3), stride=(1,1,1), padding=(1,1,1)), | |
| nn.SiLU(), | |
| nn.Conv3d(concat_dim * 4, concat_dim * 4, (3,3,3), stride=(1,1,1), padding=(1,1,1)), | |
| nn.SiLU(), | |
| nn.Conv3d(concat_dim * 4, concat_dim * 4, (3,3,3), stride=(1,2,2), padding=(1,1,1)), | |
| nn.SiLU(), | |
| nn.Conv3d(concat_dim * 4, concat_dim * 4, 3, stride=(2,2,2), padding=1), | |
| nn.SiLU(), | |
| nn.Conv3d(concat_dim * 4, concat_dim * 4, 3, stride=(2,2,2), padding=1), | |
| nn.SiLU(), | |
| nn.Conv3d(concat_dim * 4, 5120, (1,2,2), stride=(1,2,2), padding=0) | |
| ) | |
| def forward(self, x): | |
| """ | |
| x: [B, 3, T, H, W] - pose视频,T=84(81帧+前面复制的3帧) | |
| 输出: [B, 32760, 5120] | |
| """ | |
| out = self.embedding(x) # [B, 5120, 21, 52, 30] | |
| out = rearrange(out, 'b c f h w -> b (f h w) c').contiguous() # [B, 32760, 5120] | |
| return out | |
| def load_dwpose_embedding(checkpoint_path, device='cuda'): | |
| """ | |
| 从checkpoint加载dwpose_embedding模块 | |
| """ | |
| from diffsynth import load_state_dict, load_state_dict_from_folder | |
| model = DwposeEmbedding() | |
| # 加载checkpoint | |
| try: | |
| state_dict = load_state_dict(checkpoint_path) | |
| except: | |
| state_dict = load_state_dict_from_folder(checkpoint_path) | |
| # 提取dwpose_embedding的权重 | |
| state_dict_new = {} | |
| for key in state_dict.keys(): | |
| if "dwpose_embedding" in key: | |
| new_key = key.split("dwpose_embedding.")[1] | |
| new_key = "embedding." + new_key | |
| state_dict_new[new_key] = state_dict[key] | |
| if len(state_dict_new) == 0: | |
| print(f"警告: 在checkpoint中未找到dwpose_embedding的权重,使用随机初始化") | |
| else: | |
| model.load_state_dict(state_dict_new, strict=True) | |
| print(f"成功加载dwpose_embedding权重,共{len(state_dict_new)}个参数") | |
| model = model.to(device).eval() | |
| return model | |
| def parse_video_filename(video_filename): | |
| """ | |
| 解析视频文件名,提取原始pose目录名和segment索引 | |
| 例如: "movies_000005_0000_seg0.mp4" -> ("movies_000005_0000", 0) | |
| """ | |
| # 去掉.mp4后缀 | |
| name = video_filename.replace('.mp4', '') | |
| # 找到最后一个_seg | |
| parts = name.rsplit('_seg', 1) | |
| if len(parts) != 2: | |
| raise ValueError(f"无法解析文件名: {video_filename}") | |
| pose_dir_name = parts[0] | |
| seg_index = int(parts[1]) | |
| return pose_dir_name, seg_index | |
| def get_video_fps(video_path): | |
| """ | |
| 使用ffprobe获取视频帧率 | |
| Args: | |
| video_path: 视频文件路径 | |
| Returns: | |
| float: 视频帧率,获取失败返回None | |
| """ | |
| import subprocess | |
| cmd = [ | |
| 'ffprobe', '-v', 'error', | |
| '-select_streams', 'v:0', | |
| '-show_entries', 'stream=r_frame_rate', | |
| '-of', 'default=noprint_wrappers=1:nokey=1', | |
| str(video_path) | |
| ] | |
| try: | |
| result = subprocess.run(cmd, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, text=True) | |
| fps_str = result.stdout.strip() | |
| # fps_str 格式可能是 "30/1" 或 "30000/1001" | |
| if '/' in fps_str: | |
| num, den = fps_str.split('/') | |
| return float(num) / float(den) | |
| return float(fps_str) | |
| except: | |
| return None | |
| def load_pose_frames_with_sampling(pose_dir, start_time, duration, original_fps, target_fps, target_frames, height=832, width=480, augmentation='none', augmentation_ratio=1.2): | |
| """ | |
| 从pose目录加载指定时间范围的帧,并进行下采样 | |
| Args: | |
| pose_dir: pose数据目录路径 | |
| start_time: 起始时间(秒) | |
| duration: 时长(秒) | |
| original_fps: 原始视频帧率 | |
| target_fps: 目标帧率 | |
| target_frames: 目标帧数 | |
| height: 目标高度 | |
| width: 目标宽度 | |
| augmentation: 数据增强类型,可选 'none', 'fat', 'thin' | |
| - 'fat': 横向拉伸,让人变胖 | |
| - 'thin': 纵向拉伸,让人变瘦 | |
| augmentation_ratio: 拉伸比例,默认1.2表示拉伸20% | |
| Returns: | |
| torch.Tensor: shape [T, 3, H, W], 值范围 [0, 255] | |
| """ | |
| dwpose_path = os.path.join(pose_dir, 'dw_pose_with_foot_wo_face.pkl') | |
| if not os.path.exists(dwpose_path): | |
| raise FileNotFoundError(f"未找到pose文件: {dwpose_path}") | |
| with open(dwpose_path, 'rb') as f: | |
| dwpose_all = pickle.load(f) | |
| keys = list(dwpose_all.keys()) | |
| total_frames = len(keys) | |
| # 计算在原始帧率下的起始帧索引 | |
| start_frame = int(start_time * original_fps) | |
| # 计算采样步长:每隔多少帧取一帧 | |
| # 例如:original_fps=30, target_fps=16 -> step=30/16=1.875 | |
| sample_step = original_fps / target_fps | |
| # 计算需要采样的帧索引 | |
| frame_indices = [] | |
| for i in range(target_frames): | |
| frame_idx = int(start_frame + i * sample_step) | |
| # 确保不超出范围 | |
| if frame_idx >= total_frames: | |
| frame_idx = total_frames - 1 | |
| frame_indices.append(frame_idx) | |
| # 检查起始帧是否有效 | |
| if start_frame >= total_frames: | |
| raise ValueError(f"起始帧{start_frame}超出总帧数{total_frames}") | |
| # 定义最终resize transform | |
| resize_transform = T.Resize((height, width), antialias=True) | |
| dwpose_list = [] | |
| for frame_idx in frame_indices: | |
| key = keys[frame_idx] | |
| pose_img = Image.open(BytesIO(dwpose_all[key])) | |
| if pose_img.mode != 'RGB': | |
| pose_img = pose_img.convert('RGB') | |
| # 获取原始图像尺寸 | |
| orig_w, orig_h = pose_img.size # PIL的size是(width, height) | |
| if augmentation == 'fat': | |
| # fat版本:裁剪掉左右两侧,保留中间部分,再resize回原尺寸 | |
| # 例如ratio=1.2时,保留中间 1/1.2 = 83.3% 的宽度 | |
| crop_w = int(orig_w / augmentation_ratio) | |
| left = (orig_w - crop_w) // 2 | |
| right = left + crop_w | |
| pose_img = pose_img.crop((left, 0, right, orig_h)) # (left, top, right, bottom) | |
| elif augmentation == 'thin': | |
| # thin版本:裁剪掉上下两侧,保留中间部分,再resize回原尺寸 | |
| # 例如ratio=1.2时,保留中间 1/1.2 = 83.3% 的高度 | |
| crop_h = int(orig_h / augmentation_ratio) | |
| top = (orig_h - crop_h) // 2 | |
| bottom = top + crop_h | |
| pose_img = pose_img.crop((0, top, orig_w, bottom)) # (left, top, right, bottom) | |
| # Resize到目标尺寸 | |
| pose_resized = resize_transform(pose_img) | |
| # PIL Image 转换为 numpy array,再转为 tensor | |
| pose_array = np.array(pose_resized, dtype=np.uint8) # [H, W, C] | |
| pose_tensor = torch.from_numpy(pose_array.copy()) # [H, W, C] | |
| pose_tensor = pose_tensor.permute(2, 0, 1) # [C, H, W] | |
| dwpose_list.append(pose_tensor) | |
| # Stack成 [T, C, H, W] | |
| dwpose_tensor = torch.stack(dwpose_list, dim=0) | |
| return dwpose_tensor | |
| def process_single_video(model, video_filename, pose_base_dir, output_dir, | |
| target_fps=16, target_frames=81, height=832, width=480, device='cuda', | |
| pose_video_dir=None, original_video_dir=None, default_original_fps=30.0, | |
| augmentation='none', augmentation_ratio=1.2): | |
| """ | |
| 处理单个视频对应的pose数据 | |
| Args: | |
| model: dwpose_embedding模型 | |
| video_filename: 视频文件名 (如 "movies_000005_0000_seg0.mp4") | |
| pose_base_dir: pose数据根目录 | |
| output_dir: 输出目录 | |
| target_fps: 目标帧率 | |
| target_frames: 每个segment的帧数 | |
| height: 目标高度 | |
| width: 目标宽度 | |
| device: 计算设备 | |
| pose_video_dir: pose视频保存目录,如果为None则不保存pose视频 | |
| original_video_dir: 原始视频目录,用于获取帧率 | |
| default_original_fps: 默认原始视频帧率(当无法获取时使用) | |
| augmentation: 数据增强类型,可选 'none', 'fat', 'thin' | |
| augmentation_ratio: 拉伸比例 | |
| Returns: | |
| bool: 是否成功处理 | |
| """ | |
| try: | |
| # 解析文件名 | |
| pose_dir_name, seg_index = parse_video_filename(video_filename) | |
| # 构建pose目录路径 | |
| pose_dir = os.path.join(pose_base_dir, pose_dir_name) | |
| if not os.path.exists(pose_dir): | |
| print(f"警告: pose目录不存在: {pose_dir}") | |
| return False | |
| # 获取原始视频帧率 | |
| original_fps = default_original_fps | |
| if original_video_dir: | |
| # 原始视频文件名(去掉_segX后缀) | |
| original_video_name = pose_dir_name + '.mp4' | |
| original_video_path = os.path.join(original_video_dir, original_video_name) | |
| if os.path.exists(original_video_path): | |
| detected_fps = get_video_fps(original_video_path) | |
| if detected_fps: | |
| original_fps = detected_fps | |
| # 计算时间范围 | |
| # segment_duration = target_frames / target_fps = 81 / 16 = 5.0625秒 | |
| segment_duration = target_frames / target_fps | |
| start_time = seg_index * segment_duration | |
| # 加载pose帧(带下采样和可选的数据增强) | |
| dwpose_tensor = load_pose_frames_with_sampling( | |
| pose_dir, start_time, segment_duration, | |
| original_fps, target_fps, target_frames, | |
| height, width, augmentation, augmentation_ratio | |
| ) | |
| # 检查帧数是否足够 | |
| actual_frames = dwpose_tensor.shape[0] | |
| if actual_frames < target_frames: | |
| print(f"警告: {video_filename} 帧数不足 ({actual_frames} < {target_frames}),跳过") | |
| return False | |
| # 保存pose视频(如果指定了目录) | |
| if pose_video_dir is not None: | |
| pose_video_path = os.path.join(pose_video_dir, video_filename) | |
| # dwpose_tensor: [T, C, H, W] -> [T, H, W, C] for imageio | |
| video_frames = dwpose_tensor.permute(0, 2, 3, 1).numpy() # [T, H, W, C], uint8 | |
| imageio.mimwrite(pose_video_path, video_frames, fps=target_fps, codec='libx264') | |
| # 准备输入: 需要在时间维度前面复制3帧,变成84帧 | |
| # dwpose_tensor: [81, 3, H, W] | |
| first_frame = dwpose_tensor[:1] # [1, 3, H, W] | |
| first_3_frames = first_frame.repeat(3, 1, 1, 1) # [3, 3, H, W] | |
| dwpose_padded = torch.cat([first_3_frames, dwpose_tensor], dim=0) # [84, 3, H, W] | |
| # 转换为 [1, 3, 84, H, W] 并归一化到 [0, 1] | |
| dwpose_input = dwpose_padded.unsqueeze(0).permute(0, 2, 1, 3, 4).float() / 255.0 | |
| dwpose_input = dwpose_input.to(device) | |
| # 前向推理 | |
| with torch.no_grad(): | |
| embedding = model(dwpose_input) # [1, 32760, 5120] | |
| # 保存结果 | |
| output_filename = video_filename.replace('.mp4', '.pt') | |
| output_path = os.path.join(output_dir, output_filename) | |
| torch.save(embedding.cpu(), output_path) | |
| return True | |
| except Exception as e: | |
| print(f"处理 {video_filename} 时出错: {e}") | |
| return False | |
| def main(): | |
| parser = argparse.ArgumentParser(description="批量提取pose embedding") | |
| parser.add_argument( | |
| "--checkpoint_path", | |
| type=str, | |
| default="/cpfs01/VideoGen/user_data/buwendong/code/UniAnimate-DiT/wan/UniA-20260120-191336/epoch=3-step=820.ckpt/output_dir", | |
| help="训练好的checkpoint路径" | |
| ) | |
| parser.add_argument( | |
| "--video_dir", | |
| type=str, | |
| default="/cpfs01/VideoGen/user_data/buwendong/code/dance_2858_video", | |
| help="2858个视频所在目录" | |
| ) | |
| parser.add_argument( | |
| "--pose_base_dir", | |
| type=str, | |
| default="/cpfs01/VideoGen/user_data/buwendong/code/UniAnimate-DiT/dance_1723_new/training_video_pose", | |
| help="1723个pose数据所在目录" | |
| ) | |
| parser.add_argument( | |
| "--output_dir", | |
| type=str, | |
| default="/cpfs01/VideoGen/user_data/buwendong/code/dance_2858_pose_embed_v2", | |
| help="输出目录" | |
| ) | |
| parser.add_argument( | |
| "--pose_video_dir", | |
| type=str, | |
| default="/cpfs01/VideoGen/user_data/buwendong/code/dance_2858_pose_video_v2", | |
| help="pose视频保存目录,设为空字符串则不保存pose视频" | |
| ) | |
| parser.add_argument( | |
| "--original_video_dir", | |
| type=str, | |
| default="/cpfs01/VideoGen/user_data/buwendong/code/dance_1723_video", | |
| help="原始视频目录,用于获取视频帧率" | |
| ) | |
| parser.add_argument( | |
| "--default_original_fps", | |
| type=float, | |
| default=30.0, | |
| help="默认原始视频帧率(当无法从视频获取时使用)" | |
| ) | |
| parser.add_argument( | |
| "--target_fps", | |
| type=int, | |
| default=16, | |
| help="目标帧率" | |
| ) | |
| parser.add_argument( | |
| "--target_frames", | |
| type=int, | |
| default=81, | |
| help="每个segment的帧数" | |
| ) | |
| parser.add_argument( | |
| "--height", | |
| type=int, | |
| default=832, | |
| help="目标高度" | |
| ) | |
| parser.add_argument( | |
| "--width", | |
| type=int, | |
| default=480, | |
| help="目标宽度" | |
| ) | |
| parser.add_argument( | |
| "--device", | |
| type=str, | |
| default="cuda", | |
| help="计算设备" | |
| ) | |
| parser.add_argument( | |
| "--batch_size", | |
| type=int, | |
| default=1, | |
| help="批处理大小(暂未实现多batch)" | |
| ) | |
| parser.add_argument( | |
| "--augmentation", | |
| type=str, | |
| default="none", | |
| choices=["none", "fat", "thin"], | |
| help="数据增强类型:none=无增强,fat=横向拉伸(人变胖),thin=纵向拉伸(人变瘦)" | |
| ) | |
| parser.add_argument( | |
| "--augmentation_ratio", | |
| type=float, | |
| default=1.2, | |
| help="拉伸比例,默认1.2表示拉伸20%%。比例越大,fat/thin效果越明显" | |
| ) | |
| args = parser.parse_args() | |
| # 创建输出目录 | |
| os.makedirs(args.output_dir, exist_ok=True) | |
| # 处理pose_video_dir参数 | |
| pose_video_dir = args.pose_video_dir if args.pose_video_dir else None | |
| if pose_video_dir: | |
| os.makedirs(pose_video_dir, exist_ok=True) | |
| print(f"pose视频将保存到: {pose_video_dir}") | |
| # 显示数据增强设置 | |
| if args.augmentation != 'none': | |
| print(f"数据增强模式: {args.augmentation}, 拉伸比例: {args.augmentation_ratio}") | |
| else: | |
| print("数据增强模式: 无") | |
| # 加载模型 | |
| print(f"正在加载模型: {args.checkpoint_path}") | |
| model = load_dwpose_embedding(args.checkpoint_path, args.device) | |
| # 获取所有视频文件名 | |
| video_files = [f for f in os.listdir(args.video_dir) if f.endswith('.mp4')] | |
| video_files.sort() | |
| print(f"找到 {len(video_files)} 个视频文件") | |
| # 检查已处理的文件,支持断点续传(同时检查embedding和pose视频) | |
| existing_embed_files = set(f.replace('.pt', '.mp4') for f in os.listdir(args.output_dir) if f.endswith('.pt')) | |
| if pose_video_dir: | |
| existing_pose_videos = set(f for f in os.listdir(pose_video_dir) if f.endswith('.mp4')) | |
| # 只有两者都存在才算处理完成 | |
| existing_files = existing_embed_files & existing_pose_videos | |
| else: | |
| existing_files = existing_embed_files | |
| remaining_files = [f for f in video_files if f not in existing_files] | |
| print(f"已处理 {len(existing_files)} 个,还需处理 {len(remaining_files)} 个") | |
| # 处理每个视频 | |
| success_count = 0 | |
| fail_count = 0 | |
| desc = f"提取pose embedding ({args.augmentation})" if args.augmentation != 'none' else "提取pose embedding" | |
| for video_file in tqdm(remaining_files, desc=desc): | |
| success = process_single_video( | |
| model=model, | |
| video_filename=video_file, | |
| pose_base_dir=args.pose_base_dir, | |
| output_dir=args.output_dir, | |
| target_fps=args.target_fps, | |
| target_frames=args.target_frames, | |
| height=args.height, | |
| width=args.width, | |
| device=args.device, | |
| pose_video_dir=pose_video_dir, | |
| original_video_dir=args.original_video_dir, | |
| default_original_fps=args.default_original_fps, | |
| augmentation=args.augmentation, | |
| augmentation_ratio=args.augmentation_ratio | |
| ) | |
| if success: | |
| success_count += 1 | |
| else: | |
| fail_count += 1 | |
| print(f"\n处理完成!") | |
| print(f"成功: {success_count}") | |
| print(f"失败: {fail_count}") | |
| print(f"总计: {success_count + fail_count}") | |
| if __name__ == "__main__": | |
| main() | |