import sys import pickle import torch import numpy as np from torch import nn import math import torch.nn.functional as F from pytorch3d.io import load_obj import torchvision import os from flame import FLAME_mica, parse_args from utils.general_utils import Pytorch3dRasterizer, Embedder, a_in_b_torch, face_vertices_gen from model_training.model.flame import FLAMELayer, FLAME_CONSTS, FlameParams import time from dataclasses import replace def rot_mat_from_6dof(v: torch.Tensor) -> torch.Tensor: assert v.shape[-1] == 6 v = v.view(-1, 6) vx, vy = v[..., :3].clone(), v[..., 3:].clone() b1 = F.normalize(vx, dim=-1) b3 = F.normalize(torch.cross(b1, vy), dim=-1) b2 = - torch.cross(b1, b3) return torch.stack((b1, b2, b3), dim=-1) class Deform_Model(nn.Module): def __init__(self, device): super().__init__() self.device = device self.batch_size = 1 self.flame_constants = FLAME_CONSTS mica_flame_config = parse_args() # MICA预测 表情参数 目前使用的是dad3dnet预测 # self.flame_model = FLAME_mica(mica_flame_config).to(self.device) self.default_shape_code = torch.zeros(1, 300) self.default_expr_code = torch.zeros(1, 100) # positional encoding self.pts_freq = 8 self.pts_embedder = Embedder(self.pts_freq) # verts[5023, 3] verts, faces, aux = load_obj('flame/FlameMesh.obj', load_textures=False) # faces 4 # faces.verts_idx # (10006, 3),每个三角面对应 verts 里的 3 个顶点索引 # faces.textures_idx # (10006, 3),每个三角面对应 aux.verts_uvs 里的 3 个 UV 索引 # faces.normals_idx # (10006, 3),每个三角面对应 aux.normals 的法线索引 # faces.materials_idx # (10006, ), 每个 face 对应的材质索引 # aux # aux.verts_uvs # (5150, 2),UV 坐标 # aux.normals # (9317, 3),法线 # aux.material_colors # None 材质颜色 # aux.texture_images # None 贴图图片 # aux.texture_atlas # None texture atlas uv_coords = aux.verts_uvs[None, ...] # (1, 5150, 2) uv_coords = uv_coords * 2 - 1 uv_coords[..., 1] = - uv_coords[..., 1] self.uvcoords = torch.cat([uv_coords, uv_coords[:, :, 0:1] * 0. + 1.], -1).to(self.device) self.uvfaces = faces.textures_idx[None, ...].to(self.device) # (1, 10006, 3) self.tri_faces = faces.verts_idx[None, ...].to(self.device) # (1, 10006, 3) # rasterizer self.uv_size = 512 self.uv_rasterizer = Pytorch3dRasterizer(self.uv_size) # flame mask full_id = np.array(range(5023)).astype(int) neckhead_id_list = list(set(full_id)) self.neckhead_id_list = neckhead_id_list self.neckhead_id_tensor = torch.tensor(self.neckhead_id_list, dtype=torch.int64).to(self.device) # 新增:光栅化缓存标志和缓存数据 self.rasterization_cached = False self.cached_pix_to_face_ori = None # (237544, 512, 512, 1) 表示 UV 图上每个像素落在哪个三角面上 self.cached_bary_coords = None # (B, 512, 512, 1, 3) 每个有效像素在对应三角面内的重心坐标权重,后面用它把 UV 像素插值回 3D 面片上的点。 self.cached_uvmask_flaten_idx = None # (512 * 512) 缓存有效 UV 像素 mask self.cached_pix_to_face = None # (N_valid,) 缓存只保留有效 UV 像素后的 face index 每个有效 UV 像素对应哪个三角面 self.cached_pix_to_v_idx = None # (N_valid, 3) 缓存每个有效 UV 像素对应三角面的 3 个 3D 顶点索引 self.cached_uv_vertices_shape = None # (1, N_valid, 3) 缓存 canonical/default shape 下,每个有效 UV 像素插值出来的 3D 点。 self.cached_uv_vertices_shape_embeded = None # (1, N_valid, embed_dim) 缓存对 uv_vertices_shape 做 positional encoding 后的结果 self.cached_v_num = None # 缓存有效点数量 N_valid self.cached_uv_head_idx = None # (N_valid,) 缓存最终用于筛选点的 mask # 同时清除FLAME缓存 self.cached_flame_geometry = None self.cached_flame_params = None self.init_networks() def init_networks(self): ## full mica self.deformNet = MLP( input_dim=self.pts_embedder.dim_embeded+100+64,#+64,#74 output_dim=10, hidden_dim=256, hidden_layers=6 ) def example_init(self, codedict, flame): # speed up # print(f"[DEBUG] example_init called with codedict shape: {codedict.shape if hasattr(codedict, 'shape') else 'N/A'}", flush=True) # 检查是否已经缓存了光栅化结果 if self.rasterization_cached: # print(f"[INFO] Using cached rasterization results - skipping expensive UV rasterization", flush=True) # 恢复缓存的光栅化结果 self.pix_to_face_ori = self.cached_pix_to_face_ori self.bary_coords = self.cached_bary_coords self.uvmask_flaten_idx = self.cached_uvmask_flaten_idx self.pix_to_face = self.cached_pix_to_face self.pix_to_v_idx = self.cached_pix_to_v_idx self.uv_vertices_shape = self.cached_uv_vertices_shape self.uv_vertices_shape_embeded = self.cached_uv_vertices_shape_embeded self.v_num = self.cached_v_num self.uv_head_idx = self.cached_uv_head_idx # print(f"[INFO] Cached rasterization results restored successfully", flush=True) return # print(f"[INFO] First time execution - performing full UV rasterization", flush=True) # 总体计时开始 # torch.cuda.synchronize() # total_start_time = time.time() shape_code = codedict.shape.detach() batch_size = shape_code.shape[0] # 步骤1: FLAME模型前向传播 # torch.cuda.synchronize() # step1_start = time.time() # 深引用版本(新) temp_codedict = replace(codedict, expression=self.default_expr_code) # print(f"[DEBUG] About to call flame.forward", flush=True) geometry_shape = flame.forward(temp_codedict, zero_rot=True).cuda() # 浅引用版本(原来的) # codedict.expression = self.default_expr_code # # print(f"[DEBUG] About to call flame.forward", flush=True) # geometry_shape = flame.forward(codedict, zero_rot=True).cuda() # torch.cuda.synchronize() # step1_time = time.time() - step1_start # print(f"[TIMING] Step 1 - FLAME forward: {step1_time:.4f} seconds", flush=True) # print(f"[DEBUG] flame.forward completed, geometry_shape shape: {geometry_shape.shape}", flush=True) # 步骤2: 生成面部顶点 # torch.cuda.synchronize() # step2_start = time.time() # self.tri_faces是flame 的三角面索引 # 得到每个三角面片的顶点索引 face_vertices_shape = face_vertices_gen(geometry_shape, self.tri_faces.expand(batch_size, -1, -1)) # torch.cuda.synchronize() # step2_time = time.time() - step2_start # print(f"[TIMING] Step 2 - Face vertices generation: {step2_time:.4f} seconds", flush=True) # print(f"[DEBUG] face_vertices_shape created, shape: {face_vertices_shape.shape}", flush=True) # 步骤3: UV光栅化 # torch.cuda.synchronize() # step3_start = time.time() rast_out, pix_to_face, bary_coords = self.uv_rasterizer(self.uvcoords.expand(batch_size, -1, -1), self.uvfaces.expand(batch_size, -1, -1), face_vertices_shape) # torch.cuda.synchronize() # step3_time = time.time() - step3_start # print(f"[TIMING] Step 3 - UV rasterization: {step3_time:.4f} seconds", flush=True) # print(f"[DEBUG] UV rasterization completed", flush=True) # 步骤4: 数据预处理和索引计算 # torch.cuda.synchronize() # step4_start = time.time() self.pix_to_face_ori = pix_to_face self.bary_coords = bary_coords uvmask = rast_out[:, -1].unsqueeze(1) uvmask_flaten = uvmask[0].view(uvmask.shape[1], -1).permute(1, 0).squeeze(1) # batch=1 self.uvmask_flaten_idx = (uvmask_flaten[:]>0) pix_to_face_flaten = pix_to_face[0].clone().view(-1) # batch=1 self.pix_to_face = pix_to_face_flaten[self.uvmask_flaten_idx] # pix to face idx self.pix_to_v_idx = self.tri_faces[0, self.pix_to_face, :] # pix to vert idx uv_vertices_shape = rast_out[:, :3] uv_vertices_shape_flaten = uv_vertices_shape[0].view(uv_vertices_shape.shape[1], -1).permute(1, 0) # batch=1 uv_vertices_shape = uv_vertices_shape_flaten[self.uvmask_flaten_idx].unsqueeze(0) self.uv_vertices_shape = uv_vertices_shape self.uv_vertices_shape_embeded = self.pts_embedder(self.uv_vertices_shape) self.v_num = self.uv_vertices_shape_embeded.shape[1] # mask self.uv_head_idx = ( a_in_b_torch(self.pix_to_v_idx[:,0], self.neckhead_id_tensor) & a_in_b_torch(self.pix_to_v_idx[:,1], self.neckhead_id_tensor) & a_in_b_torch(self.pix_to_v_idx[:,2], self.neckhead_id_tensor) ) # 缓存光栅化结果 self.cached_pix_to_face_ori = self.pix_to_face_ori.clone() self.cached_bary_coords = self.bary_coords.clone() self.cached_uvmask_flaten_idx = self.uvmask_flaten_idx.clone() self.cached_pix_to_face = self.pix_to_face.clone() self.cached_pix_to_v_idx = self.pix_to_v_idx.clone() self.cached_uv_vertices_shape = self.uv_vertices_shape.clone() self.cached_uv_vertices_shape_embeded = self.uv_vertices_shape_embeded.clone() self.cached_v_num = self.v_num self.cached_uv_head_idx = self.uv_head_idx.clone() # 标记光栅化已完成缓存 self.rasterization_cached = True # print(f"[INFO] Rasterization results cached successfully", flush=True) # print(f"[DEBUG] example_init completed successfully", flush=True) def decode(self, codedict, flame, feature_flame, target_rotation=None): shape_code = codedict[1]['flame_params'].shape.detach() expr_code = codedict[1]['flame_params'].expression.detach() # jaw_pose = codedict['jaw_pose'].detach() # neck_pose = codedict['neck_pose'].detach() # eyes_pose = codedict['eyes_pose'].detach() # person_code = codedict['person_code'].detach() batch_size = shape_code.shape[0] # condition = torch.cat((expr_code, jaw_pose, eyes_pose, neck_pose), dim=1) # condition = expr_code.cuda() condition = torch.cat((expr_code.cuda(), feature_flame), dim=1) # MLP condition = condition.unsqueeze(1).repeat(1, self.v_num, 1) uv_vertices_shape_embeded_condition = torch.cat((self.uv_vertices_shape_embeded, condition), dim=2) deforms = self.deformNet(uv_vertices_shape_embeded_condition) deforms = torch.tanh(deforms) uv_vertices_deforms = 2.0 * deforms[..., :3] rot_delta_0 = deforms[..., 3:7] rot_delta_r = torch.exp(rot_delta_0[..., 0]).unsqueeze(-1) rot_delta_v = rot_delta_0[..., 1:] rot_delta = torch.cat((rot_delta_r, rot_delta_v), dim=-1) scale_coef = deforms[..., 7:] scale_coef = torch.exp(scale_coef) geometry = flame.forward(codedict[1]['flame_params'], zero_rot=True).cuda() face_vertices = face_vertices_gen(geometry, self.tri_faces.expand(batch_size, -1, -1)) # rasterize face_vertices to uv space D = face_vertices.shape[-1] # 3 attributes = face_vertices.clone() attributes = attributes.view(attributes.shape[0] * attributes.shape[1], 3, attributes.shape[-1]) N, H, W, K, _ = self.bary_coords.shape idx = self.pix_to_face_ori.clone().view(N * H * W * K, 1, 1).expand(N * H * W * K, 3, D) pixel_face_vals = attributes.gather(0, idx).view(N, H, W, K, 3, D) pixel_vals = (self.bary_coords[..., None] * pixel_face_vals).sum(dim=-2) uv_vertices = pixel_vals[:, :, :, 0].permute(0, 3, 1, 2) uv_vertices_flaten = uv_vertices[0].view(uv_vertices.shape[1], -1).permute(1, 0) # batch=1 uv_vertices = uv_vertices_flaten[self.uvmask_flaten_idx].unsqueeze(0) # 获取z坐标 x_coords = uv_vertices[:,:,0] # y_coords = uv_vertices[:,:,1] z_coords = uv_vertices[:,:,2] tan_xy = torch.abs(x_coords)/torch.abs((z_coords+0.01))#0.2 # 找到z坐标小于0的点的索引 indices = torch.where((z_coords >= -0.01) | (tan_xy >= 1.732)) verts_final = uv_vertices + uv_vertices_deforms # conduct mask verts_final = verts_final[:, self.uv_head_idx, :] rot_delta = rot_delta[:, self.uv_head_idx, :] scale_coef = scale_coef[:, self.uv_head_idx, :] verts_flame = uv_vertices[:, self.uv_head_idx, :] # # 根据索引提取满足条件的点 # verts_final = verts_final[:, indices[1], :] # verts_flame = verts_flame[:, indices[1], :] # rotation_mat = rot_mat_from_6dof(codedict[1]['flame_params'].rotation).type(verts_final.dtype).cuda() # vertices = torch.matmul(rotation_mat.unsqueeze(1), verts_final.unsqueeze(-1)) # vertices = vertices[..., 0] scale = torch.clamp(codedict[1]['flame_params'].scale[:, None] + 1.0, 1e-8).cuda().squeeze() # vertices *= scale # [B, 1, 1] # translation = codedict[1]['flame_params'].translation.cuda() # translation[..., 2] = 0.0 # vertices += translation[:, None] # [B, 1, 3] # vertices[..., 2] += 2183.5264 vertices = scale * verts_final # 计算偏移量的L2损失,使形变保持在合理范围 lambda_l2 = 0.01 # 降低L2正则化系数,让形变有更大的自由度 l2_loss = lambda_l2 * torch.norm(uv_vertices_deforms, p=2) # L2范数 return vertices, rot_delta, scale_coef, verts_flame, verts_final, l2_loss, scale def decode_single(self, codedict, flame, feature_flame): shape_code = codedict.shape.detach() expr_code = codedict.expression.detach() # jaw_pose = codedict['jaw_pose'].detach() # neck_pose = codedict['neck_pose'].detach() # eyes_pose = codedict['eyes_pose'].detach() # person_code = codedict['person_code'].detach() batch_size = shape_code.shape[0] # condition = torch.cat((expr_code, jaw_pose, eyes_pose, neck_pose), dim=1) # condition = expr_code.cuda() condition = torch.cat((expr_code.cuda(), feature_flame), dim=1) # MLP condition = condition.unsqueeze(1).repeat(1, self.v_num, 1) uv_vertices_shape_embeded_condition = torch.cat((self.uv_vertices_shape_embeded, condition), dim=2) deforms = self.deformNet(uv_vertices_shape_embeded_condition) deforms = torch.tanh(deforms) uv_vertices_deforms = deforms[..., :3] rot_delta_0 = deforms[..., 3:7] rot_delta_r = torch.exp(rot_delta_0[..., 0]).unsqueeze(-1) rot_delta_v = rot_delta_0[..., 1:] rot_delta = torch.cat((rot_delta_r, rot_delta_v), dim=-1) scale_coef = deforms[..., 7:] scale_coef = torch.exp(scale_coef) geometry = flame.forward(codedict, zero_rot=True).cuda() face_vertices = face_vertices_gen(geometry, self.tri_faces.expand(batch_size, -1, -1)) # rasterize face_vertices to uv space D = face_vertices.shape[-1] # 3 attributes = face_vertices.clone() attributes = attributes.view(attributes.shape[0] * attributes.shape[1], 3, attributes.shape[-1]) N, H, W, K, _ = self.bary_coords.shape idx = self.pix_to_face_ori.clone().view(N * H * W * K, 1, 1).expand(N * H * W * K, 3, D) pixel_face_vals = attributes.gather(0, idx).view(N, H, W, K, 3, D) pixel_vals = (self.bary_coords[..., None] * pixel_face_vals).sum(dim=-2) uv_vertices = pixel_vals[:, :, :, 0].permute(0, 3, 1, 2) uv_vertices_flaten = uv_vertices[0].view(uv_vertices.shape[1], -1).permute(1, 0) # batch=1 uv_vertices = uv_vertices_flaten[self.uvmask_flaten_idx].unsqueeze(0) # 获取z坐标 x_coords = uv_vertices[:,:,0] # y_coords = uv_vertices[:,:,1] z_coords = uv_vertices[:,:,2] tan_xy = torch.abs(x_coords)/torch.abs((z_coords+0.01))#0.2 # 找到z坐标小于0的点的索引 indices = torch.where((z_coords >= -0.01) | (tan_xy >= 1.732)) verts_final = uv_vertices + uv_vertices_deforms # conduct mask verts_final = verts_final[:, self.uv_head_idx, :] rot_delta = rot_delta[:, self.uv_head_idx, :] scale_coef = scale_coef[:, self.uv_head_idx, :] verts_flame = uv_vertices[:, self.uv_head_idx, :] # # 根据索引提取满足条件的点 # verts_final = verts_final[:, indices[1], :] # verts_flame = verts_flame[:, indices[1], :] # rotation_mat = rot_mat_from_6dof(codedict[1]['flame_params'].rotation).type(verts_final.dtype).cuda() # vertices = torch.matmul(rotation_mat.unsqueeze(1), verts_final.unsqueeze(-1)) # vertices = vertices[..., 0] scale = torch.clamp(codedict.scale[:, None] + 1.0, 1e-8).cuda().squeeze() # vertices *= scale # [B, 1, 1] # translation = codedict[1]['flame_params'].translation.cuda() # translation[..., 2] = 0.0 # vertices += translation[:, None] # [B, 1, 3] # vertices[..., 2] += 2183.5264 vertices = scale * verts_final # 计算偏移量的L2损失,使形变保持在合理范围 lambda_l2 = 0.03 # 降低L2正则化系数,让形变有更大的自由度 l2_loss = lambda_l2 * torch.norm(uv_vertices_deforms, p=2) # L2范数 return vertices, rot_delta, scale_coef, verts_flame, verts_final, l2_loss, scale def decode_single3(self, codedict, flame, feature_flame): shape_code = codedict.shape.detach() expr_code = codedict.expression.detach() batch_size = shape_code.shape[0] # 表情编码 + 特征 condition = torch.cat((expr_code.cuda(), feature_flame), dim=1) condition = condition.unsqueeze(1).repeat(1, self.v_num, 1) uv_vertices_shape_embeded_condition = torch.cat((self.uv_vertices_shape_embeded, condition), dim=2) # MLP 解码器 deforms = self.deformNet(uv_vertices_shape_embeded_condition) deforms = torch.tanh(deforms) # 只使用偏移信息 uv_vertices_deforms = deforms[..., :3] # 获取原始 flame 网格 geometry = flame.forward(codedict, zero_rot=True).cuda() face_vertices = face_vertices_gen(geometry, self.tri_faces.expand(batch_size, -1, -1)) # rasterize 到 UV 空间 D = face_vertices.shape[-1] attributes = face_vertices.view(-1, 3, D) N, H, W, K, _ = self.bary_coords.shape idx = self.pix_to_face_ori.clone().view(N * H * W * K, 1, 1).expand(-1, 3, D) pixel_face_vals = attributes.gather(0, idx).view(N, H, W, K, 3, D) pixel_vals = (self.bary_coords[..., None] * pixel_face_vals).sum(dim=-2) uv_vertices = pixel_vals[:, :, :, 0].permute(0, 3, 1, 2) uv_vertices_flaten = uv_vertices[0].view(uv_vertices.shape[1], -1).permute(1, 0) uv_vertices = uv_vertices_flaten[self.uvmask_flaten_idx].unsqueeze(0) # 形变 + mask 头部区域 verts_final = uv_vertices + uv_vertices_deforms # 计算缩放并应用(z 方向不加) scale = torch.clamp(codedict.scale[:, None] + 1.0, 1e-8).cuda().squeeze() vertices = scale * verts_final return vertices, verts_final, scale def decode_single2(self, codedict, flame, feature_flame): # 总体计时开始 # torch.cuda.synchronize() # total_start_time = time.time() shape_code = codedict.shape.detach() expr_code = codedict.expression.detach() batch_size = shape_code.shape[0] # 步骤1: 表情编码和特征处理 # torch.cuda.synchronize() # step1_start = time.time() condition = torch.cat((expr_code.cuda(), feature_flame), dim=1) condition = condition.unsqueeze(1).repeat(1, self.v_num, 1) uv_vertices_shape_embeded_condition = torch.cat((self.uv_vertices_shape_embeded, condition), dim=2) # torch.cuda.synchronize() # step1_time = time.time() - step1_start # print(f"[TIMING] decode_single2 Step 1 - Feature processing: {step1_time:.4f} seconds", flush=True) # # 步骤2: MLP解码器 # torch.cuda.synchronize() # step2_start = time.time() deforms = self.deformNet(uv_vertices_shape_embeded_condition) deforms = torch.tanh(deforms) uv_vertices_deforms = deforms[..., :3] # torch.cuda.synchronize() # step2_time = time.time() - step2_start # print(f"[TIMING] decode_single2 Step 2 - MLP inference: {step2_time:.4f} seconds", flush=True) # # 步骤3: FLAME几何计算(带缓存) # torch.cuda.synchronize() # step3_start = time.time() if not hasattr(self, 'cached_flame_geometry') or self.cached_flame_geometry is None: # print(f"[INFO] Computing FLAME geometry (first time)", flush=True) geometry = flame.forward(codedict, zero_rot=True).cuda() # 缓存FLAME几何结果 self.cached_flame_geometry = geometry self.cached_flame_params = { 'shape': codedict.shape.clone(), 'expression': codedict.expression.clone() } else: # 检查参数是否变化 # 在缓存参数时确保它们在GPU上 self.cached_flame_params['expression'] = self.cached_flame_params['expression'].to(codedict.expression.device) shape_changed = torch.norm(codedict.shape - self.cached_flame_params['shape']) > 1e-6 expr_changed = torch.norm(codedict.expression - self.cached_flame_params['expression']) > 1e-6 if shape_changed or expr_changed: # print(f"[INFO] FLAME parameters changed, recomputing geometry", flush=True) geometry = flame.forward(codedict, zero_rot=True).cuda() self.cached_flame_geometry = geometry self.cached_flame_params['shape'] = codedict.shape.clone() self.cached_flame_params['expression'] = codedict.expression.clone() else: # print(f"[INFO] Using cached FLAME geometry", flush=True) geometry = self.cached_flame_geometry face_vertices = face_vertices_gen(geometry, self.tri_faces.expand(batch_size, -1, -1)) # torch.cuda.synchronize() # step3_time = time.time() - step3_start # print(f"[TIMING] decode_single2 Step 3 - FLAME geometry: {step3_time:.4f} seconds", flush=True) # 步骤4: UV处理(带缓存优化) # torch.cuda.synchronize() # step4_start = time.time() if hasattr(self, 'uv_vertices_shape') and self.uv_vertices_shape is not None: # print(f"[INFO] Using cached UV vertices from example_init", flush=True) uv_vertices = self.uv_vertices_shape else: # print(f"[INFO] Computing UV rasterization (fallback)", flush=True) # rasterize 到 UV 空间 D = face_vertices.shape[-1] attributes = face_vertices.view(-1, 3, D) N, H, W, K, _ = self.bary_coords.shape idx = self.pix_to_face_ori.clone().view(N * H * W * K, 1, 1).expand(-1, 3, D) pixel_face_vals = attributes.gather(0, idx).view(N, H, W, K, 3, D) pixel_vals = (self.bary_coords[..., None] * pixel_face_vals).sum(dim=-2) uv_vertices = pixel_vals[:, :, :, 0].permute(0, 3, 1, 2) uv_vertices_flaten = uv_vertices[0].view(uv_vertices.shape[1], -1).permute(1, 0) uv_vertices = uv_vertices_flaten[self.uvmask_flaten_idx].unsqueeze(0) # torch.cuda.synchronize() # step4_time = time.time() - step4_start # print(f"[TIMING] decode_single2 Step 4 - UV processing: {step4_time:.4f} seconds", flush=True) # # 步骤5: 最终计算 # torch.cuda.synchronize() # step5_start = time.time() # 形变 + mask 头部区域 verts_final = uv_vertices + uv_vertices_deforms # 计算缩放并应用(z 方向不加) scale = torch.clamp(codedict.scale[:, None] + 1.0, 1e-8).cuda().squeeze() vertices = scale * verts_final torch.cuda.synchronize() # step5_time = time.time() - step5_start # print(f"[TIMING] decode_single2 Step 5 - Final computation: {step5_time:.4f} seconds", flush=True) # # 总体计时结束 # torch.cuda.synchronize() # total_time = time.time() - total_start_time # print(f"[TIMING] decode_single2 Total time: {total_time:.4f} seconds", flush=True) # print(f"[TIMING] decode_single2 Breakdown: Features({step1_time:.4f}s) + MLP({step2_time:.4f}s) + FLAME({step3_time:.4f}s) + UV({step4_time:.4f}s) + Final({step5_time:.4f}s)", flush=True) return vertices, verts_final, scale def capture(self): return ( self.deformNet.state_dict(), self.optimizer.state_dict(), ) def restore(self, model_args): (net_dict, opt_dict) = model_args self.deformNet.load_state_dict(net_dict) self.training_setup() self.optimizer.load_state_dict(opt_dict) def training_setup(self): params_group = [ {'params': self.deformNet.parameters(), 'lr': 1e-4}, ] self.optimizer = torch.optim.Adam(params_group, betas=(0.9, 0.999)) def get_template(self): geometry_template = self.flame_model.forward_geo( self.default_shape_code, self.default_expr_code, self.default_jaw_pose, eye_pose_params=self.default_eyes_pose, ) return geometry_template class MLP(nn.Module): def __init__(self, input_dim, output_dim, hidden_dim=256, hidden_layers=8): super().__init__() self.hidden_dim = hidden_dim self.hidden_layers = hidden_layers self.input_dim = input_dim self.output_dim = output_dim self.fcs = nn.ModuleList( [nn.Linear(input_dim, hidden_dim)] + [nn.Linear(hidden_dim, hidden_dim) for i in range(hidden_layers-1)] ) self.output_linear = nn.Linear(hidden_dim, output_dim) def forward(self, input): # input: B,V,d batch_size, N_v, input_dim = input.shape input_ori = input.reshape(batch_size*N_v, -1) h = input_ori for i, l in enumerate(self.fcs): h = self.fcs[i](h) h = F.relu(h) output = self.output_linear(h) output = output.reshape(batch_size, N_v, -1) return output class MLP2(nn.Module): def __init__(self, input_dim, condition_dim, output_dim1, output_dim2, hidden_dim=256, hidden_layers=8): super().__init__() self.hidden_dim = hidden_dim self.condition_dim = condition_dim self.hidden_layers = hidden_layers mid_layers = math.ceil(hidden_layers/2.) self.input_dim = input_dim self.output_dim1 = output_dim1 self.output_dim2 = output_dim2 self.fcs1 = nn.ModuleList( [nn.Linear(input_dim, hidden_dim)] + [nn.Linear(hidden_dim, hidden_dim) if i!=(mid_layers-1) else nn.Linear(hidden_dim, hidden_dim+output_dim1) for i in range(mid_layers)] ) self.fcs2 = nn.ModuleList( [nn.Linear(hidden_dim, hidden_dim) if i!=0 else nn.Linear(hidden_dim+condition_dim, hidden_dim) for i in range(mid_layers, hidden_layers-1)] ) self.output_linear = nn.Linear(hidden_dim, output_dim2) def forward(self, input, condition): # input: B,V,d1 # condition: B,d2 batch_size, N_v, input_dim = input.shape input_ori = input.reshape(batch_size*N_v, -1) h = input_ori for i, l in enumerate(self.fcs1): h = self.fcs1[i](h) h = F.relu(h) oup1 = h[:, -self.output_dim1:] h = h[:, :-self.output_dim1] ... for i, l in enumerate(self.fcs2): h = self.fcs1[i](h) h = F.relu(h) # input_ori = input.reshape(batch_size*N_v, -1) # h = input_ori # for i, l in enumerate(self.fcs): # h = self.fcs[i](h) # h = F.relu(h) # output = self.output_linear(h) # output = output.reshape(batch_size, N_v, -1) class SIRENMLP(nn.Module): def __init__(self, input_dim=3, output_dim=3, hidden_dim=256, hidden_layers=8, condition_dim=100, device=None): super().__init__() self.device = device self.input_dim = input_dim self.z_dim = condition_dim self.hidden_layers = hidden_layers self.hidden_dim = hidden_dim self.output_dim = output_dim self.network = nn.ModuleList( [FiLMLayer(self.input_dim, self.hidden_dim)] + [FiLMLayer(self.hidden_dim, self.hidden_dim) for i in range(self.hidden_layers - 1)] ) self.final_layer = nn.Linear(self.hidden_dim, self.output_dim) self.mapping_network = MappingNetwork(condition_dim, 256, len(self.network) * self.hidden_dim * 2) self.network.apply(frequency_init(25)) # self.final_layer.apply(frequency_init(25)) self.final_layer.weight.data.normal_(0.0, 0.) self.final_layer.bias.data.fill_(0.) self.network[0].apply(first_layer_film_sine_init) def forward_vector(self, input, z): frequencies, phase_shifts = self.mapping_network(z) return self.forward_with_frequencies_phase_shifts(input, frequencies, phase_shifts) def forward_with_frequencies_phase_shifts(self, input, frequencies, phase_shifts): frequencies = frequencies * 15 + 30 x = input for index, layer in enumerate(self.network): start = index * self.hidden_dim end = (index + 1) * self.hidden_dim x = layer(x, frequencies[..., start:end], phase_shifts[..., start:end]) sigma = self.final_layer(x) return sigma def forward(self, vertices, additional_conditioning): # vertices: in canonical space of flame # vertex eval: torch.Size([N, V, 3]) # map eval: torch.Size([N, 3, H, W]) # conditioning # torch.Size([N, C]) # vertex inputs (N, V, 3) -> (N, 3, V, 1) vertices = vertices.permute(0, 2, 1)[:, :, :, None] b, c, h, w = vertices.shape # to vector x = vertices.permute(0, 2, 3, 1).reshape(b, -1, c) z = additional_conditioning # .repeat(1, h*w, 1) # apply siren network o = self.forward_vector(x, z) # to image o = o.reshape(b, h, w, self.output_dim) output = o.permute(0, 3, 1, 2) return output[:, :, :, 0].permute(0, 2, 1) class FiLMLayer(nn.Module): def __init__(self, input_dim, hidden_dim): super().__init__() self.layer = nn.Linear(input_dim, hidden_dim) def forward(self, x, freq, phase_shift, ignore_conditions=None): x = self.layer(x) if ignore_conditions is not None: cond_freq, cond_phase_shift = freq[:-1], phase_shift[:-1] cond_freq = cond_freq.unsqueeze(1).expand_as(x).clone() cond_phase_shift = cond_phase_shift.unsqueeze(1).expand_as(x).clone() ignore_freq, ignore_phase_shift = freq[-1:], phase_shift[-1:] ignore_freq = ignore_freq.unsqueeze(1).expand_as(x) ignore_phase_shift = ignore_phase_shift.unsqueeze(1).expand_as(x) cond_freq[:, ignore_conditions] = ignore_freq[:, ignore_conditions] cond_phase_shift[:, ignore_conditions] = ignore_phase_shift[:, ignore_conditions] freq, phase_shift = cond_freq, cond_phase_shift else: freq = freq.unsqueeze(1).expand_as(x) phase_shift = phase_shift.unsqueeze(1).expand_as(x) # print('x', x.shape) # print('freq', freq.shape) # print('phase_shift', phase_shift.shape) # x torch.Size([6, 5023, 256]) # freq torch.Size([6, 5023, 256]) # phase_shift torch.Size([6, 5023, 256]) return torch.sin(freq * x + phase_shift) def frequency_init(freq): def init(m): with torch.no_grad(): if isinstance(m, nn.Linear): num_input = m.weight.size(-1) m.weight.uniform_(-np.sqrt(6 / num_input) / freq, np.sqrt(6 / num_input) / freq) elif isinstance(m, nn.Conv2d): num_input = torch.prod( torch.tensor(m.weight.shape[1:], device=m.weight.device)).cpu().item() m.weight.uniform_(-np.sqrt(6 / num_input) / freq, np.sqrt(6 / num_input) / freq) return init class MappingNetwork(nn.Module): def __init__(self, z_dim, map_hidden_dim, map_output_dim): super().__init__() self.network = nn.Sequential(nn.Linear(z_dim, map_hidden_dim), nn.LeakyReLU(0.2, inplace=True), nn.Linear(map_hidden_dim, map_hidden_dim), nn.LeakyReLU(0.2, inplace=True), nn.Linear(map_hidden_dim, map_hidden_dim), nn.LeakyReLU(0.2, inplace=True), nn.Linear(map_hidden_dim, map_output_dim)) self.network.apply(kaiming_leaky_init) with torch.no_grad(): self.network[-1].weight *= 0.25 def forward(self, z): frequencies_offsets = self.network(z) frequencies = frequencies_offsets[..., :frequencies_offsets.shape[-1] // 2] phase_shifts = frequencies_offsets[..., frequencies_offsets.shape[-1] // 2:] return frequencies, phase_shifts def kaiming_leaky_init(m): classname = m.__class__.__name__ if classname.find('Linear') != -1: torch.nn.init.kaiming_normal_(m.weight, a=0.2, mode='fan_in', nonlinearity='leaky_relu') def first_layer_film_sine_init(m): with torch.no_grad(): if isinstance(m, nn.Linear): num_input = m.weight.size(-1) m.weight.uniform_(-1 / num_input, 1 / num_input) elif isinstance(m, nn.Conv2d): num_input = torch.prod( torch.tensor(m.weight.shape[1:], device=m.weight.device)).cpu().item() m.weight.uniform_(-1 / num_input, 1 / num_input)