gha / src /deform_model.py
vegagoto123's picture
Add files using upload-large-folder tool
4e3ea85 verified
Raw History Blame Contribute Delete
35.6 kB
import sys
import pickle
import torch
import numpy as np
from torch import nn
import math
import torch.nn.functional as F
from pytorch3d.io import load_obj
import torchvision
import os
from flame import FLAME_mica, parse_args
from utils.general_utils import Pytorch3dRasterizer, Embedder, a_in_b_torch, face_vertices_gen
from model_training.model.flame import FLAMELayer, FLAME_CONSTS, FlameParams
import time
from dataclasses import replace
def rot_mat_from_6dof(v: torch.Tensor) -> torch.Tensor:
assert v.shape[-1] == 6
v = v.view(-1, 6)
vx, vy = v[..., :3].clone(), v[..., 3:].clone()
b1 = F.normalize(vx, dim=-1)
b3 = F.normalize(torch.cross(b1, vy), dim=-1)
b2 = - torch.cross(b1, b3)
return torch.stack((b1, b2, b3), dim=-1)
class Deform_Model(nn.Module):
def __init__(self, device):
super().__init__()
self.device = device
self.batch_size = 1
self.flame_constants = FLAME_CONSTS
mica_flame_config = parse_args()
# MICA预测 表情参数 目前使用的是dad3dnet预测
# self.flame_model = FLAME_mica(mica_flame_config).to(self.device)
self.default_shape_code = torch.zeros(1, 300)
self.default_expr_code = torch.zeros(1, 100)
# positional encoding
self.pts_freq = 8
self.pts_embedder = Embedder(self.pts_freq)
# verts[5023, 3]
verts, faces, aux = load_obj('flame/FlameMesh.obj', load_textures=False)
# faces 4
# faces.verts_idx # (10006, 3),每个三角面对应 verts 里的 3 个顶点索引
# faces.textures_idx # (10006, 3),每个三角面对应 aux.verts_uvs 里的 3 个 UV 索引
# faces.normals_idx # (10006, 3),每个三角面对应 aux.normals 的法线索引
# faces.materials_idx # (10006, ), 每个 face 对应的材质索引
# aux
# aux.verts_uvs # (5150, 2),UV 坐标
# aux.normals # (9317, 3),法线
# aux.material_colors # None 材质颜色
# aux.texture_images # None 贴图图片
# aux.texture_atlas # None texture atlas
uv_coords = aux.verts_uvs[None, ...] # (1, 5150, 2)
uv_coords = uv_coords * 2 - 1
uv_coords[..., 1] = - uv_coords[..., 1]
self.uvcoords = torch.cat([uv_coords, uv_coords[:, :, 0:1] * 0. + 1.], -1).to(self.device)
self.uvfaces = faces.textures_idx[None, ...].to(self.device) # (1, 10006, 3)
self.tri_faces = faces.verts_idx[None, ...].to(self.device) # (1, 10006, 3)
# rasterizer
self.uv_size = 512
self.uv_rasterizer = Pytorch3dRasterizer(self.uv_size)
# flame mask
full_id = np.array(range(5023)).astype(int)
neckhead_id_list = list(set(full_id))
self.neckhead_id_list = neckhead_id_list
self.neckhead_id_tensor = torch.tensor(self.neckhead_id_list, dtype=torch.int64).to(self.device)
# 新增:光栅化缓存标志和缓存数据
self.rasterization_cached = False
self.cached_pix_to_face_ori = None # (237544, 512, 512, 1) 表示 UV 图上每个像素落在哪个三角面上
self.cached_bary_coords = None # (B, 512, 512, 1, 3) 每个有效像素在对应三角面内的重心坐标权重,后面用它把 UV 像素插值回 3D 面片上的点。
self.cached_uvmask_flaten_idx = None # (512 * 512) 缓存有效 UV 像素 mask
self.cached_pix_to_face = None # (N_valid,) 缓存只保留有效 UV 像素后的 face index 每个有效 UV 像素对应哪个三角面
self.cached_pix_to_v_idx = None # (N_valid, 3) 缓存每个有效 UV 像素对应三角面的 3 个 3D 顶点索引
self.cached_uv_vertices_shape = None # (1, N_valid, 3) 缓存 canonical/default shape 下,每个有效 UV 像素插值出来的 3D 点。
self.cached_uv_vertices_shape_embeded = None # (1, N_valid, embed_dim) 缓存对 uv_vertices_shape 做 positional encoding 后的结果
self.cached_v_num = None # 缓存有效点数量 N_valid
self.cached_uv_head_idx = None # (N_valid,) 缓存最终用于筛选点的 mask
# 同时清除FLAME缓存
self.cached_flame_geometry = None
self.cached_flame_params = None
self.init_networks()
def init_networks(self):
## full mica
self.deformNet = MLP(
input_dim=self.pts_embedder.dim_embeded+100+64,#+64,#74
output_dim=10,
hidden_dim=256,
hidden_layers=6
)
def example_init(self, codedict, flame):
# speed up
# print(f"[DEBUG] example_init called with codedict shape: {codedict.shape if hasattr(codedict, 'shape') else 'N/A'}", flush=True)
# 检查是否已经缓存了光栅化结果
if self.rasterization_cached:
# print(f"[INFO] Using cached rasterization results - skipping expensive UV rasterization", flush=True)
# 恢复缓存的光栅化结果
self.pix_to_face_ori = self.cached_pix_to_face_ori
self.bary_coords = self.cached_bary_coords
self.uvmask_flaten_idx = self.cached_uvmask_flaten_idx
self.pix_to_face = self.cached_pix_to_face
self.pix_to_v_idx = self.cached_pix_to_v_idx
self.uv_vertices_shape = self.cached_uv_vertices_shape
self.uv_vertices_shape_embeded = self.cached_uv_vertices_shape_embeded
self.v_num = self.cached_v_num
self.uv_head_idx = self.cached_uv_head_idx
# print(f"[INFO] Cached rasterization results restored successfully", flush=True)
return
# print(f"[INFO] First time execution - performing full UV rasterization", flush=True)
# 总体计时开始
# torch.cuda.synchronize()
# total_start_time = time.time()
shape_code = codedict.shape.detach()
batch_size = shape_code.shape[0]
# 步骤1: FLAME模型前向传播
# torch.cuda.synchronize()
# step1_start = time.time()
# 深引用版本(新)
temp_codedict = replace(codedict, expression=self.default_expr_code)
# print(f"[DEBUG] About to call flame.forward", flush=True)
geometry_shape = flame.forward(temp_codedict, zero_rot=True).cuda()
# 浅引用版本(原来的)
# codedict.expression = self.default_expr_code
# # print(f"[DEBUG] About to call flame.forward", flush=True)
# geometry_shape = flame.forward(codedict, zero_rot=True).cuda()
# torch.cuda.synchronize()
# step1_time = time.time() - step1_start
# print(f"[TIMING] Step 1 - FLAME forward: {step1_time:.4f} seconds", flush=True)
# print(f"[DEBUG] flame.forward completed, geometry_shape shape: {geometry_shape.shape}", flush=True)
# 步骤2: 生成面部顶点
# torch.cuda.synchronize()
# step2_start = time.time()
# self.tri_faces是flame 的三角面索引
# 得到每个三角面片的顶点索引
face_vertices_shape = face_vertices_gen(geometry_shape, self.tri_faces.expand(batch_size, -1, -1))
# torch.cuda.synchronize()
# step2_time = time.time() - step2_start
# print(f"[TIMING] Step 2 - Face vertices generation: {step2_time:.4f} seconds", flush=True)
# print(f"[DEBUG] face_vertices_shape created, shape: {face_vertices_shape.shape}", flush=True)
# 步骤3: UV光栅化
# torch.cuda.synchronize()
# step3_start = time.time()
rast_out, pix_to_face, bary_coords = self.uv_rasterizer(self.uvcoords.expand(batch_size, -1, -1),
self.uvfaces.expand(batch_size, -1, -1),
face_vertices_shape)
# torch.cuda.synchronize()
# step3_time = time.time() - step3_start
# print(f"[TIMING] Step 3 - UV rasterization: {step3_time:.4f} seconds", flush=True)
# print(f"[DEBUG] UV rasterization completed", flush=True)
# 步骤4: 数据预处理和索引计算
# torch.cuda.synchronize()
# step4_start = time.time()
self.pix_to_face_ori = pix_to_face
self.bary_coords = bary_coords
uvmask = rast_out[:, -1].unsqueeze(1)
uvmask_flaten = uvmask[0].view(uvmask.shape[1], -1).permute(1, 0).squeeze(1) # batch=1
self.uvmask_flaten_idx = (uvmask_flaten[:]>0)
pix_to_face_flaten = pix_to_face[0].clone().view(-1) # batch=1
self.pix_to_face = pix_to_face_flaten[self.uvmask_flaten_idx] # pix to face idx
self.pix_to_v_idx = self.tri_faces[0, self.pix_to_face, :] # pix to vert idx
uv_vertices_shape = rast_out[:, :3]
uv_vertices_shape_flaten = uv_vertices_shape[0].view(uv_vertices_shape.shape[1], -1).permute(1, 0) # batch=1
uv_vertices_shape = uv_vertices_shape_flaten[self.uvmask_flaten_idx].unsqueeze(0)
self.uv_vertices_shape = uv_vertices_shape
self.uv_vertices_shape_embeded = self.pts_embedder(self.uv_vertices_shape)
self.v_num = self.uv_vertices_shape_embeded.shape[1]
# mask
self.uv_head_idx = (
a_in_b_torch(self.pix_to_v_idx[:,0], self.neckhead_id_tensor)
& a_in_b_torch(self.pix_to_v_idx[:,1], self.neckhead_id_tensor)
& a_in_b_torch(self.pix_to_v_idx[:,2], self.neckhead_id_tensor)
)
# 缓存光栅化结果
self.cached_pix_to_face_ori = self.pix_to_face_ori.clone()
self.cached_bary_coords = self.bary_coords.clone()
self.cached_uvmask_flaten_idx = self.uvmask_flaten_idx.clone()
self.cached_pix_to_face = self.pix_to_face.clone()
self.cached_pix_to_v_idx = self.pix_to_v_idx.clone()
self.cached_uv_vertices_shape = self.uv_vertices_shape.clone()
self.cached_uv_vertices_shape_embeded = self.uv_vertices_shape_embeded.clone()
self.cached_v_num = self.v_num
self.cached_uv_head_idx = self.uv_head_idx.clone()
# 标记光栅化已完成缓存
self.rasterization_cached = True
# print(f"[INFO] Rasterization results cached successfully", flush=True)
# print(f"[DEBUG] example_init completed successfully", flush=True)
def decode(self, codedict, flame, feature_flame, target_rotation=None):
shape_code = codedict[1]['flame_params'].shape.detach()
expr_code = codedict[1]['flame_params'].expression.detach()
# jaw_pose = codedict['jaw_pose'].detach()
# neck_pose = codedict['neck_pose'].detach()
# eyes_pose = codedict['eyes_pose'].detach()
# person_code = codedict['person_code'].detach()
batch_size = shape_code.shape[0]
# condition = torch.cat((expr_code, jaw_pose, eyes_pose, neck_pose), dim=1)
# condition = expr_code.cuda()
condition = torch.cat((expr_code.cuda(), feature_flame), dim=1)
# MLP
condition = condition.unsqueeze(1).repeat(1, self.v_num, 1)
uv_vertices_shape_embeded_condition = torch.cat((self.uv_vertices_shape_embeded, condition), dim=2)
deforms = self.deformNet(uv_vertices_shape_embeded_condition)
deforms = torch.tanh(deforms)
uv_vertices_deforms = 2.0 * deforms[..., :3]
rot_delta_0 = deforms[..., 3:7]
rot_delta_r = torch.exp(rot_delta_0[..., 0]).unsqueeze(-1)
rot_delta_v = rot_delta_0[..., 1:]
rot_delta = torch.cat((rot_delta_r, rot_delta_v), dim=-1)
scale_coef = deforms[..., 7:]
scale_coef = torch.exp(scale_coef)
geometry = flame.forward(codedict[1]['flame_params'], zero_rot=True).cuda()
face_vertices = face_vertices_gen(geometry, self.tri_faces.expand(batch_size, -1, -1))
# rasterize face_vertices to uv space
D = face_vertices.shape[-1] # 3
attributes = face_vertices.clone()
attributes = attributes.view(attributes.shape[0] * attributes.shape[1], 3, attributes.shape[-1])
N, H, W, K, _ = self.bary_coords.shape
idx = self.pix_to_face_ori.clone().view(N * H * W * K, 1, 1).expand(N * H * W * K, 3, D)
pixel_face_vals = attributes.gather(0, idx).view(N, H, W, K, 3, D)
pixel_vals = (self.bary_coords[..., None] * pixel_face_vals).sum(dim=-2)
uv_vertices = pixel_vals[:, :, :, 0].permute(0, 3, 1, 2)
uv_vertices_flaten = uv_vertices[0].view(uv_vertices.shape[1], -1).permute(1, 0) # batch=1
uv_vertices = uv_vertices_flaten[self.uvmask_flaten_idx].unsqueeze(0)
# 获取z坐标
x_coords = uv_vertices[:,:,0]
# y_coords = uv_vertices[:,:,1]
z_coords = uv_vertices[:,:,2]
tan_xy = torch.abs(x_coords)/torch.abs((z_coords+0.01))#0.2
# 找到z坐标小于0的点的索引
indices = torch.where((z_coords >= -0.01) | (tan_xy >= 1.732))
verts_final = uv_vertices + uv_vertices_deforms
# conduct mask
verts_final = verts_final[:, self.uv_head_idx, :]
rot_delta = rot_delta[:, self.uv_head_idx, :]
scale_coef = scale_coef[:, self.uv_head_idx, :]
verts_flame = uv_vertices[:, self.uv_head_idx, :]
# # 根据索引提取满足条件的点
# verts_final = verts_final[:, indices[1], :]
# verts_flame = verts_flame[:, indices[1], :]
# rotation_mat = rot_mat_from_6dof(codedict[1]['flame_params'].rotation).type(verts_final.dtype).cuda()
# vertices = torch.matmul(rotation_mat.unsqueeze(1), verts_final.unsqueeze(-1))
# vertices = vertices[..., 0]
scale = torch.clamp(codedict[1]['flame_params'].scale[:, None] + 1.0, 1e-8).cuda().squeeze()
# vertices *= scale # [B, 1, 1]
# translation = codedict[1]['flame_params'].translation.cuda()
# translation[..., 2] = 0.0
# vertices += translation[:, None] # [B, 1, 3]
# vertices[..., 2] += 2183.5264
vertices = scale * verts_final
# 计算偏移量的L2损失,使形变保持在合理范围
lambda_l2 = 0.01 # 降低L2正则化系数,让形变有更大的自由度
l2_loss = lambda_l2 * torch.norm(uv_vertices_deforms, p=2) # L2范数
return vertices, rot_delta, scale_coef, verts_flame, verts_final, l2_loss, scale
def decode_single(self, codedict, flame, feature_flame):
shape_code = codedict.shape.detach()
expr_code = codedict.expression.detach()
# jaw_pose = codedict['jaw_pose'].detach()
# neck_pose = codedict['neck_pose'].detach()
# eyes_pose = codedict['eyes_pose'].detach()
# person_code = codedict['person_code'].detach()
batch_size = shape_code.shape[0]
# condition = torch.cat((expr_code, jaw_pose, eyes_pose, neck_pose), dim=1)
# condition = expr_code.cuda()
condition = torch.cat((expr_code.cuda(), feature_flame), dim=1)
# MLP
condition = condition.unsqueeze(1).repeat(1, self.v_num, 1)
uv_vertices_shape_embeded_condition = torch.cat((self.uv_vertices_shape_embeded, condition), dim=2)
deforms = self.deformNet(uv_vertices_shape_embeded_condition)
deforms = torch.tanh(deforms)
uv_vertices_deforms = deforms[..., :3]
rot_delta_0 = deforms[..., 3:7]
rot_delta_r = torch.exp(rot_delta_0[..., 0]).unsqueeze(-1)
rot_delta_v = rot_delta_0[..., 1:]
rot_delta = torch.cat((rot_delta_r, rot_delta_v), dim=-1)
scale_coef = deforms[..., 7:]
scale_coef = torch.exp(scale_coef)
geometry = flame.forward(codedict, zero_rot=True).cuda()
face_vertices = face_vertices_gen(geometry, self.tri_faces.expand(batch_size, -1, -1))
# rasterize face_vertices to uv space
D = face_vertices.shape[-1] # 3
attributes = face_vertices.clone()
attributes = attributes.view(attributes.shape[0] * attributes.shape[1], 3, attributes.shape[-1])
N, H, W, K, _ = self.bary_coords.shape
idx = self.pix_to_face_ori.clone().view(N * H * W * K, 1, 1).expand(N * H * W * K, 3, D)
pixel_face_vals = attributes.gather(0, idx).view(N, H, W, K, 3, D)
pixel_vals = (self.bary_coords[..., None] * pixel_face_vals).sum(dim=-2)
uv_vertices = pixel_vals[:, :, :, 0].permute(0, 3, 1, 2)
uv_vertices_flaten = uv_vertices[0].view(uv_vertices.shape[1], -1).permute(1, 0) # batch=1
uv_vertices = uv_vertices_flaten[self.uvmask_flaten_idx].unsqueeze(0)
# 获取z坐标
x_coords = uv_vertices[:,:,0]
# y_coords = uv_vertices[:,:,1]
z_coords = uv_vertices[:,:,2]
tan_xy = torch.abs(x_coords)/torch.abs((z_coords+0.01))#0.2
# 找到z坐标小于0的点的索引
indices = torch.where((z_coords >= -0.01) | (tan_xy >= 1.732))
verts_final = uv_vertices + uv_vertices_deforms
# conduct mask
verts_final = verts_final[:, self.uv_head_idx, :]
rot_delta = rot_delta[:, self.uv_head_idx, :]
scale_coef = scale_coef[:, self.uv_head_idx, :]
verts_flame = uv_vertices[:, self.uv_head_idx, :]
# # 根据索引提取满足条件的点
# verts_final = verts_final[:, indices[1], :]
# verts_flame = verts_flame[:, indices[1], :]
# rotation_mat = rot_mat_from_6dof(codedict[1]['flame_params'].rotation).type(verts_final.dtype).cuda()
# vertices = torch.matmul(rotation_mat.unsqueeze(1), verts_final.unsqueeze(-1))
# vertices = vertices[..., 0]
scale = torch.clamp(codedict.scale[:, None] + 1.0, 1e-8).cuda().squeeze()
# vertices *= scale # [B, 1, 1]
# translation = codedict[1]['flame_params'].translation.cuda()
# translation[..., 2] = 0.0
# vertices += translation[:, None] # [B, 1, 3]
# vertices[..., 2] += 2183.5264
vertices = scale * verts_final
# 计算偏移量的L2损失,使形变保持在合理范围
lambda_l2 = 0.03 # 降低L2正则化系数,让形变有更大的自由度
l2_loss = lambda_l2 * torch.norm(uv_vertices_deforms, p=2) # L2范数
return vertices, rot_delta, scale_coef, verts_flame, verts_final, l2_loss, scale
def decode_single3(self, codedict, flame, feature_flame):
shape_code = codedict.shape.detach()
expr_code = codedict.expression.detach()
batch_size = shape_code.shape[0]
# 表情编码 + 特征
condition = torch.cat((expr_code.cuda(), feature_flame), dim=1)
condition = condition.unsqueeze(1).repeat(1, self.v_num, 1)
uv_vertices_shape_embeded_condition = torch.cat((self.uv_vertices_shape_embeded, condition), dim=2)
# MLP 解码器
deforms = self.deformNet(uv_vertices_shape_embeded_condition)
deforms = torch.tanh(deforms)
# 只使用偏移信息
uv_vertices_deforms = deforms[..., :3]
# 获取原始 flame 网格
geometry = flame.forward(codedict, zero_rot=True).cuda()
face_vertices = face_vertices_gen(geometry, self.tri_faces.expand(batch_size, -1, -1))
# rasterize 到 UV 空间
D = face_vertices.shape[-1]
attributes = face_vertices.view(-1, 3, D)
N, H, W, K, _ = self.bary_coords.shape
idx = self.pix_to_face_ori.clone().view(N * H * W * K, 1, 1).expand(-1, 3, D)
pixel_face_vals = attributes.gather(0, idx).view(N, H, W, K, 3, D)
pixel_vals = (self.bary_coords[..., None] * pixel_face_vals).sum(dim=-2)
uv_vertices = pixel_vals[:, :, :, 0].permute(0, 3, 1, 2)
uv_vertices_flaten = uv_vertices[0].view(uv_vertices.shape[1], -1).permute(1, 0)
uv_vertices = uv_vertices_flaten[self.uvmask_flaten_idx].unsqueeze(0)
# 形变 + mask 头部区域
verts_final = uv_vertices + uv_vertices_deforms
# 计算缩放并应用(z 方向不加)
scale = torch.clamp(codedict.scale[:, None] + 1.0, 1e-8).cuda().squeeze()
vertices = scale * verts_final
return vertices, verts_final, scale
def decode_single2(self, codedict, flame, feature_flame):
# 总体计时开始
# torch.cuda.synchronize()
# total_start_time = time.time()
shape_code = codedict.shape.detach()
expr_code = codedict.expression.detach()
batch_size = shape_code.shape[0]
# 步骤1: 表情编码和特征处理
# torch.cuda.synchronize()
# step1_start = time.time()
condition = torch.cat((expr_code.cuda(), feature_flame), dim=1)
condition = condition.unsqueeze(1).repeat(1, self.v_num, 1)
uv_vertices_shape_embeded_condition = torch.cat((self.uv_vertices_shape_embeded, condition), dim=2)
# torch.cuda.synchronize()
# step1_time = time.time() - step1_start
# print(f"[TIMING] decode_single2 Step 1 - Feature processing: {step1_time:.4f} seconds", flush=True)
# # 步骤2: MLP解码器
# torch.cuda.synchronize()
# step2_start = time.time()
deforms = self.deformNet(uv_vertices_shape_embeded_condition)
deforms = torch.tanh(deforms)
uv_vertices_deforms = deforms[..., :3]
# torch.cuda.synchronize()
# step2_time = time.time() - step2_start
# print(f"[TIMING] decode_single2 Step 2 - MLP inference: {step2_time:.4f} seconds", flush=True)
# # 步骤3: FLAME几何计算(带缓存)
# torch.cuda.synchronize()
# step3_start = time.time()
if not hasattr(self, 'cached_flame_geometry') or self.cached_flame_geometry is None:
# print(f"[INFO] Computing FLAME geometry (first time)", flush=True)
geometry = flame.forward(codedict, zero_rot=True).cuda()
# 缓存FLAME几何结果
self.cached_flame_geometry = geometry
self.cached_flame_params = {
'shape': codedict.shape.clone(),
'expression': codedict.expression.clone()
}
else:
# 检查参数是否变化
# 在缓存参数时确保它们在GPU上
self.cached_flame_params['expression'] = self.cached_flame_params['expression'].to(codedict.expression.device)
shape_changed = torch.norm(codedict.shape - self.cached_flame_params['shape']) > 1e-6
expr_changed = torch.norm(codedict.expression - self.cached_flame_params['expression']) > 1e-6
if shape_changed or expr_changed:
# print(f"[INFO] FLAME parameters changed, recomputing geometry", flush=True)
geometry = flame.forward(codedict, zero_rot=True).cuda()
self.cached_flame_geometry = geometry
self.cached_flame_params['shape'] = codedict.shape.clone()
self.cached_flame_params['expression'] = codedict.expression.clone()
else:
# print(f"[INFO] Using cached FLAME geometry", flush=True)
geometry = self.cached_flame_geometry
face_vertices = face_vertices_gen(geometry, self.tri_faces.expand(batch_size, -1, -1))
# torch.cuda.synchronize()
# step3_time = time.time() - step3_start
# print(f"[TIMING] decode_single2 Step 3 - FLAME geometry: {step3_time:.4f} seconds", flush=True)
# 步骤4: UV处理(带缓存优化)
# torch.cuda.synchronize()
# step4_start = time.time()
if hasattr(self, 'uv_vertices_shape') and self.uv_vertices_shape is not None:
# print(f"[INFO] Using cached UV vertices from example_init", flush=True)
uv_vertices = self.uv_vertices_shape
else:
# print(f"[INFO] Computing UV rasterization (fallback)", flush=True)
# rasterize 到 UV 空间
D = face_vertices.shape[-1]
attributes = face_vertices.view(-1, 3, D)
N, H, W, K, _ = self.bary_coords.shape
idx = self.pix_to_face_ori.clone().view(N * H * W * K, 1, 1).expand(-1, 3, D)
pixel_face_vals = attributes.gather(0, idx).view(N, H, W, K, 3, D)
pixel_vals = (self.bary_coords[..., None] * pixel_face_vals).sum(dim=-2)
uv_vertices = pixel_vals[:, :, :, 0].permute(0, 3, 1, 2)
uv_vertices_flaten = uv_vertices[0].view(uv_vertices.shape[1], -1).permute(1, 0)
uv_vertices = uv_vertices_flaten[self.uvmask_flaten_idx].unsqueeze(0)
# torch.cuda.synchronize()
# step4_time = time.time() - step4_start
# print(f"[TIMING] decode_single2 Step 4 - UV processing: {step4_time:.4f} seconds", flush=True)
# # 步骤5: 最终计算
# torch.cuda.synchronize()
# step5_start = time.time()
# 形变 + mask 头部区域
verts_final = uv_vertices + uv_vertices_deforms
# 计算缩放并应用(z 方向不加)
scale = torch.clamp(codedict.scale[:, None] + 1.0, 1e-8).cuda().squeeze()
vertices = scale * verts_final
torch.cuda.synchronize()
# step5_time = time.time() - step5_start
# print(f"[TIMING] decode_single2 Step 5 - Final computation: {step5_time:.4f} seconds", flush=True)
# # 总体计时结束
# torch.cuda.synchronize()
# total_time = time.time() - total_start_time
# print(f"[TIMING] decode_single2 Total time: {total_time:.4f} seconds", flush=True)
# print(f"[TIMING] decode_single2 Breakdown: Features({step1_time:.4f}s) + MLP({step2_time:.4f}s) + FLAME({step3_time:.4f}s) + UV({step4_time:.4f}s) + Final({step5_time:.4f}s)", flush=True)
return vertices, verts_final, scale
def capture(self):
return (
self.deformNet.state_dict(),
self.optimizer.state_dict(),
)
def restore(self, model_args):
(net_dict,
opt_dict) = model_args
self.deformNet.load_state_dict(net_dict)
self.training_setup()
self.optimizer.load_state_dict(opt_dict)
def training_setup(self):
params_group = [
{'params': self.deformNet.parameters(), 'lr': 1e-4},
]
self.optimizer = torch.optim.Adam(params_group, betas=(0.9, 0.999))
def get_template(self):
geometry_template = self.flame_model.forward_geo(
self.default_shape_code,
self.default_expr_code,
self.default_jaw_pose,
eye_pose_params=self.default_eyes_pose,
)
return geometry_template
class MLP(nn.Module):
def __init__(self, input_dim, output_dim, hidden_dim=256, hidden_layers=8):
super().__init__()
self.hidden_dim = hidden_dim
self.hidden_layers = hidden_layers
self.input_dim = input_dim
self.output_dim = output_dim
self.fcs = nn.ModuleList(
[nn.Linear(input_dim, hidden_dim)] + [nn.Linear(hidden_dim, hidden_dim) for i in range(hidden_layers-1)]
)
self.output_linear = nn.Linear(hidden_dim, output_dim)
def forward(self, input):
# input: B,V,d
batch_size, N_v, input_dim = input.shape
input_ori = input.reshape(batch_size*N_v, -1)
h = input_ori
for i, l in enumerate(self.fcs):
h = self.fcs[i](h)
h = F.relu(h)
output = self.output_linear(h)
output = output.reshape(batch_size, N_v, -1)
return output
class MLP2(nn.Module):
def __init__(self, input_dim, condition_dim, output_dim1, output_dim2, hidden_dim=256, hidden_layers=8):
super().__init__()
self.hidden_dim = hidden_dim
self.condition_dim = condition_dim
self.hidden_layers = hidden_layers
mid_layers = math.ceil(hidden_layers/2.)
self.input_dim = input_dim
self.output_dim1 = output_dim1
self.output_dim2 = output_dim2
self.fcs1 = nn.ModuleList(
[nn.Linear(input_dim, hidden_dim)] + [nn.Linear(hidden_dim, hidden_dim) if i!=(mid_layers-1) else nn.Linear(hidden_dim, hidden_dim+output_dim1) for i in range(mid_layers)]
)
self.fcs2 = nn.ModuleList(
[nn.Linear(hidden_dim, hidden_dim) if i!=0 else nn.Linear(hidden_dim+condition_dim, hidden_dim) for i in range(mid_layers, hidden_layers-1)]
)
self.output_linear = nn.Linear(hidden_dim, output_dim2)
def forward(self, input, condition):
# input: B,V,d1
# condition: B,d2
batch_size, N_v, input_dim = input.shape
input_ori = input.reshape(batch_size*N_v, -1)
h = input_ori
for i, l in enumerate(self.fcs1):
h = self.fcs1[i](h)
h = F.relu(h)
oup1 = h[:, -self.output_dim1:]
h = h[:, :-self.output_dim1]
...
for i, l in enumerate(self.fcs2):
h = self.fcs1[i](h)
h = F.relu(h)
# input_ori = input.reshape(batch_size*N_v, -1)
# h = input_ori
# for i, l in enumerate(self.fcs):
# h = self.fcs[i](h)
# h = F.relu(h)
# output = self.output_linear(h)
# output = output.reshape(batch_size, N_v, -1)
class SIRENMLP(nn.Module):
def __init__(self,
input_dim=3,
output_dim=3,
hidden_dim=256,
hidden_layers=8,
condition_dim=100,
device=None):
super().__init__()
self.device = device
self.input_dim = input_dim
self.z_dim = condition_dim
self.hidden_layers = hidden_layers
self.hidden_dim = hidden_dim
self.output_dim = output_dim
self.network = nn.ModuleList(
[FiLMLayer(self.input_dim, self.hidden_dim)] +
[FiLMLayer(self.hidden_dim, self.hidden_dim) for i in range(self.hidden_layers - 1)]
)
self.final_layer = nn.Linear(self.hidden_dim, self.output_dim)
self.mapping_network = MappingNetwork(condition_dim, 256,
len(self.network) * self.hidden_dim * 2)
self.network.apply(frequency_init(25))
# self.final_layer.apply(frequency_init(25))
self.final_layer.weight.data.normal_(0.0, 0.)
self.final_layer.bias.data.fill_(0.)
self.network[0].apply(first_layer_film_sine_init)
def forward_vector(self, input, z):
frequencies, phase_shifts = self.mapping_network(z)
return self.forward_with_frequencies_phase_shifts(input, frequencies, phase_shifts)
def forward_with_frequencies_phase_shifts(self, input, frequencies, phase_shifts):
frequencies = frequencies * 15 + 30
x = input
for index, layer in enumerate(self.network):
start = index * self.hidden_dim
end = (index + 1) * self.hidden_dim
x = layer(x, frequencies[..., start:end], phase_shifts[..., start:end])
sigma = self.final_layer(x)
return sigma
def forward(self, vertices, additional_conditioning):
# vertices: in canonical space of flame
# vertex eval: torch.Size([N, V, 3])
# map eval: torch.Size([N, 3, H, W])
# conditioning
# torch.Size([N, C])
# vertex inputs (N, V, 3) -> (N, 3, V, 1)
vertices = vertices.permute(0, 2, 1)[:, :, :, None]
b, c, h, w = vertices.shape
# to vector
x = vertices.permute(0, 2, 3, 1).reshape(b, -1, c)
z = additional_conditioning # .repeat(1, h*w, 1)
# apply siren network
o = self.forward_vector(x, z)
# to image
o = o.reshape(b, h, w, self.output_dim)
output = o.permute(0, 3, 1, 2)
return output[:, :, :, 0].permute(0, 2, 1)
class FiLMLayer(nn.Module):
def __init__(self, input_dim, hidden_dim):
super().__init__()
self.layer = nn.Linear(input_dim, hidden_dim)
def forward(self, x, freq, phase_shift, ignore_conditions=None):
x = self.layer(x)
if ignore_conditions is not None:
cond_freq, cond_phase_shift = freq[:-1], phase_shift[:-1]
cond_freq = cond_freq.unsqueeze(1).expand_as(x).clone()
cond_phase_shift = cond_phase_shift.unsqueeze(1).expand_as(x).clone()
ignore_freq, ignore_phase_shift = freq[-1:], phase_shift[-1:]
ignore_freq = ignore_freq.unsqueeze(1).expand_as(x)
ignore_phase_shift = ignore_phase_shift.unsqueeze(1).expand_as(x)
cond_freq[:, ignore_conditions] = ignore_freq[:, ignore_conditions]
cond_phase_shift[:, ignore_conditions] = ignore_phase_shift[:, ignore_conditions]
freq, phase_shift = cond_freq, cond_phase_shift
else:
freq = freq.unsqueeze(1).expand_as(x)
phase_shift = phase_shift.unsqueeze(1).expand_as(x)
# print('x', x.shape)
# print('freq', freq.shape)
# print('phase_shift', phase_shift.shape)
# x torch.Size([6, 5023, 256])
# freq torch.Size([6, 5023, 256])
# phase_shift torch.Size([6, 5023, 256])
return torch.sin(freq * x + phase_shift)
def frequency_init(freq):
def init(m):
with torch.no_grad():
if isinstance(m, nn.Linear):
num_input = m.weight.size(-1)
m.weight.uniform_(-np.sqrt(6 / num_input) / freq, np.sqrt(6 / num_input) / freq)
elif isinstance(m, nn.Conv2d):
num_input = torch.prod(
torch.tensor(m.weight.shape[1:], device=m.weight.device)).cpu().item()
m.weight.uniform_(-np.sqrt(6 / num_input) / freq, np.sqrt(6 / num_input) / freq)
return init
class MappingNetwork(nn.Module):
def __init__(self, z_dim, map_hidden_dim, map_output_dim):
super().__init__()
self.network = nn.Sequential(nn.Linear(z_dim, map_hidden_dim),
nn.LeakyReLU(0.2, inplace=True),
nn.Linear(map_hidden_dim, map_hidden_dim),
nn.LeakyReLU(0.2, inplace=True),
nn.Linear(map_hidden_dim, map_hidden_dim),
nn.LeakyReLU(0.2, inplace=True),
nn.Linear(map_hidden_dim, map_output_dim))
self.network.apply(kaiming_leaky_init)
with torch.no_grad():
self.network[-1].weight *= 0.25
def forward(self, z):
frequencies_offsets = self.network(z)
frequencies = frequencies_offsets[..., :frequencies_offsets.shape[-1] // 2]
phase_shifts = frequencies_offsets[..., frequencies_offsets.shape[-1] // 2:]
return frequencies, phase_shifts
def kaiming_leaky_init(m):
classname = m.__class__.__name__
if classname.find('Linear') != -1:
torch.nn.init.kaiming_normal_(m.weight, a=0.2, mode='fan_in', nonlinearity='leaky_relu')
def first_layer_film_sine_init(m):
with torch.no_grad():
if isinstance(m, nn.Linear):
num_input = m.weight.size(-1)
m.weight.uniform_(-1 / num_input, 1 / num_input)
elif isinstance(m, nn.Conv2d):
num_input = torch.prod(
torch.tensor(m.weight.shape[1:], device=m.weight.device)).cpu().item()
m.weight.uniform_(-1 / num_input, 1 / num_input)