Live3DPortrait / models /model.py
vegagoto123's picture
Add files using upload-large-folder tool
50681f9 verified
Raw History Blame Contribute Delete
29.6 kB
import torch
import torch.nn as nn
import torch.nn.functional as F
# import segmentation_models_pytorch
from segmentation_models_pytorch.decoders.deeplabv3 import DeepLabV3
'''
impletement of RealTimeRF including the full model and LT model
'''
from timm.models.layers import DropPath, to_2tuple, trunc_normal_
import math
import time
from einops import rearrange, repeat
import sys
# sys.path.insert(0, os.path.abspath(os.path.join(__file__, '..')))
sys.path.append(".")
sys.path.append("..")
from torch_utils.ops import upfirdn2d
from torch_utils.misc import assert_shape
from functools import reduce
from typing import Union
from segmentation_models_pytorch.encoders.mix_transformer import OverlapPatchEmbed, Block
from models.eg3d.volumetric_rendering.renderer import ImportanceRenderer, sample_from_planes
from models.eg3d.volumetric_rendering.ray_sampler import RaySampler
from models.eg3d.superresolution import SuperresolutionHybrid8XDC
from models.eg3d.networks_stylegan2 import FullyConnectedLayer
from models.eg3d.triplane import OSGDecoder
from models.eg3d.networks_stylegan2 import Generator as StyleGAN2Backbone
class TriGenerator(nn.Module):
'''
similar to TriplaneGenerator class but lack of renderer
'''
def __init__(self, # 参数表暂时不删,做占位用
z_dim, # Input latent (Z) dimensionality.
c_dim, # Conditioning label (C) dimensionality.
w_dim, # Intermediate latent (W) dimensionality.
img_resolution, # Output resolution.
img_channels, # Number of output color channels.
sr_num_fp16_res = 0,
mapping_kwargs = {}, # Arguments for MappingNetwork.
rendering_kwargs = {},
sr_kwargs = {},
**synthesis_kwargs, # Arguments for SynthesisNetwork.
):
super().__init__()
self.z_dim=z_dim
self.c_dim=c_dim
self.w_dim=w_dim
self.backbone = StyleGAN2Backbone(z_dim, c_dim, w_dim, img_resolution=256, img_channels=32*3, mapping_kwargs=mapping_kwargs, **synthesis_kwargs)
self._last_planes = None
self.rendering_kwargs = rendering_kwargs
def mapping(self, z, c, truncation_psi=1, truncation_cutoff=None, update_emas=False):
if self.rendering_kwargs['c_gen_conditioning_zero']:
c = torch.zeros_like(c)
return self.backbone.mapping(z, c * self.rendering_kwargs.get('c_scale', 0), truncation_psi=truncation_psi, truncation_cutoff=truncation_cutoff, update_emas=update_emas)
def synthesis(self, ws, update_emas=False, cache_backbone=False, use_cached_backbone=False, **synthesis_kwargs):
if use_cached_backbone and self._last_planes is not None:
planes = self._last_planes
else:
planes = self.backbone.synthesis(ws, update_emas=update_emas, **synthesis_kwargs)
if cache_backbone:
self._last_planes = planes
return planes
def forward(self, z, c, truncation_psi=1, truncation_cutoff=None, neural_rendering_resolution=None, update_emas=False, cache_backbone=False, use_cached_backbone=False, **synthesis_kwargs):
# Render a batch of generated images.
ws = self.mapping(z, c, truncation_psi=truncation_psi, truncation_cutoff=truncation_cutoff, update_emas=update_emas)
return self.synthesis(ws, update_emas=update_emas, neural_rendering_resolution=neural_rendering_resolution, cache_backbone=cache_backbone, use_cached_backbone=use_cached_backbone, **synthesis_kwargs)
class TriplaneRenderer(nn.Module):
def __init__(self, img_resolution, img_channels, rendering_kwargs={}) -> None:
'''
Triplane Renderer
Generate 2D image from triplanes representation
SuperResolution without stylecode
FullyConnected layer
'''
super(TriplaneRenderer, self).__init__()
self.img_resolution=img_resolution
self.img_channels=img_channels
self.renderer = ImportanceRenderer()
self.ray_sampler = RaySampler()
self.w_dim = 512
self.const = torch.nn.Parameter(torch.randn([1, 1, self.w_dim])) # 常数输入
self.decoder = OSGDecoder(32, {'decoder_lr_mul': rendering_kwargs.get('decoder_lr_mul', 1), 'decoder_output_dim': 32})
self.superresolution = SuperresolutionHybrid8XDC(32, img_resolution, sr_num_fp16_res=0, sr_antialias=True)
self.rendering_kwargs = rendering_kwargs
self.neural_rendering_resolution = 128 # 64
def synthesis(self, planes, c, neural_rendering_resolution=None):
cam2world_matrix = c[:, :16].view(-1, 4, 4)
intrinsics = c[:, 16:25].view(-1, 3, 3)
if neural_rendering_resolution is None:
neural_rendering_resolution = self.neural_rendering_resolution
else:
self.neural_rendering_resolution = neural_rendering_resolution
# Create a batch of rays for volume rendering
ray_origins, ray_directions = self.ray_sampler(cam2world_matrix, intrinsics, neural_rendering_resolution)
N, _, _ = ray_origins.shape
# Reshape output into three 32-channel planes
planes = planes.view(len(planes), 3, 32, planes.shape[-2], planes.shape[-1])
# Perform volume rendering
feature_samples, depth_samples, _ = self.renderer(planes, self.decoder, ray_origins, ray_directions, self.rendering_kwargs) # channels last
# Reshape into 'raw' neural-rendered image
H = W = neural_rendering_resolution
feature_image = feature_samples.permute(0, 2, 1).reshape(N, feature_samples.shape[-1], H, W).contiguous()
depth_image = depth_samples.permute(0, 2, 1).reshape(N, 1, H, W)
# Run superresolution to get final image
rgb_image = feature_image[:, :3]
# sr_image = self.superresolution(rgb_image, feature_image, ws, noise_mode='none')
const_w_input = self.const.repeat([N, 1, 1])
sr_image = self.superresolution(rgb_image, feature_image, const_w_input, noise_mode='none')
return {'image': sr_image,
'image_raw': rgb_image,
'image_depth': depth_image,
'feature_image': feature_image,
'planes': planes}
def sample_density(self, planes, coordinates, directions):
'''
给定triplanes和camera参数,生成图像并且返回
'''
sampled_features = sample_from_planes(self.renderer.plane_axes, planes, coordinates, padding_mode='zeros', box_warp=self.rendering_kwargs['box_warp'])
out = self.decoder(sampled_features, directions)
return out
def forward(self, planes, c):
'''
给定triplanes和camera参数,生成图像并且返回
'''
return self.synthesis(planes, c)
# coding self RTRF
class EncoderEhigh(nn.Module):
def __init__(self, mode="Normal") -> None:
super(EncoderEhigh, self).__init__()
self.mode = mode
self.net = nn.Sequential(
nn.Conv2d(5, 64, kernel_size=7, stride=2, padding=3),
nn.LeakyReLU(negative_slope=0.01),
nn.Conv2d(64, 96, kernel_size=3, stride=1, padding=1),
nn.LeakyReLU(negative_slope=0.01),
nn.Conv2d(96, 96, kernel_size=3, stride=1, padding=1),
nn.LeakyReLU(negative_slope=0.01),
nn.Conv2d(96, 96, kernel_size=3, stride=1, padding=1),
nn.LeakyReLU(negative_slope=0.01),
nn.Conv2d(96, 96, kernel_size=3, stride=1, padding=1),
nn.LeakyReLU(negative_slope=0.01)
) if self.mode == "Normal" else nn.Sequential(
# Input is Second layer output of DeeplabV3
nn.Conv2d(5, 64, kernel_size=7, stride=2, padding=3),
nn.LeakyReLU(negative_slope=0.01),
nn.Conv2d(64, 128, kernel_size=3, stride=2, padding=1),
nn.LeakyReLU(negative_slope=0.01),
nn.Conv2d(128, 128, kernel_size=3, stride=1, padding=1),
nn.LeakyReLU(negative_slope=0.01),
nn.Conv2d(128, 128, kernel_size=3, stride=1, padding=1),
nn.LeakyReLU(negative_slope=0.01),
nn.Conv2d(128, 128, kernel_size=3, stride=1, padding=1),
nn.LeakyReLU(negative_slope=0.01),
nn.Conv2d(128, 128, kernel_size=3, stride=1, padding=1),
nn.LeakyReLU(negative_slope=0.01),
nn.Conv2d(128, 128, kernel_size=3, stride=1, padding=1),
)
self.net.apply(self._init_weights) # 仿照timm的初始化方式
def _init_weights(self, m):
if isinstance(m, nn.Linear):
trunc_normal_(m.weight, std=0.02)
if isinstance(m, nn.Linear) and m.bias is not None:
nn.init.constant_(m.bias, 0)
elif isinstance(m, nn.Conv2d):
fan_out = m.kernel_size[0] * m.kernel_size[1] * m.out_channels
fan_out //= m.groups
m.weight.data.normal_(0, math.sqrt(2.0 / fan_out))
if m.bias is not None:
m.bias.data.zero_()
def forward(self, img):
return self.net(img)
class EncoderF(nn.Module):
def __init__(self, mode="Normal") -> None:
super(EncoderF, self).__init__()
self.mode = mode
self.blocks_num = 5 if self.mode == 'Normal' else 2
self.patchembed = OverlapPatchEmbed(img_size=64, stride=2, in_chans=256, embed_dim=1024)
self.blocks = nn.ModuleList([Block(dim=1024, num_heads=4, mlp_ratio=2, sr_ratio=1) for _ in range(self.blocks_num)])
self.upsample = nn.Sequential(
nn.PixelShuffle(upscale_factor=2),
nn.UpsamplingBilinear2d(scale_factor=2.0), # equivalent to ``nn.functional.interpolate(..., mode='bilinear', align_corners=True)``
)
self.net = nn.Sequential(
nn.Conv2d(256, 128, kernel_size=3, stride=1, padding=1),
nn.ReLU(),
nn.UpsamplingBilinear2d(scale_factor=2.0), # equivalent to ``nn.functional.interpolate(..., mode='bilinear', align_corners=True)``
nn.Conv2d(128, 128, kernel_size=3, stride=1, padding=1),
nn.ReLU(),
nn.Conv2d(128, 96, kernel_size=3, stride=1, padding=1)
) if self.mode == "Normal" else nn.Sequential(
nn.Conv2d(256, 128, kernel_size=3, stride=1, padding=1),
nn.LeakyReLU(negative_slope=0.01),
nn.Conv2d(128, 128, kernel_size=3, stride=1, padding=1),
)
self.net.apply(self._init_weights) # 仿照timm的初始化方式
def _init_weights(self, m):
if isinstance(m, nn.Linear):
trunc_normal_(m.weight, std=0.02)
if isinstance(m, nn.Linear) and m.bias is not None:
nn.init.constant_(m.bias, 0)
# elif isinstance(m, nn.LayerNorm):
# nn.init.constant_(m.bias, 0)
# nn.init.constant_(m.weight, 1.0)
elif isinstance(m, nn.Conv2d):
fan_out = m.kernel_size[0] * m.kernel_size[1] * m.out_channels
fan_out //= m.groups
m.weight.data.normal_(0, math.sqrt(2.0 / fan_out))
if m.bias is not None:
m.bias.data.zero_()
def forward(self, x):
x, H, W = self.patchembed(x)
for i in range(self.blocks_num):
x = self.blocks[i](x, H, W) # [B, N, C]
# print((H, W)) # (32, 32)
# print(x.shape) # [2, 1024, 1024]
x = rearrange(x, "b (h w) c -> b c h w", h=H, w=W)
# print(x.shape) # [2, 1024, 32, 32]
x = self.upsample(x)
x = self.net(x)
return x
class EncoderFinal(nn.Module):
def __init__(self, mode="Normal") -> None:
super(EncoderFinal, self).__init__()
self.mode = mode
self.net = nn.Sequential(
nn.Conv2d(192, 256, kernel_size=3, stride=1, padding=1),
nn.LeakyReLU(negative_slope=0.01),
nn.Conv2d(256, 128, kernel_size=3, stride=1, padding=1),
nn.LeakyReLU(negative_slope=0.01),
) if self.mode == "Normal" else nn.Sequential(
nn.Conv2d(256, 256, kernel_size=3, stride=1, padding=1),
)
if self.mode == "Normal":
self.pathembed = OverlapPatchEmbed(img_size=256, stride=2, in_chans=128, embed_dim=1024)
self.block = Block(dim=1024, num_heads=2, mlp_ratio=2, sr_ratio=2)
else:
self.pathembed = OverlapPatchEmbed(img_size=128, stride=2, in_chans=256, embed_dim=1024)
self.block = Block(dim=1024, num_heads=2, mlp_ratio=2, sr_ratio=2)
# self.pathembed = OverlapPatchEmbed(img_size=128, stride=2, in_chans=128, embed_dim=1024)
# self.block = Block(dim=256, num_heads=2, mlp_ratio=2, sr_ratio=2)
self.pixelShuffel = nn.PixelShuffle(upscale_factor=2)
# concat with output of encoder F
# self.F_feature = F_feature
if self.mode == "Normal":
self.output_net = nn.Sequential(
nn.Conv2d(352, 256, kernel_size=3, stride=1, padding=1),
nn.LeakyReLU(negative_slope=0.01),
nn.Conv2d(256, 128, kernel_size=3, stride=1, padding=1),
nn.LeakyReLU(negative_slope=0.01),
nn.Conv2d(128, 128, kernel_size=3, stride=1, padding=1),
nn.LeakyReLU(negative_slope=0.01),
nn.Conv2d(128, 96, kernel_size=3, stride=1, padding=1),
)
elif self.mode == "INS":
self.output_net = nn.Sequential(
nn.Conv2d(352, 256, kernel_size=3, stride=1, padding=1),
nn.LeakyReLU(negative_slope=0.01),
nn.Conv2d(256, 128, kernel_size=3, stride=1, padding=1),
nn.LeakyReLU(negative_slope=0.01),
nn.Conv2d(128, 128, kernel_size=3, stride=1, padding=1),
nn.LeakyReLU(negative_slope=0.01),
nn.Conv2d(128, 288, kernel_size=3, stride=1, padding=1)
)
else:
self.output_net = nn.Sequential(
nn.Conv2d(256, 256, kernel_size=3, stride=1, padding=1),
nn.LeakyReLU(negative_slope=0.01),
nn.UpsamplingBilinear2d(scale_factor=2.0),
nn.Conv2d(256, 128, kernel_size=3, stride=1, padding=1),
nn.LeakyReLU(negative_slope=0.01),
nn.Conv2d(128, 128, kernel_size=3, stride=1, padding=1),
nn.LeakyReLU(negative_slope=0.01),
nn.Conv2d(128, 96, kernel_size=3, stride=1, padding=1),
nn.LeakyReLU(negative_slope=0.01),
nn.Conv2d(96, 96, kernel_size=3, stride=1, padding=1),
)
self.net.apply(self._init_weights) # 仿照timm的初始化方式
self.output_net.apply(self._init_weights) # 仿照timm的初始化方式``
def _init_weights(self, m):
if isinstance(m, nn.Linear):
trunc_normal_(m.weight, std=0.02)
if isinstance(m, nn.Linear) and m.bias is not None:
nn.init.constant_(m.bias, 0)
# elif isinstance(m, nn.LayerNorm):
# nn.init.constant_(m.bias, 0)
# nn.init.constant_(m.weight, 1.0)
elif isinstance(m, nn.Conv2d):
fan_out = m.kernel_size[0] * m.kernel_size[1] * m.out_channels
fan_out //= m.groups
m.weight.data.normal_(0, math.sqrt(2.0 / fan_out))
if m.bias is not None:
m.bias.data.zero_()
def forward(self, x, F_feature): # debug
xx = self.net(torch.cat([x, F_feature], dim=1))
xx, H, W = self.pathembed(xx)
xx = self.block(xx, H, W)
xx = rearrange(xx, "b (h w) c -> b c h w", h=H, w=W)
xx = self.pixelShuffel(xx)
if self.mode == "Normal":
return self.output_net(torch.cat([xx, F_feature], dim=1))
else:
return self.output_net(xx)
class INSResidualEncoderFinal(nn.Module):
"""Balanced INS fusion with a direct high-resolution residual path."""
def __init__(
self,
branch_channels=96,
output_channels=288,
norm_groups=8,
initial_detail_scale=0.25,
) -> None:
super().__init__()
if branch_channels <= 0 or output_channels <= 0:
raise ValueError("INS fusion channel counts must be positive.")
if branch_channels % norm_groups != 0:
raise ValueError("branch_channels must be divisible by norm_groups.")
if not math.isfinite(initial_detail_scale) or initial_detail_scale == 0:
raise ValueError("initial_detail_scale must be finite and non-zero.")
self.mode = "INS"
self.branch_channels = int(branch_channels)
self.output_channels = int(output_channels)
self.high_norm = nn.GroupNorm(norm_groups, branch_channels)
self.global_norm = nn.GroupNorm(norm_groups, branch_channels)
self.fusion_net = nn.Sequential(
nn.Conv2d(
2 * branch_channels,
256,
kernel_size=3,
stride=1,
padding=1,
),
nn.LeakyReLU(negative_slope=0.01),
)
self.pathembed = OverlapPatchEmbed(
img_size=256,
stride=2,
in_chans=256,
embed_dim=1024,
)
self.block = Block(dim=1024, num_heads=2, mlp_ratio=2, sr_ratio=2)
self.pixelShuffel = nn.PixelShuffle(upscale_factor=2)
self.base_head = nn.Sequential(
nn.Conv2d(256, 256, kernel_size=3, stride=1, padding=1),
nn.LeakyReLU(negative_slope=0.01),
nn.Conv2d(256, 128, kernel_size=3, stride=1, padding=1),
nn.LeakyReLU(negative_slope=0.01),
nn.Conv2d(128, 128, kernel_size=3, stride=1, padding=1),
nn.LeakyReLU(negative_slope=0.01),
nn.Conv2d(128, output_channels, kernel_size=3, stride=1, padding=1),
)
self.detail_head = nn.Sequential(
nn.Conv2d(
branch_channels,
128,
kernel_size=3,
stride=1,
padding=1,
),
nn.LeakyReLU(negative_slope=0.01),
nn.Conv2d(128, 128, kernel_size=3, stride=1, padding=1),
nn.LeakyReLU(negative_slope=0.01),
nn.Conv2d(128, output_channels, kernel_size=3, stride=1, padding=1),
)
self.detail_gate = nn.Parameter(
torch.full(
(1, output_channels, 1, 1),
float(initial_detail_scale),
)
)
self.fusion_net.apply(self._init_weights)
self.base_head.apply(self._init_weights)
self.detail_head.apply(self._init_weights)
# The non-zero gate keeps this He-initialized path visible while
# limiting its initial contribution relative to the base triplane.
@staticmethod
def _init_weights(module):
if isinstance(module, nn.Linear):
trunc_normal_(module.weight, std=0.02)
if module.bias is not None:
nn.init.constant_(module.bias, 0)
elif isinstance(module, nn.Conv2d):
fan_out = (
module.kernel_size[0]
* module.kernel_size[1]
* module.out_channels
)
fan_out //= module.groups
module.weight.data.normal_(0, math.sqrt(2.0 / fan_out))
if module.bias is not None:
module.bias.data.zero_()
def forward_components(self, high_feature, global_feature):
if high_feature.ndim != 4 or high_feature.shape[1:] != (
self.branch_channels,
256,
256,
):
raise ValueError(
"INS high feature must have shape [B, "
f"{self.branch_channels}, 256, 256], got "
f"{tuple(high_feature.shape)}."
)
if global_feature.shape != high_feature.shape:
raise ValueError(
"INS global/high features must have identical shapes, got "
f"{tuple(global_feature.shape)} and "
f"{tuple(high_feature.shape)}."
)
normalized_high = self.high_norm(high_feature)
normalized_global = self.global_norm(global_feature)
fused = self.fusion_net(
torch.cat([normalized_high, normalized_global], dim=1)
)
tokens, height, width = self.pathembed(fused)
tokens = self.block(tokens, height, width)
base = rearrange(
tokens,
"b (h w) c -> b c h w",
h=height,
w=width,
)
base = self.pixelShuffel(base)
base = self.base_head(base)
detail = self.detail_head(normalized_high)
return base, detail
def forward(self, high_feature, global_feature):
base, detail = self.forward_components(high_feature, global_feature)
return base + self.detail_gate * detail
class EG3DInvEncoder(nn.Module):
def __init__(
self,
in_channels=5,
encoder_name="resnet34",
encoder_depth=3,
mode="Normal",
use_bn=False,
) -> None:
super(EG3DInvEncoder, self).__init__()
self.mode = mode
deeplab = DeepLabV3(in_channels=in_channels, encoder_name=encoder_name, encoder_depth=encoder_depth) # encoder_depth [3, 5]
self.encoder = deeplab.encoder
self.decoder = deeplab.decoder
if not use_bn: # 很漂亮的实现方式
encoder_name_modules = self.encoder.named_modules()
self.set_module(encoder_name_modules, self.encoder)
decoder_name_modules = self.decoder.named_modules() # 对于decoder也这样操作
self.set_module(decoder_name_modules, self.decoder) # 当ASPP中不使用可分离卷积时就不会有bug
self.F_encoder = EncoderF(mode=self.mode)
self.EHigh_encoder = EncoderEhigh(mode=self.mode)
self.Final_encoder = EncoderFinal(mode=self.mode)
def set_module(self, name_modules, root_module):
'''
Remove BN, Conv add bias
Original: Conv2d + BN + ReLU (Do not use separate Conv)
Now: Conv2d(add bias) + ReLU
'''
for name, m in name_modules:
if isinstance(m, nn.BatchNorm2d): # 反射机制
names = name.split(sep='.')
parents = reduce(getattr, names[:-1], root_module)
setattr(parents, names[-1], nn.Identity())
elif isinstance(m, nn.Conv2d):
if m.bias is not None: # may not enter here
m.bias.data.zero_()
else:
m.bias = nn.Parameter(torch.zeros(m.out_channels))
# optimizer utils
def get_params(self, lr):
# 可分训练阶段进行调整
params = [
{'params': self.encoder.parameters(), 'lr': lr},
{'params': self.decoder.parameters(), 'lr': lr},
{'params': self.F_encoder.parameters(), 'lr': lr},
{'params': self.EHigh_encoder.parameters(), 'lr': lr},
{'params': self.Final_encoder.parameters(), 'lr': lr},
]
return params
def forward(self, x):
# x is [B, 5, H, W]
chan_num = x.shape[1]
device = x.device
if chan_num == 3:
# use coordnate as the fourth and fifth channle
B = x.shape[0]
H, W = x.shape[2], x.shape[3]
grid_x, grid_y = torch.meshgrid(torch.arange(H), torch.arange(W), indexing='ij')
grid_x = repeat(grid_x, "h w -> b c h w", b=B, c=1).to(device)
grid_y = repeat(grid_y, "h w -> b c h w", b=B, c=1).to(device)
# print(x.shape, grid_x.shape, grid_y.shape)
x = torch.cat([x, grid_x, grid_y], dim=1)
chan_num = x.shape[1]
assert chan_num == 5, "x.shape[1] shoule be 5"
x_encode = self.encoder(x)
low_feature = self.decoder(x_encode[-1])
# low_feature = self.decoder(*x_encode)
f_feature = self.F_encoder(low_feature)
high_feature = self.EHigh_encoder(x)
# if self.mode == "Normal":
# high_feature = self.EHigh_encoder(x)
# elif self.mode == "LT":
# high_feature = self.EHigh_encoder(x_encode[1])
# else:
# raise ValueError("mode should be Normal or LT")
output = self.Final_encoder(high_feature, f_feature)
return output
def config_option():
rendering_options = {
'image_resolution': 128,
'disparity_space_sampling': False,
'clamp_mode': 'softplus',
'sr_antialias': True,
}
rendering_options.update({
'depth_resolution': 48, # number of uniform samples to take per ray.
'depth_resolution_importance': 48, # number of importance samples to take per ray.
'ray_start': 2.25, # near point along each ray to start taking samples.
'ray_end': 3.3, # far point along each ray to stop taking samples.
'box_warp': 1, # the side-length of the bounding box spanned by the tri-planes; box_warp=1 means [-0.5, -0.5, -0.5] -> [0.5, 0.5, 0.5].
'avg_camera_radius': 2.7, # used only in the visualizer to specify camera orbit radius.
'avg_camera_pivot': [0, 0, 0.2], # used only in the visualizer to control center of camera rotation.
})
return rendering_options
def debug_test():
# mode = "Normal"
# # mode = "LT"
# a = torch.rand((2, 5, 512, 512))
# layer = DeepLabV3(in_channels=5, encoder_name="resnet34", encoder_depth=3) # encoder_depth [3, 5]
# encoder = layer.encoder # 先尝试,后面再改
# decoder = layer.decoder
# x = decoder(encoder(a)[-1])
# print(x.shape) # torch.size(2, 256, 64, 64)
# F = EncoderF(mode=mode)
# f_feature = F(x)
# print(f_feature.shape) # [2, 96, 256, 256] # LT: [2, 128, 128, 128]
# EHigh = EncoderEhigh(mode=mode)
# y = EHigh(a)
# print(y.shape) # [2, 96, 256, 256] # LT: [2, 128, 128, 128]
# # 融合
# final_layer = EncoderFinal(mode=mode)
# feature = final_layer(y, f_feature)
# print(feature.shape) # [2, 96, 256, 256]
############################
device = 'cuda'
render_options = config_option()
B = 1
# plane = torch.rand((B, 96, 256, 256)) # 可以修改为其他的分辨率
a = torch.rand((B, 5, 512, 512)).to(device)
m = EG3DInvEncoder(in_channels=5, encoder_name="resnet34", encoder_depth=3, mode="LT", use_bn=False).to(device)
m.eval()
for i in range(50):
a = torch.rand((B, 5, 512, 512)).to(device)
s1 = time.time()
plane = m(a).contiguous()
s2 = time.time()
print("time: {} ms".format((s2 - s1 ) * 1000))
print(plane.shape)
# # print(plane.shape)
# c = [0.8452497124671936, 0.028885385021567345, -0.5335903167724609, 1.3744339756005164, -0.03056236356496811, -0.9942903518676758, -0.10223815590143204, 0.27722038701208845, -0.5334968566894531, 0.10272454470396042, -0.8395407795906067, 2.307396824072493, 0.0, 0.0, 0.0, 1.0, 4.2647, 0.0, 0.5, 0.0, 4.2647, 0.5, 0.0, 0.0, 1.0]
# c = torch.tensor(c).repeat((B, 1))
# print(c.shape)
print("Rendering... ")
render = TriplaneRenderer(img_resolution=512, img_channels=3, rendering_kwargs=render_options).to(device)
for i in range(50):
c = torch.rand((B, 25)).to(device)
s1 = time.time()
d = render(plane, c)
s2 = time.time()
print("time: {} ms".format((s2 - s1 ) * 1000))
# print(list(m.named_parameters()))
# print(list(m.named_modules()))
# print(d['image'].shape)
# print(d['image_raw'].shape)
# print(d['image_depth'].shape)
# print(d['feature_image'].shape)
# print(d['planes'].shape)
import torch
import torch.nn as nn
import torch.nn.functional as F
class GaussianDecoder(nn.Module):
def __init__(self, max_position_offset=None, scale_min=1e-5):
super().__init__()
self.fc1 = nn.Linear(48, 96)
self.fc2 = nn.Linear(96, 14)
self.max_position_offset = max_position_offset
self.scale_min = scale_min
# Quaternion identity: [w, x, y, z]
with torch.no_grad():
self.fc2.bias[6:10].zero_()
self.fc2.bias[6] = 1.0
# Initial opacity around 0.1
self.fc2.bias[10] = -2.197
def forward(self, features, sample_positions=None):
hidden = F.softplus(self.fc1(features))
raw = self.fc2(hidden)
raw_mu, raw_scale, raw_quat, raw_opacity, raw_color = \
raw.split([3, 3, 4, 1, 3], dim=-1)
# This parameterization is not specified by the paper.
if sample_positions is None:
mu = raw_mu
elif self.max_position_offset is None:
mu = sample_positions + raw_mu
else:
offset = self.max_position_offset * torch.tanh(raw_mu)
mu = sample_positions + offset
scale = F.softplus(raw_scale) + self.scale_min
quat = F.normalize(raw_quat, dim=-1, eps=1e-8)
opacity = torch.sigmoid(raw_opacity)
color = torch.sigmoid(raw_color)
return {
"means3D": mu,
"scales": scale,
"rotations": quat,
"opacities": opacity,
"colors": color,
}
if __name__ == '__main__':
debug_test()