import torch import torch.nn as nn import torch.nn.functional as F # import segmentation_models_pytorch from segmentation_models_pytorch.decoders.deeplabv3 import DeepLabV3 ''' impletement of RealTimeRF including the full model and LT model ''' from timm.models.layers import DropPath, to_2tuple, trunc_normal_ import math import time from einops import rearrange, repeat import sys # sys.path.insert(0, os.path.abspath(os.path.join(__file__, '..'))) sys.path.append(".") sys.path.append("..") from torch_utils.ops import upfirdn2d from torch_utils.misc import assert_shape from functools import reduce from typing import Union from segmentation_models_pytorch.encoders.mix_transformer import OverlapPatchEmbed, Block from models.eg3d.volumetric_rendering.renderer import ImportanceRenderer, sample_from_planes from models.eg3d.volumetric_rendering.ray_sampler import RaySampler from models.eg3d.superresolution import SuperresolutionHybrid8XDC from models.eg3d.networks_stylegan2 import FullyConnectedLayer from models.eg3d.triplane import OSGDecoder from models.eg3d.networks_stylegan2 import Generator as StyleGAN2Backbone class TriGenerator(nn.Module): ''' similar to TriplaneGenerator class but lack of renderer ''' def __init__(self, # 参数表暂时不删,做占位用 z_dim, # Input latent (Z) dimensionality. c_dim, # Conditioning label (C) dimensionality. w_dim, # Intermediate latent (W) dimensionality. img_resolution, # Output resolution. img_channels, # Number of output color channels. sr_num_fp16_res = 0, mapping_kwargs = {}, # Arguments for MappingNetwork. rendering_kwargs = {}, sr_kwargs = {}, **synthesis_kwargs, # Arguments for SynthesisNetwork. ): super().__init__() self.z_dim=z_dim self.c_dim=c_dim self.w_dim=w_dim self.backbone = StyleGAN2Backbone(z_dim, c_dim, w_dim, img_resolution=256, img_channels=32*3, mapping_kwargs=mapping_kwargs, **synthesis_kwargs) self._last_planes = None self.rendering_kwargs = rendering_kwargs def mapping(self, z, c, truncation_psi=1, truncation_cutoff=None, update_emas=False): if self.rendering_kwargs['c_gen_conditioning_zero']: c = torch.zeros_like(c) return self.backbone.mapping(z, c * self.rendering_kwargs.get('c_scale', 0), truncation_psi=truncation_psi, truncation_cutoff=truncation_cutoff, update_emas=update_emas) def synthesis(self, ws, update_emas=False, cache_backbone=False, use_cached_backbone=False, **synthesis_kwargs): if use_cached_backbone and self._last_planes is not None: planes = self._last_planes else: planes = self.backbone.synthesis(ws, update_emas=update_emas, **synthesis_kwargs) if cache_backbone: self._last_planes = planes return planes def forward(self, z, c, truncation_psi=1, truncation_cutoff=None, neural_rendering_resolution=None, update_emas=False, cache_backbone=False, use_cached_backbone=False, **synthesis_kwargs): # Render a batch of generated images. ws = self.mapping(z, c, truncation_psi=truncation_psi, truncation_cutoff=truncation_cutoff, update_emas=update_emas) return self.synthesis(ws, update_emas=update_emas, neural_rendering_resolution=neural_rendering_resolution, cache_backbone=cache_backbone, use_cached_backbone=use_cached_backbone, **synthesis_kwargs) class TriplaneRenderer(nn.Module): def __init__(self, img_resolution, img_channels, rendering_kwargs={}) -> None: ''' Triplane Renderer Generate 2D image from triplanes representation SuperResolution without stylecode FullyConnected layer ''' super(TriplaneRenderer, self).__init__() self.img_resolution=img_resolution self.img_channels=img_channels self.renderer = ImportanceRenderer() self.ray_sampler = RaySampler() self.w_dim = 512 self.const = torch.nn.Parameter(torch.randn([1, 1, self.w_dim])) # 常数输入 self.decoder = OSGDecoder(32, {'decoder_lr_mul': rendering_kwargs.get('decoder_lr_mul', 1), 'decoder_output_dim': 32}) self.superresolution = SuperresolutionHybrid8XDC(32, img_resolution, sr_num_fp16_res=0, sr_antialias=True) self.rendering_kwargs = rendering_kwargs self.neural_rendering_resolution = 128 # 64 def synthesis(self, planes, c, neural_rendering_resolution=None): cam2world_matrix = c[:, :16].view(-1, 4, 4) intrinsics = c[:, 16:25].view(-1, 3, 3) if neural_rendering_resolution is None: neural_rendering_resolution = self.neural_rendering_resolution else: self.neural_rendering_resolution = neural_rendering_resolution # Create a batch of rays for volume rendering ray_origins, ray_directions = self.ray_sampler(cam2world_matrix, intrinsics, neural_rendering_resolution) N, _, _ = ray_origins.shape # Reshape output into three 32-channel planes planes = planes.view(len(planes), 3, 32, planes.shape[-2], planes.shape[-1]) # Perform volume rendering feature_samples, depth_samples, _ = self.renderer(planes, self.decoder, ray_origins, ray_directions, self.rendering_kwargs) # channels last # Reshape into 'raw' neural-rendered image H = W = neural_rendering_resolution feature_image = feature_samples.permute(0, 2, 1).reshape(N, feature_samples.shape[-1], H, W).contiguous() depth_image = depth_samples.permute(0, 2, 1).reshape(N, 1, H, W) # Run superresolution to get final image rgb_image = feature_image[:, :3] # sr_image = self.superresolution(rgb_image, feature_image, ws, noise_mode='none') const_w_input = self.const.repeat([N, 1, 1]) sr_image = self.superresolution(rgb_image, feature_image, const_w_input, noise_mode='none') return {'image': sr_image, 'image_raw': rgb_image, 'image_depth': depth_image, 'feature_image': feature_image, 'planes': planes} def sample_density(self, planes, coordinates, directions): ''' 给定triplanes和camera参数,生成图像并且返回 ''' sampled_features = sample_from_planes(self.renderer.plane_axes, planes, coordinates, padding_mode='zeros', box_warp=self.rendering_kwargs['box_warp']) out = self.decoder(sampled_features, directions) return out def forward(self, planes, c): ''' 给定triplanes和camera参数,生成图像并且返回 ''' return self.synthesis(planes, c) # coding self RTRF class EncoderEhigh(nn.Module): def __init__(self, mode="Normal") -> None: super(EncoderEhigh, self).__init__() self.mode = mode self.net = nn.Sequential( nn.Conv2d(5, 64, kernel_size=7, stride=2, padding=3), nn.LeakyReLU(negative_slope=0.01), nn.Conv2d(64, 96, kernel_size=3, stride=1, padding=1), nn.LeakyReLU(negative_slope=0.01), nn.Conv2d(96, 96, kernel_size=3, stride=1, padding=1), nn.LeakyReLU(negative_slope=0.01), nn.Conv2d(96, 96, kernel_size=3, stride=1, padding=1), nn.LeakyReLU(negative_slope=0.01), nn.Conv2d(96, 96, kernel_size=3, stride=1, padding=1), nn.LeakyReLU(negative_slope=0.01) ) if self.mode == "Normal" else nn.Sequential( # Input is Second layer output of DeeplabV3 nn.Conv2d(5, 64, kernel_size=7, stride=2, padding=3), nn.LeakyReLU(negative_slope=0.01), nn.Conv2d(64, 128, kernel_size=3, stride=2, padding=1), nn.LeakyReLU(negative_slope=0.01), nn.Conv2d(128, 128, kernel_size=3, stride=1, padding=1), nn.LeakyReLU(negative_slope=0.01), nn.Conv2d(128, 128, kernel_size=3, stride=1, padding=1), nn.LeakyReLU(negative_slope=0.01), nn.Conv2d(128, 128, kernel_size=3, stride=1, padding=1), nn.LeakyReLU(negative_slope=0.01), nn.Conv2d(128, 128, kernel_size=3, stride=1, padding=1), nn.LeakyReLU(negative_slope=0.01), nn.Conv2d(128, 128, kernel_size=3, stride=1, padding=1), ) self.net.apply(self._init_weights) # 仿照timm的初始化方式 def _init_weights(self, m): if isinstance(m, nn.Linear): trunc_normal_(m.weight, std=0.02) if isinstance(m, nn.Linear) and m.bias is not None: nn.init.constant_(m.bias, 0) elif isinstance(m, nn.Conv2d): fan_out = m.kernel_size[0] * m.kernel_size[1] * m.out_channels fan_out //= m.groups m.weight.data.normal_(0, math.sqrt(2.0 / fan_out)) if m.bias is not None: m.bias.data.zero_() def forward(self, img): return self.net(img) class EncoderF(nn.Module): def __init__(self, mode="Normal") -> None: super(EncoderF, self).__init__() self.mode = mode self.blocks_num = 5 if self.mode == 'Normal' else 2 self.patchembed = OverlapPatchEmbed(img_size=64, stride=2, in_chans=256, embed_dim=1024) self.blocks = nn.ModuleList([Block(dim=1024, num_heads=4, mlp_ratio=2, sr_ratio=1) for _ in range(self.blocks_num)]) self.upsample = nn.Sequential( nn.PixelShuffle(upscale_factor=2), nn.UpsamplingBilinear2d(scale_factor=2.0), # equivalent to ``nn.functional.interpolate(..., mode='bilinear', align_corners=True)`` ) self.net = nn.Sequential( nn.Conv2d(256, 128, kernel_size=3, stride=1, padding=1), nn.ReLU(), nn.UpsamplingBilinear2d(scale_factor=2.0), # equivalent to ``nn.functional.interpolate(..., mode='bilinear', align_corners=True)`` nn.Conv2d(128, 128, kernel_size=3, stride=1, padding=1), nn.ReLU(), nn.Conv2d(128, 96, kernel_size=3, stride=1, padding=1) ) if self.mode == "Normal" else nn.Sequential( nn.Conv2d(256, 128, kernel_size=3, stride=1, padding=1), nn.LeakyReLU(negative_slope=0.01), nn.Conv2d(128, 128, kernel_size=3, stride=1, padding=1), ) self.net.apply(self._init_weights) # 仿照timm的初始化方式 def _init_weights(self, m): if isinstance(m, nn.Linear): trunc_normal_(m.weight, std=0.02) if isinstance(m, nn.Linear) and m.bias is not None: nn.init.constant_(m.bias, 0) # elif isinstance(m, nn.LayerNorm): # nn.init.constant_(m.bias, 0) # nn.init.constant_(m.weight, 1.0) elif isinstance(m, nn.Conv2d): fan_out = m.kernel_size[0] * m.kernel_size[1] * m.out_channels fan_out //= m.groups m.weight.data.normal_(0, math.sqrt(2.0 / fan_out)) if m.bias is not None: m.bias.data.zero_() def forward(self, x): x, H, W = self.patchembed(x) for i in range(self.blocks_num): x = self.blocks[i](x, H, W) # [B, N, C] # print((H, W)) # (32, 32) # print(x.shape) # [2, 1024, 1024] x = rearrange(x, "b (h w) c -> b c h w", h=H, w=W) # print(x.shape) # [2, 1024, 32, 32] x = self.upsample(x) x = self.net(x) return x class EncoderFinal(nn.Module): def __init__(self, mode="Normal") -> None: super(EncoderFinal, self).__init__() self.mode = mode self.net = nn.Sequential( nn.Conv2d(192, 256, kernel_size=3, stride=1, padding=1), nn.LeakyReLU(negative_slope=0.01), nn.Conv2d(256, 128, kernel_size=3, stride=1, padding=1), nn.LeakyReLU(negative_slope=0.01), ) if self.mode == "Normal" else nn.Sequential( nn.Conv2d(256, 256, kernel_size=3, stride=1, padding=1), ) if self.mode == "Normal": self.pathembed = OverlapPatchEmbed(img_size=256, stride=2, in_chans=128, embed_dim=1024) self.block = Block(dim=1024, num_heads=2, mlp_ratio=2, sr_ratio=2) else: self.pathembed = OverlapPatchEmbed(img_size=128, stride=2, in_chans=256, embed_dim=1024) self.block = Block(dim=1024, num_heads=2, mlp_ratio=2, sr_ratio=2) # self.pathembed = OverlapPatchEmbed(img_size=128, stride=2, in_chans=128, embed_dim=1024) # self.block = Block(dim=256, num_heads=2, mlp_ratio=2, sr_ratio=2) self.pixelShuffel = nn.PixelShuffle(upscale_factor=2) # concat with output of encoder F # self.F_feature = F_feature if self.mode == "Normal": self.output_net = nn.Sequential( nn.Conv2d(352, 256, kernel_size=3, stride=1, padding=1), nn.LeakyReLU(negative_slope=0.01), nn.Conv2d(256, 128, kernel_size=3, stride=1, padding=1), nn.LeakyReLU(negative_slope=0.01), nn.Conv2d(128, 128, kernel_size=3, stride=1, padding=1), nn.LeakyReLU(negative_slope=0.01), nn.Conv2d(128, 96, kernel_size=3, stride=1, padding=1), ) elif self.mode == "INS": self.output_net = nn.Sequential( nn.Conv2d(352, 256, kernel_size=3, stride=1, padding=1), nn.LeakyReLU(negative_slope=0.01), nn.Conv2d(256, 128, kernel_size=3, stride=1, padding=1), nn.LeakyReLU(negative_slope=0.01), nn.Conv2d(128, 128, kernel_size=3, stride=1, padding=1), nn.LeakyReLU(negative_slope=0.01), nn.Conv2d(128, 288, kernel_size=3, stride=1, padding=1) ) else: self.output_net = nn.Sequential( nn.Conv2d(256, 256, kernel_size=3, stride=1, padding=1), nn.LeakyReLU(negative_slope=0.01), nn.UpsamplingBilinear2d(scale_factor=2.0), nn.Conv2d(256, 128, kernel_size=3, stride=1, padding=1), nn.LeakyReLU(negative_slope=0.01), nn.Conv2d(128, 128, kernel_size=3, stride=1, padding=1), nn.LeakyReLU(negative_slope=0.01), nn.Conv2d(128, 96, kernel_size=3, stride=1, padding=1), nn.LeakyReLU(negative_slope=0.01), nn.Conv2d(96, 96, kernel_size=3, stride=1, padding=1), ) self.net.apply(self._init_weights) # 仿照timm的初始化方式 self.output_net.apply(self._init_weights) # 仿照timm的初始化方式`` def _init_weights(self, m): if isinstance(m, nn.Linear): trunc_normal_(m.weight, std=0.02) if isinstance(m, nn.Linear) and m.bias is not None: nn.init.constant_(m.bias, 0) # elif isinstance(m, nn.LayerNorm): # nn.init.constant_(m.bias, 0) # nn.init.constant_(m.weight, 1.0) elif isinstance(m, nn.Conv2d): fan_out = m.kernel_size[0] * m.kernel_size[1] * m.out_channels fan_out //= m.groups m.weight.data.normal_(0, math.sqrt(2.0 / fan_out)) if m.bias is not None: m.bias.data.zero_() def forward(self, x, F_feature): # debug xx = self.net(torch.cat([x, F_feature], dim=1)) xx, H, W = self.pathembed(xx) xx = self.block(xx, H, W) xx = rearrange(xx, "b (h w) c -> b c h w", h=H, w=W) xx = self.pixelShuffel(xx) if self.mode == "Normal": return self.output_net(torch.cat([xx, F_feature], dim=1)) else: return self.output_net(xx) class INSResidualEncoderFinal(nn.Module): """Balanced INS fusion with a direct high-resolution residual path.""" def __init__( self, branch_channels=96, output_channels=288, norm_groups=8, initial_detail_scale=0.25, ) -> None: super().__init__() if branch_channels <= 0 or output_channels <= 0: raise ValueError("INS fusion channel counts must be positive.") if branch_channels % norm_groups != 0: raise ValueError("branch_channels must be divisible by norm_groups.") if not math.isfinite(initial_detail_scale) or initial_detail_scale == 0: raise ValueError("initial_detail_scale must be finite and non-zero.") self.mode = "INS" self.branch_channels = int(branch_channels) self.output_channels = int(output_channels) self.high_norm = nn.GroupNorm(norm_groups, branch_channels) self.global_norm = nn.GroupNorm(norm_groups, branch_channels) self.fusion_net = nn.Sequential( nn.Conv2d( 2 * branch_channels, 256, kernel_size=3, stride=1, padding=1, ), nn.LeakyReLU(negative_slope=0.01), ) self.pathembed = OverlapPatchEmbed( img_size=256, stride=2, in_chans=256, embed_dim=1024, ) self.block = Block(dim=1024, num_heads=2, mlp_ratio=2, sr_ratio=2) self.pixelShuffel = nn.PixelShuffle(upscale_factor=2) self.base_head = nn.Sequential( nn.Conv2d(256, 256, kernel_size=3, stride=1, padding=1), nn.LeakyReLU(negative_slope=0.01), nn.Conv2d(256, 128, kernel_size=3, stride=1, padding=1), nn.LeakyReLU(negative_slope=0.01), nn.Conv2d(128, 128, kernel_size=3, stride=1, padding=1), nn.LeakyReLU(negative_slope=0.01), nn.Conv2d(128, output_channels, kernel_size=3, stride=1, padding=1), ) self.detail_head = nn.Sequential( nn.Conv2d( branch_channels, 128, kernel_size=3, stride=1, padding=1, ), nn.LeakyReLU(negative_slope=0.01), nn.Conv2d(128, 128, kernel_size=3, stride=1, padding=1), nn.LeakyReLU(negative_slope=0.01), nn.Conv2d(128, output_channels, kernel_size=3, stride=1, padding=1), ) self.detail_gate = nn.Parameter( torch.full( (1, output_channels, 1, 1), float(initial_detail_scale), ) ) self.fusion_net.apply(self._init_weights) self.base_head.apply(self._init_weights) self.detail_head.apply(self._init_weights) # The non-zero gate keeps this He-initialized path visible while # limiting its initial contribution relative to the base triplane. @staticmethod def _init_weights(module): if isinstance(module, nn.Linear): trunc_normal_(module.weight, std=0.02) if module.bias is not None: nn.init.constant_(module.bias, 0) elif isinstance(module, nn.Conv2d): fan_out = ( module.kernel_size[0] * module.kernel_size[1] * module.out_channels ) fan_out //= module.groups module.weight.data.normal_(0, math.sqrt(2.0 / fan_out)) if module.bias is not None: module.bias.data.zero_() def forward_components(self, high_feature, global_feature): if high_feature.ndim != 4 or high_feature.shape[1:] != ( self.branch_channels, 256, 256, ): raise ValueError( "INS high feature must have shape [B, " f"{self.branch_channels}, 256, 256], got " f"{tuple(high_feature.shape)}." ) if global_feature.shape != high_feature.shape: raise ValueError( "INS global/high features must have identical shapes, got " f"{tuple(global_feature.shape)} and " f"{tuple(high_feature.shape)}." ) normalized_high = self.high_norm(high_feature) normalized_global = self.global_norm(global_feature) fused = self.fusion_net( torch.cat([normalized_high, normalized_global], dim=1) ) tokens, height, width = self.pathembed(fused) tokens = self.block(tokens, height, width) base = rearrange( tokens, "b (h w) c -> b c h w", h=height, w=width, ) base = self.pixelShuffel(base) base = self.base_head(base) detail = self.detail_head(normalized_high) return base, detail def forward(self, high_feature, global_feature): base, detail = self.forward_components(high_feature, global_feature) return base + self.detail_gate * detail class EG3DInvEncoder(nn.Module): def __init__( self, in_channels=5, encoder_name="resnet34", encoder_depth=3, mode="Normal", use_bn=False, ) -> None: super(EG3DInvEncoder, self).__init__() self.mode = mode deeplab = DeepLabV3(in_channels=in_channels, encoder_name=encoder_name, encoder_depth=encoder_depth) # encoder_depth [3, 5] self.encoder = deeplab.encoder self.decoder = deeplab.decoder if not use_bn: # 很漂亮的实现方式 encoder_name_modules = self.encoder.named_modules() self.set_module(encoder_name_modules, self.encoder) decoder_name_modules = self.decoder.named_modules() # 对于decoder也这样操作 self.set_module(decoder_name_modules, self.decoder) # 当ASPP中不使用可分离卷积时就不会有bug self.F_encoder = EncoderF(mode=self.mode) self.EHigh_encoder = EncoderEhigh(mode=self.mode) self.Final_encoder = EncoderFinal(mode=self.mode) def set_module(self, name_modules, root_module): ''' Remove BN, Conv add bias Original: Conv2d + BN + ReLU (Do not use separate Conv) Now: Conv2d(add bias) + ReLU ''' for name, m in name_modules: if isinstance(m, nn.BatchNorm2d): # 反射机制 names = name.split(sep='.') parents = reduce(getattr, names[:-1], root_module) setattr(parents, names[-1], nn.Identity()) elif isinstance(m, nn.Conv2d): if m.bias is not None: # may not enter here m.bias.data.zero_() else: m.bias = nn.Parameter(torch.zeros(m.out_channels)) # optimizer utils def get_params(self, lr): # 可分训练阶段进行调整 params = [ {'params': self.encoder.parameters(), 'lr': lr}, {'params': self.decoder.parameters(), 'lr': lr}, {'params': self.F_encoder.parameters(), 'lr': lr}, {'params': self.EHigh_encoder.parameters(), 'lr': lr}, {'params': self.Final_encoder.parameters(), 'lr': lr}, ] return params def forward(self, x): # x is [B, 5, H, W] chan_num = x.shape[1] device = x.device if chan_num == 3: # use coordnate as the fourth and fifth channle B = x.shape[0] H, W = x.shape[2], x.shape[3] grid_x, grid_y = torch.meshgrid(torch.arange(H), torch.arange(W), indexing='ij') grid_x = repeat(grid_x, "h w -> b c h w", b=B, c=1).to(device) grid_y = repeat(grid_y, "h w -> b c h w", b=B, c=1).to(device) # print(x.shape, grid_x.shape, grid_y.shape) x = torch.cat([x, grid_x, grid_y], dim=1) chan_num = x.shape[1] assert chan_num == 5, "x.shape[1] shoule be 5" x_encode = self.encoder(x) low_feature = self.decoder(x_encode[-1]) # low_feature = self.decoder(*x_encode) f_feature = self.F_encoder(low_feature) high_feature = self.EHigh_encoder(x) # if self.mode == "Normal": # high_feature = self.EHigh_encoder(x) # elif self.mode == "LT": # high_feature = self.EHigh_encoder(x_encode[1]) # else: # raise ValueError("mode should be Normal or LT") output = self.Final_encoder(high_feature, f_feature) return output def config_option(): rendering_options = { 'image_resolution': 128, 'disparity_space_sampling': False, 'clamp_mode': 'softplus', 'sr_antialias': True, } rendering_options.update({ 'depth_resolution': 48, # number of uniform samples to take per ray. 'depth_resolution_importance': 48, # number of importance samples to take per ray. 'ray_start': 2.25, # near point along each ray to start taking samples. 'ray_end': 3.3, # far point along each ray to stop taking samples. 'box_warp': 1, # the side-length of the bounding box spanned by the tri-planes; box_warp=1 means [-0.5, -0.5, -0.5] -> [0.5, 0.5, 0.5]. 'avg_camera_radius': 2.7, # used only in the visualizer to specify camera orbit radius. 'avg_camera_pivot': [0, 0, 0.2], # used only in the visualizer to control center of camera rotation. }) return rendering_options def debug_test(): # mode = "Normal" # # mode = "LT" # a = torch.rand((2, 5, 512, 512)) # layer = DeepLabV3(in_channels=5, encoder_name="resnet34", encoder_depth=3) # encoder_depth [3, 5] # encoder = layer.encoder # 先尝试,后面再改 # decoder = layer.decoder # x = decoder(encoder(a)[-1]) # print(x.shape) # torch.size(2, 256, 64, 64) # F = EncoderF(mode=mode) # f_feature = F(x) # print(f_feature.shape) # [2, 96, 256, 256] # LT: [2, 128, 128, 128] # EHigh = EncoderEhigh(mode=mode) # y = EHigh(a) # print(y.shape) # [2, 96, 256, 256] # LT: [2, 128, 128, 128] # # 融合 # final_layer = EncoderFinal(mode=mode) # feature = final_layer(y, f_feature) # print(feature.shape) # [2, 96, 256, 256] ############################ device = 'cuda' render_options = config_option() B = 1 # plane = torch.rand((B, 96, 256, 256)) # 可以修改为其他的分辨率 a = torch.rand((B, 5, 512, 512)).to(device) m = EG3DInvEncoder(in_channels=5, encoder_name="resnet34", encoder_depth=3, mode="LT", use_bn=False).to(device) m.eval() for i in range(50): a = torch.rand((B, 5, 512, 512)).to(device) s1 = time.time() plane = m(a).contiguous() s2 = time.time() print("time: {} ms".format((s2 - s1 ) * 1000)) print(plane.shape) # # print(plane.shape) # c = [0.8452497124671936, 0.028885385021567345, -0.5335903167724609, 1.3744339756005164, -0.03056236356496811, -0.9942903518676758, -0.10223815590143204, 0.27722038701208845, -0.5334968566894531, 0.10272454470396042, -0.8395407795906067, 2.307396824072493, 0.0, 0.0, 0.0, 1.0, 4.2647, 0.0, 0.5, 0.0, 4.2647, 0.5, 0.0, 0.0, 1.0] # c = torch.tensor(c).repeat((B, 1)) # print(c.shape) print("Rendering... ") render = TriplaneRenderer(img_resolution=512, img_channels=3, rendering_kwargs=render_options).to(device) for i in range(50): c = torch.rand((B, 25)).to(device) s1 = time.time() d = render(plane, c) s2 = time.time() print("time: {} ms".format((s2 - s1 ) * 1000)) # print(list(m.named_parameters())) # print(list(m.named_modules())) # print(d['image'].shape) # print(d['image_raw'].shape) # print(d['image_depth'].shape) # print(d['feature_image'].shape) # print(d['planes'].shape) import torch import torch.nn as nn import torch.nn.functional as F class GaussianDecoder(nn.Module): def __init__(self, max_position_offset=None, scale_min=1e-5): super().__init__() self.fc1 = nn.Linear(48, 96) self.fc2 = nn.Linear(96, 14) self.max_position_offset = max_position_offset self.scale_min = scale_min # Quaternion identity: [w, x, y, z] with torch.no_grad(): self.fc2.bias[6:10].zero_() self.fc2.bias[6] = 1.0 # Initial opacity around 0.1 self.fc2.bias[10] = -2.197 def forward(self, features, sample_positions=None): hidden = F.softplus(self.fc1(features)) raw = self.fc2(hidden) raw_mu, raw_scale, raw_quat, raw_opacity, raw_color = \ raw.split([3, 3, 4, 1, 3], dim=-1) # This parameterization is not specified by the paper. if sample_positions is None: mu = raw_mu elif self.max_position_offset is None: mu = sample_positions + raw_mu else: offset = self.max_position_offset * torch.tanh(raw_mu) mu = sample_positions + offset scale = F.softplus(raw_scale) + self.scale_min quat = F.normalize(raw_quat, dim=-1, eps=1e-8) opacity = torch.sigmoid(raw_opacity) color = torch.sigmoid(raw_color) return { "means3D": mu, "scales": scale, "rotations": quat, "opacities": opacity, "colors": color, } if __name__ == '__main__': debug_test()