Spaces:
Running on Zero
Running on Zero
File size: 3,568 Bytes
83c6c2d | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 | # Copyright (c) Meta Platforms, Inc. and affiliates.
# SPDX-License-Identifier: LicenseRef-SAM-License
# From SAM 3D Body (https://github.com/facebookresearch/sam-3d-body), modified for FAB; licence: third_party_licenses/SAM-License.txt
import einops
import numpy as np
import torch
import torch.nn.functional as F
from fab.models.modules.transformer import LayerNorm2d
from torch import nn
class CameraEncoder(nn.Module):
def __init__(self, embed_dim, patch_size=14):
super().__init__()
self.patch_size = patch_size
self.embed_dim = embed_dim
self.camera = FourierPositionEncoding(n=3, num_bands=16, max_resolution=64)
self.conv = nn.Conv2d(embed_dim + 99, embed_dim, kernel_size=1, bias=False)
self.norm = LayerNorm2d(embed_dim)
def forward(self, img_embeddings, rays):
B, D, _h, _w = img_embeddings.shape
with torch.no_grad():
scale = 1 / self.patch_size
rays = F.interpolate(
rays,
scale_factor=(scale, scale),
mode="bilinear",
align_corners=False,
antialias=True,
)
rays = rays.permute(0, 2, 3, 1).contiguous() # [b, h, w, 2]
rays = torch.cat([rays, torch.ones_like(rays[..., :1])], dim=-1)
rays_embeddings = self.camera(
pos=rays.reshape(B, -1, 3)
) # (bs, N, 99): rays fourier embedding
rays_embeddings = einops.rearrange(
rays_embeddings, "b (h w) c -> b c h w", h=_h, w=_w
).contiguous()
z = torch.concat([img_embeddings, rays_embeddings], dim=1)
z = self.norm(self.conv(z))
return z
class FourierPositionEncoding(nn.Module):
def __init__(self, n, num_bands, max_resolution):
"""
Module that generate Fourier encoding - no learning involved
"""
super().__init__()
self.num_bands = num_bands
self.max_resolution = [max_resolution] * n
@property
def channels(self):
"""
Return the output dimension
"""
num_dims = len(self.max_resolution)
encoding_size = self.num_bands * num_dims
encoding_size *= 2 # sin-cos
encoding_size += num_dims # concat
return encoding_size
def forward(self, pos):
"""
Forward pass that take rays as input and generate Fourier positional encodings
"""
fourier_pos_enc = _generate_fourier_features(
pos, num_bands=self.num_bands, max_resolution=self.max_resolution
)
return fourier_pos_enc
def _generate_fourier_features(pos, num_bands, max_resolution):
"""Generate fourier features from a given set of positions and frequencies"""
b, n = pos.shape[:2]
device = pos.device
# Linear frequency sampling
min_freq = 1.0
freq_bands = torch.stack(
[
torch.linspace(start=min_freq, end=res / 2, steps=num_bands, device=device)
for res in max_resolution
],
dim=0,
)
# Stacking
per_pos_features = torch.stack(
[pos[i, :, :][:, :, None] * freq_bands[None, :, :] for i in range(b)], 0
)
per_pos_features = per_pos_features.reshape(b, n, -1)
# Sin-Cos
per_pos_features = torch.cat(
[torch.sin(np.pi * per_pos_features), torch.cos(np.pi * per_pos_features)],
dim=-1,
)
# Concat with initial pos
per_pos_features = torch.cat([pos, per_pos_features], dim=-1)
return per_pos_features
|