Spaces:
Running on Zero
Running on Zero
Download fab/models/modules/camera_embed.py from hugging-apps/fab-body-estimation: direct link, hf CLI and curl.
- Browser
- Download file 3.57 kB
-
https://huggingface.co/spaces/hugging-apps/fab-body-estimation/resolve/main/fab/models/modules/camera_embed.py
- Command line
-
hf download hf://spaces/hugging-apps/fab-body-estimation/fab/models/modules/camera_embed.py
-
curl -L -o camera_embed.py https://huggingface.co/spaces/hugging-apps/fab-body-estimation/resolve/main/fab/models/modules/camera_embed.py
3.57 kB
| # Copyright (c) Meta Platforms, Inc. and affiliates. | |
| # SPDX-License-Identifier: LicenseRef-SAM-License | |
| # From SAM 3D Body (https://github.com/facebookresearch/sam-3d-body), modified for FAB; licence: third_party_licenses/SAM-License.txt | |
| import einops | |
| import numpy as np | |
| import torch | |
| import torch.nn.functional as F | |
| from fab.models.modules.transformer import LayerNorm2d | |
| from torch import nn | |
| class CameraEncoder(nn.Module): | |
| def __init__(self, embed_dim, patch_size=14): | |
| super().__init__() | |
| self.patch_size = patch_size | |
| self.embed_dim = embed_dim | |
| self.camera = FourierPositionEncoding(n=3, num_bands=16, max_resolution=64) | |
| self.conv = nn.Conv2d(embed_dim + 99, embed_dim, kernel_size=1, bias=False) | |
| self.norm = LayerNorm2d(embed_dim) | |
| def forward(self, img_embeddings, rays): | |
| B, D, _h, _w = img_embeddings.shape | |
| with torch.no_grad(): | |
| scale = 1 / self.patch_size | |
| rays = F.interpolate( | |
| rays, | |
| scale_factor=(scale, scale), | |
| mode="bilinear", | |
| align_corners=False, | |
| antialias=True, | |
| ) | |
| rays = rays.permute(0, 2, 3, 1).contiguous() # [b, h, w, 2] | |
| rays = torch.cat([rays, torch.ones_like(rays[..., :1])], dim=-1) | |
| rays_embeddings = self.camera( | |
| pos=rays.reshape(B, -1, 3) | |
| ) # (bs, N, 99): rays fourier embedding | |
| rays_embeddings = einops.rearrange( | |
| rays_embeddings, "b (h w) c -> b c h w", h=_h, w=_w | |
| ).contiguous() | |
| z = torch.concat([img_embeddings, rays_embeddings], dim=1) | |
| z = self.norm(self.conv(z)) | |
| return z | |
| class FourierPositionEncoding(nn.Module): | |
| def __init__(self, n, num_bands, max_resolution): | |
| """ | |
| Module that generate Fourier encoding - no learning involved | |
| """ | |
| super().__init__() | |
| self.num_bands = num_bands | |
| self.max_resolution = [max_resolution] * n | |
| def channels(self): | |
| """ | |
| Return the output dimension | |
| """ | |
| num_dims = len(self.max_resolution) | |
| encoding_size = self.num_bands * num_dims | |
| encoding_size *= 2 # sin-cos | |
| encoding_size += num_dims # concat | |
| return encoding_size | |
| def forward(self, pos): | |
| """ | |
| Forward pass that take rays as input and generate Fourier positional encodings | |
| """ | |
| fourier_pos_enc = _generate_fourier_features( | |
| pos, num_bands=self.num_bands, max_resolution=self.max_resolution | |
| ) | |
| return fourier_pos_enc | |
| def _generate_fourier_features(pos, num_bands, max_resolution): | |
| """Generate fourier features from a given set of positions and frequencies""" | |
| b, n = pos.shape[:2] | |
| device = pos.device | |
| # Linear frequency sampling | |
| min_freq = 1.0 | |
| freq_bands = torch.stack( | |
| [ | |
| torch.linspace(start=min_freq, end=res / 2, steps=num_bands, device=device) | |
| for res in max_resolution | |
| ], | |
| dim=0, | |
| ) | |
| # Stacking | |
| per_pos_features = torch.stack( | |
| [pos[i, :, :][:, :, None] * freq_bands[None, :, :] for i in range(b)], 0 | |
| ) | |
| per_pos_features = per_pos_features.reshape(b, n, -1) | |
| # Sin-Cos | |
| per_pos_features = torch.cat( | |
| [torch.sin(np.pi * per_pos_features), torch.cos(np.pi * per_pos_features)], | |
| dim=-1, | |
| ) | |
| # Concat with initial pos | |
| per_pos_features = torch.cat([pos, per_pos_features], dim=-1) | |
| return per_pos_features | |