Spaces:
Running on Zero
Running on Zero
Upload folder using huggingface_hub
Browse files- gaussianformer/__init__.py +5 -0
- gaussianformer/encodings/__init__.py +0 -0
- gaussianformer/encodings/rope.py +65 -0
- gaussianformer/layers/__init__.py +0 -0
- gaussianformer/layers/attention.py +212 -0
- gaussianformer/layers/dpt.py +244 -0
- gaussianformer/layers/window.py +100 -0
- gaussianformer/models/__init__.py +0 -0
- gaussianformer/models/config.py +84 -0
- gaussianformer/models/gaussianformer.py +65 -0
- gaussianformer/models/view_transformer.py +117 -0
- gaussianformer/pipelines/__init__.py +0 -0
- gaussianformer/pipelines/rendering_pipeline.py +59 -0
- gaussianformer/splat.py +153 -0
- gaussianformer/utils/__init__.py +0 -0
- gaussianformer/utils/cameras.py +44 -0
- gaussianformer/utils/checkpoint.py +34 -0
- gaussianformer/utils/ray_generator.py +22 -0
- gaussianformer/utils/transform.py +30 -0
gaussianformer/__init__.py
ADDED
|
@@ -0,0 +1,5 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
from gaussianformer.models.gaussianformer import GaussianFormer
|
| 2 |
+
from gaussianformer.pipelines.rendering_pipeline import GaussianFormerRenderingPipeline
|
| 3 |
+
from gaussianformer.splat import load_ply
|
| 4 |
+
|
| 5 |
+
__all__ = ["GaussianFormer", "GaussianFormerRenderingPipeline", "load_ply"]
|
gaussianformer/encodings/__init__.py
ADDED
|
File without changes
|
gaussianformer/encodings/rope.py
ADDED
|
@@ -0,0 +1,65 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# Modified from https://github.com/lucidrains/rotary-embedding-torch/blob/main/rotary_embedding_torch/rotary_embedding_torch.py
|
| 2 |
+
|
| 3 |
+
# Copyright (c) 2021 Phil Wang
|
| 4 |
+
#
|
| 5 |
+
# Permission is hereby granted, free of charge, to any person obtaining a copy
|
| 6 |
+
# of this software and associated documentation files (the "Software"), to deal
|
| 7 |
+
# in the Software without restriction, including without limitation the rights
|
| 8 |
+
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
| 9 |
+
# copies of the Software, and to permit persons to whom the Software is
|
| 10 |
+
# furnished to do so, subject to the following conditions:
|
| 11 |
+
#
|
| 12 |
+
# The above copyright notice and this permission notice shall be included in all
|
| 13 |
+
# copies or substantial portions of the Software.
|
| 14 |
+
#
|
| 15 |
+
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
| 16 |
+
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
| 17 |
+
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
| 18 |
+
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
| 19 |
+
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
| 20 |
+
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
| 21 |
+
# SOFTWARE.
|
| 22 |
+
|
| 23 |
+
from math import log2
|
| 24 |
+
|
| 25 |
+
import torch
|
| 26 |
+
from einops import rearrange
|
| 27 |
+
from torch import Tensor, einsum, nn
|
| 28 |
+
from torch.amp import autocast
|
| 29 |
+
|
| 30 |
+
|
| 31 |
+
def rotate_half(x: Tensor) -> Tensor:
|
| 32 |
+
x1, x2 = x[..., : x.shape[-1] // 2], x[..., x.shape[-1] // 2:]
|
| 33 |
+
return torch.cat((-x2, x1), dim=-1)
|
| 34 |
+
|
| 35 |
+
|
| 36 |
+
def freqs_to_cos_sin(freqs: Tensor, head_dim: int) -> tuple[Tensor, Tensor]:
|
| 37 |
+
"""[..., 2F] frequencies (duplicated halves) -> cos, sin padded with zero angles to [..., head_dim]."""
|
| 38 |
+
freqs = freqs[..., : freqs.shape[-1] // 2]
|
| 39 |
+
pad = torch.zeros((*freqs.shape[:-1], head_dim // 2 - freqs.shape[-1]), device=freqs.device)
|
| 40 |
+
freqs = torch.cat((freqs, pad), dim=-1)
|
| 41 |
+
freqs = torch.cat([freqs, freqs], dim=-1)
|
| 42 |
+
return freqs.cos(), freqs.sin()
|
| 43 |
+
|
| 44 |
+
|
| 45 |
+
@autocast("cuda", enabled=False)
|
| 46 |
+
def apply_rotary_emb(t: Tensor, cos: Tensor, sin: Tensor) -> Tensor:
|
| 47 |
+
"""t [B, H, N, head_dim], cos/sin [B, 1, N, head_dim]."""
|
| 48 |
+
return ((t * cos) + (rotate_half(t) * sin)).type(t.dtype)
|
| 49 |
+
|
| 50 |
+
|
| 51 |
+
class SpatialRotaryEmbedding(nn.Module):
|
| 52 |
+
"""Rotary embedding of `pos_dim`-D positions: dim // 2 log-spaced frequencies per axis."""
|
| 53 |
+
|
| 54 |
+
def __init__(self, dim: int, pos_dim: int = 3, pos_scale: float = 1.0):
|
| 55 |
+
super().__init__()
|
| 56 |
+
self.pos_dim = pos_dim
|
| 57 |
+
self.pos_scale = pos_scale
|
| 58 |
+
self.register_buffer("freqs", 2 ** torch.linspace(0, log2(dim // 2 - 1), dim // 2))
|
| 59 |
+
|
| 60 |
+
@autocast("cuda", enabled=False)
|
| 61 |
+
def get_spatial_freqs(self, pos: Tensor) -> Tensor:
|
| 62 |
+
"""pos [B, N, pos_dim] -> angles [B, 1, N, 2 * pos_dim * dim // 2] (halves duplicated)."""
|
| 63 |
+
freqs = einsum("... i, f -> ... i f", (pos * self.pos_scale).type(self.freqs.dtype), self.freqs)
|
| 64 |
+
freqs = rearrange(freqs, "b n i f -> b 1 n (i f)")
|
| 65 |
+
return torch.cat([freqs, freqs], dim=-1)
|
gaussianformer/layers/__init__.py
ADDED
|
File without changes
|
gaussianformer/layers/attention.py
ADDED
|
@@ -0,0 +1,212 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
import os
|
| 2 |
+
|
| 3 |
+
import torch
|
| 4 |
+
import torch.nn.functional as F
|
| 5 |
+
import torch.utils.checkpoint
|
| 6 |
+
from einops import rearrange
|
| 7 |
+
from torch import nn
|
| 8 |
+
|
| 9 |
+
from gaussianformer.encodings.rope import SpatialRotaryEmbedding, apply_rotary_emb, freqs_to_cos_sin
|
| 10 |
+
from gaussianformer.layers.window import windowed_attention
|
| 11 |
+
|
| 12 |
+
EPS = 1e-6
|
| 13 |
+
|
| 14 |
+
ATTN = os.environ.get("ATTN_IMPL", "flash_attn")
|
| 15 |
+
assert ATTN in ("flash_attn", "sdpa"), "ATTN_IMPL must be 'flash_attn' or 'sdpa'"
|
| 16 |
+
if ATTN == "flash_attn":
|
| 17 |
+
try:
|
| 18 |
+
from flash_attn import (
|
| 19 |
+
flash_attn_qkvpacked_func,
|
| 20 |
+
flash_attn_varlen_kvpacked_func,
|
| 21 |
+
flash_attn_varlen_qkvpacked_func,
|
| 22 |
+
)
|
| 23 |
+
from flash_attn.bert_padding import pad_input, unpad_input
|
| 24 |
+
except ImportError:
|
| 25 |
+
print("flash_attn is not installed, falling back to PyTorch SDPA.")
|
| 26 |
+
ATTN = "sdpa"
|
| 27 |
+
|
| 28 |
+
|
| 29 |
+
class FeedForwardSwiGLU(nn.Module):
|
| 30 |
+
def __init__(self, dim: int, hidden_dim: int):
|
| 31 |
+
super().__init__()
|
| 32 |
+
self.w1 = nn.Linear(dim, hidden_dim, bias=False)
|
| 33 |
+
self.w2 = nn.Linear(hidden_dim, dim, bias=False)
|
| 34 |
+
self.w3 = nn.Linear(dim, hidden_dim, bias=False)
|
| 35 |
+
|
| 36 |
+
def forward(self, x):
|
| 37 |
+
return self.w2(F.silu(self.w1(x)) * self.w3(x))
|
| 38 |
+
|
| 39 |
+
|
| 40 |
+
class MultiHeadAttention(nn.Module):
|
| 41 |
+
"""Self-attention (kv_dim None) or cross-attention with QK-norm and RoPE."""
|
| 42 |
+
|
| 43 |
+
def __init__(self, query_dim: int, num_heads: int, kv_dim: int | None = None):
|
| 44 |
+
super().__init__()
|
| 45 |
+
self.num_heads = num_heads
|
| 46 |
+
self.is_self_attn = kv_dim is None
|
| 47 |
+
if self.is_self_attn:
|
| 48 |
+
self.in_proj = nn.Linear(query_dim, 3 * query_dim, bias=False)
|
| 49 |
+
else:
|
| 50 |
+
self.q_proj = nn.Linear(query_dim, query_dim, bias=False)
|
| 51 |
+
self.k_proj = nn.Linear(kv_dim, query_dim, bias=False)
|
| 52 |
+
self.v_proj = nn.Linear(kv_dim, query_dim, bias=False)
|
| 53 |
+
self.out_proj = nn.Linear(query_dim, query_dim, bias=False)
|
| 54 |
+
self.q_norm = nn.RMSNorm(query_dim, eps=EPS)
|
| 55 |
+
self.k_norm = nn.RMSNorm(query_dim, eps=EPS)
|
| 56 |
+
|
| 57 |
+
def forward(self, q, kv, key_mask=None, rope_q=None, rope_k=None, force_sdpa=False, window=None, k_add=None,
|
| 58 |
+
v_add=None):
|
| 59 |
+
"""
|
| 60 |
+
q [B, S, D], kv [B, N, D_kv]; key_mask [B, N] (True = attend); rope_q / rope_k (cos, sin) tables
|
| 61 |
+
[B, 1, S|N, head_dim] (rope_k defaults to rope_q); window from layers.window.build_window;
|
| 62 |
+
k_add / v_add [B, N, D] added to the projected keys / values (cross-attention only).
|
| 63 |
+
"""
|
| 64 |
+
bs, src_len, ctx_len = q.shape[0], q.shape[1], kv.shape[1]
|
| 65 |
+
if self.is_self_attn:
|
| 66 |
+
q, k, v = self.in_proj(q).chunk(3, dim=-1)
|
| 67 |
+
else:
|
| 68 |
+
q, k, v = self.q_proj(q), self.k_proj(kv), self.v_proj(kv)
|
| 69 |
+
if k_add is not None:
|
| 70 |
+
k = k + k_add.to(k.dtype)
|
| 71 |
+
if v_add is not None:
|
| 72 |
+
v = v + v_add.to(v.dtype)
|
| 73 |
+
q = self.q_norm(q).type(v.dtype)
|
| 74 |
+
k = self.k_norm(k).type(v.dtype)
|
| 75 |
+
|
| 76 |
+
q = q.view(bs, src_len, self.num_heads, -1).transpose(1, 2) # [B, H, S, d]
|
| 77 |
+
k = k.view(bs, ctx_len, self.num_heads, -1).transpose(1, 2)
|
| 78 |
+
v = v.view(bs, ctx_len, self.num_heads, -1).transpose(1, 2)
|
| 79 |
+
if rope_q is not None:
|
| 80 |
+
rope_k = rope_q if rope_k is None else rope_k
|
| 81 |
+
q = apply_rotary_emb(q, *rope_q)
|
| 82 |
+
k = apply_rotary_emb(k, *rope_k)
|
| 83 |
+
q, k = q.type(v.dtype), k.type(v.dtype)
|
| 84 |
+
|
| 85 |
+
if window is not None:
|
| 86 |
+
out = windowed_attention(q, k, v, window, flash_available=ATTN == "flash_attn")
|
| 87 |
+
elif ATTN == "sdpa" or force_sdpa:
|
| 88 |
+
mask = None
|
| 89 |
+
if key_mask is not None:
|
| 90 |
+
mask = key_mask.view(bs, 1, 1, ctx_len).expand(-1, self.num_heads, -1, -1).reshape(
|
| 91 |
+
bs, self.num_heads, 1, ctx_len)
|
| 92 |
+
out = F.scaled_dot_product_attention(q, k, v, attn_mask=mask)
|
| 93 |
+
out = out.transpose(1, 2).reshape(bs, src_len, -1)
|
| 94 |
+
elif self.is_self_attn and key_mask is None:
|
| 95 |
+
qkv = torch.stack([q.transpose(1, 2), k.transpose(1, 2), v.transpose(1, 2)], dim=2)
|
| 96 |
+
out = flash_attn_qkvpacked_func(qkv).reshape(bs, src_len, -1)
|
| 97 |
+
elif self.is_self_attn:
|
| 98 |
+
q_u, idx, cu, max_len, _ = unpad_input(q.transpose(1, 2), key_mask)
|
| 99 |
+
k_u = unpad_input(k.transpose(1, 2), key_mask)[0]
|
| 100 |
+
v_u = unpad_input(v.transpose(1, 2), key_mask)[0]
|
| 101 |
+
out = flash_attn_varlen_qkvpacked_func(torch.stack([q_u, k_u, v_u], dim=1), cu, max_len)
|
| 102 |
+
out = pad_input(out, idx, bs, src_len).reshape(bs, src_len, -1)
|
| 103 |
+
else:
|
| 104 |
+
q_u = rearrange(q, "b h s d -> (b s) h d")
|
| 105 |
+
cu_q = torch.arange(0, (bs + 1) * src_len, step=src_len, dtype=torch.int32, device=q.device)
|
| 106 |
+
k_u, _, cu_k, max_k, _ = unpad_input(k.transpose(1, 2), key_mask)
|
| 107 |
+
v_u = unpad_input(v.transpose(1, 2), key_mask)[0]
|
| 108 |
+
out = flash_attn_varlen_kvpacked_func(q_u, torch.stack([k_u, v_u], dim=1), cu_q, cu_k, src_len, max_k)
|
| 109 |
+
out = rearrange(out, "(b s) h d -> b s (h d)", b=bs)
|
| 110 |
+
return self.out_proj(out)
|
| 111 |
+
|
| 112 |
+
|
| 113 |
+
class AttentionLayer(nn.Module):
|
| 114 |
+
"""Pre-norm block: attention (self, or cross + patch self-attention), then a SwiGLU FFN."""
|
| 115 |
+
|
| 116 |
+
def __init__(self, dim: int, num_heads: int, ffn_hidden_dim: int, kv_dim: int | None = None):
|
| 117 |
+
super().__init__()
|
| 118 |
+
self.query_norm = nn.RMSNorm(dim, eps=EPS)
|
| 119 |
+
self.multihead_attn = MultiHeadAttention(dim, num_heads, kv_dim)
|
| 120 |
+
if kv_dim is not None:
|
| 121 |
+
self.kv_norm = nn.RMSNorm(kv_dim, eps=EPS)
|
| 122 |
+
self.self_attn_norm = nn.RMSNorm(dim, eps=EPS)
|
| 123 |
+
self.self_attn = MultiHeadAttention(dim, num_heads)
|
| 124 |
+
self.ffn_norm = nn.RMSNorm(dim, eps=EPS)
|
| 125 |
+
self.ffn = FeedForwardSwiGLU(dim, ffn_hidden_dim)
|
| 126 |
+
|
| 127 |
+
def forward(self, x, ctx=None, key_mask=None, rope_q=None, rope_k=None, force_sdpa=False, window=None, k_add=None,
|
| 128 |
+
v_add=None):
|
| 129 |
+
q = self.query_norm(x)
|
| 130 |
+
if ctx is None:
|
| 131 |
+
x = x + self.multihead_attn(q, q, key_mask, rope_q, force_sdpa=force_sdpa)
|
| 132 |
+
else:
|
| 133 |
+
x = x + self.multihead_attn(q, self.kv_norm(ctx), key_mask, rope_q, rope_k, force_sdpa, window, k_add,
|
| 134 |
+
v_add)
|
| 135 |
+
q = self.self_attn_norm(x)
|
| 136 |
+
x = x + self.self_attn(q, q, None, rope_q, force_sdpa=force_sdpa)
|
| 137 |
+
return x + self.ffn(self.ffn_norm(x))
|
| 138 |
+
|
| 139 |
+
|
| 140 |
+
class TransformerEncoder(nn.Module):
|
| 141 |
+
"""Scene encoder: self-attention over the Gaussian and register tokens with 3-D RoPE."""
|
| 142 |
+
|
| 143 |
+
def __init__(self, num_layers: int, num_heads: int, dim: int, ffn_hidden_dim: int, rope_dim: int):
|
| 144 |
+
super().__init__()
|
| 145 |
+
self.head_dim = dim // num_heads
|
| 146 |
+
self.layers = nn.ModuleList([AttentionLayer(dim, num_heads, ffn_hidden_dim) for _ in range(num_layers)])
|
| 147 |
+
self.rope_emb = SpatialRotaryEmbedding(rope_dim)
|
| 148 |
+
|
| 149 |
+
def forward(self, x, key_mask, pos):
|
| 150 |
+
rope = freqs_to_cos_sin(self.rope_emb.get_spatial_freqs(pos), self.head_dim)
|
| 151 |
+
for layer in self.layers:
|
| 152 |
+
x = layer(x, key_mask=key_mask, rope_q=rope)
|
| 153 |
+
return x
|
| 154 |
+
|
| 155 |
+
|
| 156 |
+
class TransformerDecoder(nn.Module):
|
| 157 |
+
"""View decoder: patch tokens cross-attend to the scene tokens, then self-attend.
|
| 158 |
+
|
| 159 |
+
RoPE per head: the pretrained 3-D band (camera origin for patches, Gaussian position for
|
| 160 |
+
Gaussians), then with proj_rope_2d a 2-D band in the channel pairs right after it
|
| 161 |
+
(patch center for patches, projected position for Gaussians).
|
| 162 |
+
"""
|
| 163 |
+
|
| 164 |
+
def __init__(self, num_layers: int, num_heads: int, dim: int, ctx_dim: int, ffn_hidden_dim: int,
|
| 165 |
+
rope_dim: int, proj_rope_2d: bool, ray_rope_2d_dim: int, ray_rope_2d_scale: float,
|
| 166 |
+
grad_checkpoint: bool, key_feat_dim: int = 0, value_feat: bool = False):
|
| 167 |
+
super().__init__()
|
| 168 |
+
self.head_dim = dim // num_heads
|
| 169 |
+
self.layers = nn.ModuleList([AttentionLayer(dim, num_heads, ffn_hidden_dim, ctx_dim)
|
| 170 |
+
for _ in range(num_layers)])
|
| 171 |
+
self.rope_emb = SpatialRotaryEmbedding(rope_dim)
|
| 172 |
+
if proj_rope_2d:
|
| 173 |
+
assert 3 * rope_dim // 2 + ray_rope_2d_dim <= self.head_dim // 2, "not enough head channels for 2-D RoPE"
|
| 174 |
+
self.rope_uv = SpatialRotaryEmbedding(ray_rope_2d_dim, pos_dim=2, pos_scale=ray_rope_2d_scale)
|
| 175 |
+
self.grad_checkpoint = grad_checkpoint
|
| 176 |
+
self.key_embed = None # cov_key_embed: per-layer zero-initialized embedding of the footprint on the keys
|
| 177 |
+
if key_feat_dim:
|
| 178 |
+
self.key_embed = nn.ModuleList([nn.Linear(key_feat_dim, dim) for _ in range(num_layers)])
|
| 179 |
+
for lin in self.key_embed:
|
| 180 |
+
nn.init.zeros_(lin.weight)
|
| 181 |
+
nn.init.zeros_(lin.bias)
|
| 182 |
+
self.value_embed = None # cov_key_values: the footprint on the values too, through its own zero-initialized layers
|
| 183 |
+
if key_feat_dim and value_feat:
|
| 184 |
+
self.value_embed = nn.ModuleList([nn.Linear(key_feat_dim, dim) for _ in range(num_layers)])
|
| 185 |
+
for lin in self.value_embed:
|
| 186 |
+
nn.init.zeros_(lin.weight)
|
| 187 |
+
nn.init.zeros_(lin.bias)
|
| 188 |
+
|
| 189 |
+
def _rope(self, pos, uv=None):
|
| 190 |
+
bands = [self.rope_emb.get_spatial_freqs(pos)]
|
| 191 |
+
if uv is not None:
|
| 192 |
+
bands.append(self.rope_uv.get_spatial_freqs(uv))
|
| 193 |
+
f = torch.cat([b[..., : b.shape[-1] // 2] for b in bands], -1)
|
| 194 |
+
return freqs_to_cos_sin(torch.cat([f, f], -1), self.head_dim)
|
| 195 |
+
|
| 196 |
+
def forward(self, x, ctx, key_mask, ctx_pos, ray_pos, out_layers, force_sdpa=False,
|
| 197 |
+
uv_q=None, uv_k=None, window=None, key_feat=None):
|
| 198 |
+
kw = {"key_mask": key_mask, "rope_q": self._rope(ray_pos, uv_q), "rope_k": self._rope(ctx_pos, uv_k),
|
| 199 |
+
"force_sdpa": force_sdpa, "window": window}
|
| 200 |
+
out = []
|
| 201 |
+
for i, layer in enumerate(self.layers):
|
| 202 |
+
kw_i = kw if key_feat is None else dict(kw, k_add=self.key_embed[i](key_feat))
|
| 203 |
+
if key_feat is not None and self.value_embed is not None:
|
| 204 |
+
kw_i = dict(kw_i, v_add=self.value_embed[i](key_feat))
|
| 205 |
+
if self.grad_checkpoint and self.training and torch.is_grad_enabled():
|
| 206 |
+
x = torch.utils.checkpoint.checkpoint(lambda x_, layer_=layer, kw_=kw_i: layer_(x_, ctx, **kw_), x,
|
| 207 |
+
use_reentrant=False)
|
| 208 |
+
else:
|
| 209 |
+
x = layer(x, ctx, **kw_i)
|
| 210 |
+
if i in out_layers:
|
| 211 |
+
out.append(x)
|
| 212 |
+
return out
|
gaussianformer/layers/dpt.py
ADDED
|
@@ -0,0 +1,244 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# Modified from https://github.com/isl-org/DPT/blob/main/dpt/models.py
|
| 2 |
+
|
| 3 |
+
# Copyright (c) 2021 Intel ISL (Intel Intelligent Systems Lab)
|
| 4 |
+
#
|
| 5 |
+
# Permission is hereby granted, free of charge, to any person obtaining a copy
|
| 6 |
+
# of this software and associated documentation files (the "Software"), to deal
|
| 7 |
+
# in the Software without restriction, including without limitation the rights
|
| 8 |
+
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
| 9 |
+
# copies of the Software, and to permit persons to whom the Software is
|
| 10 |
+
# furnished to do so, subject to the following conditions:
|
| 11 |
+
#
|
| 12 |
+
# The above copyright notice and this permission notice shall be included in all
|
| 13 |
+
# copies or substantial portions of the Software.
|
| 14 |
+
#
|
| 15 |
+
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
| 16 |
+
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
| 17 |
+
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
| 18 |
+
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
| 19 |
+
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
| 20 |
+
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
| 21 |
+
# SOFTWARE.
|
| 22 |
+
|
| 23 |
+
import torch.nn.functional as F
|
| 24 |
+
from torch import nn
|
| 25 |
+
|
| 26 |
+
|
| 27 |
+
def _make_scratch(in_shape, out_shape, groups=1, expand=False):
|
| 28 |
+
scratch = nn.Module()
|
| 29 |
+
|
| 30 |
+
out_shape1 = out_shape
|
| 31 |
+
out_shape2 = out_shape
|
| 32 |
+
out_shape3 = out_shape
|
| 33 |
+
if len(in_shape) >= 4:
|
| 34 |
+
out_shape4 = out_shape
|
| 35 |
+
|
| 36 |
+
if expand:
|
| 37 |
+
out_shape1 = out_shape
|
| 38 |
+
out_shape2 = out_shape * 2
|
| 39 |
+
out_shape3 = out_shape * 4
|
| 40 |
+
if len(in_shape) >= 4:
|
| 41 |
+
out_shape4 = out_shape * 8
|
| 42 |
+
|
| 43 |
+
scratch.layer1_rn = nn.Conv2d(
|
| 44 |
+
in_shape[0], out_shape1, kernel_size=3, stride=1, padding=1, bias=False, groups=groups)
|
| 45 |
+
scratch.layer2_rn = nn.Conv2d(
|
| 46 |
+
in_shape[1], out_shape2, kernel_size=3, stride=1, padding=1, bias=False, groups=groups)
|
| 47 |
+
scratch.layer3_rn = nn.Conv2d(
|
| 48 |
+
in_shape[2], out_shape3, kernel_size=3, stride=1, padding=1, bias=False, groups=groups)
|
| 49 |
+
if len(in_shape) >= 4:
|
| 50 |
+
scratch.layer4_rn = nn.Conv2d(
|
| 51 |
+
in_shape[3], out_shape4, kernel_size=3, stride=1, padding=1, bias=False, groups=groups)
|
| 52 |
+
|
| 53 |
+
return scratch
|
| 54 |
+
|
| 55 |
+
|
| 56 |
+
class ResidualConvUnit(nn.Module):
|
| 57 |
+
|
| 58 |
+
def __init__(self, features, activation):
|
| 59 |
+
super().__init__()
|
| 60 |
+
|
| 61 |
+
self.conv1 = nn.Conv2d(
|
| 62 |
+
features, features, kernel_size=3, stride=1, padding=1, bias=True)
|
| 63 |
+
self.conv2 = nn.Conv2d(
|
| 64 |
+
features, features, kernel_size=3, stride=1, padding=1, bias=True)
|
| 65 |
+
|
| 66 |
+
self.activation = activation
|
| 67 |
+
|
| 68 |
+
def forward(self, x):
|
| 69 |
+
|
| 70 |
+
out = self.activation(x)
|
| 71 |
+
out = self.conv1(out)
|
| 72 |
+
|
| 73 |
+
out = self.activation(out)
|
| 74 |
+
out = self.conv2(out)
|
| 75 |
+
|
| 76 |
+
return out + x
|
| 77 |
+
|
| 78 |
+
|
| 79 |
+
class FeatureFusionBlock(nn.Module):
|
| 80 |
+
|
| 81 |
+
def __init__(
|
| 82 |
+
self,
|
| 83 |
+
features,
|
| 84 |
+
activation,
|
| 85 |
+
deconv=False,
|
| 86 |
+
expand=False,
|
| 87 |
+
align_corners=True,
|
| 88 |
+
size=None,
|
| 89 |
+
no_resconv1=False,
|
| 90 |
+
):
|
| 91 |
+
super().__init__()
|
| 92 |
+
|
| 93 |
+
self.deconv = deconv
|
| 94 |
+
self.align_corners = align_corners
|
| 95 |
+
|
| 96 |
+
self.expand = expand
|
| 97 |
+
out_features = features
|
| 98 |
+
if self.expand == True:
|
| 99 |
+
out_features = features // 2
|
| 100 |
+
|
| 101 |
+
self.out_conv = nn.Conv2d(
|
| 102 |
+
features, out_features, kernel_size=1, stride=1, padding=0, bias=True)
|
| 103 |
+
|
| 104 |
+
if not no_resconv1:
|
| 105 |
+
self.resConvUnit1 = ResidualConvUnit(features, activation)
|
| 106 |
+
self.resConvUnit2 = ResidualConvUnit(features, activation)
|
| 107 |
+
|
| 108 |
+
self.size = size
|
| 109 |
+
|
| 110 |
+
def forward(self, *xs, size=None):
|
| 111 |
+
output = xs[0]
|
| 112 |
+
|
| 113 |
+
if len(xs) == 2:
|
| 114 |
+
res = self.resConvUnit1(xs[1])
|
| 115 |
+
output = output + res
|
| 116 |
+
|
| 117 |
+
output = self.resConvUnit2(output)
|
| 118 |
+
|
| 119 |
+
if (size is None) and (self.size is None):
|
| 120 |
+
modifier = {"scale_factor": 2}
|
| 121 |
+
elif size is None:
|
| 122 |
+
modifier = {"size": self.size}
|
| 123 |
+
else:
|
| 124 |
+
modifier = {"size": size}
|
| 125 |
+
|
| 126 |
+
output = nn.functional.interpolate(
|
| 127 |
+
output, **modifier, mode="bilinear", align_corners=self.align_corners)
|
| 128 |
+
|
| 129 |
+
output = self.out_conv(output)
|
| 130 |
+
|
| 131 |
+
return output
|
| 132 |
+
|
| 133 |
+
|
| 134 |
+
def _make_fusion_block(features, size=None, no_resconv1=False):
|
| 135 |
+
return FeatureFusionBlock(
|
| 136 |
+
features,
|
| 137 |
+
nn.SiLU(False),
|
| 138 |
+
deconv=False,
|
| 139 |
+
expand=False,
|
| 140 |
+
align_corners=True,
|
| 141 |
+
size=size,
|
| 142 |
+
no_resconv1=no_resconv1,
|
| 143 |
+
)
|
| 144 |
+
|
| 145 |
+
|
| 146 |
+
class DPTHead(nn.Module):
|
| 147 |
+
def __init__(
|
| 148 |
+
self,
|
| 149 |
+
in_channels,
|
| 150 |
+
features=256,
|
| 151 |
+
out_channels=(256, 512, 1024, 1024),
|
| 152 |
+
out_dim=3,
|
| 153 |
+
):
|
| 154 |
+
super().__init__()
|
| 155 |
+
|
| 156 |
+
self.projects = nn.ModuleList([
|
| 157 |
+
nn.Conv2d(
|
| 158 |
+
in_channels=in_channels,
|
| 159 |
+
out_channels=out_channel,
|
| 160 |
+
kernel_size=1,
|
| 161 |
+
stride=1,
|
| 162 |
+
padding=0,
|
| 163 |
+
) for out_channel in out_channels
|
| 164 |
+
])
|
| 165 |
+
|
| 166 |
+
self.resize_layers = nn.ModuleList([
|
| 167 |
+
nn.ConvTranspose2d(
|
| 168 |
+
in_channels=out_channels[0],
|
| 169 |
+
out_channels=out_channels[0],
|
| 170 |
+
kernel_size=4,
|
| 171 |
+
stride=4,
|
| 172 |
+
padding=0),
|
| 173 |
+
nn.ConvTranspose2d(
|
| 174 |
+
in_channels=out_channels[1],
|
| 175 |
+
out_channels=out_channels[1],
|
| 176 |
+
kernel_size=2,
|
| 177 |
+
stride=2,
|
| 178 |
+
padding=0),
|
| 179 |
+
nn.Identity(),
|
| 180 |
+
nn.Conv2d(
|
| 181 |
+
in_channels=out_channels[3],
|
| 182 |
+
out_channels=out_channels[3],
|
| 183 |
+
kernel_size=3,
|
| 184 |
+
stride=2,
|
| 185 |
+
padding=1)
|
| 186 |
+
])
|
| 187 |
+
|
| 188 |
+
self.scratch = _make_scratch(
|
| 189 |
+
out_channels,
|
| 190 |
+
features,
|
| 191 |
+
groups=1,
|
| 192 |
+
expand=False,
|
| 193 |
+
)
|
| 194 |
+
|
| 195 |
+
self.scratch.refinenet1 = _make_fusion_block(features)
|
| 196 |
+
self.scratch.refinenet2 = _make_fusion_block(features)
|
| 197 |
+
self.scratch.refinenet3 = _make_fusion_block(features)
|
| 198 |
+
self.scratch.refinenet4 = _make_fusion_block(
|
| 199 |
+
features, no_resconv1=True)
|
| 200 |
+
|
| 201 |
+
head_features_1 = features
|
| 202 |
+
head_features_2 = 32
|
| 203 |
+
|
| 204 |
+
self.scratch.output_conv1 = nn.Conv2d(
|
| 205 |
+
head_features_1, head_features_1 // 2, kernel_size=3, stride=1, padding=1)
|
| 206 |
+
self.scratch.output_conv2 = nn.Sequential(
|
| 207 |
+
nn.Conv2d(head_features_1 // 2, head_features_2,
|
| 208 |
+
kernel_size=3, stride=1, padding=1),
|
| 209 |
+
nn.SiLU(True),
|
| 210 |
+
nn.Conv2d(head_features_2, out_dim,
|
| 211 |
+
kernel_size=1, stride=1, padding=0),
|
| 212 |
+
)
|
| 213 |
+
|
| 214 |
+
def forward(self, out_features, patch_h, patch_w, patch_size=16):
|
| 215 |
+
out = []
|
| 216 |
+
for i, x in enumerate(out_features):
|
| 217 |
+
x = x.permute(0, 2, 1).reshape(
|
| 218 |
+
(x.shape[0], x.shape[-1], patch_h, patch_w))
|
| 219 |
+
|
| 220 |
+
x = self.projects[i](x)
|
| 221 |
+
x = self.resize_layers[i](x)
|
| 222 |
+
|
| 223 |
+
out.append(x)
|
| 224 |
+
|
| 225 |
+
layer_1, layer_2, layer_3, layer_4 = out
|
| 226 |
+
|
| 227 |
+
layer_1_rn = self.scratch.layer1_rn(layer_1)
|
| 228 |
+
layer_2_rn = self.scratch.layer2_rn(layer_2)
|
| 229 |
+
layer_3_rn = self.scratch.layer3_rn(layer_3)
|
| 230 |
+
layer_4_rn = self.scratch.layer4_rn(layer_4)
|
| 231 |
+
|
| 232 |
+
path_4 = self.scratch.refinenet4(layer_4_rn, size=layer_3_rn.shape[2:])
|
| 233 |
+
path_3 = self.scratch.refinenet3(
|
| 234 |
+
path_4, layer_3_rn, size=layer_2_rn.shape[2:])
|
| 235 |
+
path_2 = self.scratch.refinenet2(
|
| 236 |
+
path_3, layer_2_rn, size=layer_1_rn.shape[2:])
|
| 237 |
+
path_1 = self.scratch.refinenet1(path_2, layer_1_rn)
|
| 238 |
+
|
| 239 |
+
out = self.scratch.output_conv1(path_1)
|
| 240 |
+
out = F.interpolate(out, (int(patch_h * patch_size),
|
| 241 |
+
int(patch_w * patch_size)), mode="bilinear", align_corners=True)
|
| 242 |
+
out = self.scratch.output_conv2(out)
|
| 243 |
+
|
| 244 |
+
return out
|
gaussianformer/layers/window.py
ADDED
|
@@ -0,0 +1,100 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Projection-windowed cross-attention: each ray token attends only to the Gaussians whose
|
| 2 |
+
projected footprint reaches its image tile (plus the register tokens), the way a rasterizer
|
| 3 |
+
bins splats by tile. Cost goes from queries x all Gaussians to queries x local Gaussians.
|
| 4 |
+
|
| 5 |
+
`build_window` turns the per-view projection into a flat (tile -> key indices) structure that
|
| 6 |
+
`windowed_attention` consumes with flash-attn varlen (one "sequence" per tile) or, on CPU /
|
| 7 |
+
SDPA, with a padded gather. Built once per forward and shared by every decoder layer.
|
| 8 |
+
"""
|
| 9 |
+
|
| 10 |
+
import torch
|
| 11 |
+
import torch.nn.functional as F
|
| 12 |
+
from torch import Tensor
|
| 13 |
+
|
| 14 |
+
|
| 15 |
+
def build_window(uv: Tensor, radius: Tensor, key_ok: Tensor, n_reg: int, patch_h: int, patch_w: int,
|
| 16 |
+
tile: int, margin: float) -> dict:
|
| 17 |
+
"""
|
| 18 |
+
uv [B, N, 2] projected key centers in patch units (registers included, ignored)
|
| 19 |
+
radius [B, N] projected footprint radius in patch units
|
| 20 |
+
key_ok [B, N] valid Gaussian keys (False for padding, behind-camera, registers)
|
| 21 |
+
n_reg register tokens (keys 0..n_reg-1), attended by every tile
|
| 22 |
+
Returns a dict: q_perm/q_inv [S] query permutation to tile-major order, kv_index [total_k]
|
| 23 |
+
flat (b*N + n) key indices in tile order, cu_q/cu_k [B*T+1] int32 segment offsets,
|
| 24 |
+
max_k, plus tiles/tile_q for reshaping.
|
| 25 |
+
"""
|
| 26 |
+
B, N = uv.shape[:2]
|
| 27 |
+
assert patch_h % tile == 0 and patch_w % tile == 0, "the ray grid must be a whole number of tiles"
|
| 28 |
+
th, tw = patch_h // tile, patch_w // tile
|
| 29 |
+
T = th * tw
|
| 30 |
+
dev = uv.device
|
| 31 |
+
# queries (raster order h1 w1) -> tile-major (tile, y, x)
|
| 32 |
+
ii, jj = torch.meshgrid(torch.arange(patch_h, device=dev), torch.arange(patch_w, device=dev), indexing="ij")
|
| 33 |
+
tile_of_q = (ii // tile) * tw + (jj // tile)
|
| 34 |
+
q_perm = torch.argsort(tile_of_q.reshape(-1), stable=True)
|
| 35 |
+
q_inv = torch.empty_like(q_perm)
|
| 36 |
+
q_inv[q_perm] = torch.arange(q_perm.numel(), device=dev)
|
| 37 |
+
# per-Gaussian tile range from the footprint bbox (+ margin), clipped to the grid
|
| 38 |
+
lo = (uv - (radius + margin)[..., None]).div(tile).floor().clamp_min(0).long()
|
| 39 |
+
hi = (uv + (radius + margin)[..., None]).div(tile).floor().long()
|
| 40 |
+
hi = torch.minimum(hi, torch.tensor([tw - 1, th - 1], device=dev))
|
| 41 |
+
ok = key_ok & (hi >= lo).all(-1)
|
| 42 |
+
lo = lo.masked_fill(~ok[..., None], 0)
|
| 43 |
+
hi = hi.masked_fill(~ok[..., None], -1)
|
| 44 |
+
w = (hi[..., 0] - lo[..., 0] + 1).clamp_min(0)
|
| 45 |
+
h = (hi[..., 1] - lo[..., 1] + 1).clamp_min(0)
|
| 46 |
+
cnt = (w * h).reshape(-1) # [B*N] tiles touched per key
|
| 47 |
+
key = torch.arange(B * N, device=dev).repeat_interleave(cnt)
|
| 48 |
+
local = torch.arange(key.numel(), device=dev) - torch.repeat_interleave(cnt.cumsum(0) - cnt, cnt)
|
| 49 |
+
wk = w.reshape(-1)[key]
|
| 50 |
+
dx, dy = local % wk, local // wk
|
| 51 |
+
tx = lo[..., 0].reshape(-1)[key] + dx
|
| 52 |
+
ty = lo[..., 1].reshape(-1)[key] + dy
|
| 53 |
+
seg = (key // N) * T + ty * tw + tx # (b, tile) segment
|
| 54 |
+
# register tokens in every tile
|
| 55 |
+
reg_seg = torch.arange(B * T, device=dev).repeat_interleave(n_reg)
|
| 56 |
+
reg_key = (reg_seg // T) * N + torch.arange(n_reg, device=dev).repeat(B * T)
|
| 57 |
+
seg = torch.cat([reg_seg, seg])
|
| 58 |
+
key = torch.cat([reg_key, key])
|
| 59 |
+
order = torch.argsort(seg, stable=True)
|
| 60 |
+
seg, key = seg[order], key[order]
|
| 61 |
+
counts = torch.bincount(seg, minlength=B * T)
|
| 62 |
+
cu_k = F.pad(counts.cumsum(0), (1, 0)).to(torch.int32)
|
| 63 |
+
cu_q = torch.arange(0, B * T + 1, device=dev, dtype=torch.int32) * (tile * tile)
|
| 64 |
+
return {"q_perm": q_perm, "q_inv": q_inv, "kv_index": key, "cu_q": cu_q, "cu_k": cu_k,
|
| 65 |
+
"max_k": int(counts.max()), "tiles": T, "tile_q": tile * tile, "B": B, "N": N}
|
| 66 |
+
|
| 67 |
+
|
| 68 |
+
def windowed_attention(q: Tensor, k: Tensor, v: Tensor, win: dict, flash_available: bool) -> Tensor:
|
| 69 |
+
"""q [B, H, S, d] (raster order), k/v [B, H, N, d] (RoPE already applied) -> [B, S, H*d].
|
| 70 |
+
flash-attn varlen (one sequence per tile, no padding) whenever it is installed and the tensors
|
| 71 |
+
are on the GPU -- in bf16, since flash has no fp32 kernel and the view stage runs under tf32;
|
| 72 |
+
otherwise a padded gather + SDPA (CPU). The padded path costs tiles x max_k, so it is only
|
| 73 |
+
for tests: a single tile can hold most of a small object's Gaussians."""
|
| 74 |
+
B, H, S, d = q.shape
|
| 75 |
+
N, T, tq = win["N"], win["tiles"], win["tile_q"]
|
| 76 |
+
q_t = q[:, :, win["q_perm"]] # tile-major queries
|
| 77 |
+
if flash_available and q.is_cuda:
|
| 78 |
+
from flash_attn import flash_attn_varlen_kvpacked_func
|
| 79 |
+
ht = q.dtype if q.dtype in (torch.float16, torch.bfloat16) else torch.bfloat16
|
| 80 |
+
kv = torch.stack([k, v], 2).permute(0, 3, 2, 1, 4).reshape(B * N, 2, H, d)[win["kv_index"]].to(ht) # [total_k, 2, H, d]
|
| 81 |
+
q_flat = q_t.permute(0, 2, 1, 3).reshape(B * S, H, d).to(ht)
|
| 82 |
+
out = flash_attn_varlen_kvpacked_func(q_flat, kv, win["cu_q"], win["cu_k"], tq, win["max_k"])
|
| 83 |
+
out = out.reshape(B, S, H * d).to(q.dtype)
|
| 84 |
+
else:
|
| 85 |
+
k_flat = k.transpose(1, 2).reshape(B * N, H, d)[win["kv_index"]] # [total_k, H, d]
|
| 86 |
+
v_flat = v.transpose(1, 2).reshape(B * N, H, d)[win["kv_index"]]
|
| 87 |
+
# padded gather: one row per (b, tile), keys padded to max_k and masked
|
| 88 |
+
Kmax = win["max_k"]
|
| 89 |
+
counts = (win["cu_k"][1:] - win["cu_k"][:-1]).long()
|
| 90 |
+
pos = torch.arange(win["kv_index"].numel(), device=q.device) - torch.repeat_interleave(win["cu_k"][:-1].long(), counts)
|
| 91 |
+
seg = torch.repeat_interleave(torch.arange(B * T, device=q.device), counts)
|
| 92 |
+
kp = k_flat.new_zeros(B * T, Kmax, H, d)
|
| 93 |
+
vp = v_flat.new_zeros(B * T, Kmax, H, d)
|
| 94 |
+
mask = torch.zeros(B * T, Kmax, dtype=torch.bool, device=q.device)
|
| 95 |
+
kp[seg, pos], vp[seg, pos], mask[seg, pos] = k_flat, v_flat, True
|
| 96 |
+
qt = q_t.reshape(B, H, T, tq, d).permute(0, 2, 1, 3, 4).reshape(B * T, H, tq, d)
|
| 97 |
+
out = F.scaled_dot_product_attention(qt, kp.transpose(1, 2), vp.transpose(1, 2),
|
| 98 |
+
attn_mask=mask[:, None, None, :]) # [B*T, H, tq, d]
|
| 99 |
+
out = out.reshape(B, T, H, tq, d).permute(0, 1, 3, 2, 4).reshape(B, S, H * d)
|
| 100 |
+
return out[:, win["q_inv"]]
|
gaussianformer/models/__init__.py
ADDED
|
File without changes
|
gaussianformer/models/config.py
ADDED
|
@@ -0,0 +1,84 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
import types
|
| 2 |
+
import typing
|
| 3 |
+
from dataclasses import dataclass, field, replace
|
| 4 |
+
|
| 5 |
+
|
| 6 |
+
@dataclass(frozen=True)
|
| 7 |
+
class GaussianFormerConfig:
|
| 8 |
+
# Scene encoder: one token per Gaussian, 3-D RoPE on its position.
|
| 9 |
+
latent_dim: int = 768
|
| 10 |
+
num_layers: int = 12
|
| 11 |
+
num_heads: int = 6
|
| 12 |
+
dim_feedforward: int = 768 * 4
|
| 13 |
+
num_register_tokens: int = 16
|
| 14 |
+
pos_pe_num_freqs: int = 12
|
| 15 |
+
"""Rotary dimension of the 3-D position RoPE (shared with the view decoder)."""
|
| 16 |
+
|
| 17 |
+
# View decoder: one token per image patch, cross-attention to the scene tokens, DPT head.
|
| 18 |
+
view_transformer_latent_dim: int = 768
|
| 19 |
+
view_transformer_ffn_hidden_dim: int = 768 * 4
|
| 20 |
+
view_transformer_n_heads: int = 6
|
| 21 |
+
view_transformer_n_layers: int = 6
|
| 22 |
+
patch_size: int = 8
|
| 23 |
+
ray_embed_patch: int = 0
|
| 24 |
+
"""If larger than patch_size, each patch's ray directions are upsampled to this size before the
|
| 25 |
+
ray embedding, so a finer patch grid reuses an embedding pretrained at this patch size."""
|
| 26 |
+
dpt_features: int = 128
|
| 27 |
+
dpt_out_channels: list[int] = field(default_factory=lambda: [96, 192, 384, 768])
|
| 28 |
+
|
| 29 |
+
proj_rope_2d: bool = False
|
| 30 |
+
"""2-D RoPE in the cross-attention: keys carry each Gaussian's projected image position, queries
|
| 31 |
+
their patch center; patch self-attention uses the same 2-D band."""
|
| 32 |
+
ray_rope_2d_dim: int = 16
|
| 33 |
+
ray_rope_2d_scale: float = 0.25
|
| 34 |
+
"""Patch coordinates are multiplied by this before the 2-D RoPE."""
|
| 35 |
+
|
| 36 |
+
xattn_window: int = 0
|
| 37 |
+
"""Tile size in patches for windowed cross-attention: each tile attends only to the Gaussians
|
| 38 |
+
whose projected footprint reaches it, plus the register tokens. 0 = dense."""
|
| 39 |
+
xattn_margin: float = 1.0
|
| 40 |
+
"""Tile expansion in patches on top of the footprint."""
|
| 41 |
+
xattn_sigmas: float = 3.0
|
| 42 |
+
"""Footprint radius in units of the Gaussian's largest scale."""
|
| 43 |
+
|
| 44 |
+
cov_key_embed: bool = False
|
| 45 |
+
"""Projected footprint on the cross-attention keys: each Gaussian's covariance projected into the view (Jacobian
|
| 46 |
+
of the pinhole projection at its center, in patch units, plus gsplat's 0.3 px^2 low-pass term), described by
|
| 47 |
+
(log sigma_1, log sigma_2, cos 2t, sin 2t) and added to its key through a zero-initialized Linear per decoder
|
| 48 |
+
layer."""
|
| 49 |
+
cov_key_eps2d: float = 0.3
|
| 50 |
+
"""Isotropic term added to the projected covariance, in square pixels (gsplat's eps2d low-pass filter); 0 removes
|
| 51 |
+
it (ablation)."""
|
| 52 |
+
cov_key_values: bool = False
|
| 53 |
+
"""Also add the footprint to the cross-attention values, through its own zero-initialized Linear per decoder
|
| 54 |
+
layer (ablation; the released models use the keys only)."""
|
| 55 |
+
|
| 56 |
+
view_bf16: bool = False
|
| 57 |
+
"""Run the view decoder in bf16 (Flash Attention for the patch self-attention)."""
|
| 58 |
+
view_grad_checkpoint: bool = False
|
| 59 |
+
"""Recompute view-decoder layers in the backward pass to save activation memory."""
|
| 60 |
+
|
| 61 |
+
def with_overrides(self, pairs: list[str] | None) -> "GaussianFormerConfig":
|
| 62 |
+
"""Apply `key=value` overrides (the `--model_cfg` option)."""
|
| 63 |
+
if not pairs:
|
| 64 |
+
return self
|
| 65 |
+
hints = typing.get_type_hints(type(self))
|
| 66 |
+
out = {}
|
| 67 |
+
for p in pairs:
|
| 68 |
+
key, _, raw = p.partition("=")
|
| 69 |
+
if key not in hints:
|
| 70 |
+
raise KeyError(f"unknown GaussianFormerConfig field {key!r}")
|
| 71 |
+
out[key] = _coerce(hints[key], raw)
|
| 72 |
+
return replace(self, **out)
|
| 73 |
+
|
| 74 |
+
|
| 75 |
+
def _coerce(tp, raw: str):
|
| 76 |
+
if isinstance(tp, types.UnionType):
|
| 77 |
+
inner = [a for a in typing.get_args(tp) if a is not type(None)]
|
| 78 |
+
return None if raw.lower() in ("none", "null") else _coerce(inner[0], raw)
|
| 79 |
+
if typing.get_origin(tp) is list:
|
| 80 |
+
(item,) = typing.get_args(tp)
|
| 81 |
+
return [_coerce(item, x) for x in raw.split(",") if x]
|
| 82 |
+
if tp is bool:
|
| 83 |
+
return raw.lower() in ("1", "true", "yes")
|
| 84 |
+
return tp(raw)
|
gaussianformer/models/gaussianformer.py
ADDED
|
@@ -0,0 +1,65 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
import torch
|
| 2 |
+
from huggingface_hub import PyTorchModelHubMixin
|
| 3 |
+
from torch import nn
|
| 4 |
+
|
| 5 |
+
from gaussianformer.layers.attention import TransformerEncoder
|
| 6 |
+
from gaussianformer.models.config import GaussianFormerConfig
|
| 7 |
+
from gaussianformer.models.view_transformer import ViewTransformer
|
| 8 |
+
|
| 9 |
+
VIEW_CHUNK = 7 # views decoded per pass; the DPT upsampling of more 512 px views exceeds int32 indexing
|
| 10 |
+
|
| 11 |
+
|
| 12 |
+
class GaussianFormer(nn.Module, PyTorchModelHubMixin):
|
| 13 |
+
"""Scene encoder over one token per Gaussian [pos(3), scale(3), quat wxyz(4), rgb(3), opacity(1)],
|
| 14 |
+
followed by the view decoder. Output: log10(HDR + 1) images."""
|
| 15 |
+
|
| 16 |
+
def __init__(self, config: GaussianFormerConfig | dict):
|
| 17 |
+
super().__init__()
|
| 18 |
+
if isinstance(config, dict):
|
| 19 |
+
config = GaussianFormerConfig(**config)
|
| 20 |
+
self.config = config
|
| 21 |
+
self.gaussian_encoder = nn.Linear(14, config.latent_dim)
|
| 22 |
+
self.gaussian_encoder_norm = nn.RMSNorm(config.latent_dim)
|
| 23 |
+
self.gaussian_token = nn.Parameter(torch.randn(1, 1, config.latent_dim))
|
| 24 |
+
self.reg_tokens = nn.Parameter(torch.randn(1, config.num_register_tokens, config.latent_dim))
|
| 25 |
+
self.transformer = TransformerEncoder(config.num_layers, config.num_heads, config.latent_dim,
|
| 26 |
+
config.dim_feedforward, config.pos_pe_num_freqs)
|
| 27 |
+
self.view_transformer = ViewTransformer(config)
|
| 28 |
+
|
| 29 |
+
@property
|
| 30 |
+
def device(self):
|
| 31 |
+
return next(self.parameters()).device
|
| 32 |
+
|
| 33 |
+
def _with_registers(self, pos, mask):
|
| 34 |
+
"""Prepend register tokens positioned at the center of the valid Gaussians."""
|
| 35 |
+
n = self.config.num_register_tokens
|
| 36 |
+
w = (mask.float() / (mask.sum(dim=1, keepdim=True) + 1e-5))[..., None]
|
| 37 |
+
center = (w * pos).sum(dim=1, keepdim=True)
|
| 38 |
+
pos = torch.cat([center.repeat(1, n, 1), pos], dim=1)
|
| 39 |
+
mask = torch.cat([torch.ones((mask.size(0), n), dtype=torch.bool, device=mask.device), mask], dim=1)
|
| 40 |
+
return pos, mask
|
| 41 |
+
|
| 42 |
+
def forward(self, gaussians, valid_mask, rays_o, rays_d, gaussians_cam, fov, tf32_view=False):
|
| 43 |
+
"""
|
| 44 |
+
gaussians [B, N, 14] padded, valid_mask [B, N]; rays_o [B, V, 3], rays_d [B, V, H, W, 3];
|
| 45 |
+
gaussians_cam [B, V, N, 14] the Gaussians in each view's camera frame; fov [B, V] in radians.
|
| 46 |
+
Returns [B, V, 3, H, W].
|
| 47 |
+
"""
|
| 48 |
+
B, V = rays_o.shape[:2]
|
| 49 |
+
tokens = self.gaussian_token + self.gaussian_encoder_norm(self.gaussian_encoder(gaussians))
|
| 50 |
+
seq = torch.cat([self.reg_tokens.expand(B, -1, -1), tokens], dim=1)
|
| 51 |
+
pos, mask = self._with_registers(gaussians[..., :3], valid_mask)
|
| 52 |
+
seq = self.transformer(seq, mask, pos)
|
| 53 |
+
|
| 54 |
+
seq = seq.repeat_interleave(V, dim=0)
|
| 55 |
+
gaussians_cam = gaussians_cam.reshape(B * V, *gaussians_cam.shape[2:])
|
| 56 |
+
pos_cam, mask_cam = self._with_registers(gaussians_cam[..., :3], valid_mask.repeat_interleave(V, dim=0))
|
| 57 |
+
shape = gaussians_cam[..., 3:10] # scale, rotation; registers get zeros (no footprint)
|
| 58 |
+
shape_cam = torch.cat([shape.new_zeros(B * V, self.config.num_register_tokens, 7), shape], dim=1)
|
| 59 |
+
rays_o, rays_d, fov = rays_o.reshape(B * V, 3), rays_d.reshape(B * V, *rays_d.shape[2:]), fov.reshape(-1)
|
| 60 |
+
out = torch.cat([
|
| 61 |
+
self.view_transformer(rays_o[i:i + VIEW_CHUNK], rays_d[i:i + VIEW_CHUNK], seq[i:i + VIEW_CHUNK],
|
| 62 |
+
pos_cam[i:i + VIEW_CHUNK], shape_cam[i:i + VIEW_CHUNK], mask_cam[i:i + VIEW_CHUNK],
|
| 63 |
+
fov[i:i + VIEW_CHUNK], tf32_mode=tf32_view)
|
| 64 |
+
for i in range(0, B * V, VIEW_CHUNK)])
|
| 65 |
+
return out.view(B, V, *out.shape[1:])
|
gaussianformer/models/view_transformer.py
ADDED
|
@@ -0,0 +1,117 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
import roma
|
| 2 |
+
import torch
|
| 3 |
+
from einops import rearrange
|
| 4 |
+
from torch import nn
|
| 5 |
+
|
| 6 |
+
from gaussianformer.layers.attention import TransformerDecoder
|
| 7 |
+
from gaussianformer.layers.dpt import DPTHead
|
| 8 |
+
from gaussianformer.layers.window import build_window
|
| 9 |
+
from gaussianformer.models.config import GaussianFormerConfig
|
| 10 |
+
|
| 11 |
+
|
| 12 |
+
class ViewTransformer(nn.Module):
|
| 13 |
+
"""View decoder: ray-direction patch tokens attend to the scene tokens; a DPT head decodes pixels."""
|
| 14 |
+
|
| 15 |
+
def __init__(self, config: GaussianFormerConfig):
|
| 16 |
+
super().__init__()
|
| 17 |
+
self.config = config
|
| 18 |
+
assert not config.ray_embed_patch or config.ray_embed_patch > config.patch_size, \
|
| 19 |
+
"ray_embed_patch must exceed patch_size"
|
| 20 |
+
dim = config.view_transformer_latent_dim
|
| 21 |
+
embed_patch = config.ray_embed_patch or config.patch_size
|
| 22 |
+
self.ray_map_patch_token = nn.Parameter(torch.randn(1, 1, dim))
|
| 23 |
+
self.ray_map_encoder = nn.Linear(3 * embed_patch ** 2, dim)
|
| 24 |
+
self.ray_map_encoder_norm = nn.RMSNorm(dim)
|
| 25 |
+
self.transformer = TransformerDecoder(
|
| 26 |
+
num_layers=config.view_transformer_n_layers,
|
| 27 |
+
num_heads=config.view_transformer_n_heads,
|
| 28 |
+
dim=dim,
|
| 29 |
+
ctx_dim=config.latent_dim,
|
| 30 |
+
ffn_hidden_dim=config.view_transformer_ffn_hidden_dim,
|
| 31 |
+
rope_dim=config.pos_pe_num_freqs,
|
| 32 |
+
proj_rope_2d=config.proj_rope_2d,
|
| 33 |
+
ray_rope_2d_dim=config.ray_rope_2d_dim,
|
| 34 |
+
ray_rope_2d_scale=config.ray_rope_2d_scale,
|
| 35 |
+
grad_checkpoint=config.view_grad_checkpoint,
|
| 36 |
+
key_feat_dim=4 if config.cov_key_embed else 0,
|
| 37 |
+
value_feat=config.cov_key_values,
|
| 38 |
+
)
|
| 39 |
+
self.out_dpt = DPTHead(dim, features=config.dpt_features, out_channels=config.dpt_out_channels)
|
| 40 |
+
self.out_layers = list(range(config.view_transformer_n_layers - 4, config.view_transformer_n_layers))
|
| 41 |
+
self.out_proj_act = nn.ELU(alpha=1e-3)
|
| 42 |
+
|
| 43 |
+
def project(self, pos, fov, H, W):
|
| 44 |
+
"""Camera-frame positions -> image position in patch units, depth, behind-camera flag, focal length.
|
| 45 |
+
Camera at the origin looking down -Z (as in RayGenerator)."""
|
| 46 |
+
focal = 0.5 * W / torch.tan(0.5 * fov.to(pos.dtype)).view(-1, 1)
|
| 47 |
+
X, Y, Z = pos.unbind(-1)
|
| 48 |
+
depth = (-Z).clamp_min(1e-3)
|
| 49 |
+
u = (W / 2 + focal * X / depth) / self.config.patch_size
|
| 50 |
+
v = (H / 2 - focal * Y / depth) / self.config.patch_size
|
| 51 |
+
return torch.stack([u, v], -1), depth, Z > -1e-3, focal
|
| 52 |
+
|
| 53 |
+
def footprint(self, pos, shape, fov, H, W):
|
| 54 |
+
"""Projected covariance of each Gaussian in patch units -> [B, N, 4] = (log sigma_1, log sigma_2, cos 2t, sin 2t)
|
| 55 |
+
of its image-space axes (sigma_1 >= sigma_2). shape = camera-frame [scale(3), quat wxyz(4)]. Zero for register
|
| 56 |
+
tokens and Gaussians behind the camera. fp32 with autocast off: in bf16 / fp16 the minor axis cancels away."""
|
| 57 |
+
ps = self.config.patch_size
|
| 58 |
+
with torch.autocast(pos.device.type, enabled=False):
|
| 59 |
+
pos = pos.float()
|
| 60 |
+
_, depth, behind, focal = self.project(pos, fov.float(), H, W)
|
| 61 |
+
f = focal / ps
|
| 62 |
+
scale, quat = shape[..., :3].float(), shape[..., 3:7].float()
|
| 63 |
+
R = roma.unitquat_to_rotmat(quat[..., [1, 2, 3, 0]])
|
| 64 |
+
cov = (R * scale[..., None, :] ** 2) @ R.transpose(-1, -2)
|
| 65 |
+
X, Y = pos[..., 0], pos[..., 1]
|
| 66 |
+
zero = torch.zeros_like(depth)
|
| 67 |
+
J = torch.stack([torch.stack([f / depth, zero, f * X / depth ** 2], -1),
|
| 68 |
+
torch.stack([zero, -f / depth, -f * Y / depth ** 2], -1)], -2)
|
| 69 |
+
c2 = J @ cov @ J.transpose(-1, -2) + self.config.cov_key_eps2d / ps ** 2 * torch.eye(2, device=cov.device)
|
| 70 |
+
a, b, c = c2[..., 0, 0], c2[..., 0, 1], c2[..., 1, 1]
|
| 71 |
+
half = torch.hypot((a - c) / 2, b)
|
| 72 |
+
l1, l2 = (a + c) / 2 + half, ((a + c) / 2 - half).clamp_min(1e-8)
|
| 73 |
+
feat = torch.stack([0.5 * l1.log(), 0.5 * l2.log(), (a - c) / (2 * half + 1e-8), 2 * b / (2 * half + 1e-8)], -1)
|
| 74 |
+
ok = (scale.amax(-1) > 0) & ~behind & torch.isfinite(feat).all(-1)
|
| 75 |
+
return torch.where(ok[..., None], feat.clamp(-6, 6), 0.0)
|
| 76 |
+
|
| 77 |
+
def forward(self, camera_o, ray_map, ctx_tokens, ctx_pos, ctx_shape, key_mask, fov, tf32_mode=False):
|
| 78 |
+
"""
|
| 79 |
+
camera_o [B, 3]; ray_map [B, H, W, 3] ray directions; ctx_tokens [B, N, D] scene tokens;
|
| 80 |
+
ctx_pos [B, N, 3] camera-frame positions and ctx_shape [B, N, 7] camera-frame scales and rotations
|
| 81 |
+
(registers first, with zeros);
|
| 82 |
+
key_mask [B, N]; fov [B] in radians. Returns log-HDR images [B, 3, H, W].
|
| 83 |
+
"""
|
| 84 |
+
cfg = self.config
|
| 85 |
+
B, H, W, _ = ray_map.shape
|
| 86 |
+
patch_h, patch_w = H // cfg.patch_size, W // cfg.patch_size
|
| 87 |
+
embed_patch = cfg.ray_embed_patch or cfg.patch_size
|
| 88 |
+
rays = ray_map
|
| 89 |
+
if embed_patch != cfg.patch_size: # nearest-upsample each patch to the embedding's patch size
|
| 90 |
+
r = embed_patch // cfg.patch_size
|
| 91 |
+
rays = rays.permute(0, 3, 1, 2).repeat_interleave(r, 2).repeat_interleave(r, 3).permute(0, 2, 3, 1)
|
| 92 |
+
rays = rearrange(rays, "b (h1 p1) (w1 p2) c -> b (h1 w1) (c p1 p2)", p1=embed_patch, p2=embed_patch)
|
| 93 |
+
ray_tokens = self.ray_map_patch_token + self.ray_map_encoder_norm(self.ray_map_encoder(rays))
|
| 94 |
+
ray_pos = camera_o[:, None].repeat(1, ray_tokens.size(1), 1)
|
| 95 |
+
|
| 96 |
+
uv_q = uv_k = window = None
|
| 97 |
+
if cfg.proj_rope_2d or cfg.xattn_window:
|
| 98 |
+
uv_k, depth, behind, focal = self.project(ctx_pos, fov, H, W)
|
| 99 |
+
ii, jj = torch.meshgrid(torch.arange(patch_h, device=ray_map.device),
|
| 100 |
+
torch.arange(patch_w, device=ray_map.device), indexing="ij")
|
| 101 |
+
uv_q = (torch.stack([jj, ii], -1).reshape(1, -1, 2).float() + 0.5).expand(B, -1, -1)
|
| 102 |
+
if cfg.xattn_window:
|
| 103 |
+
radius = cfg.xattn_sigmas * ctx_shape[..., :3].amax(-1) * focal / depth / cfg.patch_size
|
| 104 |
+
key_ok = key_mask & ~behind
|
| 105 |
+
key_ok[:, :cfg.num_register_tokens] = False
|
| 106 |
+
window = build_window(uv_k, radius, key_ok, cfg.num_register_tokens, patch_h, patch_w,
|
| 107 |
+
cfg.xattn_window, cfg.xattn_margin)
|
| 108 |
+
if not cfg.proj_rope_2d:
|
| 109 |
+
uv_q = uv_k = None
|
| 110 |
+
key_feat = self.footprint(ctx_pos, ctx_shape, fov, H, W) if cfg.cov_key_embed else None
|
| 111 |
+
|
| 112 |
+
fp32 = tf32_mode and not cfg.view_bf16
|
| 113 |
+
with torch.autocast(device_type="cuda", dtype=torch.float32 if fp32 else torch.bfloat16):
|
| 114 |
+
features = self.transformer(ray_tokens, ctx_tokens, key_mask, ctx_pos, ray_pos, self.out_layers,
|
| 115 |
+
force_sdpa=fp32, uv_q=uv_q, uv_k=uv_k, window=window,
|
| 116 |
+
key_feat=key_feat)
|
| 117 |
+
return self.out_proj_act(self.out_dpt(features, patch_h, patch_w, patch_size=cfg.patch_size))
|
gaussianformer/pipelines/__init__.py
ADDED
|
File without changes
|
gaussianformer/pipelines/rendering_pipeline.py
ADDED
|
@@ -0,0 +1,59 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
import torch
|
| 2 |
+
|
| 3 |
+
from gaussianformer.models.gaussianformer import GaussianFormer
|
| 4 |
+
from gaussianformer.utils.ray_generator import RayGenerator
|
| 5 |
+
from gaussianformer.utils.transform import transform_gaussians_to_cam_coord
|
| 6 |
+
|
| 7 |
+
|
| 8 |
+
def model_forward(model: GaussianFormer, gaussians, mask, c2w, fov, resolution: int, tf32_view: bool = False):
|
| 9 |
+
"""Render log-HDR images [B, V, H, W, 3]; gaussians [B, N, 14], mask [B, N], c2w [B, V, 4, 4],
|
| 10 |
+
fov [B, V] in degrees. The decoder works in each camera's frame, so its rays start at the origin."""
|
| 11 |
+
B, V = c2w.shape[:2]
|
| 12 |
+
with torch.no_grad():
|
| 13 |
+
gaussians_cam = transform_gaussians_to_cam_coord(
|
| 14 |
+
c2w.reshape(-1, 4, 4), gaussians.repeat_interleave(V, dim=0)).reshape(B, V, *gaussians.shape[1:])
|
| 15 |
+
eye = torch.eye(4, device=c2w.device, dtype=c2w.dtype).expand(B, V, 4, 4)
|
| 16 |
+
fov = fov / 180.0 * torch.pi
|
| 17 |
+
rays_o, rays_d = RayGenerator()(eye, fov[..., None], resolution)
|
| 18 |
+
out = model(gaussians, mask, rays_o, rays_d, gaussians_cam, fov, tf32_view=tf32_view)
|
| 19 |
+
return out.permute(0, 1, 3, 4, 2)
|
| 20 |
+
|
| 21 |
+
|
| 22 |
+
class GaussianFormerRenderingPipeline:
|
| 23 |
+
def __init__(self, model: GaussianFormer):
|
| 24 |
+
self.model = model
|
| 25 |
+
|
| 26 |
+
@classmethod
|
| 27 |
+
def from_pretrained(cls, model_id: str):
|
| 28 |
+
return cls(GaussianFormer.from_pretrained(model_id).eval())
|
| 29 |
+
|
| 30 |
+
@property
|
| 31 |
+
def device(self):
|
| 32 |
+
return self.model.device
|
| 33 |
+
|
| 34 |
+
def to(self, device: str | torch.device):
|
| 35 |
+
self.model.to(device)
|
| 36 |
+
return self
|
| 37 |
+
|
| 38 |
+
@torch.no_grad()
|
| 39 |
+
def __call__(self, gaussians, c2w, fov=45.0, mask=None, resolution: int = 512,
|
| 40 |
+
torch_dtype: torch.dtype = torch.bfloat16):
|
| 41 |
+
"""
|
| 42 |
+
One object: gaussians [N, 14] = pos(3) | scale(3) | quat wxyz(4) | rgb(3) | opacity(1), c2w [V, 4, 4]
|
| 43 |
+
camera-to-world (Blender convention) -> images [V, H, W, 3]. A padded batch: gaussians [B, N, 14],
|
| 44 |
+
c2w [B, V, 4, 4], mask [B, N] (True = real Gaussian; default all) -> images [B, V, H, W, 3].
|
| 45 |
+
fov in degrees: a number or a tensor broadcastable to [V] / [B, V]. Inputs are moved to the model's device.
|
| 46 |
+
Images are display-referred (clip to [0, 1] for 8-bit output).
|
| 47 |
+
"""
|
| 48 |
+
single = gaussians.dim() == 2
|
| 49 |
+
if single:
|
| 50 |
+
gaussians, c2w = gaussians[None], c2w[None]
|
| 51 |
+
gaussians, c2w = gaussians.to(self.device), c2w.to(self.device)
|
| 52 |
+
fov = torch.as_tensor(fov, dtype=torch.float32, device=self.device).expand(c2w.shape[:2])
|
| 53 |
+
mask = (torch.ones(gaussians.shape[:2], dtype=torch.bool, device=self.device) if mask is None
|
| 54 |
+
else mask.to(self.device))
|
| 55 |
+
with torch.autocast(device_type=self.device.type, dtype=torch_dtype):
|
| 56 |
+
log_img = model_forward(self.model, gaussians, mask, c2w, fov, resolution,
|
| 57 |
+
tf32_view=torch_dtype != torch.float32)
|
| 58 |
+
images = torch.pow(10.0, log_img) - 1.0
|
| 59 |
+
return images[0] if single else images
|
gaussianformer/splat.py
ADDED
|
@@ -0,0 +1,153 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""3D Gaussian Splatting PLY files -> model inputs.
|
| 2 |
+
|
| 3 |
+
A 3DGS PLY stores the optimizer's raw parameters; GaussianFormer takes activated values in a normalized object
|
| 4 |
+
frame, [N, 14] = pos(3) | scale(3) | quat wxyz(4) | rgb(3) | opacity(1):
|
| 5 |
+
|
| 6 |
+
scale exp(scale_0..2)
|
| 7 |
+
quat rot_0..3 (w, x, y, z), normalized
|
| 8 |
+
rgb clip(0.5 + SH_C0 * f_dc_0..2, 0, 1) (higher spherical-harmonic orders are dropped)
|
| 9 |
+
opacity sigmoid(opacity)
|
| 10 |
+
|
| 11 |
+
The object is centered on its median position, turned Y-up and scaled so its largest coordinate is 0.45, and
|
| 12 |
+
pruned to about 20k Gaussians (the size the model is trained on) by an importance score adapted from LightGaussian,
|
| 13 |
+
followed by a short fine-tune of the kept Gaussians against the full splat.
|
| 14 |
+
|
| 15 |
+
from gaussianformer.splat import load_ply
|
| 16 |
+
gaussians = load_ply("object.ply", up="z") # [N, 14] tensor on the GPU
|
| 17 |
+
"""
|
| 18 |
+
import io
|
| 19 |
+
from pathlib import Path
|
| 20 |
+
|
| 21 |
+
import numpy as np
|
| 22 |
+
import roma
|
| 23 |
+
import torch
|
| 24 |
+
import torch.nn.functional as F
|
| 25 |
+
from plyfile import PlyData
|
| 26 |
+
|
| 27 |
+
from gaussianformer.utils.cameras import RESOLUTION, orbit, to_gsplat
|
| 28 |
+
|
| 29 |
+
SH_C0 = 0.28209479177387814
|
| 30 |
+
TO_Y_UP = {
|
| 31 |
+
"y": np.eye(3, dtype=np.float32),
|
| 32 |
+
"z": np.array([[1, 0, 0], [0, 0, 1], [0, -1, 0]], dtype=np.float32), # Z-up (e.g. Objaverse_Splats)
|
| 33 |
+
"-y": np.diag([1.0, -1.0, -1.0]).astype(np.float32), # Y-down (e.g. COLMAP captures)
|
| 34 |
+
}
|
| 35 |
+
KEEP = 20_000
|
| 36 |
+
RECOVERY_VIEWS = 64
|
| 37 |
+
RECOVERY_ITERS = 1500
|
| 38 |
+
FIELDS = ("means", "scales", "rotations", "colors", "opacities")
|
| 39 |
+
|
| 40 |
+
|
| 41 |
+
def read_ply(src: str | Path | bytes) -> dict[str, np.ndarray]:
|
| 42 |
+
"""3DGS PLY (path or bytes) -> activated Gaussians: means, scales, rotations (wxyz), colors, opacities."""
|
| 43 |
+
v = PlyData.read(io.BytesIO(src) if isinstance(src, bytes) else str(src))["vertex"]
|
| 44 |
+
quats = np.stack([v[f"rot_{i}"] for i in range(4)], -1).astype(np.float32)
|
| 45 |
+
return {
|
| 46 |
+
"means": np.stack([v["x"], v["y"], v["z"]], -1).astype(np.float32),
|
| 47 |
+
"scales": np.exp(np.stack([v[f"scale_{i}"] for i in range(3)], -1)).astype(np.float32),
|
| 48 |
+
"rotations": quats / (np.linalg.norm(quats, axis=-1, keepdims=True) + 1e-9),
|
| 49 |
+
"colors": np.clip(0.5 + SH_C0 * np.stack([v[f"f_dc_{i}"] for i in range(3)], -1), 0, 1).astype(np.float32),
|
| 50 |
+
"opacities": (1.0 / (1.0 + np.exp(-np.asarray(v["opacity"], np.float32)))).astype(np.float32),
|
| 51 |
+
}
|
| 52 |
+
|
| 53 |
+
|
| 54 |
+
def normalize(g: dict[str, np.ndarray], up: str = "y") -> dict[str, np.ndarray] | None:
|
| 55 |
+
"""Center on the median position, rotate `up` to +Y and scale into [-0.45, 0.45]^3; None if degenerate."""
|
| 56 |
+
R = TO_Y_UP[up]
|
| 57 |
+
means = ((g["means"] - np.median(g["means"], axis=0)) @ R.T).astype(np.float32)
|
| 58 |
+
rot = roma.unitquat_to_rotmat(torch.from_numpy(g["rotations"])[:, [1, 2, 3, 0]])
|
| 59 |
+
rotations = roma.rotmat_to_unitquat(torch.from_numpy(R) @ rot)[:, [3, 0, 1, 2]].numpy()
|
| 60 |
+
extent = float(np.abs(means).max())
|
| 61 |
+
if extent < 1e-4:
|
| 62 |
+
return None
|
| 63 |
+
s = 0.45 / extent
|
| 64 |
+
return {"means": means * s, "scales": g["scales"] * s, "rotations": rotations, "colors": g["colors"],
|
| 65 |
+
"opacities": g["opacities"]}
|
| 66 |
+
|
| 67 |
+
|
| 68 |
+
def rasterize(g: dict[str, torch.Tensor], viewmats: torch.Tensor, Ks: torch.Tensor):
|
| 69 |
+
"""gsplat render of Gaussians `g` (activated values, opacities [N]) -> images [V, H, W, 3], render info."""
|
| 70 |
+
import gsplat
|
| 71 |
+
|
| 72 |
+
return gsplat.rasterization(
|
| 73 |
+
means=g["means"], quats=g["rotations"] / g["rotations"].norm(dim=-1, keepdim=True), scales=g["scales"],
|
| 74 |
+
opacities=g["opacities"], colors=g["colors"], viewmats=viewmats, Ks=Ks, width=RESOLUTION,
|
| 75 |
+
height=RESOLUTION, sh_degree=None, eps2d=0.3, render_mode="RGB", near_plane=0.01, packed=True)[::2]
|
| 76 |
+
|
| 77 |
+
|
| 78 |
+
def importance_topk(g: dict[str, torch.Tensor], viewmats, Ks, keep: int) -> torch.Tensor:
|
| 79 |
+
"""Indices of the `keep` Gaussians with the highest importance, a score adapted from LightGaussian: opacity x
|
| 80 |
+
projected area summed over the views, times the largest scale^0.1."""
|
| 81 |
+
score = torch.zeros(len(g["means"]), device=g["means"].device, dtype=torch.float64)
|
| 82 |
+
for v in range(len(viewmats)):
|
| 83 |
+
with torch.no_grad():
|
| 84 |
+
_, info = rasterize(g, viewmats[v:v + 1], Ks[v:v + 1])
|
| 85 |
+
area = info["radii"][:, 0].double() * info["radii"][:, 1].double()
|
| 86 |
+
score.scatter_add_(0, info["gaussian_ids"], info["opacities"].double() * area)
|
| 87 |
+
score = score * g["scales"].max(1).values.double() ** 0.1
|
| 88 |
+
return torch.topk(score, min(keep, len(score))).indices
|
| 89 |
+
|
| 90 |
+
|
| 91 |
+
def ssim(x: torch.Tensor, y: torch.Tensor) -> torch.Tensor:
|
| 92 |
+
g = torch.exp(-(torch.arange(11, dtype=torch.float32, device=x.device) - 5) ** 2 / (2 * 1.5 ** 2))
|
| 93 |
+
g = g / g.sum()
|
| 94 |
+
w = (g[:, None] * g[None, :]).expand(3, 1, 11, 11).contiguous()
|
| 95 |
+
conv = lambda t: F.conv2d(t, w, padding=5, groups=3)
|
| 96 |
+
mx, my = conv(x), conv(y)
|
| 97 |
+
sx, sy, sxy = conv(x * x) - mx * mx, conv(y * y) - my * my, conv(x * y) - mx * my
|
| 98 |
+
c1, c2 = 0.01 ** 2, 0.03 ** 2
|
| 99 |
+
return (((2 * mx * my + c1) * (2 * sxy + c2)) / ((mx * mx + my * my + c1) * (sx + sy + c2))).mean()
|
| 100 |
+
|
| 101 |
+
|
| 102 |
+
def recover(g: dict[str, torch.Tensor], target: torch.Tensor, viewmats, Ks) -> dict[str, torch.Tensor]:
|
| 103 |
+
"""Fine-tune pruned Gaussians (L1 + SSIM, no densification) against the full splat's renders `target`."""
|
| 104 |
+
p = {"means": g["means"], "log_scales": g["scales"].clamp_min(1e-8).log(), "rotations": g["rotations"],
|
| 105 |
+
"opacity_logits": torch.logit(g["opacities"].clamp(1e-4, 1 - 1e-4)),
|
| 106 |
+
"color_logits": torch.logit(g["colors"].clamp(1e-4, 1 - 1e-4))}
|
| 107 |
+
p = {k: torch.nn.Parameter(v.clone()) for k, v in p.items()}
|
| 108 |
+
lrs = {"means": 1.6e-4, "log_scales": 5e-3, "rotations": 1e-3, "opacity_logits": 5e-2, "color_logits": 2.5e-3}
|
| 109 |
+
opt = torch.optim.Adam([{"params": [p[k]], "lr": lr} for k, lr in lrs.items()])
|
| 110 |
+
sched = torch.optim.lr_scheduler.ExponentialLR(opt, gamma=0.95 ** (1 / 400))
|
| 111 |
+
current = lambda: {"means": p["means"], "scales": p["log_scales"].exp(), "rotations": p["rotations"],
|
| 112 |
+
"opacities": p["opacity_logits"].sigmoid(), "colors": p["color_logits"].sigmoid()}
|
| 113 |
+
for _ in range(RECOVERY_ITERS):
|
| 114 |
+
v = torch.randint(0, len(viewmats), (4,), device=target.device)
|
| 115 |
+
img = rasterize(current(), viewmats[v], Ks[v])[0].clamp(0, 1)
|
| 116 |
+
loss = 0.8 * (img - target[v]).abs().mean() + 0.2 * (1 - ssim(img.permute(0, 3, 1, 2), target[v].permute(0, 3, 1, 2)))
|
| 117 |
+
loss.backward()
|
| 118 |
+
opt.step()
|
| 119 |
+
opt.zero_grad()
|
| 120 |
+
sched.step()
|
| 121 |
+
with torch.no_grad():
|
| 122 |
+
out = current()
|
| 123 |
+
out["rotations"] = out["rotations"] / out["rotations"].norm(dim=-1, keepdim=True)
|
| 124 |
+
return {k: v.detach() for k, v in out.items()}
|
| 125 |
+
|
| 126 |
+
|
| 127 |
+
def prune(full: dict[str, torch.Tensor], keep: int = KEEP, recovery: bool = True) -> dict[str, torch.Tensor]:
|
| 128 |
+
"""Keep the `keep` most important Gaussians (scored from a 64-view orbit), then optionally fine-tune them."""
|
| 129 |
+
views = tuple(torch.from_numpy(x).to(full["means"].device) for x in to_gsplat(orbit(RECOVERY_VIEWS, 1.7)))
|
| 130 |
+
with torch.no_grad():
|
| 131 |
+
target = torch.cat([rasterize(full, views[0][i:i + 1], views[1][i:i + 1])[0].clamp(0, 1)
|
| 132 |
+
for i in range(RECOVERY_VIEWS)]) if recovery else None
|
| 133 |
+
kept = {k: v[importance_topk(full, *views, keep)] for k, v in full.items()}
|
| 134 |
+
return recover(kept, target, *views) if recovery else kept
|
| 135 |
+
|
| 136 |
+
|
| 137 |
+
def to_tensor(g: dict[str, torch.Tensor | np.ndarray]) -> torch.Tensor:
|
| 138 |
+
"""Gaussians as a dict -> the model's [N, 14] layout."""
|
| 139 |
+
parts = [torch.as_tensor(g[k]).float().reshape(len(g["means"]), -1) for k in FIELDS]
|
| 140 |
+
return torch.cat(parts, dim=-1)
|
| 141 |
+
|
| 142 |
+
|
| 143 |
+
def load_ply(path: str | Path, up: str = "y", keep: int = KEEP, recovery: bool = True,
|
| 144 |
+
device: str | torch.device = "cuda") -> torch.Tensor:
|
| 145 |
+
"""3DGS PLY -> model input [N, 14] on `device`: activated, normalized and pruned to at most `keep` Gaussians.
|
| 146 |
+
`up` is the file's up axis ('y', 'z' or '-y'). `recovery` fine-tunes the kept Gaussians against the full
|
| 147 |
+
splat (about a minute on a GPU); without it a large splat pruned to 20k loses coverage and turns dark."""
|
| 148 |
+
g = normalize(read_ply(path), up)
|
| 149 |
+
assert g is not None, f"{path}: all Gaussians at one point"
|
| 150 |
+
g = {k: torch.from_numpy(np.ascontiguousarray(v)).to(device) for k, v in g.items()}
|
| 151 |
+
if len(g["means"]) > keep:
|
| 152 |
+
g = prune(g, keep, recovery)
|
| 153 |
+
return to_tensor(g)
|
gaussianformer/utils/__init__.py
ADDED
|
File without changes
|
gaussianformer/utils/cameras.py
ADDED
|
@@ -0,0 +1,44 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Orbit cameras around the origin in the Blender convention (-Z forward, +Y up, +X right).
|
| 2 |
+
|
| 3 |
+
Views alternate between a high (+0.4 r) and a low (-0.1 r) elevation to cover the top and the bottom of the object.
|
| 4 |
+
"""
|
| 5 |
+
import numpy as np
|
| 6 |
+
|
| 7 |
+
FOV = 45.0
|
| 8 |
+
RESOLUTION = 512
|
| 9 |
+
DISTANCES = ((1.7, 14), (1.15, 7), (2.45, 7)) # (orbit radius, views): 28 views per object
|
| 10 |
+
|
| 11 |
+
|
| 12 |
+
def look_at(eye: np.ndarray, target: np.ndarray, up: np.ndarray) -> np.ndarray:
|
| 13 |
+
forward = target - eye
|
| 14 |
+
forward /= np.linalg.norm(forward) + 1e-9
|
| 15 |
+
right = np.cross(forward, up)
|
| 16 |
+
right /= np.linalg.norm(right) + 1e-9
|
| 17 |
+
c2w = np.eye(4, dtype=np.float32)
|
| 18 |
+
c2w[:3, 0], c2w[:3, 1], c2w[:3, 2], c2w[:3, 3] = right, np.cross(right, forward), -forward, eye
|
| 19 |
+
return c2w
|
| 20 |
+
|
| 21 |
+
|
| 22 |
+
def orbit(n_views: int, radius: float) -> np.ndarray:
|
| 23 |
+
"""Camera-to-world matrices [n_views, 4, 4]."""
|
| 24 |
+
c2w = []
|
| 25 |
+
for i in range(n_views):
|
| 26 |
+
theta = 2 * np.pi * i / n_views
|
| 27 |
+
elevation = 0.4 * radius if i % 2 == 0 else -0.1 * radius
|
| 28 |
+
eye = np.array([radius * np.cos(theta), elevation, radius * np.sin(theta)], np.float32)
|
| 29 |
+
c2w.append(look_at(eye, np.zeros(3, np.float32), np.array([0, 1, 0], np.float32)))
|
| 30 |
+
return np.stack(c2w)
|
| 31 |
+
|
| 32 |
+
|
| 33 |
+
def all_views() -> np.ndarray:
|
| 34 |
+
"""The 28 training cameras: 14 at distance 1.7, then 7 each at 1.15 and 2.45."""
|
| 35 |
+
return np.concatenate([orbit(n, r) for r, n in DISTANCES])
|
| 36 |
+
|
| 37 |
+
|
| 38 |
+
def to_gsplat(c2w: np.ndarray, fov: float = FOV, resolution: int = RESOLUTION) -> tuple[np.ndarray, np.ndarray]:
|
| 39 |
+
"""Camera-to-world matrices [V, 4, 4] -> gsplat view matrices (OpenCV convention) and intrinsics [V, 3, 3]."""
|
| 40 |
+
flip = np.diag([1.0, -1.0, -1.0, 1.0]).astype(np.float32)
|
| 41 |
+
viewmats = (flip @ np.linalg.inv(c2w)).astype(np.float32)
|
| 42 |
+
f = 0.5 * resolution / np.tan(0.5 * np.radians(fov))
|
| 43 |
+
K = np.array([[f, 0, resolution / 2], [0, f, resolution / 2], [0, 0, 1]], np.float32)
|
| 44 |
+
return viewmats, np.tile(K, (len(c2w), 1, 1))
|
gaussianformer/utils/checkpoint.py
ADDED
|
@@ -0,0 +1,34 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
from pathlib import Path
|
| 2 |
+
|
| 3 |
+
import torch
|
| 4 |
+
from torch import nn
|
| 5 |
+
|
| 6 |
+
from gaussianformer.models.config import GaussianFormerConfig
|
| 7 |
+
from gaussianformer.models.gaussianformer import GaussianFormer
|
| 8 |
+
|
| 9 |
+
|
| 10 |
+
def load_seed(module: nn.Module, state_dict: dict, allow_new: bool = True) -> list[str]:
|
| 11 |
+
"""Warm-start `module` from `state_dict`; returns the keys the state dict lacked.
|
| 12 |
+
|
| 13 |
+
New tensors (`allow_new`) must be zero-initialized so the model computes the same function as the
|
| 14 |
+
seed at load; RoPE frequency tables are derived from the config and may be new or change shape.
|
| 15 |
+
"""
|
| 16 |
+
own = module.state_dict()
|
| 17 |
+
sd = {k: v for k, v in state_dict.items() if not (k.endswith(".freqs") and k in own and own[k].shape != v.shape)}
|
| 18 |
+
missing, unexpected = module.load_state_dict(sd, strict=False)
|
| 19 |
+
assert not unexpected, f"state dict has keys the model lacks: {list(unexpected)[:5]}"
|
| 20 |
+
new = [k for k in missing if not k.endswith(".freqs")]
|
| 21 |
+
assert allow_new or not new, f"state dict lacks tensors: {new[:5]}"
|
| 22 |
+
nonzero = [k for k in new if own[k].abs().sum() > 0]
|
| 23 |
+
assert not nonzero, f"new tensors must be zero-initialized: {nonzero[:5]}"
|
| 24 |
+
return list(missing)
|
| 25 |
+
|
| 26 |
+
|
| 27 |
+
def load_checkpoint(path: Path, overrides: list[str] | None = None,
|
| 28 |
+
allow_new: bool = False) -> tuple[GaussianFormer, dict]:
|
| 29 |
+
"""Model built from the checkpoint's stored config (plus `key=value` overrides), weights loaded."""
|
| 30 |
+
ckpt = torch.load(path, map_location="cpu", weights_only=True)
|
| 31 |
+
config = GaussianFormerConfig(**ckpt["config"]).with_overrides(overrides)
|
| 32 |
+
model = GaussianFormer(config)
|
| 33 |
+
load_seed(model, ckpt["model_state_dict"], allow_new=allow_new)
|
| 34 |
+
return model, ckpt
|
gaussianformer/utils/ray_generator.py
ADDED
|
@@ -0,0 +1,22 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
import torch
|
| 2 |
+
import torch.nn.functional as F
|
| 3 |
+
from torch import nn
|
| 4 |
+
|
| 5 |
+
|
| 6 |
+
class RayGenerator(nn.Module):
|
| 7 |
+
"""Pinhole camera rays through pixel centers (Blender convention: -Z forward, +Y up)."""
|
| 8 |
+
|
| 9 |
+
def forward(self, c2w: torch.Tensor, fov: torch.Tensor, img_res: int = 256):
|
| 10 |
+
"""c2w [..., 4, 4], fov [..., 1] in radians -> origins [..., 3], unit directions [..., H, W, 3]."""
|
| 11 |
+
batch_shape = c2w.shape[:-2]
|
| 12 |
+
x, y = torch.meshgrid(
|
| 13 |
+
torch.linspace(0.5, img_res - 0.5, img_res, device=c2w.device, dtype=c2w.dtype),
|
| 14 |
+
torch.linspace(0.5, img_res - 0.5, img_res, device=c2w.device, dtype=c2w.dtype),
|
| 15 |
+
indexing="xy")
|
| 16 |
+
c = img_res / 2
|
| 17 |
+
f = img_res / 2 / torch.tan(0.5 * fov[..., 0, None, None])
|
| 18 |
+
x = x[None].repeat(*batch_shape, 1, 1)
|
| 19 |
+
y = y[None].repeat(*batch_shape, 1, 1)
|
| 20 |
+
dirs = torch.stack([(x - c) / f, -(y - c) / f, -torch.ones_like(x)], dim=-1)
|
| 21 |
+
rays_d = torch.sum(dirs[..., None, :] * c2w[..., None, None, :3, :3], dim=-1)
|
| 22 |
+
return c2w[..., :3, 3], F.normalize(rays_d, dim=-1, p=2)
|
gaussianformer/utils/transform.py
ADDED
|
@@ -0,0 +1,30 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
import roma
|
| 2 |
+
import torch
|
| 3 |
+
import torch.nn.functional as F
|
| 4 |
+
from torch.amp import autocast
|
| 5 |
+
|
| 6 |
+
|
| 7 |
+
def quaternion_multiply(a: torch.Tensor, b: torch.Tensor) -> torch.Tensor:
|
| 8 |
+
"""Composition of rotations given as wxyz quaternions, returned with a non-negative real part."""
|
| 9 |
+
aw, ax, ay, az = torch.unbind(a, -1)
|
| 10 |
+
bw, bx, by, bz = torch.unbind(b, -1)
|
| 11 |
+
ab = torch.stack((
|
| 12 |
+
aw * bw - ax * bx - ay * by - az * bz,
|
| 13 |
+
aw * bx + ax * bw + ay * bz - az * by,
|
| 14 |
+
aw * by - ax * bz + ay * bw + az * bx,
|
| 15 |
+
aw * bz + ax * by - ay * bx + az * bw,
|
| 16 |
+
), -1)
|
| 17 |
+
return torch.where(ab[..., 0:1] < 0, -ab, ab)
|
| 18 |
+
|
| 19 |
+
|
| 20 |
+
@torch.no_grad()
|
| 21 |
+
@autocast("cuda", enabled=False)
|
| 22 |
+
def transform_gaussians_to_cam_coord(c2w: torch.Tensor, gaussians: torch.Tensor) -> torch.Tensor:
|
| 23 |
+
"""Gaussians [B, N, 14] (pos, scale, quat wxyz, ...) -> the same Gaussians in the camera frame of
|
| 24 |
+
the rigid camera-to-world matrices c2w [B, 4, 4]."""
|
| 25 |
+
w2c = roma.Rigid.from_homogeneous(c2w).inverse()
|
| 26 |
+
pos = w2c[:, None].apply(gaussians[..., :3])
|
| 27 |
+
w2c_quat = roma.rotmat_to_unitquat(w2c.linear)[..., [3, 0, 1, 2]] # roma is xyzw
|
| 28 |
+
rot = F.normalize(gaussians[..., 6:10], dim=-1) # padded all-zero rows stay zero instead of NaN
|
| 29 |
+
rot = quaternion_multiply(w2c_quat[:, None], rot)
|
| 30 |
+
return torch.cat([pos, gaussians[..., 3:6], rot, gaussians[..., 10:]], dim=-1)
|