multimodalart HF Staff commited on
Commit
91a49a5
·
verified ·
1 Parent(s): 181fe17

Upload folder using huggingface_hub

Browse files
gaussianformer/__init__.py ADDED
@@ -0,0 +1,5 @@
 
 
 
 
 
 
1
+ from gaussianformer.models.gaussianformer import GaussianFormer
2
+ from gaussianformer.pipelines.rendering_pipeline import GaussianFormerRenderingPipeline
3
+ from gaussianformer.splat import load_ply
4
+
5
+ __all__ = ["GaussianFormer", "GaussianFormerRenderingPipeline", "load_ply"]
gaussianformer/encodings/__init__.py ADDED
File without changes
gaussianformer/encodings/rope.py ADDED
@@ -0,0 +1,65 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # Modified from https://github.com/lucidrains/rotary-embedding-torch/blob/main/rotary_embedding_torch/rotary_embedding_torch.py
2
+
3
+ # Copyright (c) 2021 Phil Wang
4
+ #
5
+ # Permission is hereby granted, free of charge, to any person obtaining a copy
6
+ # of this software and associated documentation files (the "Software"), to deal
7
+ # in the Software without restriction, including without limitation the rights
8
+ # to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
9
+ # copies of the Software, and to permit persons to whom the Software is
10
+ # furnished to do so, subject to the following conditions:
11
+ #
12
+ # The above copyright notice and this permission notice shall be included in all
13
+ # copies or substantial portions of the Software.
14
+ #
15
+ # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16
+ # IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17
+ # FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
18
+ # AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19
+ # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
20
+ # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
21
+ # SOFTWARE.
22
+
23
+ from math import log2
24
+
25
+ import torch
26
+ from einops import rearrange
27
+ from torch import Tensor, einsum, nn
28
+ from torch.amp import autocast
29
+
30
+
31
+ def rotate_half(x: Tensor) -> Tensor:
32
+ x1, x2 = x[..., : x.shape[-1] // 2], x[..., x.shape[-1] // 2:]
33
+ return torch.cat((-x2, x1), dim=-1)
34
+
35
+
36
+ def freqs_to_cos_sin(freqs: Tensor, head_dim: int) -> tuple[Tensor, Tensor]:
37
+ """[..., 2F] frequencies (duplicated halves) -> cos, sin padded with zero angles to [..., head_dim]."""
38
+ freqs = freqs[..., : freqs.shape[-1] // 2]
39
+ pad = torch.zeros((*freqs.shape[:-1], head_dim // 2 - freqs.shape[-1]), device=freqs.device)
40
+ freqs = torch.cat((freqs, pad), dim=-1)
41
+ freqs = torch.cat([freqs, freqs], dim=-1)
42
+ return freqs.cos(), freqs.sin()
43
+
44
+
45
+ @autocast("cuda", enabled=False)
46
+ def apply_rotary_emb(t: Tensor, cos: Tensor, sin: Tensor) -> Tensor:
47
+ """t [B, H, N, head_dim], cos/sin [B, 1, N, head_dim]."""
48
+ return ((t * cos) + (rotate_half(t) * sin)).type(t.dtype)
49
+
50
+
51
+ class SpatialRotaryEmbedding(nn.Module):
52
+ """Rotary embedding of `pos_dim`-D positions: dim // 2 log-spaced frequencies per axis."""
53
+
54
+ def __init__(self, dim: int, pos_dim: int = 3, pos_scale: float = 1.0):
55
+ super().__init__()
56
+ self.pos_dim = pos_dim
57
+ self.pos_scale = pos_scale
58
+ self.register_buffer("freqs", 2 ** torch.linspace(0, log2(dim // 2 - 1), dim // 2))
59
+
60
+ @autocast("cuda", enabled=False)
61
+ def get_spatial_freqs(self, pos: Tensor) -> Tensor:
62
+ """pos [B, N, pos_dim] -> angles [B, 1, N, 2 * pos_dim * dim // 2] (halves duplicated)."""
63
+ freqs = einsum("... i, f -> ... i f", (pos * self.pos_scale).type(self.freqs.dtype), self.freqs)
64
+ freqs = rearrange(freqs, "b n i f -> b 1 n (i f)")
65
+ return torch.cat([freqs, freqs], dim=-1)
gaussianformer/layers/__init__.py ADDED
File without changes
gaussianformer/layers/attention.py ADDED
@@ -0,0 +1,212 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import os
2
+
3
+ import torch
4
+ import torch.nn.functional as F
5
+ import torch.utils.checkpoint
6
+ from einops import rearrange
7
+ from torch import nn
8
+
9
+ from gaussianformer.encodings.rope import SpatialRotaryEmbedding, apply_rotary_emb, freqs_to_cos_sin
10
+ from gaussianformer.layers.window import windowed_attention
11
+
12
+ EPS = 1e-6
13
+
14
+ ATTN = os.environ.get("ATTN_IMPL", "flash_attn")
15
+ assert ATTN in ("flash_attn", "sdpa"), "ATTN_IMPL must be 'flash_attn' or 'sdpa'"
16
+ if ATTN == "flash_attn":
17
+ try:
18
+ from flash_attn import (
19
+ flash_attn_qkvpacked_func,
20
+ flash_attn_varlen_kvpacked_func,
21
+ flash_attn_varlen_qkvpacked_func,
22
+ )
23
+ from flash_attn.bert_padding import pad_input, unpad_input
24
+ except ImportError:
25
+ print("flash_attn is not installed, falling back to PyTorch SDPA.")
26
+ ATTN = "sdpa"
27
+
28
+
29
+ class FeedForwardSwiGLU(nn.Module):
30
+ def __init__(self, dim: int, hidden_dim: int):
31
+ super().__init__()
32
+ self.w1 = nn.Linear(dim, hidden_dim, bias=False)
33
+ self.w2 = nn.Linear(hidden_dim, dim, bias=False)
34
+ self.w3 = nn.Linear(dim, hidden_dim, bias=False)
35
+
36
+ def forward(self, x):
37
+ return self.w2(F.silu(self.w1(x)) * self.w3(x))
38
+
39
+
40
+ class MultiHeadAttention(nn.Module):
41
+ """Self-attention (kv_dim None) or cross-attention with QK-norm and RoPE."""
42
+
43
+ def __init__(self, query_dim: int, num_heads: int, kv_dim: int | None = None):
44
+ super().__init__()
45
+ self.num_heads = num_heads
46
+ self.is_self_attn = kv_dim is None
47
+ if self.is_self_attn:
48
+ self.in_proj = nn.Linear(query_dim, 3 * query_dim, bias=False)
49
+ else:
50
+ self.q_proj = nn.Linear(query_dim, query_dim, bias=False)
51
+ self.k_proj = nn.Linear(kv_dim, query_dim, bias=False)
52
+ self.v_proj = nn.Linear(kv_dim, query_dim, bias=False)
53
+ self.out_proj = nn.Linear(query_dim, query_dim, bias=False)
54
+ self.q_norm = nn.RMSNorm(query_dim, eps=EPS)
55
+ self.k_norm = nn.RMSNorm(query_dim, eps=EPS)
56
+
57
+ def forward(self, q, kv, key_mask=None, rope_q=None, rope_k=None, force_sdpa=False, window=None, k_add=None,
58
+ v_add=None):
59
+ """
60
+ q [B, S, D], kv [B, N, D_kv]; key_mask [B, N] (True = attend); rope_q / rope_k (cos, sin) tables
61
+ [B, 1, S|N, head_dim] (rope_k defaults to rope_q); window from layers.window.build_window;
62
+ k_add / v_add [B, N, D] added to the projected keys / values (cross-attention only).
63
+ """
64
+ bs, src_len, ctx_len = q.shape[0], q.shape[1], kv.shape[1]
65
+ if self.is_self_attn:
66
+ q, k, v = self.in_proj(q).chunk(3, dim=-1)
67
+ else:
68
+ q, k, v = self.q_proj(q), self.k_proj(kv), self.v_proj(kv)
69
+ if k_add is not None:
70
+ k = k + k_add.to(k.dtype)
71
+ if v_add is not None:
72
+ v = v + v_add.to(v.dtype)
73
+ q = self.q_norm(q).type(v.dtype)
74
+ k = self.k_norm(k).type(v.dtype)
75
+
76
+ q = q.view(bs, src_len, self.num_heads, -1).transpose(1, 2) # [B, H, S, d]
77
+ k = k.view(bs, ctx_len, self.num_heads, -1).transpose(1, 2)
78
+ v = v.view(bs, ctx_len, self.num_heads, -1).transpose(1, 2)
79
+ if rope_q is not None:
80
+ rope_k = rope_q if rope_k is None else rope_k
81
+ q = apply_rotary_emb(q, *rope_q)
82
+ k = apply_rotary_emb(k, *rope_k)
83
+ q, k = q.type(v.dtype), k.type(v.dtype)
84
+
85
+ if window is not None:
86
+ out = windowed_attention(q, k, v, window, flash_available=ATTN == "flash_attn")
87
+ elif ATTN == "sdpa" or force_sdpa:
88
+ mask = None
89
+ if key_mask is not None:
90
+ mask = key_mask.view(bs, 1, 1, ctx_len).expand(-1, self.num_heads, -1, -1).reshape(
91
+ bs, self.num_heads, 1, ctx_len)
92
+ out = F.scaled_dot_product_attention(q, k, v, attn_mask=mask)
93
+ out = out.transpose(1, 2).reshape(bs, src_len, -1)
94
+ elif self.is_self_attn and key_mask is None:
95
+ qkv = torch.stack([q.transpose(1, 2), k.transpose(1, 2), v.transpose(1, 2)], dim=2)
96
+ out = flash_attn_qkvpacked_func(qkv).reshape(bs, src_len, -1)
97
+ elif self.is_self_attn:
98
+ q_u, idx, cu, max_len, _ = unpad_input(q.transpose(1, 2), key_mask)
99
+ k_u = unpad_input(k.transpose(1, 2), key_mask)[0]
100
+ v_u = unpad_input(v.transpose(1, 2), key_mask)[0]
101
+ out = flash_attn_varlen_qkvpacked_func(torch.stack([q_u, k_u, v_u], dim=1), cu, max_len)
102
+ out = pad_input(out, idx, bs, src_len).reshape(bs, src_len, -1)
103
+ else:
104
+ q_u = rearrange(q, "b h s d -> (b s) h d")
105
+ cu_q = torch.arange(0, (bs + 1) * src_len, step=src_len, dtype=torch.int32, device=q.device)
106
+ k_u, _, cu_k, max_k, _ = unpad_input(k.transpose(1, 2), key_mask)
107
+ v_u = unpad_input(v.transpose(1, 2), key_mask)[0]
108
+ out = flash_attn_varlen_kvpacked_func(q_u, torch.stack([k_u, v_u], dim=1), cu_q, cu_k, src_len, max_k)
109
+ out = rearrange(out, "(b s) h d -> b s (h d)", b=bs)
110
+ return self.out_proj(out)
111
+
112
+
113
+ class AttentionLayer(nn.Module):
114
+ """Pre-norm block: attention (self, or cross + patch self-attention), then a SwiGLU FFN."""
115
+
116
+ def __init__(self, dim: int, num_heads: int, ffn_hidden_dim: int, kv_dim: int | None = None):
117
+ super().__init__()
118
+ self.query_norm = nn.RMSNorm(dim, eps=EPS)
119
+ self.multihead_attn = MultiHeadAttention(dim, num_heads, kv_dim)
120
+ if kv_dim is not None:
121
+ self.kv_norm = nn.RMSNorm(kv_dim, eps=EPS)
122
+ self.self_attn_norm = nn.RMSNorm(dim, eps=EPS)
123
+ self.self_attn = MultiHeadAttention(dim, num_heads)
124
+ self.ffn_norm = nn.RMSNorm(dim, eps=EPS)
125
+ self.ffn = FeedForwardSwiGLU(dim, ffn_hidden_dim)
126
+
127
+ def forward(self, x, ctx=None, key_mask=None, rope_q=None, rope_k=None, force_sdpa=False, window=None, k_add=None,
128
+ v_add=None):
129
+ q = self.query_norm(x)
130
+ if ctx is None:
131
+ x = x + self.multihead_attn(q, q, key_mask, rope_q, force_sdpa=force_sdpa)
132
+ else:
133
+ x = x + self.multihead_attn(q, self.kv_norm(ctx), key_mask, rope_q, rope_k, force_sdpa, window, k_add,
134
+ v_add)
135
+ q = self.self_attn_norm(x)
136
+ x = x + self.self_attn(q, q, None, rope_q, force_sdpa=force_sdpa)
137
+ return x + self.ffn(self.ffn_norm(x))
138
+
139
+
140
+ class TransformerEncoder(nn.Module):
141
+ """Scene encoder: self-attention over the Gaussian and register tokens with 3-D RoPE."""
142
+
143
+ def __init__(self, num_layers: int, num_heads: int, dim: int, ffn_hidden_dim: int, rope_dim: int):
144
+ super().__init__()
145
+ self.head_dim = dim // num_heads
146
+ self.layers = nn.ModuleList([AttentionLayer(dim, num_heads, ffn_hidden_dim) for _ in range(num_layers)])
147
+ self.rope_emb = SpatialRotaryEmbedding(rope_dim)
148
+
149
+ def forward(self, x, key_mask, pos):
150
+ rope = freqs_to_cos_sin(self.rope_emb.get_spatial_freqs(pos), self.head_dim)
151
+ for layer in self.layers:
152
+ x = layer(x, key_mask=key_mask, rope_q=rope)
153
+ return x
154
+
155
+
156
+ class TransformerDecoder(nn.Module):
157
+ """View decoder: patch tokens cross-attend to the scene tokens, then self-attend.
158
+
159
+ RoPE per head: the pretrained 3-D band (camera origin for patches, Gaussian position for
160
+ Gaussians), then with proj_rope_2d a 2-D band in the channel pairs right after it
161
+ (patch center for patches, projected position for Gaussians).
162
+ """
163
+
164
+ def __init__(self, num_layers: int, num_heads: int, dim: int, ctx_dim: int, ffn_hidden_dim: int,
165
+ rope_dim: int, proj_rope_2d: bool, ray_rope_2d_dim: int, ray_rope_2d_scale: float,
166
+ grad_checkpoint: bool, key_feat_dim: int = 0, value_feat: bool = False):
167
+ super().__init__()
168
+ self.head_dim = dim // num_heads
169
+ self.layers = nn.ModuleList([AttentionLayer(dim, num_heads, ffn_hidden_dim, ctx_dim)
170
+ for _ in range(num_layers)])
171
+ self.rope_emb = SpatialRotaryEmbedding(rope_dim)
172
+ if proj_rope_2d:
173
+ assert 3 * rope_dim // 2 + ray_rope_2d_dim <= self.head_dim // 2, "not enough head channels for 2-D RoPE"
174
+ self.rope_uv = SpatialRotaryEmbedding(ray_rope_2d_dim, pos_dim=2, pos_scale=ray_rope_2d_scale)
175
+ self.grad_checkpoint = grad_checkpoint
176
+ self.key_embed = None # cov_key_embed: per-layer zero-initialized embedding of the footprint on the keys
177
+ if key_feat_dim:
178
+ self.key_embed = nn.ModuleList([nn.Linear(key_feat_dim, dim) for _ in range(num_layers)])
179
+ for lin in self.key_embed:
180
+ nn.init.zeros_(lin.weight)
181
+ nn.init.zeros_(lin.bias)
182
+ self.value_embed = None # cov_key_values: the footprint on the values too, through its own zero-initialized layers
183
+ if key_feat_dim and value_feat:
184
+ self.value_embed = nn.ModuleList([nn.Linear(key_feat_dim, dim) for _ in range(num_layers)])
185
+ for lin in self.value_embed:
186
+ nn.init.zeros_(lin.weight)
187
+ nn.init.zeros_(lin.bias)
188
+
189
+ def _rope(self, pos, uv=None):
190
+ bands = [self.rope_emb.get_spatial_freqs(pos)]
191
+ if uv is not None:
192
+ bands.append(self.rope_uv.get_spatial_freqs(uv))
193
+ f = torch.cat([b[..., : b.shape[-1] // 2] for b in bands], -1)
194
+ return freqs_to_cos_sin(torch.cat([f, f], -1), self.head_dim)
195
+
196
+ def forward(self, x, ctx, key_mask, ctx_pos, ray_pos, out_layers, force_sdpa=False,
197
+ uv_q=None, uv_k=None, window=None, key_feat=None):
198
+ kw = {"key_mask": key_mask, "rope_q": self._rope(ray_pos, uv_q), "rope_k": self._rope(ctx_pos, uv_k),
199
+ "force_sdpa": force_sdpa, "window": window}
200
+ out = []
201
+ for i, layer in enumerate(self.layers):
202
+ kw_i = kw if key_feat is None else dict(kw, k_add=self.key_embed[i](key_feat))
203
+ if key_feat is not None and self.value_embed is not None:
204
+ kw_i = dict(kw_i, v_add=self.value_embed[i](key_feat))
205
+ if self.grad_checkpoint and self.training and torch.is_grad_enabled():
206
+ x = torch.utils.checkpoint.checkpoint(lambda x_, layer_=layer, kw_=kw_i: layer_(x_, ctx, **kw_), x,
207
+ use_reentrant=False)
208
+ else:
209
+ x = layer(x, ctx, **kw_i)
210
+ if i in out_layers:
211
+ out.append(x)
212
+ return out
gaussianformer/layers/dpt.py ADDED
@@ -0,0 +1,244 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # Modified from https://github.com/isl-org/DPT/blob/main/dpt/models.py
2
+
3
+ # Copyright (c) 2021 Intel ISL (Intel Intelligent Systems Lab)
4
+ #
5
+ # Permission is hereby granted, free of charge, to any person obtaining a copy
6
+ # of this software and associated documentation files (the "Software"), to deal
7
+ # in the Software without restriction, including without limitation the rights
8
+ # to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
9
+ # copies of the Software, and to permit persons to whom the Software is
10
+ # furnished to do so, subject to the following conditions:
11
+ #
12
+ # The above copyright notice and this permission notice shall be included in all
13
+ # copies or substantial portions of the Software.
14
+ #
15
+ # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16
+ # IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17
+ # FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
18
+ # AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19
+ # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
20
+ # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
21
+ # SOFTWARE.
22
+
23
+ import torch.nn.functional as F
24
+ from torch import nn
25
+
26
+
27
+ def _make_scratch(in_shape, out_shape, groups=1, expand=False):
28
+ scratch = nn.Module()
29
+
30
+ out_shape1 = out_shape
31
+ out_shape2 = out_shape
32
+ out_shape3 = out_shape
33
+ if len(in_shape) >= 4:
34
+ out_shape4 = out_shape
35
+
36
+ if expand:
37
+ out_shape1 = out_shape
38
+ out_shape2 = out_shape * 2
39
+ out_shape3 = out_shape * 4
40
+ if len(in_shape) >= 4:
41
+ out_shape4 = out_shape * 8
42
+
43
+ scratch.layer1_rn = nn.Conv2d(
44
+ in_shape[0], out_shape1, kernel_size=3, stride=1, padding=1, bias=False, groups=groups)
45
+ scratch.layer2_rn = nn.Conv2d(
46
+ in_shape[1], out_shape2, kernel_size=3, stride=1, padding=1, bias=False, groups=groups)
47
+ scratch.layer3_rn = nn.Conv2d(
48
+ in_shape[2], out_shape3, kernel_size=3, stride=1, padding=1, bias=False, groups=groups)
49
+ if len(in_shape) >= 4:
50
+ scratch.layer4_rn = nn.Conv2d(
51
+ in_shape[3], out_shape4, kernel_size=3, stride=1, padding=1, bias=False, groups=groups)
52
+
53
+ return scratch
54
+
55
+
56
+ class ResidualConvUnit(nn.Module):
57
+
58
+ def __init__(self, features, activation):
59
+ super().__init__()
60
+
61
+ self.conv1 = nn.Conv2d(
62
+ features, features, kernel_size=3, stride=1, padding=1, bias=True)
63
+ self.conv2 = nn.Conv2d(
64
+ features, features, kernel_size=3, stride=1, padding=1, bias=True)
65
+
66
+ self.activation = activation
67
+
68
+ def forward(self, x):
69
+
70
+ out = self.activation(x)
71
+ out = self.conv1(out)
72
+
73
+ out = self.activation(out)
74
+ out = self.conv2(out)
75
+
76
+ return out + x
77
+
78
+
79
+ class FeatureFusionBlock(nn.Module):
80
+
81
+ def __init__(
82
+ self,
83
+ features,
84
+ activation,
85
+ deconv=False,
86
+ expand=False,
87
+ align_corners=True,
88
+ size=None,
89
+ no_resconv1=False,
90
+ ):
91
+ super().__init__()
92
+
93
+ self.deconv = deconv
94
+ self.align_corners = align_corners
95
+
96
+ self.expand = expand
97
+ out_features = features
98
+ if self.expand == True:
99
+ out_features = features // 2
100
+
101
+ self.out_conv = nn.Conv2d(
102
+ features, out_features, kernel_size=1, stride=1, padding=0, bias=True)
103
+
104
+ if not no_resconv1:
105
+ self.resConvUnit1 = ResidualConvUnit(features, activation)
106
+ self.resConvUnit2 = ResidualConvUnit(features, activation)
107
+
108
+ self.size = size
109
+
110
+ def forward(self, *xs, size=None):
111
+ output = xs[0]
112
+
113
+ if len(xs) == 2:
114
+ res = self.resConvUnit1(xs[1])
115
+ output = output + res
116
+
117
+ output = self.resConvUnit2(output)
118
+
119
+ if (size is None) and (self.size is None):
120
+ modifier = {"scale_factor": 2}
121
+ elif size is None:
122
+ modifier = {"size": self.size}
123
+ else:
124
+ modifier = {"size": size}
125
+
126
+ output = nn.functional.interpolate(
127
+ output, **modifier, mode="bilinear", align_corners=self.align_corners)
128
+
129
+ output = self.out_conv(output)
130
+
131
+ return output
132
+
133
+
134
+ def _make_fusion_block(features, size=None, no_resconv1=False):
135
+ return FeatureFusionBlock(
136
+ features,
137
+ nn.SiLU(False),
138
+ deconv=False,
139
+ expand=False,
140
+ align_corners=True,
141
+ size=size,
142
+ no_resconv1=no_resconv1,
143
+ )
144
+
145
+
146
+ class DPTHead(nn.Module):
147
+ def __init__(
148
+ self,
149
+ in_channels,
150
+ features=256,
151
+ out_channels=(256, 512, 1024, 1024),
152
+ out_dim=3,
153
+ ):
154
+ super().__init__()
155
+
156
+ self.projects = nn.ModuleList([
157
+ nn.Conv2d(
158
+ in_channels=in_channels,
159
+ out_channels=out_channel,
160
+ kernel_size=1,
161
+ stride=1,
162
+ padding=0,
163
+ ) for out_channel in out_channels
164
+ ])
165
+
166
+ self.resize_layers = nn.ModuleList([
167
+ nn.ConvTranspose2d(
168
+ in_channels=out_channels[0],
169
+ out_channels=out_channels[0],
170
+ kernel_size=4,
171
+ stride=4,
172
+ padding=0),
173
+ nn.ConvTranspose2d(
174
+ in_channels=out_channels[1],
175
+ out_channels=out_channels[1],
176
+ kernel_size=2,
177
+ stride=2,
178
+ padding=0),
179
+ nn.Identity(),
180
+ nn.Conv2d(
181
+ in_channels=out_channels[3],
182
+ out_channels=out_channels[3],
183
+ kernel_size=3,
184
+ stride=2,
185
+ padding=1)
186
+ ])
187
+
188
+ self.scratch = _make_scratch(
189
+ out_channels,
190
+ features,
191
+ groups=1,
192
+ expand=False,
193
+ )
194
+
195
+ self.scratch.refinenet1 = _make_fusion_block(features)
196
+ self.scratch.refinenet2 = _make_fusion_block(features)
197
+ self.scratch.refinenet3 = _make_fusion_block(features)
198
+ self.scratch.refinenet4 = _make_fusion_block(
199
+ features, no_resconv1=True)
200
+
201
+ head_features_1 = features
202
+ head_features_2 = 32
203
+
204
+ self.scratch.output_conv1 = nn.Conv2d(
205
+ head_features_1, head_features_1 // 2, kernel_size=3, stride=1, padding=1)
206
+ self.scratch.output_conv2 = nn.Sequential(
207
+ nn.Conv2d(head_features_1 // 2, head_features_2,
208
+ kernel_size=3, stride=1, padding=1),
209
+ nn.SiLU(True),
210
+ nn.Conv2d(head_features_2, out_dim,
211
+ kernel_size=1, stride=1, padding=0),
212
+ )
213
+
214
+ def forward(self, out_features, patch_h, patch_w, patch_size=16):
215
+ out = []
216
+ for i, x in enumerate(out_features):
217
+ x = x.permute(0, 2, 1).reshape(
218
+ (x.shape[0], x.shape[-1], patch_h, patch_w))
219
+
220
+ x = self.projects[i](x)
221
+ x = self.resize_layers[i](x)
222
+
223
+ out.append(x)
224
+
225
+ layer_1, layer_2, layer_3, layer_4 = out
226
+
227
+ layer_1_rn = self.scratch.layer1_rn(layer_1)
228
+ layer_2_rn = self.scratch.layer2_rn(layer_2)
229
+ layer_3_rn = self.scratch.layer3_rn(layer_3)
230
+ layer_4_rn = self.scratch.layer4_rn(layer_4)
231
+
232
+ path_4 = self.scratch.refinenet4(layer_4_rn, size=layer_3_rn.shape[2:])
233
+ path_3 = self.scratch.refinenet3(
234
+ path_4, layer_3_rn, size=layer_2_rn.shape[2:])
235
+ path_2 = self.scratch.refinenet2(
236
+ path_3, layer_2_rn, size=layer_1_rn.shape[2:])
237
+ path_1 = self.scratch.refinenet1(path_2, layer_1_rn)
238
+
239
+ out = self.scratch.output_conv1(path_1)
240
+ out = F.interpolate(out, (int(patch_h * patch_size),
241
+ int(patch_w * patch_size)), mode="bilinear", align_corners=True)
242
+ out = self.scratch.output_conv2(out)
243
+
244
+ return out
gaussianformer/layers/window.py ADDED
@@ -0,0 +1,100 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """Projection-windowed cross-attention: each ray token attends only to the Gaussians whose
2
+ projected footprint reaches its image tile (plus the register tokens), the way a rasterizer
3
+ bins splats by tile. Cost goes from queries x all Gaussians to queries x local Gaussians.
4
+
5
+ `build_window` turns the per-view projection into a flat (tile -> key indices) structure that
6
+ `windowed_attention` consumes with flash-attn varlen (one "sequence" per tile) or, on CPU /
7
+ SDPA, with a padded gather. Built once per forward and shared by every decoder layer.
8
+ """
9
+
10
+ import torch
11
+ import torch.nn.functional as F
12
+ from torch import Tensor
13
+
14
+
15
+ def build_window(uv: Tensor, radius: Tensor, key_ok: Tensor, n_reg: int, patch_h: int, patch_w: int,
16
+ tile: int, margin: float) -> dict:
17
+ """
18
+ uv [B, N, 2] projected key centers in patch units (registers included, ignored)
19
+ radius [B, N] projected footprint radius in patch units
20
+ key_ok [B, N] valid Gaussian keys (False for padding, behind-camera, registers)
21
+ n_reg register tokens (keys 0..n_reg-1), attended by every tile
22
+ Returns a dict: q_perm/q_inv [S] query permutation to tile-major order, kv_index [total_k]
23
+ flat (b*N + n) key indices in tile order, cu_q/cu_k [B*T+1] int32 segment offsets,
24
+ max_k, plus tiles/tile_q for reshaping.
25
+ """
26
+ B, N = uv.shape[:2]
27
+ assert patch_h % tile == 0 and patch_w % tile == 0, "the ray grid must be a whole number of tiles"
28
+ th, tw = patch_h // tile, patch_w // tile
29
+ T = th * tw
30
+ dev = uv.device
31
+ # queries (raster order h1 w1) -> tile-major (tile, y, x)
32
+ ii, jj = torch.meshgrid(torch.arange(patch_h, device=dev), torch.arange(patch_w, device=dev), indexing="ij")
33
+ tile_of_q = (ii // tile) * tw + (jj // tile)
34
+ q_perm = torch.argsort(tile_of_q.reshape(-1), stable=True)
35
+ q_inv = torch.empty_like(q_perm)
36
+ q_inv[q_perm] = torch.arange(q_perm.numel(), device=dev)
37
+ # per-Gaussian tile range from the footprint bbox (+ margin), clipped to the grid
38
+ lo = (uv - (radius + margin)[..., None]).div(tile).floor().clamp_min(0).long()
39
+ hi = (uv + (radius + margin)[..., None]).div(tile).floor().long()
40
+ hi = torch.minimum(hi, torch.tensor([tw - 1, th - 1], device=dev))
41
+ ok = key_ok & (hi >= lo).all(-1)
42
+ lo = lo.masked_fill(~ok[..., None], 0)
43
+ hi = hi.masked_fill(~ok[..., None], -1)
44
+ w = (hi[..., 0] - lo[..., 0] + 1).clamp_min(0)
45
+ h = (hi[..., 1] - lo[..., 1] + 1).clamp_min(0)
46
+ cnt = (w * h).reshape(-1) # [B*N] tiles touched per key
47
+ key = torch.arange(B * N, device=dev).repeat_interleave(cnt)
48
+ local = torch.arange(key.numel(), device=dev) - torch.repeat_interleave(cnt.cumsum(0) - cnt, cnt)
49
+ wk = w.reshape(-1)[key]
50
+ dx, dy = local % wk, local // wk
51
+ tx = lo[..., 0].reshape(-1)[key] + dx
52
+ ty = lo[..., 1].reshape(-1)[key] + dy
53
+ seg = (key // N) * T + ty * tw + tx # (b, tile) segment
54
+ # register tokens in every tile
55
+ reg_seg = torch.arange(B * T, device=dev).repeat_interleave(n_reg)
56
+ reg_key = (reg_seg // T) * N + torch.arange(n_reg, device=dev).repeat(B * T)
57
+ seg = torch.cat([reg_seg, seg])
58
+ key = torch.cat([reg_key, key])
59
+ order = torch.argsort(seg, stable=True)
60
+ seg, key = seg[order], key[order]
61
+ counts = torch.bincount(seg, minlength=B * T)
62
+ cu_k = F.pad(counts.cumsum(0), (1, 0)).to(torch.int32)
63
+ cu_q = torch.arange(0, B * T + 1, device=dev, dtype=torch.int32) * (tile * tile)
64
+ return {"q_perm": q_perm, "q_inv": q_inv, "kv_index": key, "cu_q": cu_q, "cu_k": cu_k,
65
+ "max_k": int(counts.max()), "tiles": T, "tile_q": tile * tile, "B": B, "N": N}
66
+
67
+
68
+ def windowed_attention(q: Tensor, k: Tensor, v: Tensor, win: dict, flash_available: bool) -> Tensor:
69
+ """q [B, H, S, d] (raster order), k/v [B, H, N, d] (RoPE already applied) -> [B, S, H*d].
70
+ flash-attn varlen (one sequence per tile, no padding) whenever it is installed and the tensors
71
+ are on the GPU -- in bf16, since flash has no fp32 kernel and the view stage runs under tf32;
72
+ otherwise a padded gather + SDPA (CPU). The padded path costs tiles x max_k, so it is only
73
+ for tests: a single tile can hold most of a small object's Gaussians."""
74
+ B, H, S, d = q.shape
75
+ N, T, tq = win["N"], win["tiles"], win["tile_q"]
76
+ q_t = q[:, :, win["q_perm"]] # tile-major queries
77
+ if flash_available and q.is_cuda:
78
+ from flash_attn import flash_attn_varlen_kvpacked_func
79
+ ht = q.dtype if q.dtype in (torch.float16, torch.bfloat16) else torch.bfloat16
80
+ kv = torch.stack([k, v], 2).permute(0, 3, 2, 1, 4).reshape(B * N, 2, H, d)[win["kv_index"]].to(ht) # [total_k, 2, H, d]
81
+ q_flat = q_t.permute(0, 2, 1, 3).reshape(B * S, H, d).to(ht)
82
+ out = flash_attn_varlen_kvpacked_func(q_flat, kv, win["cu_q"], win["cu_k"], tq, win["max_k"])
83
+ out = out.reshape(B, S, H * d).to(q.dtype)
84
+ else:
85
+ k_flat = k.transpose(1, 2).reshape(B * N, H, d)[win["kv_index"]] # [total_k, H, d]
86
+ v_flat = v.transpose(1, 2).reshape(B * N, H, d)[win["kv_index"]]
87
+ # padded gather: one row per (b, tile), keys padded to max_k and masked
88
+ Kmax = win["max_k"]
89
+ counts = (win["cu_k"][1:] - win["cu_k"][:-1]).long()
90
+ pos = torch.arange(win["kv_index"].numel(), device=q.device) - torch.repeat_interleave(win["cu_k"][:-1].long(), counts)
91
+ seg = torch.repeat_interleave(torch.arange(B * T, device=q.device), counts)
92
+ kp = k_flat.new_zeros(B * T, Kmax, H, d)
93
+ vp = v_flat.new_zeros(B * T, Kmax, H, d)
94
+ mask = torch.zeros(B * T, Kmax, dtype=torch.bool, device=q.device)
95
+ kp[seg, pos], vp[seg, pos], mask[seg, pos] = k_flat, v_flat, True
96
+ qt = q_t.reshape(B, H, T, tq, d).permute(0, 2, 1, 3, 4).reshape(B * T, H, tq, d)
97
+ out = F.scaled_dot_product_attention(qt, kp.transpose(1, 2), vp.transpose(1, 2),
98
+ attn_mask=mask[:, None, None, :]) # [B*T, H, tq, d]
99
+ out = out.reshape(B, T, H, tq, d).permute(0, 1, 3, 2, 4).reshape(B, S, H * d)
100
+ return out[:, win["q_inv"]]
gaussianformer/models/__init__.py ADDED
File without changes
gaussianformer/models/config.py ADDED
@@ -0,0 +1,84 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import types
2
+ import typing
3
+ from dataclasses import dataclass, field, replace
4
+
5
+
6
+ @dataclass(frozen=True)
7
+ class GaussianFormerConfig:
8
+ # Scene encoder: one token per Gaussian, 3-D RoPE on its position.
9
+ latent_dim: int = 768
10
+ num_layers: int = 12
11
+ num_heads: int = 6
12
+ dim_feedforward: int = 768 * 4
13
+ num_register_tokens: int = 16
14
+ pos_pe_num_freqs: int = 12
15
+ """Rotary dimension of the 3-D position RoPE (shared with the view decoder)."""
16
+
17
+ # View decoder: one token per image patch, cross-attention to the scene tokens, DPT head.
18
+ view_transformer_latent_dim: int = 768
19
+ view_transformer_ffn_hidden_dim: int = 768 * 4
20
+ view_transformer_n_heads: int = 6
21
+ view_transformer_n_layers: int = 6
22
+ patch_size: int = 8
23
+ ray_embed_patch: int = 0
24
+ """If larger than patch_size, each patch's ray directions are upsampled to this size before the
25
+ ray embedding, so a finer patch grid reuses an embedding pretrained at this patch size."""
26
+ dpt_features: int = 128
27
+ dpt_out_channels: list[int] = field(default_factory=lambda: [96, 192, 384, 768])
28
+
29
+ proj_rope_2d: bool = False
30
+ """2-D RoPE in the cross-attention: keys carry each Gaussian's projected image position, queries
31
+ their patch center; patch self-attention uses the same 2-D band."""
32
+ ray_rope_2d_dim: int = 16
33
+ ray_rope_2d_scale: float = 0.25
34
+ """Patch coordinates are multiplied by this before the 2-D RoPE."""
35
+
36
+ xattn_window: int = 0
37
+ """Tile size in patches for windowed cross-attention: each tile attends only to the Gaussians
38
+ whose projected footprint reaches it, plus the register tokens. 0 = dense."""
39
+ xattn_margin: float = 1.0
40
+ """Tile expansion in patches on top of the footprint."""
41
+ xattn_sigmas: float = 3.0
42
+ """Footprint radius in units of the Gaussian's largest scale."""
43
+
44
+ cov_key_embed: bool = False
45
+ """Projected footprint on the cross-attention keys: each Gaussian's covariance projected into the view (Jacobian
46
+ of the pinhole projection at its center, in patch units, plus gsplat's 0.3 px^2 low-pass term), described by
47
+ (log sigma_1, log sigma_2, cos 2t, sin 2t) and added to its key through a zero-initialized Linear per decoder
48
+ layer."""
49
+ cov_key_eps2d: float = 0.3
50
+ """Isotropic term added to the projected covariance, in square pixels (gsplat's eps2d low-pass filter); 0 removes
51
+ it (ablation)."""
52
+ cov_key_values: bool = False
53
+ """Also add the footprint to the cross-attention values, through its own zero-initialized Linear per decoder
54
+ layer (ablation; the released models use the keys only)."""
55
+
56
+ view_bf16: bool = False
57
+ """Run the view decoder in bf16 (Flash Attention for the patch self-attention)."""
58
+ view_grad_checkpoint: bool = False
59
+ """Recompute view-decoder layers in the backward pass to save activation memory."""
60
+
61
+ def with_overrides(self, pairs: list[str] | None) -> "GaussianFormerConfig":
62
+ """Apply `key=value` overrides (the `--model_cfg` option)."""
63
+ if not pairs:
64
+ return self
65
+ hints = typing.get_type_hints(type(self))
66
+ out = {}
67
+ for p in pairs:
68
+ key, _, raw = p.partition("=")
69
+ if key not in hints:
70
+ raise KeyError(f"unknown GaussianFormerConfig field {key!r}")
71
+ out[key] = _coerce(hints[key], raw)
72
+ return replace(self, **out)
73
+
74
+
75
+ def _coerce(tp, raw: str):
76
+ if isinstance(tp, types.UnionType):
77
+ inner = [a for a in typing.get_args(tp) if a is not type(None)]
78
+ return None if raw.lower() in ("none", "null") else _coerce(inner[0], raw)
79
+ if typing.get_origin(tp) is list:
80
+ (item,) = typing.get_args(tp)
81
+ return [_coerce(item, x) for x in raw.split(",") if x]
82
+ if tp is bool:
83
+ return raw.lower() in ("1", "true", "yes")
84
+ return tp(raw)
gaussianformer/models/gaussianformer.py ADDED
@@ -0,0 +1,65 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import torch
2
+ from huggingface_hub import PyTorchModelHubMixin
3
+ from torch import nn
4
+
5
+ from gaussianformer.layers.attention import TransformerEncoder
6
+ from gaussianformer.models.config import GaussianFormerConfig
7
+ from gaussianformer.models.view_transformer import ViewTransformer
8
+
9
+ VIEW_CHUNK = 7 # views decoded per pass; the DPT upsampling of more 512 px views exceeds int32 indexing
10
+
11
+
12
+ class GaussianFormer(nn.Module, PyTorchModelHubMixin):
13
+ """Scene encoder over one token per Gaussian [pos(3), scale(3), quat wxyz(4), rgb(3), opacity(1)],
14
+ followed by the view decoder. Output: log10(HDR + 1) images."""
15
+
16
+ def __init__(self, config: GaussianFormerConfig | dict):
17
+ super().__init__()
18
+ if isinstance(config, dict):
19
+ config = GaussianFormerConfig(**config)
20
+ self.config = config
21
+ self.gaussian_encoder = nn.Linear(14, config.latent_dim)
22
+ self.gaussian_encoder_norm = nn.RMSNorm(config.latent_dim)
23
+ self.gaussian_token = nn.Parameter(torch.randn(1, 1, config.latent_dim))
24
+ self.reg_tokens = nn.Parameter(torch.randn(1, config.num_register_tokens, config.latent_dim))
25
+ self.transformer = TransformerEncoder(config.num_layers, config.num_heads, config.latent_dim,
26
+ config.dim_feedforward, config.pos_pe_num_freqs)
27
+ self.view_transformer = ViewTransformer(config)
28
+
29
+ @property
30
+ def device(self):
31
+ return next(self.parameters()).device
32
+
33
+ def _with_registers(self, pos, mask):
34
+ """Prepend register tokens positioned at the center of the valid Gaussians."""
35
+ n = self.config.num_register_tokens
36
+ w = (mask.float() / (mask.sum(dim=1, keepdim=True) + 1e-5))[..., None]
37
+ center = (w * pos).sum(dim=1, keepdim=True)
38
+ pos = torch.cat([center.repeat(1, n, 1), pos], dim=1)
39
+ mask = torch.cat([torch.ones((mask.size(0), n), dtype=torch.bool, device=mask.device), mask], dim=1)
40
+ return pos, mask
41
+
42
+ def forward(self, gaussians, valid_mask, rays_o, rays_d, gaussians_cam, fov, tf32_view=False):
43
+ """
44
+ gaussians [B, N, 14] padded, valid_mask [B, N]; rays_o [B, V, 3], rays_d [B, V, H, W, 3];
45
+ gaussians_cam [B, V, N, 14] the Gaussians in each view's camera frame; fov [B, V] in radians.
46
+ Returns [B, V, 3, H, W].
47
+ """
48
+ B, V = rays_o.shape[:2]
49
+ tokens = self.gaussian_token + self.gaussian_encoder_norm(self.gaussian_encoder(gaussians))
50
+ seq = torch.cat([self.reg_tokens.expand(B, -1, -1), tokens], dim=1)
51
+ pos, mask = self._with_registers(gaussians[..., :3], valid_mask)
52
+ seq = self.transformer(seq, mask, pos)
53
+
54
+ seq = seq.repeat_interleave(V, dim=0)
55
+ gaussians_cam = gaussians_cam.reshape(B * V, *gaussians_cam.shape[2:])
56
+ pos_cam, mask_cam = self._with_registers(gaussians_cam[..., :3], valid_mask.repeat_interleave(V, dim=0))
57
+ shape = gaussians_cam[..., 3:10] # scale, rotation; registers get zeros (no footprint)
58
+ shape_cam = torch.cat([shape.new_zeros(B * V, self.config.num_register_tokens, 7), shape], dim=1)
59
+ rays_o, rays_d, fov = rays_o.reshape(B * V, 3), rays_d.reshape(B * V, *rays_d.shape[2:]), fov.reshape(-1)
60
+ out = torch.cat([
61
+ self.view_transformer(rays_o[i:i + VIEW_CHUNK], rays_d[i:i + VIEW_CHUNK], seq[i:i + VIEW_CHUNK],
62
+ pos_cam[i:i + VIEW_CHUNK], shape_cam[i:i + VIEW_CHUNK], mask_cam[i:i + VIEW_CHUNK],
63
+ fov[i:i + VIEW_CHUNK], tf32_mode=tf32_view)
64
+ for i in range(0, B * V, VIEW_CHUNK)])
65
+ return out.view(B, V, *out.shape[1:])
gaussianformer/models/view_transformer.py ADDED
@@ -0,0 +1,117 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import roma
2
+ import torch
3
+ from einops import rearrange
4
+ from torch import nn
5
+
6
+ from gaussianformer.layers.attention import TransformerDecoder
7
+ from gaussianformer.layers.dpt import DPTHead
8
+ from gaussianformer.layers.window import build_window
9
+ from gaussianformer.models.config import GaussianFormerConfig
10
+
11
+
12
+ class ViewTransformer(nn.Module):
13
+ """View decoder: ray-direction patch tokens attend to the scene tokens; a DPT head decodes pixels."""
14
+
15
+ def __init__(self, config: GaussianFormerConfig):
16
+ super().__init__()
17
+ self.config = config
18
+ assert not config.ray_embed_patch or config.ray_embed_patch > config.patch_size, \
19
+ "ray_embed_patch must exceed patch_size"
20
+ dim = config.view_transformer_latent_dim
21
+ embed_patch = config.ray_embed_patch or config.patch_size
22
+ self.ray_map_patch_token = nn.Parameter(torch.randn(1, 1, dim))
23
+ self.ray_map_encoder = nn.Linear(3 * embed_patch ** 2, dim)
24
+ self.ray_map_encoder_norm = nn.RMSNorm(dim)
25
+ self.transformer = TransformerDecoder(
26
+ num_layers=config.view_transformer_n_layers,
27
+ num_heads=config.view_transformer_n_heads,
28
+ dim=dim,
29
+ ctx_dim=config.latent_dim,
30
+ ffn_hidden_dim=config.view_transformer_ffn_hidden_dim,
31
+ rope_dim=config.pos_pe_num_freqs,
32
+ proj_rope_2d=config.proj_rope_2d,
33
+ ray_rope_2d_dim=config.ray_rope_2d_dim,
34
+ ray_rope_2d_scale=config.ray_rope_2d_scale,
35
+ grad_checkpoint=config.view_grad_checkpoint,
36
+ key_feat_dim=4 if config.cov_key_embed else 0,
37
+ value_feat=config.cov_key_values,
38
+ )
39
+ self.out_dpt = DPTHead(dim, features=config.dpt_features, out_channels=config.dpt_out_channels)
40
+ self.out_layers = list(range(config.view_transformer_n_layers - 4, config.view_transformer_n_layers))
41
+ self.out_proj_act = nn.ELU(alpha=1e-3)
42
+
43
+ def project(self, pos, fov, H, W):
44
+ """Camera-frame positions -> image position in patch units, depth, behind-camera flag, focal length.
45
+ Camera at the origin looking down -Z (as in RayGenerator)."""
46
+ focal = 0.5 * W / torch.tan(0.5 * fov.to(pos.dtype)).view(-1, 1)
47
+ X, Y, Z = pos.unbind(-1)
48
+ depth = (-Z).clamp_min(1e-3)
49
+ u = (W / 2 + focal * X / depth) / self.config.patch_size
50
+ v = (H / 2 - focal * Y / depth) / self.config.patch_size
51
+ return torch.stack([u, v], -1), depth, Z > -1e-3, focal
52
+
53
+ def footprint(self, pos, shape, fov, H, W):
54
+ """Projected covariance of each Gaussian in patch units -> [B, N, 4] = (log sigma_1, log sigma_2, cos 2t, sin 2t)
55
+ of its image-space axes (sigma_1 >= sigma_2). shape = camera-frame [scale(3), quat wxyz(4)]. Zero for register
56
+ tokens and Gaussians behind the camera. fp32 with autocast off: in bf16 / fp16 the minor axis cancels away."""
57
+ ps = self.config.patch_size
58
+ with torch.autocast(pos.device.type, enabled=False):
59
+ pos = pos.float()
60
+ _, depth, behind, focal = self.project(pos, fov.float(), H, W)
61
+ f = focal / ps
62
+ scale, quat = shape[..., :3].float(), shape[..., 3:7].float()
63
+ R = roma.unitquat_to_rotmat(quat[..., [1, 2, 3, 0]])
64
+ cov = (R * scale[..., None, :] ** 2) @ R.transpose(-1, -2)
65
+ X, Y = pos[..., 0], pos[..., 1]
66
+ zero = torch.zeros_like(depth)
67
+ J = torch.stack([torch.stack([f / depth, zero, f * X / depth ** 2], -1),
68
+ torch.stack([zero, -f / depth, -f * Y / depth ** 2], -1)], -2)
69
+ c2 = J @ cov @ J.transpose(-1, -2) + self.config.cov_key_eps2d / ps ** 2 * torch.eye(2, device=cov.device)
70
+ a, b, c = c2[..., 0, 0], c2[..., 0, 1], c2[..., 1, 1]
71
+ half = torch.hypot((a - c) / 2, b)
72
+ l1, l2 = (a + c) / 2 + half, ((a + c) / 2 - half).clamp_min(1e-8)
73
+ feat = torch.stack([0.5 * l1.log(), 0.5 * l2.log(), (a - c) / (2 * half + 1e-8), 2 * b / (2 * half + 1e-8)], -1)
74
+ ok = (scale.amax(-1) > 0) & ~behind & torch.isfinite(feat).all(-1)
75
+ return torch.where(ok[..., None], feat.clamp(-6, 6), 0.0)
76
+
77
+ def forward(self, camera_o, ray_map, ctx_tokens, ctx_pos, ctx_shape, key_mask, fov, tf32_mode=False):
78
+ """
79
+ camera_o [B, 3]; ray_map [B, H, W, 3] ray directions; ctx_tokens [B, N, D] scene tokens;
80
+ ctx_pos [B, N, 3] camera-frame positions and ctx_shape [B, N, 7] camera-frame scales and rotations
81
+ (registers first, with zeros);
82
+ key_mask [B, N]; fov [B] in radians. Returns log-HDR images [B, 3, H, W].
83
+ """
84
+ cfg = self.config
85
+ B, H, W, _ = ray_map.shape
86
+ patch_h, patch_w = H // cfg.patch_size, W // cfg.patch_size
87
+ embed_patch = cfg.ray_embed_patch or cfg.patch_size
88
+ rays = ray_map
89
+ if embed_patch != cfg.patch_size: # nearest-upsample each patch to the embedding's patch size
90
+ r = embed_patch // cfg.patch_size
91
+ rays = rays.permute(0, 3, 1, 2).repeat_interleave(r, 2).repeat_interleave(r, 3).permute(0, 2, 3, 1)
92
+ rays = rearrange(rays, "b (h1 p1) (w1 p2) c -> b (h1 w1) (c p1 p2)", p1=embed_patch, p2=embed_patch)
93
+ ray_tokens = self.ray_map_patch_token + self.ray_map_encoder_norm(self.ray_map_encoder(rays))
94
+ ray_pos = camera_o[:, None].repeat(1, ray_tokens.size(1), 1)
95
+
96
+ uv_q = uv_k = window = None
97
+ if cfg.proj_rope_2d or cfg.xattn_window:
98
+ uv_k, depth, behind, focal = self.project(ctx_pos, fov, H, W)
99
+ ii, jj = torch.meshgrid(torch.arange(patch_h, device=ray_map.device),
100
+ torch.arange(patch_w, device=ray_map.device), indexing="ij")
101
+ uv_q = (torch.stack([jj, ii], -1).reshape(1, -1, 2).float() + 0.5).expand(B, -1, -1)
102
+ if cfg.xattn_window:
103
+ radius = cfg.xattn_sigmas * ctx_shape[..., :3].amax(-1) * focal / depth / cfg.patch_size
104
+ key_ok = key_mask & ~behind
105
+ key_ok[:, :cfg.num_register_tokens] = False
106
+ window = build_window(uv_k, radius, key_ok, cfg.num_register_tokens, patch_h, patch_w,
107
+ cfg.xattn_window, cfg.xattn_margin)
108
+ if not cfg.proj_rope_2d:
109
+ uv_q = uv_k = None
110
+ key_feat = self.footprint(ctx_pos, ctx_shape, fov, H, W) if cfg.cov_key_embed else None
111
+
112
+ fp32 = tf32_mode and not cfg.view_bf16
113
+ with torch.autocast(device_type="cuda", dtype=torch.float32 if fp32 else torch.bfloat16):
114
+ features = self.transformer(ray_tokens, ctx_tokens, key_mask, ctx_pos, ray_pos, self.out_layers,
115
+ force_sdpa=fp32, uv_q=uv_q, uv_k=uv_k, window=window,
116
+ key_feat=key_feat)
117
+ return self.out_proj_act(self.out_dpt(features, patch_h, patch_w, patch_size=cfg.patch_size))
gaussianformer/pipelines/__init__.py ADDED
File without changes
gaussianformer/pipelines/rendering_pipeline.py ADDED
@@ -0,0 +1,59 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import torch
2
+
3
+ from gaussianformer.models.gaussianformer import GaussianFormer
4
+ from gaussianformer.utils.ray_generator import RayGenerator
5
+ from gaussianformer.utils.transform import transform_gaussians_to_cam_coord
6
+
7
+
8
+ def model_forward(model: GaussianFormer, gaussians, mask, c2w, fov, resolution: int, tf32_view: bool = False):
9
+ """Render log-HDR images [B, V, H, W, 3]; gaussians [B, N, 14], mask [B, N], c2w [B, V, 4, 4],
10
+ fov [B, V] in degrees. The decoder works in each camera's frame, so its rays start at the origin."""
11
+ B, V = c2w.shape[:2]
12
+ with torch.no_grad():
13
+ gaussians_cam = transform_gaussians_to_cam_coord(
14
+ c2w.reshape(-1, 4, 4), gaussians.repeat_interleave(V, dim=0)).reshape(B, V, *gaussians.shape[1:])
15
+ eye = torch.eye(4, device=c2w.device, dtype=c2w.dtype).expand(B, V, 4, 4)
16
+ fov = fov / 180.0 * torch.pi
17
+ rays_o, rays_d = RayGenerator()(eye, fov[..., None], resolution)
18
+ out = model(gaussians, mask, rays_o, rays_d, gaussians_cam, fov, tf32_view=tf32_view)
19
+ return out.permute(0, 1, 3, 4, 2)
20
+
21
+
22
+ class GaussianFormerRenderingPipeline:
23
+ def __init__(self, model: GaussianFormer):
24
+ self.model = model
25
+
26
+ @classmethod
27
+ def from_pretrained(cls, model_id: str):
28
+ return cls(GaussianFormer.from_pretrained(model_id).eval())
29
+
30
+ @property
31
+ def device(self):
32
+ return self.model.device
33
+
34
+ def to(self, device: str | torch.device):
35
+ self.model.to(device)
36
+ return self
37
+
38
+ @torch.no_grad()
39
+ def __call__(self, gaussians, c2w, fov=45.0, mask=None, resolution: int = 512,
40
+ torch_dtype: torch.dtype = torch.bfloat16):
41
+ """
42
+ One object: gaussians [N, 14] = pos(3) | scale(3) | quat wxyz(4) | rgb(3) | opacity(1), c2w [V, 4, 4]
43
+ camera-to-world (Blender convention) -> images [V, H, W, 3]. A padded batch: gaussians [B, N, 14],
44
+ c2w [B, V, 4, 4], mask [B, N] (True = real Gaussian; default all) -> images [B, V, H, W, 3].
45
+ fov in degrees: a number or a tensor broadcastable to [V] / [B, V]. Inputs are moved to the model's device.
46
+ Images are display-referred (clip to [0, 1] for 8-bit output).
47
+ """
48
+ single = gaussians.dim() == 2
49
+ if single:
50
+ gaussians, c2w = gaussians[None], c2w[None]
51
+ gaussians, c2w = gaussians.to(self.device), c2w.to(self.device)
52
+ fov = torch.as_tensor(fov, dtype=torch.float32, device=self.device).expand(c2w.shape[:2])
53
+ mask = (torch.ones(gaussians.shape[:2], dtype=torch.bool, device=self.device) if mask is None
54
+ else mask.to(self.device))
55
+ with torch.autocast(device_type=self.device.type, dtype=torch_dtype):
56
+ log_img = model_forward(self.model, gaussians, mask, c2w, fov, resolution,
57
+ tf32_view=torch_dtype != torch.float32)
58
+ images = torch.pow(10.0, log_img) - 1.0
59
+ return images[0] if single else images
gaussianformer/splat.py ADDED
@@ -0,0 +1,153 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """3D Gaussian Splatting PLY files -> model inputs.
2
+
3
+ A 3DGS PLY stores the optimizer's raw parameters; GaussianFormer takes activated values in a normalized object
4
+ frame, [N, 14] = pos(3) | scale(3) | quat wxyz(4) | rgb(3) | opacity(1):
5
+
6
+ scale exp(scale_0..2)
7
+ quat rot_0..3 (w, x, y, z), normalized
8
+ rgb clip(0.5 + SH_C0 * f_dc_0..2, 0, 1) (higher spherical-harmonic orders are dropped)
9
+ opacity sigmoid(opacity)
10
+
11
+ The object is centered on its median position, turned Y-up and scaled so its largest coordinate is 0.45, and
12
+ pruned to about 20k Gaussians (the size the model is trained on) by an importance score adapted from LightGaussian,
13
+ followed by a short fine-tune of the kept Gaussians against the full splat.
14
+
15
+ from gaussianformer.splat import load_ply
16
+ gaussians = load_ply("object.ply", up="z") # [N, 14] tensor on the GPU
17
+ """
18
+ import io
19
+ from pathlib import Path
20
+
21
+ import numpy as np
22
+ import roma
23
+ import torch
24
+ import torch.nn.functional as F
25
+ from plyfile import PlyData
26
+
27
+ from gaussianformer.utils.cameras import RESOLUTION, orbit, to_gsplat
28
+
29
+ SH_C0 = 0.28209479177387814
30
+ TO_Y_UP = {
31
+ "y": np.eye(3, dtype=np.float32),
32
+ "z": np.array([[1, 0, 0], [0, 0, 1], [0, -1, 0]], dtype=np.float32), # Z-up (e.g. Objaverse_Splats)
33
+ "-y": np.diag([1.0, -1.0, -1.0]).astype(np.float32), # Y-down (e.g. COLMAP captures)
34
+ }
35
+ KEEP = 20_000
36
+ RECOVERY_VIEWS = 64
37
+ RECOVERY_ITERS = 1500
38
+ FIELDS = ("means", "scales", "rotations", "colors", "opacities")
39
+
40
+
41
+ def read_ply(src: str | Path | bytes) -> dict[str, np.ndarray]:
42
+ """3DGS PLY (path or bytes) -> activated Gaussians: means, scales, rotations (wxyz), colors, opacities."""
43
+ v = PlyData.read(io.BytesIO(src) if isinstance(src, bytes) else str(src))["vertex"]
44
+ quats = np.stack([v[f"rot_{i}"] for i in range(4)], -1).astype(np.float32)
45
+ return {
46
+ "means": np.stack([v["x"], v["y"], v["z"]], -1).astype(np.float32),
47
+ "scales": np.exp(np.stack([v[f"scale_{i}"] for i in range(3)], -1)).astype(np.float32),
48
+ "rotations": quats / (np.linalg.norm(quats, axis=-1, keepdims=True) + 1e-9),
49
+ "colors": np.clip(0.5 + SH_C0 * np.stack([v[f"f_dc_{i}"] for i in range(3)], -1), 0, 1).astype(np.float32),
50
+ "opacities": (1.0 / (1.0 + np.exp(-np.asarray(v["opacity"], np.float32)))).astype(np.float32),
51
+ }
52
+
53
+
54
+ def normalize(g: dict[str, np.ndarray], up: str = "y") -> dict[str, np.ndarray] | None:
55
+ """Center on the median position, rotate `up` to +Y and scale into [-0.45, 0.45]^3; None if degenerate."""
56
+ R = TO_Y_UP[up]
57
+ means = ((g["means"] - np.median(g["means"], axis=0)) @ R.T).astype(np.float32)
58
+ rot = roma.unitquat_to_rotmat(torch.from_numpy(g["rotations"])[:, [1, 2, 3, 0]])
59
+ rotations = roma.rotmat_to_unitquat(torch.from_numpy(R) @ rot)[:, [3, 0, 1, 2]].numpy()
60
+ extent = float(np.abs(means).max())
61
+ if extent < 1e-4:
62
+ return None
63
+ s = 0.45 / extent
64
+ return {"means": means * s, "scales": g["scales"] * s, "rotations": rotations, "colors": g["colors"],
65
+ "opacities": g["opacities"]}
66
+
67
+
68
+ def rasterize(g: dict[str, torch.Tensor], viewmats: torch.Tensor, Ks: torch.Tensor):
69
+ """gsplat render of Gaussians `g` (activated values, opacities [N]) -> images [V, H, W, 3], render info."""
70
+ import gsplat
71
+
72
+ return gsplat.rasterization(
73
+ means=g["means"], quats=g["rotations"] / g["rotations"].norm(dim=-1, keepdim=True), scales=g["scales"],
74
+ opacities=g["opacities"], colors=g["colors"], viewmats=viewmats, Ks=Ks, width=RESOLUTION,
75
+ height=RESOLUTION, sh_degree=None, eps2d=0.3, render_mode="RGB", near_plane=0.01, packed=True)[::2]
76
+
77
+
78
+ def importance_topk(g: dict[str, torch.Tensor], viewmats, Ks, keep: int) -> torch.Tensor:
79
+ """Indices of the `keep` Gaussians with the highest importance, a score adapted from LightGaussian: opacity x
80
+ projected area summed over the views, times the largest scale^0.1."""
81
+ score = torch.zeros(len(g["means"]), device=g["means"].device, dtype=torch.float64)
82
+ for v in range(len(viewmats)):
83
+ with torch.no_grad():
84
+ _, info = rasterize(g, viewmats[v:v + 1], Ks[v:v + 1])
85
+ area = info["radii"][:, 0].double() * info["radii"][:, 1].double()
86
+ score.scatter_add_(0, info["gaussian_ids"], info["opacities"].double() * area)
87
+ score = score * g["scales"].max(1).values.double() ** 0.1
88
+ return torch.topk(score, min(keep, len(score))).indices
89
+
90
+
91
+ def ssim(x: torch.Tensor, y: torch.Tensor) -> torch.Tensor:
92
+ g = torch.exp(-(torch.arange(11, dtype=torch.float32, device=x.device) - 5) ** 2 / (2 * 1.5 ** 2))
93
+ g = g / g.sum()
94
+ w = (g[:, None] * g[None, :]).expand(3, 1, 11, 11).contiguous()
95
+ conv = lambda t: F.conv2d(t, w, padding=5, groups=3)
96
+ mx, my = conv(x), conv(y)
97
+ sx, sy, sxy = conv(x * x) - mx * mx, conv(y * y) - my * my, conv(x * y) - mx * my
98
+ c1, c2 = 0.01 ** 2, 0.03 ** 2
99
+ return (((2 * mx * my + c1) * (2 * sxy + c2)) / ((mx * mx + my * my + c1) * (sx + sy + c2))).mean()
100
+
101
+
102
+ def recover(g: dict[str, torch.Tensor], target: torch.Tensor, viewmats, Ks) -> dict[str, torch.Tensor]:
103
+ """Fine-tune pruned Gaussians (L1 + SSIM, no densification) against the full splat's renders `target`."""
104
+ p = {"means": g["means"], "log_scales": g["scales"].clamp_min(1e-8).log(), "rotations": g["rotations"],
105
+ "opacity_logits": torch.logit(g["opacities"].clamp(1e-4, 1 - 1e-4)),
106
+ "color_logits": torch.logit(g["colors"].clamp(1e-4, 1 - 1e-4))}
107
+ p = {k: torch.nn.Parameter(v.clone()) for k, v in p.items()}
108
+ lrs = {"means": 1.6e-4, "log_scales": 5e-3, "rotations": 1e-3, "opacity_logits": 5e-2, "color_logits": 2.5e-3}
109
+ opt = torch.optim.Adam([{"params": [p[k]], "lr": lr} for k, lr in lrs.items()])
110
+ sched = torch.optim.lr_scheduler.ExponentialLR(opt, gamma=0.95 ** (1 / 400))
111
+ current = lambda: {"means": p["means"], "scales": p["log_scales"].exp(), "rotations": p["rotations"],
112
+ "opacities": p["opacity_logits"].sigmoid(), "colors": p["color_logits"].sigmoid()}
113
+ for _ in range(RECOVERY_ITERS):
114
+ v = torch.randint(0, len(viewmats), (4,), device=target.device)
115
+ img = rasterize(current(), viewmats[v], Ks[v])[0].clamp(0, 1)
116
+ loss = 0.8 * (img - target[v]).abs().mean() + 0.2 * (1 - ssim(img.permute(0, 3, 1, 2), target[v].permute(0, 3, 1, 2)))
117
+ loss.backward()
118
+ opt.step()
119
+ opt.zero_grad()
120
+ sched.step()
121
+ with torch.no_grad():
122
+ out = current()
123
+ out["rotations"] = out["rotations"] / out["rotations"].norm(dim=-1, keepdim=True)
124
+ return {k: v.detach() for k, v in out.items()}
125
+
126
+
127
+ def prune(full: dict[str, torch.Tensor], keep: int = KEEP, recovery: bool = True) -> dict[str, torch.Tensor]:
128
+ """Keep the `keep` most important Gaussians (scored from a 64-view orbit), then optionally fine-tune them."""
129
+ views = tuple(torch.from_numpy(x).to(full["means"].device) for x in to_gsplat(orbit(RECOVERY_VIEWS, 1.7)))
130
+ with torch.no_grad():
131
+ target = torch.cat([rasterize(full, views[0][i:i + 1], views[1][i:i + 1])[0].clamp(0, 1)
132
+ for i in range(RECOVERY_VIEWS)]) if recovery else None
133
+ kept = {k: v[importance_topk(full, *views, keep)] for k, v in full.items()}
134
+ return recover(kept, target, *views) if recovery else kept
135
+
136
+
137
+ def to_tensor(g: dict[str, torch.Tensor | np.ndarray]) -> torch.Tensor:
138
+ """Gaussians as a dict -> the model's [N, 14] layout."""
139
+ parts = [torch.as_tensor(g[k]).float().reshape(len(g["means"]), -1) for k in FIELDS]
140
+ return torch.cat(parts, dim=-1)
141
+
142
+
143
+ def load_ply(path: str | Path, up: str = "y", keep: int = KEEP, recovery: bool = True,
144
+ device: str | torch.device = "cuda") -> torch.Tensor:
145
+ """3DGS PLY -> model input [N, 14] on `device`: activated, normalized and pruned to at most `keep` Gaussians.
146
+ `up` is the file's up axis ('y', 'z' or '-y'). `recovery` fine-tunes the kept Gaussians against the full
147
+ splat (about a minute on a GPU); without it a large splat pruned to 20k loses coverage and turns dark."""
148
+ g = normalize(read_ply(path), up)
149
+ assert g is not None, f"{path}: all Gaussians at one point"
150
+ g = {k: torch.from_numpy(np.ascontiguousarray(v)).to(device) for k, v in g.items()}
151
+ if len(g["means"]) > keep:
152
+ g = prune(g, keep, recovery)
153
+ return to_tensor(g)
gaussianformer/utils/__init__.py ADDED
File without changes
gaussianformer/utils/cameras.py ADDED
@@ -0,0 +1,44 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """Orbit cameras around the origin in the Blender convention (-Z forward, +Y up, +X right).
2
+
3
+ Views alternate between a high (+0.4 r) and a low (-0.1 r) elevation to cover the top and the bottom of the object.
4
+ """
5
+ import numpy as np
6
+
7
+ FOV = 45.0
8
+ RESOLUTION = 512
9
+ DISTANCES = ((1.7, 14), (1.15, 7), (2.45, 7)) # (orbit radius, views): 28 views per object
10
+
11
+
12
+ def look_at(eye: np.ndarray, target: np.ndarray, up: np.ndarray) -> np.ndarray:
13
+ forward = target - eye
14
+ forward /= np.linalg.norm(forward) + 1e-9
15
+ right = np.cross(forward, up)
16
+ right /= np.linalg.norm(right) + 1e-9
17
+ c2w = np.eye(4, dtype=np.float32)
18
+ c2w[:3, 0], c2w[:3, 1], c2w[:3, 2], c2w[:3, 3] = right, np.cross(right, forward), -forward, eye
19
+ return c2w
20
+
21
+
22
+ def orbit(n_views: int, radius: float) -> np.ndarray:
23
+ """Camera-to-world matrices [n_views, 4, 4]."""
24
+ c2w = []
25
+ for i in range(n_views):
26
+ theta = 2 * np.pi * i / n_views
27
+ elevation = 0.4 * radius if i % 2 == 0 else -0.1 * radius
28
+ eye = np.array([radius * np.cos(theta), elevation, radius * np.sin(theta)], np.float32)
29
+ c2w.append(look_at(eye, np.zeros(3, np.float32), np.array([0, 1, 0], np.float32)))
30
+ return np.stack(c2w)
31
+
32
+
33
+ def all_views() -> np.ndarray:
34
+ """The 28 training cameras: 14 at distance 1.7, then 7 each at 1.15 and 2.45."""
35
+ return np.concatenate([orbit(n, r) for r, n in DISTANCES])
36
+
37
+
38
+ def to_gsplat(c2w: np.ndarray, fov: float = FOV, resolution: int = RESOLUTION) -> tuple[np.ndarray, np.ndarray]:
39
+ """Camera-to-world matrices [V, 4, 4] -> gsplat view matrices (OpenCV convention) and intrinsics [V, 3, 3]."""
40
+ flip = np.diag([1.0, -1.0, -1.0, 1.0]).astype(np.float32)
41
+ viewmats = (flip @ np.linalg.inv(c2w)).astype(np.float32)
42
+ f = 0.5 * resolution / np.tan(0.5 * np.radians(fov))
43
+ K = np.array([[f, 0, resolution / 2], [0, f, resolution / 2], [0, 0, 1]], np.float32)
44
+ return viewmats, np.tile(K, (len(c2w), 1, 1))
gaussianformer/utils/checkpoint.py ADDED
@@ -0,0 +1,34 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ from pathlib import Path
2
+
3
+ import torch
4
+ from torch import nn
5
+
6
+ from gaussianformer.models.config import GaussianFormerConfig
7
+ from gaussianformer.models.gaussianformer import GaussianFormer
8
+
9
+
10
+ def load_seed(module: nn.Module, state_dict: dict, allow_new: bool = True) -> list[str]:
11
+ """Warm-start `module` from `state_dict`; returns the keys the state dict lacked.
12
+
13
+ New tensors (`allow_new`) must be zero-initialized so the model computes the same function as the
14
+ seed at load; RoPE frequency tables are derived from the config and may be new or change shape.
15
+ """
16
+ own = module.state_dict()
17
+ sd = {k: v for k, v in state_dict.items() if not (k.endswith(".freqs") and k in own and own[k].shape != v.shape)}
18
+ missing, unexpected = module.load_state_dict(sd, strict=False)
19
+ assert not unexpected, f"state dict has keys the model lacks: {list(unexpected)[:5]}"
20
+ new = [k for k in missing if not k.endswith(".freqs")]
21
+ assert allow_new or not new, f"state dict lacks tensors: {new[:5]}"
22
+ nonzero = [k for k in new if own[k].abs().sum() > 0]
23
+ assert not nonzero, f"new tensors must be zero-initialized: {nonzero[:5]}"
24
+ return list(missing)
25
+
26
+
27
+ def load_checkpoint(path: Path, overrides: list[str] | None = None,
28
+ allow_new: bool = False) -> tuple[GaussianFormer, dict]:
29
+ """Model built from the checkpoint's stored config (plus `key=value` overrides), weights loaded."""
30
+ ckpt = torch.load(path, map_location="cpu", weights_only=True)
31
+ config = GaussianFormerConfig(**ckpt["config"]).with_overrides(overrides)
32
+ model = GaussianFormer(config)
33
+ load_seed(model, ckpt["model_state_dict"], allow_new=allow_new)
34
+ return model, ckpt
gaussianformer/utils/ray_generator.py ADDED
@@ -0,0 +1,22 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import torch
2
+ import torch.nn.functional as F
3
+ from torch import nn
4
+
5
+
6
+ class RayGenerator(nn.Module):
7
+ """Pinhole camera rays through pixel centers (Blender convention: -Z forward, +Y up)."""
8
+
9
+ def forward(self, c2w: torch.Tensor, fov: torch.Tensor, img_res: int = 256):
10
+ """c2w [..., 4, 4], fov [..., 1] in radians -> origins [..., 3], unit directions [..., H, W, 3]."""
11
+ batch_shape = c2w.shape[:-2]
12
+ x, y = torch.meshgrid(
13
+ torch.linspace(0.5, img_res - 0.5, img_res, device=c2w.device, dtype=c2w.dtype),
14
+ torch.linspace(0.5, img_res - 0.5, img_res, device=c2w.device, dtype=c2w.dtype),
15
+ indexing="xy")
16
+ c = img_res / 2
17
+ f = img_res / 2 / torch.tan(0.5 * fov[..., 0, None, None])
18
+ x = x[None].repeat(*batch_shape, 1, 1)
19
+ y = y[None].repeat(*batch_shape, 1, 1)
20
+ dirs = torch.stack([(x - c) / f, -(y - c) / f, -torch.ones_like(x)], dim=-1)
21
+ rays_d = torch.sum(dirs[..., None, :] * c2w[..., None, None, :3, :3], dim=-1)
22
+ return c2w[..., :3, 3], F.normalize(rays_d, dim=-1, p=2)
gaussianformer/utils/transform.py ADDED
@@ -0,0 +1,30 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import roma
2
+ import torch
3
+ import torch.nn.functional as F
4
+ from torch.amp import autocast
5
+
6
+
7
+ def quaternion_multiply(a: torch.Tensor, b: torch.Tensor) -> torch.Tensor:
8
+ """Composition of rotations given as wxyz quaternions, returned with a non-negative real part."""
9
+ aw, ax, ay, az = torch.unbind(a, -1)
10
+ bw, bx, by, bz = torch.unbind(b, -1)
11
+ ab = torch.stack((
12
+ aw * bw - ax * bx - ay * by - az * bz,
13
+ aw * bx + ax * bw + ay * bz - az * by,
14
+ aw * by - ax * bz + ay * bw + az * bx,
15
+ aw * bz + ax * by - ay * bx + az * bw,
16
+ ), -1)
17
+ return torch.where(ab[..., 0:1] < 0, -ab, ab)
18
+
19
+
20
+ @torch.no_grad()
21
+ @autocast("cuda", enabled=False)
22
+ def transform_gaussians_to_cam_coord(c2w: torch.Tensor, gaussians: torch.Tensor) -> torch.Tensor:
23
+ """Gaussians [B, N, 14] (pos, scale, quat wxyz, ...) -> the same Gaussians in the camera frame of
24
+ the rigid camera-to-world matrices c2w [B, 4, 4]."""
25
+ w2c = roma.Rigid.from_homogeneous(c2w).inverse()
26
+ pos = w2c[:, None].apply(gaussians[..., :3])
27
+ w2c_quat = roma.rotmat_to_unitquat(w2c.linear)[..., [3, 0, 1, 2]] # roma is xyzw
28
+ rot = F.normalize(gaussians[..., 6:10], dim=-1) # padded all-zero rows stay zero instead of NaN
29
+ rot = quaternion_multiply(w2c_quat[:, None], rot)
30
+ return torch.cat([pos, gaussians[..., 3:6], rot, gaussians[..., 10:]], dim=-1)