import torch import torch.nn as nn import torch.nn.functional as F from .utils import split_feature, merge_splits def gather_view_tensor(tensor, view_indices): view_indices = view_indices.to(device=tensor.device, dtype=torch.long) batch_indices = torch.arange(tensor.size(0), device=tensor.device) batch_indices = batch_indices.reshape( tensor.size(0), *([1] * (view_indices.dim() - 1)) ) return tensor[batch_indices, view_indices] def non_self_view_indices(num_views, device): view_ids = torch.arange(num_views, device=device) return torch.stack( [torch.cat((view_ids[:i], view_ids[i + 1 :])) for i in range(num_views)], dim=0, ) def target_view_indices(batch_size, num_views, nn_matrix, device): if nn_matrix is not None: return nn_matrix[..., 1:].to(device=device, dtype=torch.long) return non_self_view_indices(num_views, device).unsqueeze(0).expand( batch_size, -1, -1 ) def flatten_batch_view(tensor): # Batch-major flatten: [B, V, ...] -> [B*V, ...]. return tensor.flatten(0, 1) def flatten_view_batch(tensor): # View-major flatten: [B, V, ...] -> [V*B, ...]. return tensor.transpose(0, 1).reshape( tensor.size(0) * tensor.size(1), *tensor.shape[2:] ) def unflatten_view_batch(tensor, batch_size, num_views): # Inverse of flatten_view_batch. return tensor.reshape(num_views, batch_size, *tensor.shape[1:]).transpose(0, 1) def single_head_full_attention(q, k, v): # q, k, v: [B, L, C] assert q.dim() == k.dim() == v.dim() == 3 return F.scaled_dot_product_attention( q.unsqueeze(1), k.unsqueeze(1), v.unsqueeze(1) ).squeeze(1) def generate_shift_window_attn_mask( input_resolution, window_size_h, window_size_w, shift_size_h, shift_size_w, device=torch.device("cuda"), ): # Ref: https://github.com/microsoft/Swin-Transformer/blob/main/models/swin_transformer.py # Build the shifted-window attention mask used by Swin-style local attention. h, w = input_resolution img_mask = torch.zeros((1, h, w, 1)).to(device) # 1 H W 1 h_slices = ( slice(0, -window_size_h), slice(-window_size_h, -shift_size_h), slice(-shift_size_h, None), ) w_slices = ( slice(0, -window_size_w), slice(-window_size_w, -shift_size_w), slice(-shift_size_w, None), ) cnt = 0 for h in h_slices: for w in w_slices: img_mask[:, h, w, :] = cnt cnt += 1 mask_windows = split_feature( img_mask, num_splits=input_resolution[-1] // window_size_w, channel_last=True ) mask_windows = mask_windows.view(-1, window_size_h * window_size_w) attn_mask = mask_windows.unsqueeze(1) - mask_windows.unsqueeze(2) attn_mask = attn_mask.masked_fill(attn_mask != 0, float(-100.0)).masked_fill( attn_mask == 0, float(0.0) ) return attn_mask def single_head_split_window_attention( q, k, v, num_splits=1, with_shift=False, h=None, w=None, attn_mask=None, ): # Ref: https://github.com/microsoft/Swin-Transformer/blob/main/models/swin_transformer.py # q, k, v: [B, L, C] for self-attention or pairwise cross-attention. # For multi-view cross-attention, q: [B, L, C], k/v: [B, M, L, C]. # Multi-view cross-attention, where M is the number of target views per query. if not (q.dim() == k.dim() == v.dim() == 3): assert k.dim() == v.dim() == 4 assert h is not None and w is not None assert q.size(1) == h * w m = k.size(1) b, _, c = q.size() b_new = b * num_splits * num_splits window_size_h = h // num_splits window_size_w = w // num_splits q = q.view(b, h, w, c) # [B, H, W, C] k = k.view(b, m, h, w, c) # [B, M, H, W, C] v = v.view(b, m, h, w, c) if with_shift: assert attn_mask is not None # compute once shift_size_h = window_size_h // 2 shift_size_w = window_size_w // 2 q = torch.roll(q, shifts=(-shift_size_h, -shift_size_w), dims=(1, 2)) k = torch.roll(k, shifts=(-shift_size_h, -shift_size_w), dims=(2, 3)) v = torch.roll(v, shifts=(-shift_size_h, -shift_size_w), dims=(2, 3)) q = split_feature( q, num_splits=num_splits, channel_last=True ) # [B*K*K, H/K, W/K, C] k = split_feature( k.permute(0, 2, 3, 4, 1).reshape(b, h, w, -1), num_splits=num_splits, channel_last=True, ) # [B*K*K, H/K, W/K, C*M] v = split_feature( v.permute(0, 2, 3, 4, 1).reshape(b, h, w, -1), num_splits=num_splits, channel_last=True, ) # [B*K*K, H/K, W/K, C*M] k = ( k.view(b_new, h // num_splits, w // num_splits, c, m) .permute(0, 1, 2, 4, 3) .reshape(b_new, -1, c) ) # [B*K*K, H/K*W/K*M, C] v = ( v.view(b_new, h // num_splits, w // num_splits, c, m) .permute(0, 1, 2, 4, 3) .reshape(b_new, -1, c) ) # [B*K*K, H/K*W/K*M, C] attn_bias = None if with_shift: attn_bias = attn_mask.repeat(b, 1, m).unsqueeze(1).to(q.dtype) out = F.scaled_dot_product_attention( q.view(b_new, -1, c).unsqueeze(1), k.unsqueeze(1), v.unsqueeze(1), attn_mask=attn_bias, ).squeeze(1) out = merge_splits( out.view(b_new, h // num_splits, w // num_splits, c), num_splits=num_splits, channel_last=True, ) # [B, H, W, C] # shift back if with_shift: out = torch.roll(out, shifts=(shift_size_h, shift_size_w), dims=(1, 2)) out = out.view(b, -1, c) else: # Self-attention or pairwise cross-attention. assert q.dim() == k.dim() == v.dim() == 3 assert h is not None and w is not None assert q.size(1) == h * w b, _, c = q.size() b_new = b * num_splits * num_splits window_size_h = h // num_splits window_size_w = w // num_splits q = q.view(b, h, w, c) # [B, H, W, C] k = k.view(b, h, w, c) v = v.view(b, h, w, c) if with_shift: assert attn_mask is not None # compute once shift_size_h = window_size_h // 2 shift_size_w = window_size_w // 2 q = torch.roll(q, shifts=(-shift_size_h, -shift_size_w), dims=(1, 2)) k = torch.roll(k, shifts=(-shift_size_h, -shift_size_w), dims=(1, 2)) v = torch.roll(v, shifts=(-shift_size_h, -shift_size_w), dims=(1, 2)) q = split_feature( q, num_splits=num_splits, channel_last=True ) # [B*K*K, H/K, W/K, C] k = split_feature(k, num_splits=num_splits, channel_last=True) v = split_feature(v, num_splits=num_splits, channel_last=True) attn_bias = None if with_shift: attn_bias = attn_mask.repeat(b, 1, 1).unsqueeze(1).to(q.dtype) out = F.scaled_dot_product_attention( q.view(b_new, -1, c).unsqueeze(1), k.view(b_new, -1, c).unsqueeze(1), v.view(b_new, -1, c).unsqueeze(1), attn_mask=attn_bias, ).squeeze(1) out = merge_splits( out.view(b_new, h // num_splits, w // num_splits, c), num_splits=num_splits, channel_last=True, ) # [B, H, W, C] # shift back if with_shift: out = torch.roll(out, shifts=(shift_size_h, shift_size_w), dims=(1, 2)) out = out.view(b, -1, c) return out def multi_head_split_window_attention( q, k, v, num_splits=1, with_shift=False, h=None, w=None, attn_mask=None, num_head=1, ): """Windowed multi-head scaled dot-product attention. Args: q: [B, L, C] k: [B, S, C] v: [B, S, C] Returns: out: [B, L, C] """ assert h is not None and w is not None assert q.size(1) == h * w b, _, c = q.size() b_new = b * num_splits * num_splits window_size_h = h // num_splits window_size_w = w // num_splits q = q.view(b, h, w, c) # [B, H, W, C] k = k.view(b, h, w, c) v = v.view(b, h, w, c) assert c % num_head == 0 if with_shift: assert attn_mask is not None # compute once shift_size_h = window_size_h // 2 shift_size_w = window_size_w // 2 q = torch.roll(q, shifts=(-shift_size_h, -shift_size_w), dims=(1, 2)) k = torch.roll(k, shifts=(-shift_size_h, -shift_size_w), dims=(1, 2)) v = torch.roll(v, shifts=(-shift_size_h, -shift_size_w), dims=(1, 2)) q = split_feature(q, num_splits=num_splits) # [B*K*K, H/K, W/K, C] k = split_feature(k, num_splits=num_splits) v = split_feature(v, num_splits=num_splits) # Split channels into attention heads for PyTorch SDPA. head_dim = c // num_head q = q.view(b_new, -1, num_head, head_dim).transpose(1, 2) k = k.view(b_new, -1, num_head, head_dim).transpose(1, 2) v = v.view(b_new, -1, num_head, head_dim).transpose(1, 2) attn_bias = None if with_shift: attn_bias = attn_mask.unsqueeze(1).repeat(b, num_head, 1, 1).to(q.dtype) out = F.scaled_dot_product_attention(q, k, v, attn_mask=attn_bias) out = merge_splits( out.transpose(1, 2).reshape(b_new, h // num_splits, w // num_splits, c), num_splits=num_splits, ) # [B, H, W, C] # shift back if with_shift: out = torch.roll(out, shifts=(shift_size_h, shift_size_w), dims=(1, 2)) out = out.view(b, -1, c) return out class TransformerLayer(nn.Module): def __init__( self, d_model=256, nhead=1, attention_type="swin", no_ffn=False, ffn_dim_expansion=4, with_shift=False, add_per_view_attn=False, **kwargs, ): super(TransformerLayer, self).__init__() self.dim = d_model self.nhead = nhead self.attention_type = attention_type self.no_ffn = no_ffn self.add_per_view_attn = add_per_view_attn self.with_shift = with_shift # Linear projections used by both single-head and multi-head attention. self.q_proj = nn.Linear(d_model, d_model, bias=False) self.k_proj = nn.Linear(d_model, d_model, bias=False) self.v_proj = nn.Linear(d_model, d_model, bias=False) self.merge = nn.Linear(d_model, d_model, bias=False) self.norm1 = nn.LayerNorm(d_model) # no ffn after self-attn, with ffn after cross-attn if not self.no_ffn: in_channels = d_model * 2 self.mlp = nn.Sequential( nn.Linear(in_channels, in_channels * ffn_dim_expansion, bias=False), nn.GELU(), nn.Linear(in_channels * ffn_dim_expansion, d_model, bias=False), ) self.norm2 = nn.LayerNorm(d_model) def forward( self, source, target, height=None, width=None, shifted_window_attn_mask=None, attn_num_splits=None, **kwargs, ): if "attn_type" in kwargs: attn_type = kwargs["attn_type"] else: attn_type = self.attention_type # source: [B, L, C] # target: [B, L, C] for pairwise attention or [B, M, L, C] for multi-view. query, key, value = source, target, target # Project tokens before dispatching to local/global attention. query = self.q_proj(query) # [B, L, C] key = self.k_proj(key) # [B, L, C] or [B, M, L, C] value = self.v_proj(value) # [B, L, C] or [B, M, L, C] if attn_type == "swin" and attn_num_splits > 1: if self.nhead > 1: message = multi_head_split_window_attention( query, key, value, num_splits=attn_num_splits, with_shift=self.with_shift, h=height, w=width, attn_mask=shifted_window_attn_mask, num_head=self.nhead, ) else: if self.add_per_view_attn: assert query.dim() == 3 and key.dim() == 4 and value.dim() == 4 b, l, c = query.size() query = query.unsqueeze(1).expand( -1, key.size(1), -1, -1 ).reshape(-1, l, c) # [B*M, L, C] key = key.reshape(-1, l, c) value = value.reshape(-1, l, c) message = single_head_split_window_attention( query, key, value, num_splits=attn_num_splits, with_shift=self.with_shift, h=height, w=width, attn_mask=shifted_window_attn_mask, ) # Sum per-target-view messages back to [B, L, C]. message = message.view(b, -1, l, c).sum(1) else: message = single_head_split_window_attention( query, key, value, num_splits=attn_num_splits, with_shift=self.with_shift, h=height, w=width, attn_mask=shifted_window_attn_mask, ) else: message = single_head_full_attention(query, key, value) # [B, L, C] message = self.merge(message) # [B, L, C] message = self.norm1(message) if not self.no_ffn: message = self.mlp(torch.cat([source, message], dim=-1)) message = self.norm2(message) return source + message class TransformerBlock(nn.Module): """self attention + cross attention + FFN""" def __init__( self, d_model=256, nhead=1, attention_type="swin", ffn_dim_expansion=4, with_shift=False, add_per_view_attn=False, no_cross_attn=False, **kwargs, ): super(TransformerBlock, self).__init__() self.no_cross_attn = no_cross_attn if no_cross_attn: self.self_attn = TransformerLayer( d_model=d_model, nhead=nhead, attention_type=attention_type, ffn_dim_expansion=ffn_dim_expansion, with_shift=with_shift, add_per_view_attn=add_per_view_attn, ) else: self.self_attn = TransformerLayer( d_model=d_model, nhead=nhead, attention_type=attention_type, no_ffn=True, ffn_dim_expansion=ffn_dim_expansion, with_shift=with_shift, ) self.cross_attn_ffn = TransformerLayer( d_model=d_model, nhead=nhead, attention_type=attention_type, ffn_dim_expansion=ffn_dim_expansion, with_shift=with_shift, add_per_view_attn=add_per_view_attn, ) def forward( self, source, target, height=None, width=None, shifted_window_attn_mask=None, attn_num_splits=None, **kwargs, ): # source, target: [B, L, C] # self attention source = self.self_attn( source, source, height=height, width=width, shifted_window_attn_mask=shifted_window_attn_mask, attn_num_splits=attn_num_splits, **kwargs, ) if self.no_cross_attn: return source # cross attention and ffn source = self.cross_attn_ffn( source, target, height=height, width=width, shifted_window_attn_mask=shifted_window_attn_mask, attn_num_splits=attn_num_splits, **kwargs, ) return source def batch_features(features, nn_matrix=None): # Construct transformer inputs from tensor views. # features: [B, V, C, H, W] or [B, V, H*W, C]. # Returns view-major query/target tensors: [V*B, ...] and [V*B, M, ...]. assert features.dim() in (4, 5) batch_size, num_views = features.shape[:2] indices = target_view_indices( batch_size, num_views, nn_matrix, features.device ) q = flatten_view_batch(features) kv = gather_view_tensor(features, indices) kv = flatten_view_batch(kv) return q, kv class MultiViewFeatureTransformer(nn.Module): def __init__( self, num_layers=6, d_model=128, nhead=1, attention_type="swin", ffn_dim_expansion=4, add_per_view_attn=False, no_cross_attn=False, **kwargs, ): super(MultiViewFeatureTransformer, self).__init__() self.attention_type = attention_type self.d_model = d_model self.nhead = nhead self.layers = nn.ModuleList( [ TransformerBlock( d_model=d_model, nhead=nhead, attention_type=attention_type, ffn_dim_expansion=ffn_dim_expansion, with_shift=( True if attention_type == "swin" and i % 2 == 1 else False ), add_per_view_attn=add_per_view_attn, no_cross_attn=no_cross_attn, ) for i in range(num_layers) ] ) for p in self.parameters(): if p.dim() > 1: nn.init.xavier_uniform_(p) # zero init layers beyond 6 if num_layers > 6: for i in range(6, num_layers): self.layers[i].self_attn.norm1.weight.data.zero_() self.layers[i].self_attn.norm1.bias.data.zero_() self.layers[i].cross_attn_ffn.norm2.weight.data.zero_() self.layers[i].cross_attn_ffn.norm2.bias.data.zero_() def forward( self, multi_view_features, attn_num_splits=None, **kwargs, ): nn_matrix = kwargs.pop("nn_matrix", None) # multi_view_features: [B, V, C, H, W]. assert multi_view_features.dim() == 5 b, num_views, c, h, w = multi_view_features.shape assert self.d_model == c if self.attention_type == "swin" and attn_num_splits > 1: # global and refine use different number of splits window_size_h = h // attn_num_splits window_size_w = w // attn_num_splits # compute attn mask once shifted_window_attn_mask = generate_shift_window_attn_mask( input_resolution=(h, w), window_size_h=window_size_h, window_size_w=window_size_w, shift_size_h=window_size_h // 2, shift_size_w=window_size_w // 2, device=multi_view_features.device, ) # [K*K, H/K*W/K, H/K*W/K] else: shifted_window_attn_mask = None # View-major query/target tensors: [V*B, C, H, W], [V*B, M, C, H, W]. concat0, concat1 = batch_features(multi_view_features, nn_matrix=nn_matrix) concat0 = concat0.reshape(num_views * b, c, -1).permute( 0, 2, 1 ) # [V*B, H*W, C] c1_v = num_views - 1 if nn_matrix is None else nn_matrix.shape[-1] - 1 concat1 = concat1.reshape(num_views * b, c1_v, c, -1).permute( 0, 1, 3, 2 ) # [V*B, M, H*W, C] for i, layer in enumerate(self.layers): concat0 = layer( concat0, concat1, height=h, width=w, shifted_window_attn_mask=shifted_window_attn_mask, attn_num_splits=attn_num_splits, ) if i < len(self.layers) - 1: # Rebuild target-view groups after each layer. features = unflatten_view_batch(concat0, b, num_views) concat0, concat1 = batch_features(features, nn_matrix=nn_matrix) features = unflatten_view_batch(concat0, b, num_views) features = features.reshape(b, num_views, h, w, c) features = features.permute(0, 1, 4, 2, 3).contiguous() return features def batch_features_camera_parameters( features, intrinsics, extrinsics, nn_matrix=None, ): # Construct batch-major inputs for plane-sweep warping. # features: [B, V, C, H, W]. # intrinsics/extrinsics: [B, V, 3, 3] and [B, V, 4, 4]. # Returns [B*V, ...] query tensors and [B*V, M, ...] target tensors. assert features.dim() == 5 assert intrinsics.shape[-2:] == (3, 3) assert extrinsics.shape[-2:] == (4, 4) batch_size, num_views, c, h, w = features.shape indices = target_view_indices( batch_size, num_views, nn_matrix, features.device ) num_selected_views = indices.size(-1) q = features q_intrinsics = intrinsics q_extrinsics = extrinsics kv = gather_view_tensor(features, indices) kv_intrinsics = gather_view_tensor(intrinsics, indices) kv_extrinsics = gather_view_tensor(extrinsics, indices) q = flatten_batch_view(q) # [BV, C, H, W] q_intrinsics = flatten_batch_view(q_intrinsics) # [BV, 3, 3] q_extrinsics = flatten_batch_view(q_extrinsics) # [BV, 4, 4] kv = kv.reshape(-1, num_selected_views, c, h, w) # [BV, V-1, C, H, W] kv_intrinsics = kv_intrinsics.reshape(-1, num_selected_views, 3, 3) kv_extrinsics = kv_extrinsics.reshape(-1, num_selected_views, 4, 4) return q, q_intrinsics, q_extrinsics, kv, kv_intrinsics, kv_extrinsics