"""Fuse at each source grid, then learned transposed-convolution upsampling.""" import torch from torch import nn import torch.nn.functional as F from selected_reference import GazePyramidConv3RGBWaveletDecoder from base_model import prototype_contrast,segmentation_loss class TransposeGazePyramidDecoder(GazePyramidConv3RGBWaveletDecoder): def __init__(self): super().__init__() self.decoder=nn.ModuleList([nn.Sequential(nn.ConvTranspose2d(cin,32,3,stride=stride,padding=1,output_padding=stride-1,bias=False),nn.GroupNorm(8,32),nn.GELU()) for cin,stride in zip([268,172,108,76,35],[2,2,2,2,1])]) for layer in self.decoder.modules(): if isinstance(layer,nn.ConvTranspose2d):nn.init.kaiming_normal_(layer.weight,mode='fan_out',nonlinearity='relu') def forward(self,image,features,heatmap,return_intermediates=False): if image.ndim!=4 or image.shape[1]!=3 or any(s%16 for s in image.shape[-2:]):raise ValueError('Expected RGB, dimensions divisible by16') if heatmap.shape!=(image.shape[0],1,*image.shape[-2:]):raise ValueError('Heatmap must match input size') if len(features)!=4 or any(f.ndim!=4 or f.shape[:2]!=(image.shape[0],1024) for f in features):raise ValueError('Expected four frozen1024-channel DINO maps') sizes=[(image.shape[-2]//s,image.shape[-1]//s) for s in [2,4,8,16]] skips=[];trace=[] for f,size,project,condition in zip(features,sizes,self.dino_projection,self.gaze_conditioning): p=F.interpolate(project(f.detach().float()),size=size,mode='bilinear',align_corners=False) h=F.adaptive_avg_pool2d(heatmap.float(),size);c=prototype_contrast(p,h) s=condition(torch.cat([p,h,c],1));skips.append(s) if return_intermediates:trace.append(dict(dino=p,gaze=h,contrast=c,conditioned=s)) waves=self.wavelet(image);decoder_trace=[] z=torch.cat([skips[3],waves[3]],1);x=self.decoder[0](z) if return_intermediates:decoder_trace.append(dict(level=3,input=z,output=x)) for decoder,level in zip(self.decoder[1:4],[2,1,0]): assert x.shape[-2:]==sizes[level] merged=torch.cat([x,skips[level]],1) z=torch.cat([merged,waves[level]],1);x=decoder(z) if return_intermediates:decoder_trace.append(dict(level=level,input=z,output=x,merged=merged)) assert x.shape[-2:]==image.shape[-2:] up=x z=torch.cat([up,image.float()],1);x=self.decoder[4](z) if return_intermediates:decoder_trace.append(dict(level='full',input=z,output=x)) logits=self.mask_head(x) return (logits,trace,waves,decoder_trace) if return_intermediates else logits