Spaces:
Sleeping
Sleeping
Download model_code/selected_reference.py from oussamab2n/GazeLIFT: direct link, hf CLI and curl.
- Browser
- Download file 2.87 kB
-
https://huggingface.co/spaces/oussamab2n/GazeLIFT/resolve/main/model_code/selected_reference.py
- Command line
-
hf download hf://spaces/oussamab2n/GazeLIFT/model_code/selected_reference.py
-
curl -L -o selected_reference.py https://huggingface.co/spaces/oussamab2n/GazeLIFT/resolve/main/model_code/selected_reference.py
2.87 kB
| """DINO and gaze widths32/64/128/256; all 3x3 decoder outputs32.""" | |
| import torch | |
| from torch import nn | |
| import torch.nn.functional as F | |
| from conv3_reference import PyramidConv3RGBWaveletDecoder | |
| from base_model import prototype_contrast,segmentation_loss | |
| class GazePyramidConv3RGBWaveletDecoder(PyramidConv3RGBWaveletDecoder): | |
| def __init__(self): | |
| super().__init__() | |
| for i,width in enumerate([64,128,256],1): | |
| self.gaze_conditioning[i]=nn.Sequential(nn.Conv2d(width+2,width,1,bias=False),nn.GroupNorm(8,width),nn.GELU()) | |
| nn.init.kaiming_normal_(self.gaze_conditioning[i][0].weight,mode='fan_out',nonlinearity='relu') | |
| for i,cin in enumerate([268,172,108,76]): | |
| self.decoder[i]=nn.Sequential(nn.Conv2d(cin,32,3,padding=1,bias=False),nn.GroupNorm(8,32),nn.GELU()) | |
| nn.init.kaiming_normal_(self.decoder[i][0].weight,mode='fan_out',nonlinearity='relu') | |
| def forward(self,image,features,heatmap,return_intermediates=False): | |
| if image.ndim!=4 or image.shape[1]!=3 or any(s%16 for s in image.shape[-2:]):raise ValueError('Expected RGB, dimensions divisible by16') | |
| if heatmap.shape!=(image.shape[0],1,*image.shape[-2:]):raise ValueError('Heatmap must match input size') | |
| if len(features)!=4 or any(f.ndim!=4 or f.shape[:2]!=(image.shape[0],1024) for f in features):raise ValueError('Expected four frozen1024-channel DINO maps') | |
| sizes=[(image.shape[-2]//s,image.shape[-1]//s) for s in [2,4,8,16]] | |
| skips=[];trace=[] | |
| for f,size,project,condition in zip(features,sizes,self.dino_projection,self.gaze_conditioning): | |
| p=F.interpolate(project(f.detach().float()),size=size,mode='bilinear',align_corners=False) | |
| h=F.adaptive_avg_pool2d(heatmap.float(),size);c=prototype_contrast(p,h) | |
| s=condition(torch.cat([p,h,c],1));skips.append(s) | |
| if return_intermediates:trace.append(dict(dino=p,gaze=h,contrast=c,conditioned=s)) | |
| waves=self.wavelet(image);decoder_trace=[] | |
| z=torch.cat([skips[3],waves[3]],1);x=self.decoder[0](z) | |
| if return_intermediates:decoder_trace.append(dict(level=3,input=z,output=x)) | |
| for decoder,level in zip(self.decoder[1:4],[2,1,0]): | |
| merged=torch.cat([F.interpolate(x,size=sizes[level],mode='bilinear',align_corners=False),skips[level]],1) | |
| z=torch.cat([merged,waves[level]],1);x=decoder(z) | |
| if return_intermediates:decoder_trace.append(dict(level=level,input=z,output=x,merged=merged)) | |
| up=F.interpolate(x,size=image.shape[-2:],mode='bilinear',align_corners=False) | |
| z=torch.cat([up,image.float()],1);x=self.decoder[4](z) | |
| if return_intermediates:decoder_trace.append(dict(level='full',input=z,output=x)) | |
| logits=self.mask_head(x) | |
| return (logits,trace,waves,decoder_trace) if return_intermediates else logits | |