Download models/mask_decoder.py from Leo-Lyu/MCP-MedSAM: direct link, hf CLI and curl.
- Browser
- Download file 17.6 kB
-
https://huggingface.co/Leo-Lyu/MCP-MedSAM/resolve/refs%2Fpr%2F2/models/mask_decoder.py
- Command line
-
hf download hf://Leo-Lyu/MCP-MedSAM@refs/pr/2/models/mask_decoder.py
-
curl -L -o mask_decoder.py https://huggingface.co/Leo-Lyu/MCP-MedSAM/resolve/refs%2Fpr%2F2/models/mask_decoder.py
17.6 kB
| # -*- coding: utf-8 -*- | |
| # Copyright (c) Meta Platforms, Inc. and affiliates. | |
| # All rights reserved. | |
| # This source code is licensed under the license found in the | |
| # LICENSE file in the root directory of this source tree. | |
| import torch | |
| from torch import nn | |
| from torch.nn import functional as F | |
| from typing import List, Tuple, Type | |
| from .common import LayerNorm2d | |
| from .transformer import TwoWayTransformer | |
| class Classifier(nn.Module): | |
| def __init__(self, in_dim, hid_dim=None, out_dim=None, act=nn.GELU, drop=0.): | |
| super().__init__() | |
| out_dim = out_dim or in_dim | |
| hid_dim = hid_dim or in_dim | |
| self.fc1 = nn.Linear(in_dim, hid_dim) | |
| self.act = act() | |
| self.fc2 = nn.Linear(hid_dim, out_dim) | |
| self.drop = nn.Dropout(drop) | |
| def forward(self, x): | |
| x = self.fc1(x) | |
| x = self.act(x) | |
| x = self.drop(x) | |
| x = self.fc2(x) | |
| return x | |
| class Block(nn.Module): | |
| def __init__(self, in_channels, out_channels, i_downsample=None, stride=1): | |
| super(Block, self).__init__() | |
| self.conv1 = nn.Conv2d(in_channels, out_channels, kernel_size=3, padding=1, stride=stride, bias=False) | |
| self.batch_norm1 = nn.BatchNorm2d(out_channels) | |
| self.conv2 = nn.Conv2d(out_channels, out_channels, kernel_size=3, padding=1, stride=stride, bias=False) | |
| self.i_downsample = i_downsample | |
| self.stride = stride | |
| self.relu = nn.LeakyReLU(negative_slope=0.1, inplace=True) | |
| def forward(self, x): | |
| identity = x.clone() | |
| x = self.relu(self.batch_norm1(self.conv1(x))) | |
| x = self.conv2(x) | |
| if self.i_downsample is not None: | |
| identity = self.i_downsample(identity) | |
| x += identity | |
| return x | |
| class MaskDecoder(nn.Module): | |
| def __init__( | |
| self, | |
| *, | |
| transformer_dim: int, | |
| transformer: nn.Module, | |
| modality, | |
| contents, | |
| num_multimask_outputs: int = 3, | |
| activation: Type[nn.Module] = nn.GELU, | |
| iou_head_depth: int = 3, | |
| iou_head_hidden_dim: int = 256, | |
| category_num = 11 | |
| ) -> None: | |
| """ | |
| Predicts masks given an image and prompt embeddings, using a | |
| transformer architecture. | |
| Arguments: | |
| transformer_dim (int): the channel dimension of the transformer | |
| transformer (nn.Module): the transformer used to predict masks | |
| num_multimask_outputs (int): the number of masks to predict | |
| when disambiguating masks | |
| activation (nn.Module): the type of activation to use when | |
| upscaling masks | |
| iou_head_depth (int): the depth of the MLP used to predict | |
| mask quality | |
| iou_head_hidden_dim (int): the hidden dimension of the MLP | |
| used to predict mask quality | |
| """ | |
| super().__init__() | |
| self.transformer_dim = transformer_dim | |
| self.transformer = transformer | |
| self.category_num = category_num | |
| self.modality = modality | |
| self.contents = contents | |
| self.num_multimask_outputs = num_multimask_outputs | |
| self.iou_token = nn.Embedding(1, transformer_dim) | |
| self.num_mask_tokens = num_multimask_outputs + 1 | |
| self.mask_tokens = nn.Embedding(self.num_mask_tokens, transformer_dim) | |
| self.convs = Block(transformer_dim, transformer_dim) | |
| self.w_lin = nn.Linear(transformer_dim, transformer_dim) | |
| self.b_lin = nn.Linear(transformer_dim, transformer_dim) | |
| self.output_upscaling = nn.Sequential( | |
| nn.ConvTranspose2d( | |
| transformer_dim, transformer_dim // 4, kernel_size=2, stride=2 | |
| ), | |
| LayerNorm2d(transformer_dim // 4), | |
| activation(), | |
| nn.ConvTranspose2d( | |
| transformer_dim // 4, transformer_dim // 8, kernel_size=2, stride=2 | |
| ), | |
| activation(), | |
| ) | |
| self.output_hypernetworks_mlps = nn.ModuleList( | |
| [ | |
| MLP(transformer_dim, transformer_dim, transformer_dim // 8, 3) | |
| for i in range(self.num_mask_tokens) | |
| ] | |
| ) | |
| self.iou_prediction_head = MLP( | |
| transformer_dim, iou_head_hidden_dim, self.num_mask_tokens, iou_head_depth | |
| ) | |
| self.category_prediction_head = Classifier( | |
| transformer_dim, transformer_dim//4, category_num | |
| ) | |
| def forward( | |
| self, | |
| image_embeddings: torch.Tensor, | |
| image_pe: torch.Tensor, | |
| sparse_prompt_embeddings: torch.Tensor, | |
| dense_prompt_embeddings: torch.Tensor, | |
| multimask_output: bool, | |
| ) -> Tuple[torch.Tensor, torch.Tensor]: | |
| """ | |
| Predict masks given image and prompt embeddings. | |
| Arguments: | |
| image_embeddings (torch.Tensor): the embeddings from the image encoder | |
| image_pe (torch.Tensor): positional encoding with the shape of image_embeddings | |
| sparse_prompt_embeddings (torch.Tensor): the embeddings of the points and boxes | |
| dense_prompt_embeddings (torch.Tensor): the embeddings of the mask inputs | |
| multimask_output (bool): Whether to return multiple masks or a single | |
| mask. | |
| Returns: | |
| torch.Tensor: batched predicted masks | |
| torch.Tensor: batched predictions of mask quality | |
| """ | |
| masks, iou_pred, category_pred, clip_tokens_out, image_tokens_out = self.predict_masks( | |
| image_embeddings=image_embeddings, | |
| image_pe=image_pe, | |
| sparse_prompt_embeddings=sparse_prompt_embeddings, | |
| dense_prompt_embeddings=dense_prompt_embeddings, | |
| ) | |
| # Select the correct mask or masks for output | |
| if multimask_output: | |
| mask_slice = slice(1, None) | |
| else: | |
| mask_slice = slice(0, 1) | |
| masks = masks[:, mask_slice, :, :] | |
| iou_pred = iou_pred[:, mask_slice] | |
| # Prepare output | |
| return masks, iou_pred, category_pred, clip_tokens_out, image_tokens_out | |
| def predict_masks( | |
| self, | |
| image_embeddings: torch.Tensor, | |
| image_pe: torch.Tensor, | |
| sparse_prompt_embeddings: torch.Tensor, | |
| dense_prompt_embeddings: torch.Tensor, | |
| ) -> Tuple[torch.Tensor, torch.Tensor]: | |
| """Predicts masks. See 'forward' for more details.""" | |
| # Concatenate output tokens | |
| output_tokens = torch.cat( | |
| [self.iou_token.weight, self.mask_tokens.weight], dim=0 | |
| ) | |
| output_tokens = output_tokens.unsqueeze(0).expand( | |
| sparse_prompt_embeddings.size(0), -1, -1 | |
| ) | |
| tokens = torch.cat((output_tokens, sparse_prompt_embeddings), dim=1) | |
| # Expand per-image data in batch direction to be per-mask | |
| if image_embeddings.shape[0] != tokens.shape[0]: | |
| src = torch.repeat_interleave(image_embeddings, tokens.shape[0], dim=0) | |
| else: | |
| src = image_embeddings | |
| src = src + dense_prompt_embeddings | |
| pos_src = torch.repeat_interleave(image_pe, tokens.shape[0], dim=0) | |
| b, c, h, w = src.shape | |
| # Run the transformer | |
| hs, src = self.transformer(src, pos_src, tokens) | |
| iou_token_out = hs[:, 0, :] | |
| mask_tokens_out = hs[:, 1 : (1 + self.num_mask_tokens), :] | |
| # Upscale mask embeddings and predict masks using the mask tokens | |
| src = src.transpose(1, 2).view(b, c, h, w) | |
| if self.contents: | |
| clip_tokens_out = tokens[:,-2,:] | |
| image_tokens_out = F.adaptive_avg_pool2d(dense_prompt_embeddings, output_size=(1, 1)).squeeze(-1).squeeze(-1) | |
| clip_new_out = hs[:,-2,:].unsqueeze(-1).unsqueeze(-1) | |
| src = dense_prompt_embeddings+src+clip_new_out | |
| src = self.convs(src) | |
| else: | |
| clip_tokens_out = None | |
| image_tokens_out = None | |
| if self.modality: | |
| category_tokens_out = hs[:,-1,:] | |
| wc = self.w_lin(category_tokens_out).unsqueeze(-1).unsqueeze(-1) | |
| bc = self.b_lin(category_tokens_out).unsqueeze(-1).unsqueeze(-1) | |
| src = wc*src+bc+src | |
| category_pred = self.category_prediction_head(category_tokens_out) | |
| else: | |
| category_pred = None | |
| upscaled_embedding = self.output_upscaling(src) | |
| hyper_in_list: List[torch.Tensor] = [] | |
| for i in range(self.num_mask_tokens): | |
| hyper_in_list.append( | |
| self.output_hypernetworks_mlps[i](mask_tokens_out[:, i, :]) | |
| ) | |
| hyper_in = torch.stack(hyper_in_list, dim=1) | |
| b, c, h, w = upscaled_embedding.shape | |
| masks = (hyper_in @ upscaled_embedding.view(b, c, h * w)).view(b, -1, h, w) | |
| # Generate mask quality predictions | |
| iou_pred = self.iou_prediction_head(iou_token_out) | |
| return masks, iou_pred, category_pred, clip_tokens_out, image_tokens_out | |
| # Lightly adapted from | |
| # https://github.com/facebookresearch/MaskFormer/blob/main/mask_former/modeling/transformer/transformer_predictor.py # noqa | |
| class MLP(nn.Module): | |
| def __init__( | |
| self, | |
| input_dim: int, | |
| hidden_dim: int, | |
| output_dim: int, | |
| num_layers: int, | |
| sigmoid_output: bool = False, | |
| ) -> None: | |
| super().__init__() | |
| self.num_layers = num_layers | |
| h = [hidden_dim] * (num_layers - 1) | |
| self.layers = nn.ModuleList( | |
| nn.Linear(n, k) for n, k in zip([input_dim] + h, h + [output_dim]) | |
| ) | |
| self.sigmoid_output = sigmoid_output | |
| def forward(self, x): | |
| for i, layer in enumerate(self.layers): | |
| x = F.relu(layer(x)) if i < self.num_layers - 1 else layer(x) | |
| if self.sigmoid_output: | |
| x = F.sigmoid(x) | |
| return x | |
| class MaskDecoder_F4(nn.Module): | |
| def __init__( | |
| self, | |
| *, | |
| transformer_dim: int, | |
| transformer: nn.Module, | |
| modality, | |
| contents, | |
| num_multimask_outputs: int = 3, | |
| activation: Type[nn.Module] = nn.GELU, | |
| iou_head_depth: int = 3, | |
| iou_head_hidden_dim: int = 256, | |
| category_num = 11 | |
| ) -> None: | |
| """ | |
| Predicts masks given an image and prompt embeddings, using a | |
| transformer architecture. | |
| Arguments: | |
| transformer_dim (int): the channel dimension of the transformer | |
| transformer (nn.Module): the transformer used to predict masks | |
| num_multimask_outputs (int): the number of masks to predict | |
| when disambiguating masks | |
| activation (nn.Module): the type of activation to use when | |
| upscaling masks | |
| iou_head_depth (int): the depth of the MLP used to predict | |
| mask quality | |
| iou_head_hidden_dim (int): the hidden dimension of the MLP | |
| used to predict mask quality | |
| """ | |
| super().__init__() | |
| self.transformer_dim = transformer_dim | |
| self.transformer = transformer | |
| self.category_num = category_num | |
| self.modality = modality | |
| self.contents = contents | |
| self.num_multimask_outputs = num_multimask_outputs | |
| self.iou_token = nn.Embedding(1, transformer_dim) | |
| self.num_mask_tokens = num_multimask_outputs + 1 | |
| self.mask_tokens = nn.Embedding(self.num_mask_tokens, transformer_dim) | |
| self.convs = Block(transformer_dim, transformer_dim) | |
| self.conv1 = nn.Conv2d(transformer_dim*2, transformer_dim, 1) | |
| self.c_conv = Block(transformer_dim, transformer_dim) | |
| self.w_lin = nn.Linear(transformer_dim, transformer_dim) | |
| self.b_lin = nn.Linear(transformer_dim, transformer_dim) | |
| self.m_conv = Block(transformer_dim, transformer_dim) | |
| self.output_upscaling = nn.Sequential( | |
| nn.ConvTranspose2d( | |
| transformer_dim, transformer_dim // 4, kernel_size=2, stride=2 | |
| ), | |
| LayerNorm2d(transformer_dim // 4), | |
| activation(), | |
| nn.ConvTranspose2d( | |
| transformer_dim // 4, transformer_dim // 8, kernel_size=2, stride=2 | |
| ), | |
| activation(), | |
| ) | |
| self.output_hypernetworks_mlps = nn.ModuleList( | |
| [ | |
| MLP(transformer_dim, transformer_dim, transformer_dim // 8, 3) | |
| for i in range(self.num_mask_tokens) | |
| ] | |
| ) | |
| self.iou_prediction_head = MLP( | |
| transformer_dim, iou_head_hidden_dim, self.num_mask_tokens, iou_head_depth | |
| ) | |
| # self.category_prediction_head = Classifier( | |
| # transformer_dim, transformer_dim//4, category_num | |
| # ) | |
| self.category_prediction_head = Classifier( | |
| transformer_dim, transformer_dim//4, category_num | |
| ) | |
| def forward( | |
| self, | |
| image_embeddings: torch.Tensor, | |
| image_pe: torch.Tensor, | |
| sparse_prompt_embeddings: torch.Tensor, | |
| dense_prompt_embeddings: torch.Tensor, | |
| multimask_output: bool, | |
| ) -> Tuple[torch.Tensor, torch.Tensor]: | |
| """ | |
| Predict masks given image and prompt embeddings. | |
| Arguments: | |
| image_embeddings (torch.Tensor): the embeddings from the image encoder | |
| image_pe (torch.Tensor): positional encoding with the shape of image_embeddings | |
| sparse_prompt_embeddings (torch.Tensor): the embeddings of the points and boxes | |
| dense_prompt_embeddings (torch.Tensor): the embeddings of the mask inputs | |
| multimask_output (bool): Whether to return multiple masks or a single | |
| mask. | |
| Returns: | |
| torch.Tensor: batched predicted masks | |
| torch.Tensor: batched predictions of mask quality | |
| """ | |
| masks, iou_pred, category_pred, clip_tokens_out, image_tokens_out = self.predict_masks( | |
| image_embeddings=image_embeddings, | |
| image_pe=image_pe, | |
| sparse_prompt_embeddings=sparse_prompt_embeddings, | |
| dense_prompt_embeddings=dense_prompt_embeddings, | |
| ) | |
| # Select the correct mask or masks for output | |
| if multimask_output: | |
| mask_slice = slice(1, None) | |
| else: | |
| mask_slice = slice(0, 1) | |
| masks = masks[:, mask_slice, :, :] | |
| iou_pred = iou_pred[:, mask_slice] | |
| # Prepare output | |
| return masks, iou_pred, category_pred, clip_tokens_out, image_tokens_out | |
| def predict_masks( | |
| self, | |
| image_embeddings: torch.Tensor, | |
| image_pe: torch.Tensor, | |
| sparse_prompt_embeddings: torch.Tensor, | |
| dense_prompt_embeddings: torch.Tensor, | |
| ) -> Tuple[torch.Tensor, torch.Tensor]: | |
| """Predicts masks. See 'forward' for more details.""" | |
| # Concatenate output tokens | |
| output_tokens = torch.cat( | |
| [self.iou_token.weight, self.mask_tokens.weight], dim=0 | |
| ) | |
| output_tokens = output_tokens.unsqueeze(0).expand( | |
| sparse_prompt_embeddings.size(0), -1, -1 | |
| ) | |
| tokens = torch.cat((output_tokens, sparse_prompt_embeddings), dim=1) | |
| m_token = tokens[:,-1,:] | |
| # Expand per-image data in batch direction to be per-mask | |
| if image_embeddings.shape[0] != tokens.shape[0]: | |
| src = torch.repeat_interleave(image_embeddings, tokens.shape[0], dim=0) | |
| else: | |
| src = image_embeddings | |
| src = src + dense_prompt_embeddings | |
| pos_src = torch.repeat_interleave(image_pe, tokens.shape[0], dim=0) | |
| b, c, h, w = src.shape | |
| # Run the transformer | |
| hs, src = self.transformer(src, pos_src, tokens) | |
| iou_token_out = hs[:, 0, :] | |
| mask_tokens_out = hs[:, 1 : (1 + self.num_mask_tokens), :] | |
| # Upscale mask embeddings and predict masks using the mask tokens | |
| src = src.transpose(1, 2).view(b, c, h, w) | |
| if self.modality: | |
| category_tokens_out = hs[:,-1,:] | |
| wc = self.w_lin(category_tokens_out).unsqueeze(-1).unsqueeze(-1) | |
| bc = self.b_lin(category_tokens_out).unsqueeze(-1).unsqueeze(-1) | |
| src_m = wc*src+bc+src | |
| m_info = wc.squeeze(-1).squeeze(-1)+bc.squeeze(-1).squeeze(-1)+category_tokens_out | |
| category_pred = self.category_prediction_head(m_info) | |
| src_m = self.m_conv(src_m) | |
| else: | |
| category_pred = None | |
| if self.contents: | |
| clip_tokens_out = tokens[:,-2,:] | |
| image_tokens_out = F.adaptive_avg_pool2d(dense_prompt_embeddings, output_size=(1, 1)).squeeze(-1).squeeze(-1) | |
| clip_new_out = hs[:,-2,:].unsqueeze(-1).unsqueeze(-1) | |
| src_vp = dense_prompt_embeddings+src+clip_new_out | |
| src_vp = self.convs(src_vp) | |
| else: | |
| clip_tokens_out = None | |
| image_tokens_out = None | |
| if self.contents and self.modality: | |
| src = torch.cat((src_m, src_vp), dim=1) | |
| src = self.conv1(src) | |
| src = self.c_conv(src) | |
| elif self.contents: | |
| src = src_vp | |
| elif self.modality: | |
| src = src_m | |
| upscaled_embedding = self.output_upscaling(src) | |
| hyper_in_list: List[torch.Tensor] = [] | |
| for i in range(self.num_mask_tokens): | |
| hyper_in_list.append( | |
| self.output_hypernetworks_mlps[i](mask_tokens_out[:, i, :]) | |
| ) | |
| hyper_in = torch.stack(hyper_in_list, dim=1) | |
| b, c, h, w = upscaled_embedding.shape | |
| masks = (hyper_in @ upscaled_embedding.view(b, c, h * w)).view(b, -1, h, w) | |
| # Generate mask quality predictions | |
| iou_pred = self.iou_prediction_head(iou_token_out) | |
| return masks, iou_pred, category_pred, clip_tokens_out, image_tokens_out | |