Spaces:
Runtime error
Runtime error
🍻feat: Upload models
Browse files- model/__init__.py +1 -0
- model/decoder.py +51 -0
- model/mobilenet.py +56 -0
- model/model.py +196 -0
- model/refiner.py +282 -0
- model/resnet.py +48 -0
- model/utils.py +14 -0
model/__init__.py
ADDED
|
@@ -0,0 +1 @@
|
|
|
|
|
|
|
| 1 |
+
from .model import Base, MattingBase, MattingRefine
|
model/decoder.py
ADDED
|
@@ -0,0 +1,51 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
import torch
|
| 2 |
+
import torch.nn as nn
|
| 3 |
+
import torch.nn.functional as F
|
| 4 |
+
|
| 5 |
+
|
| 6 |
+
class Decoder(nn.Module):
|
| 7 |
+
"""
|
| 8 |
+
Decoder upsamples the image by combining the feature maps at all resolutions from the encoder.
|
| 9 |
+
|
| 10 |
+
Input:
|
| 11 |
+
x4: (B, C, H/16, W/16) feature map at 1/16 resolution.
|
| 12 |
+
x3: (B, C, H/8, W/8) feature map at 1/8 resolution.
|
| 13 |
+
x2: (B, C, H/4, W/4) feature map at 1/4 resolution.
|
| 14 |
+
x1: (B, C, H/2, W/2) feature map at 1/2 resolution.
|
| 15 |
+
x0: (B, C, H, W) feature map at full resolution.
|
| 16 |
+
|
| 17 |
+
Output:
|
| 18 |
+
x: (B, C, H, W) upsampled output at full resolution.
|
| 19 |
+
"""
|
| 20 |
+
|
| 21 |
+
def __init__(self, channels, feature_channels):
|
| 22 |
+
super().__init__()
|
| 23 |
+
self.conv1 = nn.Conv2d(feature_channels[0] + channels[0], channels[1], 3, padding=1, bias=False)
|
| 24 |
+
self.bn1 = nn.BatchNorm2d(channels[1])
|
| 25 |
+
self.conv2 = nn.Conv2d(feature_channels[1] + channels[1], channels[2], 3, padding=1, bias=False)
|
| 26 |
+
self.bn2 = nn.BatchNorm2d(channels[2])
|
| 27 |
+
self.conv3 = nn.Conv2d(feature_channels[2] + channels[2], channels[3], 3, padding=1, bias=False)
|
| 28 |
+
self.bn3 = nn.BatchNorm2d(channels[3])
|
| 29 |
+
self.conv4 = nn.Conv2d(feature_channels[3] + channels[3], channels[4], 3, padding=1)
|
| 30 |
+
self.relu = nn.ReLU(True)
|
| 31 |
+
|
| 32 |
+
def forward(self, x4, x3, x2, x1, x0):
|
| 33 |
+
x = F.interpolate(x4, size=x3.shape[2:], mode='bilinear', align_corners=False)
|
| 34 |
+
x = torch.cat([x, x3], dim=1)
|
| 35 |
+
x = self.conv1(x)
|
| 36 |
+
x = self.bn1(x)
|
| 37 |
+
x = self.relu(x)
|
| 38 |
+
x = F.interpolate(x, size=x2.shape[2:], mode='bilinear', align_corners=False)
|
| 39 |
+
x = torch.cat([x, x2], dim=1)
|
| 40 |
+
x = self.conv2(x)
|
| 41 |
+
x = self.bn2(x)
|
| 42 |
+
x = self.relu(x)
|
| 43 |
+
x = F.interpolate(x, size=x1.shape[2:], mode='bilinear', align_corners=False)
|
| 44 |
+
x = torch.cat([x, x1], dim=1)
|
| 45 |
+
x = self.conv3(x)
|
| 46 |
+
x = self.bn3(x)
|
| 47 |
+
x = self.relu(x)
|
| 48 |
+
x = F.interpolate(x, size=x0.shape[2:], mode='bilinear', align_corners=False)
|
| 49 |
+
x = torch.cat([x, x0], dim=1)
|
| 50 |
+
x = self.conv4(x)
|
| 51 |
+
return x
|
model/mobilenet.py
ADDED
|
@@ -0,0 +1,56 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
from torch import nn
|
| 2 |
+
from torchvision.models import MobileNetV2
|
| 3 |
+
|
| 4 |
+
|
| 5 |
+
class MobileNetV2Encoder(MobileNetV2):
|
| 6 |
+
"""
|
| 7 |
+
MobileNetV2Encoder inherits from torchvision's official MobileNetV2. It is modified to
|
| 8 |
+
use dilation on the last block to maintain output stride 16, and deleted the
|
| 9 |
+
classifier block that was originally used for classification. The forward method
|
| 10 |
+
additionally returns the feature maps at all resolutions for decoder's use.
|
| 11 |
+
"""
|
| 12 |
+
|
| 13 |
+
def __init__(self, in_channels, norm_layer=None):
|
| 14 |
+
super().__init__()
|
| 15 |
+
|
| 16 |
+
# Replace first conv layer if in_channels doesn't match.
|
| 17 |
+
if in_channels != 3:
|
| 18 |
+
self.features[0][0] = nn.Conv2d(in_channels, 32, 3, 2, 1, bias=False)
|
| 19 |
+
|
| 20 |
+
# Remove last block
|
| 21 |
+
self.features = self.features[:-1]
|
| 22 |
+
|
| 23 |
+
# Change to use dilation to maintain output stride = 16
|
| 24 |
+
self.features[14].conv[1][0].stride = (1, 1)
|
| 25 |
+
for feature in self.features[15:]:
|
| 26 |
+
feature.conv[1][0].dilation = (2, 2)
|
| 27 |
+
feature.conv[1][0].padding = (2, 2)
|
| 28 |
+
|
| 29 |
+
# Delete classifier
|
| 30 |
+
del self.classifier
|
| 31 |
+
|
| 32 |
+
def forward(self, x):
|
| 33 |
+
x0 = x # 1/1
|
| 34 |
+
x = self.features[0](x)
|
| 35 |
+
x = self.features[1](x)
|
| 36 |
+
x1 = x # 1/2
|
| 37 |
+
x = self.features[2](x)
|
| 38 |
+
x = self.features[3](x)
|
| 39 |
+
x2 = x # 1/4
|
| 40 |
+
x = self.features[4](x)
|
| 41 |
+
x = self.features[5](x)
|
| 42 |
+
x = self.features[6](x)
|
| 43 |
+
x3 = x # 1/8
|
| 44 |
+
x = self.features[7](x)
|
| 45 |
+
x = self.features[8](x)
|
| 46 |
+
x = self.features[9](x)
|
| 47 |
+
x = self.features[10](x)
|
| 48 |
+
x = self.features[11](x)
|
| 49 |
+
x = self.features[12](x)
|
| 50 |
+
x = self.features[13](x)
|
| 51 |
+
x = self.features[14](x)
|
| 52 |
+
x = self.features[15](x)
|
| 53 |
+
x = self.features[16](x)
|
| 54 |
+
x = self.features[17](x)
|
| 55 |
+
x4 = x # 1/16
|
| 56 |
+
return x4, x3, x2, x1, x0
|
model/model.py
ADDED
|
@@ -0,0 +1,196 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
import torch
|
| 2 |
+
from torch import nn
|
| 3 |
+
from torch.nn import functional as F
|
| 4 |
+
from torchvision.models.segmentation.deeplabv3 import ASPP
|
| 5 |
+
|
| 6 |
+
from .decoder import Decoder
|
| 7 |
+
from .mobilenet import MobileNetV2Encoder
|
| 8 |
+
from .refiner import Refiner
|
| 9 |
+
from .resnet import ResNetEncoder
|
| 10 |
+
from .utils import load_matched_state_dict
|
| 11 |
+
|
| 12 |
+
|
| 13 |
+
class Base(nn.Module):
|
| 14 |
+
"""
|
| 15 |
+
A generic implementation of the base encoder-decoder network inspired by DeepLab.
|
| 16 |
+
Accepts arbitrary channels for input and output.
|
| 17 |
+
"""
|
| 18 |
+
|
| 19 |
+
def __init__(self, backbone: str, in_channels: int, out_channels: int):
|
| 20 |
+
super().__init__()
|
| 21 |
+
assert backbone in ["resnet50", "resnet101", "mobilenetv2"]
|
| 22 |
+
if backbone in ['resnet50', 'resnet101']:
|
| 23 |
+
self.backbone = ResNetEncoder(in_channels, variant=backbone)
|
| 24 |
+
self.aspp = ASPP(2048, [3, 6, 9])
|
| 25 |
+
self.decoder = Decoder([256, 128, 64, 48, out_channels], [512, 256, 64, in_channels])
|
| 26 |
+
else:
|
| 27 |
+
self.backbone = MobileNetV2Encoder(in_channels)
|
| 28 |
+
self.aspp = ASPP(320, [3, 6, 9])
|
| 29 |
+
self.decoder = Decoder([256, 128, 64, 48, out_channels], [32, 24, 16, in_channels])
|
| 30 |
+
|
| 31 |
+
def forward(self, x):
|
| 32 |
+
x, *shortcuts = self.backbone(x)
|
| 33 |
+
x = self.aspp(x)
|
| 34 |
+
x = self.decoder(x, *shortcuts)
|
| 35 |
+
return x
|
| 36 |
+
|
| 37 |
+
def load_pretrained_deeplabv3_state_dict(self, state_dict, print_stats=True):
|
| 38 |
+
# Pretrained DeepLabV3 models are provided by <https://github.com/VainF/DeepLabV3Plus-Pytorch>.
|
| 39 |
+
# This method converts and loads their pretrained state_dict to match with our model structure.
|
| 40 |
+
# This method is not needed if you are not planning to train from deeplab weights.
|
| 41 |
+
# Use load_state_dict() for normal weight loading.
|
| 42 |
+
|
| 43 |
+
# Convert state_dict naming for aspp module
|
| 44 |
+
state_dict = {k.replace('classifier.classifier.0', 'aspp'): v for k, v in state_dict.items()}
|
| 45 |
+
|
| 46 |
+
if isinstance(self.backbone, ResNetEncoder):
|
| 47 |
+
# ResNet backbone does not need change.
|
| 48 |
+
load_matched_state_dict(self, state_dict, print_stats)
|
| 49 |
+
else:
|
| 50 |
+
# Change MobileNetV2 backbone to state_dict format, then change back after loading.
|
| 51 |
+
backbone_features = self.backbone.features
|
| 52 |
+
self.backbone.low_level_features = backbone_features[:4]
|
| 53 |
+
self.backbone.high_level_features = backbone_features[4:]
|
| 54 |
+
del self.backbone.features
|
| 55 |
+
load_matched_state_dict(self, state_dict, print_stats)
|
| 56 |
+
self.backbone.features = backbone_features
|
| 57 |
+
del self.backbone.low_level_features
|
| 58 |
+
del self.backbone.high_level_features
|
| 59 |
+
|
| 60 |
+
|
| 61 |
+
class MattingBase(Base):
|
| 62 |
+
"""
|
| 63 |
+
MattingBase is used to produce coarse global results at a lower resolution.
|
| 64 |
+
MattingBase extends Base.
|
| 65 |
+
|
| 66 |
+
Args:
|
| 67 |
+
backbone: ["resnet50", "resnet101", "mobilenetv2"]
|
| 68 |
+
|
| 69 |
+
Input:
|
| 70 |
+
src: (B, 3, H, W) the source image. Channels are RGB values normalized to 0 ~ 1.
|
| 71 |
+
bgr: (B, 3, H, W) the background image . Channels are RGB values normalized to 0 ~ 1.
|
| 72 |
+
|
| 73 |
+
Output:
|
| 74 |
+
pha: (B, 1, H, W) the alpha prediction. Normalized to 0 ~ 1.
|
| 75 |
+
fgr: (B, 3, H, W) the foreground prediction. Channels are RGB values normalized to 0 ~ 1.
|
| 76 |
+
err: (B, 1, H, W) the error prediction. Normalized to 0 ~ 1.
|
| 77 |
+
hid: (B, 32, H, W) the hidden encoding. Used for connecting refiner module.
|
| 78 |
+
|
| 79 |
+
Example:
|
| 80 |
+
model = MattingBase(backbone='resnet50')
|
| 81 |
+
|
| 82 |
+
pha, fgr, err, hid = model(src, bgr) # for training
|
| 83 |
+
pha, fgr = model(src, bgr)[:2] # for inference
|
| 84 |
+
"""
|
| 85 |
+
|
| 86 |
+
def __init__(self, backbone: str):
|
| 87 |
+
super().__init__(backbone, in_channels=6, out_channels=(1 + 3 + 1 + 32))
|
| 88 |
+
|
| 89 |
+
def forward(self, src, bgr):
|
| 90 |
+
x = torch.cat([src, bgr], dim=1)
|
| 91 |
+
x, *shortcuts = self.backbone(x)
|
| 92 |
+
x = self.aspp(x)
|
| 93 |
+
x = self.decoder(x, *shortcuts)
|
| 94 |
+
pha = x[:, 0:1].clamp_(0., 1.)
|
| 95 |
+
fgr = x[:, 1:4].add(src).clamp_(0., 1.)
|
| 96 |
+
err = x[:, 4:5].clamp_(0., 1.)
|
| 97 |
+
hid = x[:, 5: ].relu_()
|
| 98 |
+
return pha, fgr, err, hid
|
| 99 |
+
|
| 100 |
+
|
| 101 |
+
class MattingRefine(MattingBase):
|
| 102 |
+
"""
|
| 103 |
+
MattingRefine includes the refiner module to upsample coarse result to full resolution.
|
| 104 |
+
MattingRefine extends MattingBase.
|
| 105 |
+
|
| 106 |
+
Args:
|
| 107 |
+
backbone: ["resnet50", "resnet101", "mobilenetv2"]
|
| 108 |
+
backbone_scale: The image downsample scale for passing through backbone, default 1/4 or 0.25.
|
| 109 |
+
Must not be greater than 1/2.
|
| 110 |
+
refine_mode: refine area selection mode. Options:
|
| 111 |
+
"full" - No area selection, refine everywhere using regular Conv2d.
|
| 112 |
+
"sampling" - Refine fixed amount of pixels ranked by the top most errors.
|
| 113 |
+
"thresholding" - Refine varying amount of pixels that has more error than the threshold.
|
| 114 |
+
refine_sample_pixels: number of pixels to refine. Only used when mode == "sampling".
|
| 115 |
+
refine_threshold: error threshold ranged from 0 ~ 1. Refine where err > threshold. Only used when mode == "thresholding".
|
| 116 |
+
refine_kernel_size: the refiner's convolutional kernel size. Options: [1, 3]
|
| 117 |
+
refine_prevent_oversampling: prevent sampling more pixels than needed for sampling mode. Set False only for speedtest.
|
| 118 |
+
|
| 119 |
+
Input:
|
| 120 |
+
src: (B, 3, H, W) the source image. Channels are RGB values normalized to 0 ~ 1.
|
| 121 |
+
bgr: (B, 3, H, W) the background image. Channels are RGB values normalized to 0 ~ 1.
|
| 122 |
+
|
| 123 |
+
Output:
|
| 124 |
+
pha: (B, 1, H, W) the alpha prediction. Normalized to 0 ~ 1.
|
| 125 |
+
fgr: (B, 3, H, W) the foreground prediction. Channels are RGB values normalized to 0 ~ 1.
|
| 126 |
+
pha_sm: (B, 1, Hc, Wc) the coarse alpha prediction from matting base. Normalized to 0 ~ 1.
|
| 127 |
+
fgr_sm: (B, 3, Hc, Hc) the coarse foreground prediction from matting base. Normalized to 0 ~ 1.
|
| 128 |
+
err_sm: (B, 1, Hc, Wc) the coarse error prediction from matting base. Normalized to 0 ~ 1.
|
| 129 |
+
ref_sm: (B, 1, H/4, H/4) the quarter resolution refinement map. 1 indicates refined 4x4 patch locations.
|
| 130 |
+
|
| 131 |
+
Example:
|
| 132 |
+
model = MattingRefine(backbone='resnet50', backbone_scale=1/4, refine_mode='sampling', refine_sample_pixels=80_000)
|
| 133 |
+
model = MattingRefine(backbone='resnet50', backbone_scale=1/4, refine_mode='thresholding', refine_threshold=0.1)
|
| 134 |
+
model = MattingRefine(backbone='resnet50', backbone_scale=1/4, refine_mode='full')
|
| 135 |
+
|
| 136 |
+
pha, fgr, pha_sm, fgr_sm, err_sm, ref_sm = model(src, bgr) # for training
|
| 137 |
+
pha, fgr = model(src, bgr)[:2] # for inference
|
| 138 |
+
"""
|
| 139 |
+
|
| 140 |
+
def __init__(self,
|
| 141 |
+
backbone: str,
|
| 142 |
+
backbone_scale: float = 1/4,
|
| 143 |
+
refine_mode: str = 'sampling',
|
| 144 |
+
refine_sample_pixels: int = 80_000,
|
| 145 |
+
refine_threshold: float = 0.1,
|
| 146 |
+
refine_kernel_size: int = 3,
|
| 147 |
+
refine_prevent_oversampling: bool = True,
|
| 148 |
+
refine_patch_crop_method: str = 'unfold',
|
| 149 |
+
refine_patch_replace_method: str = 'scatter_nd'):
|
| 150 |
+
assert backbone_scale <= 1/2, 'backbone_scale should not be greater than 1/2'
|
| 151 |
+
super().__init__(backbone)
|
| 152 |
+
self.backbone_scale = backbone_scale
|
| 153 |
+
self.refiner = Refiner(refine_mode,
|
| 154 |
+
refine_sample_pixels,
|
| 155 |
+
refine_threshold,
|
| 156 |
+
refine_kernel_size,
|
| 157 |
+
refine_prevent_oversampling,
|
| 158 |
+
refine_patch_crop_method,
|
| 159 |
+
refine_patch_replace_method)
|
| 160 |
+
|
| 161 |
+
def forward(self, src, bgr):
|
| 162 |
+
assert src.size() == bgr.size(), 'src and bgr must have the same shape'
|
| 163 |
+
assert src.size(2) // 4 * 4 == src.size(2) and src.size(3) // 4 * 4 == src.size(3), \
|
| 164 |
+
'src and bgr must have width and height that are divisible by 4'
|
| 165 |
+
|
| 166 |
+
# Downsample src and bgr for backbone
|
| 167 |
+
src_sm = F.interpolate(src,
|
| 168 |
+
scale_factor=self.backbone_scale,
|
| 169 |
+
mode='bilinear',
|
| 170 |
+
align_corners=False,
|
| 171 |
+
recompute_scale_factor=True)
|
| 172 |
+
bgr_sm = F.interpolate(bgr,
|
| 173 |
+
scale_factor=self.backbone_scale,
|
| 174 |
+
mode='bilinear',
|
| 175 |
+
align_corners=False,
|
| 176 |
+
recompute_scale_factor=True)
|
| 177 |
+
|
| 178 |
+
# Base
|
| 179 |
+
x = torch.cat([src_sm, bgr_sm], dim=1)
|
| 180 |
+
x, *shortcuts = self.backbone(x)
|
| 181 |
+
x = self.aspp(x)
|
| 182 |
+
x = self.decoder(x, *shortcuts)
|
| 183 |
+
pha_sm = x[:, 0:1].clamp_(0., 1.)
|
| 184 |
+
fgr_sm = x[:, 1:4]
|
| 185 |
+
err_sm = x[:, 4:5].clamp_(0., 1.)
|
| 186 |
+
hid_sm = x[:, 5: ].relu_()
|
| 187 |
+
|
| 188 |
+
# Refiner
|
| 189 |
+
pha, fgr, ref_sm = self.refiner(src, bgr, pha_sm, fgr_sm, err_sm, hid_sm)
|
| 190 |
+
|
| 191 |
+
# Clamp outputs
|
| 192 |
+
pha = pha.clamp_(0., 1.)
|
| 193 |
+
fgr = fgr.add_(src).clamp_(0., 1.)
|
| 194 |
+
fgr_sm = src_sm.add_(fgr_sm).clamp_(0., 1.)
|
| 195 |
+
|
| 196 |
+
return pha, fgr, pha_sm, fgr_sm, err_sm, ref_sm
|
model/refiner.py
ADDED
|
@@ -0,0 +1,282 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
import torch
|
| 2 |
+
import torchvision
|
| 3 |
+
from torch import nn
|
| 4 |
+
from torch.nn import functional as F
|
| 5 |
+
from typing import Tuple
|
| 6 |
+
|
| 7 |
+
|
| 8 |
+
class Refiner(nn.Module):
|
| 9 |
+
"""
|
| 10 |
+
Refiner refines the coarse output to full resolution.
|
| 11 |
+
|
| 12 |
+
Args:
|
| 13 |
+
mode: area selection mode. Options:
|
| 14 |
+
"full" - No area selection, refine everywhere using regular Conv2d.
|
| 15 |
+
"sampling" - Refine fixed amount of pixels ranked by the top most errors.
|
| 16 |
+
"thresholding" - Refine varying amount of pixels that have greater error than the threshold.
|
| 17 |
+
sample_pixels: number of pixels to refine. Only used when mode == "sampling".
|
| 18 |
+
threshold: error threshold ranged from 0 ~ 1. Refine where err > threshold. Only used when mode == "thresholding".
|
| 19 |
+
kernel_size: The convolution kernel_size. Options: [1, 3]
|
| 20 |
+
prevent_oversampling: True for regular cases, False for speedtest.
|
| 21 |
+
|
| 22 |
+
Compatibility Args:
|
| 23 |
+
patch_crop_method: the method for cropping patches. Options:
|
| 24 |
+
"unfold" - Best performance for PyTorch and TorchScript.
|
| 25 |
+
"roi_align" - Another way for croping patches.
|
| 26 |
+
"gather" - Another way for croping patches.
|
| 27 |
+
patch_replace_method: the method for replacing patches. Options:
|
| 28 |
+
"scatter_nd" - Best performance for PyTorch and TorchScript.
|
| 29 |
+
"scatter_element" - Another way for replacing patches.
|
| 30 |
+
|
| 31 |
+
Input:
|
| 32 |
+
src: (B, 3, H, W) full resolution source image.
|
| 33 |
+
bgr: (B, 3, H, W) full resolution background image.
|
| 34 |
+
pha: (B, 1, Hc, Wc) coarse alpha prediction.
|
| 35 |
+
fgr: (B, 3, Hc, Wc) coarse foreground residual prediction.
|
| 36 |
+
err: (B, 1, Hc, Hc) coarse error prediction.
|
| 37 |
+
hid: (B, 32, Hc, Hc) coarse hidden encoding.
|
| 38 |
+
|
| 39 |
+
Output:
|
| 40 |
+
pha: (B, 1, H, W) full resolution alpha prediction.
|
| 41 |
+
fgr: (B, 3, H, W) full resolution foreground residual prediction.
|
| 42 |
+
ref: (B, 1, H/4, W/4) quarter resolution refinement selection map. 1 indicates refined 4x4 patch locations.
|
| 43 |
+
"""
|
| 44 |
+
|
| 45 |
+
# For TorchScript export optimization.
|
| 46 |
+
__constants__ = ['kernel_size', 'patch_crop_method', 'patch_replace_method']
|
| 47 |
+
|
| 48 |
+
def __init__(self,
|
| 49 |
+
mode: str,
|
| 50 |
+
sample_pixels: int,
|
| 51 |
+
threshold: float,
|
| 52 |
+
kernel_size: int = 3,
|
| 53 |
+
prevent_oversampling: bool = True,
|
| 54 |
+
patch_crop_method: str = 'unfold',
|
| 55 |
+
patch_replace_method: str = 'scatter_nd'):
|
| 56 |
+
super().__init__()
|
| 57 |
+
assert mode in ['full', 'sampling', 'thresholding']
|
| 58 |
+
assert kernel_size in [1, 3]
|
| 59 |
+
assert patch_crop_method in ['unfold', 'roi_align', 'gather']
|
| 60 |
+
assert patch_replace_method in ['scatter_nd', 'scatter_element']
|
| 61 |
+
|
| 62 |
+
self.mode = mode
|
| 63 |
+
self.sample_pixels = sample_pixels
|
| 64 |
+
self.threshold = threshold
|
| 65 |
+
self.kernel_size = kernel_size
|
| 66 |
+
self.prevent_oversampling = prevent_oversampling
|
| 67 |
+
self.patch_crop_method = patch_crop_method
|
| 68 |
+
self.patch_replace_method = patch_replace_method
|
| 69 |
+
|
| 70 |
+
channels = [32, 24, 16, 12, 4]
|
| 71 |
+
self.conv1 = nn.Conv2d(channels[0] + 6 + 4, channels[1], kernel_size, bias=False)
|
| 72 |
+
self.bn1 = nn.BatchNorm2d(channels[1])
|
| 73 |
+
self.conv2 = nn.Conv2d(channels[1], channels[2], kernel_size, bias=False)
|
| 74 |
+
self.bn2 = nn.BatchNorm2d(channels[2])
|
| 75 |
+
self.conv3 = nn.Conv2d(channels[2] + 6, channels[3], kernel_size, bias=False)
|
| 76 |
+
self.bn3 = nn.BatchNorm2d(channels[3])
|
| 77 |
+
self.conv4 = nn.Conv2d(channels[3], channels[4], kernel_size, bias=True)
|
| 78 |
+
self.relu = nn.ReLU(True)
|
| 79 |
+
|
| 80 |
+
def forward(self,
|
| 81 |
+
src: torch.Tensor,
|
| 82 |
+
bgr: torch.Tensor,
|
| 83 |
+
pha: torch.Tensor,
|
| 84 |
+
fgr: torch.Tensor,
|
| 85 |
+
err: torch.Tensor,
|
| 86 |
+
hid: torch.Tensor):
|
| 87 |
+
H_full, W_full = src.shape[2:]
|
| 88 |
+
H_half, W_half = H_full // 2, W_full // 2
|
| 89 |
+
H_quat, W_quat = H_full // 4, W_full // 4
|
| 90 |
+
|
| 91 |
+
src_bgr = torch.cat([src, bgr], dim=1)
|
| 92 |
+
|
| 93 |
+
if self.mode != 'full':
|
| 94 |
+
err = F.interpolate(err, (H_quat, W_quat), mode='bilinear', align_corners=False)
|
| 95 |
+
ref = self.select_refinement_regions(err)
|
| 96 |
+
idx = torch.nonzero(ref.squeeze(1))
|
| 97 |
+
idx = idx[:, 0], idx[:, 1], idx[:, 2]
|
| 98 |
+
|
| 99 |
+
if idx[0].size(0) > 0:
|
| 100 |
+
x = torch.cat([hid, pha, fgr], dim=1)
|
| 101 |
+
x = F.interpolate(x, (H_half, W_half), mode='bilinear', align_corners=False)
|
| 102 |
+
x = self.crop_patch(x, idx, 2, 3 if self.kernel_size == 3 else 0)
|
| 103 |
+
|
| 104 |
+
y = F.interpolate(src_bgr, (H_half, W_half), mode='bilinear', align_corners=False)
|
| 105 |
+
y = self.crop_patch(y, idx, 2, 3 if self.kernel_size == 3 else 0)
|
| 106 |
+
|
| 107 |
+
x = self.conv1(torch.cat([x, y], dim=1))
|
| 108 |
+
x = self.bn1(x)
|
| 109 |
+
x = self.relu(x)
|
| 110 |
+
x = self.conv2(x)
|
| 111 |
+
x = self.bn2(x)
|
| 112 |
+
x = self.relu(x)
|
| 113 |
+
|
| 114 |
+
x = F.interpolate(x, 8 if self.kernel_size == 3 else 4, mode='nearest')
|
| 115 |
+
y = self.crop_patch(src_bgr, idx, 4, 2 if self.kernel_size == 3 else 0)
|
| 116 |
+
|
| 117 |
+
x = self.conv3(torch.cat([x, y], dim=1))
|
| 118 |
+
x = self.bn3(x)
|
| 119 |
+
x = self.relu(x)
|
| 120 |
+
x = self.conv4(x)
|
| 121 |
+
|
| 122 |
+
out = torch.cat([pha, fgr], dim=1)
|
| 123 |
+
out = F.interpolate(out, (H_full, W_full), mode='bilinear', align_corners=False)
|
| 124 |
+
out = self.replace_patch(out, x, idx)
|
| 125 |
+
pha = out[:, :1]
|
| 126 |
+
fgr = out[:, 1:]
|
| 127 |
+
else:
|
| 128 |
+
pha = F.interpolate(pha, (H_full, W_full), mode='bilinear', align_corners=False)
|
| 129 |
+
fgr = F.interpolate(fgr, (H_full, W_full), mode='bilinear', align_corners=False)
|
| 130 |
+
else:
|
| 131 |
+
x = torch.cat([hid, pha, fgr], dim=1)
|
| 132 |
+
x = F.interpolate(x, (H_half, W_half), mode='bilinear', align_corners=False)
|
| 133 |
+
y = F.interpolate(src_bgr, (H_half, W_half), mode='bilinear', align_corners=False)
|
| 134 |
+
if self.kernel_size == 3:
|
| 135 |
+
x = F.pad(x, (3, 3, 3, 3))
|
| 136 |
+
y = F.pad(y, (3, 3, 3, 3))
|
| 137 |
+
|
| 138 |
+
x = self.conv1(torch.cat([x, y], dim=1))
|
| 139 |
+
x = self.bn1(x)
|
| 140 |
+
x = self.relu(x)
|
| 141 |
+
x = self.conv2(x)
|
| 142 |
+
x = self.bn2(x)
|
| 143 |
+
x = self.relu(x)
|
| 144 |
+
|
| 145 |
+
if self.kernel_size == 3:
|
| 146 |
+
x = F.interpolate(x, (H_full + 4, W_full + 4))
|
| 147 |
+
y = F.pad(src_bgr, (2, 2, 2, 2))
|
| 148 |
+
else:
|
| 149 |
+
x = F.interpolate(x, (H_full, W_full), mode='nearest')
|
| 150 |
+
y = src_bgr
|
| 151 |
+
|
| 152 |
+
x = self.conv3(torch.cat([x, y], dim=1))
|
| 153 |
+
x = self.bn3(x)
|
| 154 |
+
x = self.relu(x)
|
| 155 |
+
x = self.conv4(x)
|
| 156 |
+
|
| 157 |
+
pha = x[:, :1]
|
| 158 |
+
fgr = x[:, 1:]
|
| 159 |
+
ref = torch.ones((src.size(0), 1, H_quat, W_quat), device=src.device, dtype=src.dtype)
|
| 160 |
+
|
| 161 |
+
return pha, fgr, ref
|
| 162 |
+
|
| 163 |
+
def select_refinement_regions(self, err: torch.Tensor):
|
| 164 |
+
"""
|
| 165 |
+
Select refinement regions.
|
| 166 |
+
Input:
|
| 167 |
+
err: error map (B, 1, H, W)
|
| 168 |
+
Output:
|
| 169 |
+
ref: refinement regions (B, 1, H, W). FloatTensor. 1 is selected, 0 is not.
|
| 170 |
+
"""
|
| 171 |
+
if self.mode == 'sampling':
|
| 172 |
+
# Sampling mode.
|
| 173 |
+
b, _, h, w = err.shape
|
| 174 |
+
err = err.view(b, -1)
|
| 175 |
+
idx = err.topk(self.sample_pixels // 16, dim=1, sorted=False).indices
|
| 176 |
+
ref = torch.zeros_like(err)
|
| 177 |
+
ref.scatter_(1, idx, 1.)
|
| 178 |
+
if self.prevent_oversampling:
|
| 179 |
+
ref.mul_(err.gt(0).float())
|
| 180 |
+
ref = ref.view(b, 1, h, w)
|
| 181 |
+
else:
|
| 182 |
+
# Thresholding mode.
|
| 183 |
+
ref = err.gt(self.threshold).float()
|
| 184 |
+
return ref
|
| 185 |
+
|
| 186 |
+
def crop_patch(self,
|
| 187 |
+
x: torch.Tensor,
|
| 188 |
+
idx: Tuple[torch.Tensor, torch.Tensor, torch.Tensor],
|
| 189 |
+
size: int,
|
| 190 |
+
padding: int):
|
| 191 |
+
"""
|
| 192 |
+
Crops selected patches from image given indices.
|
| 193 |
+
|
| 194 |
+
Inputs:
|
| 195 |
+
x: image (B, C, H, W).
|
| 196 |
+
idx: selection indices Tuple[(P,), (P,), (P,),], where the 3 values are (B, H, W) index.
|
| 197 |
+
size: center size of the patch, also stride of the crop.
|
| 198 |
+
padding: expansion size of the patch.
|
| 199 |
+
Output:
|
| 200 |
+
patch: (P, C, h, w), where h = w = size + 2 * padding.
|
| 201 |
+
"""
|
| 202 |
+
if padding != 0:
|
| 203 |
+
x = F.pad(x, (padding,) * 4)
|
| 204 |
+
|
| 205 |
+
if self.patch_crop_method == 'unfold':
|
| 206 |
+
# Use unfold. Best performance for PyTorch and TorchScript.
|
| 207 |
+
return x.permute(0, 2, 3, 1) \
|
| 208 |
+
.unfold(1, size + 2 * padding, size) \
|
| 209 |
+
.unfold(2, size + 2 * padding, size)[idx[0], idx[1], idx[2]]
|
| 210 |
+
elif self.patch_crop_method == 'roi_align':
|
| 211 |
+
# Use roi_align. Best compatibility for ONNX.
|
| 212 |
+
idx = idx[0].type_as(x), idx[1].type_as(x), idx[2].type_as(x)
|
| 213 |
+
b = idx[0]
|
| 214 |
+
x1 = idx[2] * size - 0.5
|
| 215 |
+
y1 = idx[1] * size - 0.5
|
| 216 |
+
x2 = idx[2] * size + size + 2 * padding - 0.5
|
| 217 |
+
y2 = idx[1] * size + size + 2 * padding - 0.5
|
| 218 |
+
boxes = torch.stack([b, x1, y1, x2, y2], dim=1)
|
| 219 |
+
return torchvision.ops.roi_align(x, boxes, size + 2 * padding, sampling_ratio=1)
|
| 220 |
+
else:
|
| 221 |
+
# Use gather. Crops out patches pixel by pixel.
|
| 222 |
+
idx_pix = self.compute_pixel_indices(x, idx, size, padding)
|
| 223 |
+
pat = torch.gather(x.view(-1), 0, idx_pix.view(-1))
|
| 224 |
+
pat = pat.view(-1, x.size(1), size + 2 * padding, size + 2 * padding)
|
| 225 |
+
return pat
|
| 226 |
+
|
| 227 |
+
def replace_patch(self,
|
| 228 |
+
x: torch.Tensor,
|
| 229 |
+
y: torch.Tensor,
|
| 230 |
+
idx: Tuple[torch.Tensor, torch.Tensor, torch.Tensor]):
|
| 231 |
+
"""
|
| 232 |
+
Replaces patches back into image given index.
|
| 233 |
+
|
| 234 |
+
Inputs:
|
| 235 |
+
x: image (B, C, H, W)
|
| 236 |
+
y: patches (P, C, h, w)
|
| 237 |
+
idx: selection indices Tuple[(P,), (P,), (P,)] where the 3 values are (B, H, W) index.
|
| 238 |
+
|
| 239 |
+
Output:
|
| 240 |
+
image: (B, C, H, W), where patches at idx locations are replaced with y.
|
| 241 |
+
"""
|
| 242 |
+
xB, xC, xH, xW = x.shape
|
| 243 |
+
yB, yC, yH, yW = y.shape
|
| 244 |
+
if self.patch_replace_method == 'scatter_nd':
|
| 245 |
+
# Use scatter_nd. Best performance for PyTorch and TorchScript. Replacing patch by patch.
|
| 246 |
+
x = x.view(xB, xC, xH // yH, yH, xW // yW, yW).permute(0, 2, 4, 1, 3, 5)
|
| 247 |
+
x[idx[0], idx[1], idx[2]] = y
|
| 248 |
+
x = x.permute(0, 3, 1, 4, 2, 5).view(xB, xC, xH, xW)
|
| 249 |
+
return x
|
| 250 |
+
else:
|
| 251 |
+
# Use scatter_element. Best compatibility for ONNX. Replacing pixel by pixel.
|
| 252 |
+
idx_pix = self.compute_pixel_indices(x, idx, size=4, padding=0)
|
| 253 |
+
return x.view(-1).scatter_(0, idx_pix.view(-1), y.view(-1)).view(x.shape)
|
| 254 |
+
|
| 255 |
+
def compute_pixel_indices(self,
|
| 256 |
+
x: torch.Tensor,
|
| 257 |
+
idx: Tuple[torch.Tensor, torch.Tensor, torch.Tensor],
|
| 258 |
+
size: int,
|
| 259 |
+
padding: int):
|
| 260 |
+
"""
|
| 261 |
+
Compute selected pixel indices in the tensor.
|
| 262 |
+
Used for crop_method == 'gather' and replace_method == 'scatter_element', which crop and replace pixel by pixel.
|
| 263 |
+
Input:
|
| 264 |
+
x: image: (B, C, H, W)
|
| 265 |
+
idx: selection indices Tuple[(P,), (P,), (P,),], where the 3 values are (B, H, W) index.
|
| 266 |
+
size: center size of the patch, also stride of the crop.
|
| 267 |
+
padding: expansion size of the patch.
|
| 268 |
+
Output:
|
| 269 |
+
idx: (P, C, O, O) long tensor where O is the output size: size + 2 * padding, P is number of patches.
|
| 270 |
+
the element are indices pointing to the input x.view(-1).
|
| 271 |
+
"""
|
| 272 |
+
B, C, H, W = x.shape
|
| 273 |
+
S, P = size, padding
|
| 274 |
+
O = S + 2 * P
|
| 275 |
+
b, y, x = idx
|
| 276 |
+
n = b.size(0)
|
| 277 |
+
c = torch.arange(C)
|
| 278 |
+
o = torch.arange(O)
|
| 279 |
+
idx_pat = (c * H * W).view(C, 1, 1).expand([C, O, O]) + (o * W).view(1, O, 1).expand([C, O, O]) + o.view(1, 1, O).expand([C, O, O])
|
| 280 |
+
idx_loc = b * W * H + y * W * S + x * S
|
| 281 |
+
idx_pix = idx_loc.view(-1, 1, 1, 1).expand([n, C, O, O]) + idx_pat.view(1, C, O, O).expand([n, C, O, O])
|
| 282 |
+
return idx_pix
|
model/resnet.py
ADDED
|
@@ -0,0 +1,48 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
from torch import nn
|
| 2 |
+
from torchvision.models.resnet import ResNet, Bottleneck
|
| 3 |
+
|
| 4 |
+
|
| 5 |
+
class ResNetEncoder(ResNet):
|
| 6 |
+
"""
|
| 7 |
+
ResNetEncoder inherits from torchvision's official ResNet. It is modified to
|
| 8 |
+
use dilation on the last block to maintain output stride 16, and deleted the
|
| 9 |
+
global average pooling layer and the fully connected layer that was originally
|
| 10 |
+
used for classification. The forward method additionally returns the feature
|
| 11 |
+
maps at all resolutions for decoder's use.
|
| 12 |
+
"""
|
| 13 |
+
|
| 14 |
+
layers = {
|
| 15 |
+
'resnet50': [3, 4, 6, 3],
|
| 16 |
+
'resnet101': [3, 4, 23, 3],
|
| 17 |
+
}
|
| 18 |
+
|
| 19 |
+
def __init__(self, in_channels, variant='resnet101', norm_layer=None):
|
| 20 |
+
super().__init__(
|
| 21 |
+
block=Bottleneck,
|
| 22 |
+
layers=self.layers[variant],
|
| 23 |
+
replace_stride_with_dilation=[False, False, True],
|
| 24 |
+
norm_layer=norm_layer)
|
| 25 |
+
|
| 26 |
+
# Replace first conv layer if in_channels doesn't match.
|
| 27 |
+
if in_channels != 3:
|
| 28 |
+
self.conv1 = nn.Conv2d(in_channels, 64, 7, 2, 3, bias=False)
|
| 29 |
+
|
| 30 |
+
# Delete fully-connected layer
|
| 31 |
+
del self.avgpool
|
| 32 |
+
del self.fc
|
| 33 |
+
|
| 34 |
+
def forward(self, x):
|
| 35 |
+
x0 = x # 1/1
|
| 36 |
+
x = self.conv1(x)
|
| 37 |
+
x = self.bn1(x)
|
| 38 |
+
x = self.relu(x)
|
| 39 |
+
x1 = x # 1/2
|
| 40 |
+
x = self.maxpool(x)
|
| 41 |
+
x = self.layer1(x)
|
| 42 |
+
x2 = x # 1/4
|
| 43 |
+
x = self.layer2(x)
|
| 44 |
+
x3 = x # 1/8
|
| 45 |
+
x = self.layer3(x)
|
| 46 |
+
x = self.layer4(x)
|
| 47 |
+
x4 = x # 1/16
|
| 48 |
+
return x4, x3, x2, x1, x0
|
model/utils.py
ADDED
|
@@ -0,0 +1,14 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
def load_matched_state_dict(model, state_dict, print_stats=True):
|
| 2 |
+
"""
|
| 3 |
+
Only loads weights that matched in key and shape. Ignore other weights.
|
| 4 |
+
"""
|
| 5 |
+
num_matched, num_total = 0, 0
|
| 6 |
+
curr_state_dict = model.state_dict()
|
| 7 |
+
for key in curr_state_dict.keys():
|
| 8 |
+
num_total += 1
|
| 9 |
+
if key in state_dict and curr_state_dict[key].shape == state_dict[key].shape:
|
| 10 |
+
curr_state_dict[key] = state_dict[key]
|
| 11 |
+
num_matched += 1
|
| 12 |
+
model.load_state_dict(curr_state_dict)
|
| 13 |
+
if print_stats:
|
| 14 |
+
print(f'Loaded state_dict: {num_matched}/{num_total} matched')
|