Shangyunle commited on
Commit
70b3486
·
verified ·
1 Parent(s): dd49ace

🍻feat: Upload models

Browse files
model/__init__.py ADDED
@@ -0,0 +1 @@
 
 
1
+ from .model import Base, MattingBase, MattingRefine
model/decoder.py ADDED
@@ -0,0 +1,51 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import torch
2
+ import torch.nn as nn
3
+ import torch.nn.functional as F
4
+
5
+
6
+ class Decoder(nn.Module):
7
+ """
8
+ Decoder upsamples the image by combining the feature maps at all resolutions from the encoder.
9
+
10
+ Input:
11
+ x4: (B, C, H/16, W/16) feature map at 1/16 resolution.
12
+ x3: (B, C, H/8, W/8) feature map at 1/8 resolution.
13
+ x2: (B, C, H/4, W/4) feature map at 1/4 resolution.
14
+ x1: (B, C, H/2, W/2) feature map at 1/2 resolution.
15
+ x0: (B, C, H, W) feature map at full resolution.
16
+
17
+ Output:
18
+ x: (B, C, H, W) upsampled output at full resolution.
19
+ """
20
+
21
+ def __init__(self, channels, feature_channels):
22
+ super().__init__()
23
+ self.conv1 = nn.Conv2d(feature_channels[0] + channels[0], channels[1], 3, padding=1, bias=False)
24
+ self.bn1 = nn.BatchNorm2d(channels[1])
25
+ self.conv2 = nn.Conv2d(feature_channels[1] + channels[1], channels[2], 3, padding=1, bias=False)
26
+ self.bn2 = nn.BatchNorm2d(channels[2])
27
+ self.conv3 = nn.Conv2d(feature_channels[2] + channels[2], channels[3], 3, padding=1, bias=False)
28
+ self.bn3 = nn.BatchNorm2d(channels[3])
29
+ self.conv4 = nn.Conv2d(feature_channels[3] + channels[3], channels[4], 3, padding=1)
30
+ self.relu = nn.ReLU(True)
31
+
32
+ def forward(self, x4, x3, x2, x1, x0):
33
+ x = F.interpolate(x4, size=x3.shape[2:], mode='bilinear', align_corners=False)
34
+ x = torch.cat([x, x3], dim=1)
35
+ x = self.conv1(x)
36
+ x = self.bn1(x)
37
+ x = self.relu(x)
38
+ x = F.interpolate(x, size=x2.shape[2:], mode='bilinear', align_corners=False)
39
+ x = torch.cat([x, x2], dim=1)
40
+ x = self.conv2(x)
41
+ x = self.bn2(x)
42
+ x = self.relu(x)
43
+ x = F.interpolate(x, size=x1.shape[2:], mode='bilinear', align_corners=False)
44
+ x = torch.cat([x, x1], dim=1)
45
+ x = self.conv3(x)
46
+ x = self.bn3(x)
47
+ x = self.relu(x)
48
+ x = F.interpolate(x, size=x0.shape[2:], mode='bilinear', align_corners=False)
49
+ x = torch.cat([x, x0], dim=1)
50
+ x = self.conv4(x)
51
+ return x
model/mobilenet.py ADDED
@@ -0,0 +1,56 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ from torch import nn
2
+ from torchvision.models import MobileNetV2
3
+
4
+
5
+ class MobileNetV2Encoder(MobileNetV2):
6
+ """
7
+ MobileNetV2Encoder inherits from torchvision's official MobileNetV2. It is modified to
8
+ use dilation on the last block to maintain output stride 16, and deleted the
9
+ classifier block that was originally used for classification. The forward method
10
+ additionally returns the feature maps at all resolutions for decoder's use.
11
+ """
12
+
13
+ def __init__(self, in_channels, norm_layer=None):
14
+ super().__init__()
15
+
16
+ # Replace first conv layer if in_channels doesn't match.
17
+ if in_channels != 3:
18
+ self.features[0][0] = nn.Conv2d(in_channels, 32, 3, 2, 1, bias=False)
19
+
20
+ # Remove last block
21
+ self.features = self.features[:-1]
22
+
23
+ # Change to use dilation to maintain output stride = 16
24
+ self.features[14].conv[1][0].stride = (1, 1)
25
+ for feature in self.features[15:]:
26
+ feature.conv[1][0].dilation = (2, 2)
27
+ feature.conv[1][0].padding = (2, 2)
28
+
29
+ # Delete classifier
30
+ del self.classifier
31
+
32
+ def forward(self, x):
33
+ x0 = x # 1/1
34
+ x = self.features[0](x)
35
+ x = self.features[1](x)
36
+ x1 = x # 1/2
37
+ x = self.features[2](x)
38
+ x = self.features[3](x)
39
+ x2 = x # 1/4
40
+ x = self.features[4](x)
41
+ x = self.features[5](x)
42
+ x = self.features[6](x)
43
+ x3 = x # 1/8
44
+ x = self.features[7](x)
45
+ x = self.features[8](x)
46
+ x = self.features[9](x)
47
+ x = self.features[10](x)
48
+ x = self.features[11](x)
49
+ x = self.features[12](x)
50
+ x = self.features[13](x)
51
+ x = self.features[14](x)
52
+ x = self.features[15](x)
53
+ x = self.features[16](x)
54
+ x = self.features[17](x)
55
+ x4 = x # 1/16
56
+ return x4, x3, x2, x1, x0
model/model.py ADDED
@@ -0,0 +1,196 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import torch
2
+ from torch import nn
3
+ from torch.nn import functional as F
4
+ from torchvision.models.segmentation.deeplabv3 import ASPP
5
+
6
+ from .decoder import Decoder
7
+ from .mobilenet import MobileNetV2Encoder
8
+ from .refiner import Refiner
9
+ from .resnet import ResNetEncoder
10
+ from .utils import load_matched_state_dict
11
+
12
+
13
+ class Base(nn.Module):
14
+ """
15
+ A generic implementation of the base encoder-decoder network inspired by DeepLab.
16
+ Accepts arbitrary channels for input and output.
17
+ """
18
+
19
+ def __init__(self, backbone: str, in_channels: int, out_channels: int):
20
+ super().__init__()
21
+ assert backbone in ["resnet50", "resnet101", "mobilenetv2"]
22
+ if backbone in ['resnet50', 'resnet101']:
23
+ self.backbone = ResNetEncoder(in_channels, variant=backbone)
24
+ self.aspp = ASPP(2048, [3, 6, 9])
25
+ self.decoder = Decoder([256, 128, 64, 48, out_channels], [512, 256, 64, in_channels])
26
+ else:
27
+ self.backbone = MobileNetV2Encoder(in_channels)
28
+ self.aspp = ASPP(320, [3, 6, 9])
29
+ self.decoder = Decoder([256, 128, 64, 48, out_channels], [32, 24, 16, in_channels])
30
+
31
+ def forward(self, x):
32
+ x, *shortcuts = self.backbone(x)
33
+ x = self.aspp(x)
34
+ x = self.decoder(x, *shortcuts)
35
+ return x
36
+
37
+ def load_pretrained_deeplabv3_state_dict(self, state_dict, print_stats=True):
38
+ # Pretrained DeepLabV3 models are provided by <https://github.com/VainF/DeepLabV3Plus-Pytorch>.
39
+ # This method converts and loads their pretrained state_dict to match with our model structure.
40
+ # This method is not needed if you are not planning to train from deeplab weights.
41
+ # Use load_state_dict() for normal weight loading.
42
+
43
+ # Convert state_dict naming for aspp module
44
+ state_dict = {k.replace('classifier.classifier.0', 'aspp'): v for k, v in state_dict.items()}
45
+
46
+ if isinstance(self.backbone, ResNetEncoder):
47
+ # ResNet backbone does not need change.
48
+ load_matched_state_dict(self, state_dict, print_stats)
49
+ else:
50
+ # Change MobileNetV2 backbone to state_dict format, then change back after loading.
51
+ backbone_features = self.backbone.features
52
+ self.backbone.low_level_features = backbone_features[:4]
53
+ self.backbone.high_level_features = backbone_features[4:]
54
+ del self.backbone.features
55
+ load_matched_state_dict(self, state_dict, print_stats)
56
+ self.backbone.features = backbone_features
57
+ del self.backbone.low_level_features
58
+ del self.backbone.high_level_features
59
+
60
+
61
+ class MattingBase(Base):
62
+ """
63
+ MattingBase is used to produce coarse global results at a lower resolution.
64
+ MattingBase extends Base.
65
+
66
+ Args:
67
+ backbone: ["resnet50", "resnet101", "mobilenetv2"]
68
+
69
+ Input:
70
+ src: (B, 3, H, W) the source image. Channels are RGB values normalized to 0 ~ 1.
71
+ bgr: (B, 3, H, W) the background image . Channels are RGB values normalized to 0 ~ 1.
72
+
73
+ Output:
74
+ pha: (B, 1, H, W) the alpha prediction. Normalized to 0 ~ 1.
75
+ fgr: (B, 3, H, W) the foreground prediction. Channels are RGB values normalized to 0 ~ 1.
76
+ err: (B, 1, H, W) the error prediction. Normalized to 0 ~ 1.
77
+ hid: (B, 32, H, W) the hidden encoding. Used for connecting refiner module.
78
+
79
+ Example:
80
+ model = MattingBase(backbone='resnet50')
81
+
82
+ pha, fgr, err, hid = model(src, bgr) # for training
83
+ pha, fgr = model(src, bgr)[:2] # for inference
84
+ """
85
+
86
+ def __init__(self, backbone: str):
87
+ super().__init__(backbone, in_channels=6, out_channels=(1 + 3 + 1 + 32))
88
+
89
+ def forward(self, src, bgr):
90
+ x = torch.cat([src, bgr], dim=1)
91
+ x, *shortcuts = self.backbone(x)
92
+ x = self.aspp(x)
93
+ x = self.decoder(x, *shortcuts)
94
+ pha = x[:, 0:1].clamp_(0., 1.)
95
+ fgr = x[:, 1:4].add(src).clamp_(0., 1.)
96
+ err = x[:, 4:5].clamp_(0., 1.)
97
+ hid = x[:, 5: ].relu_()
98
+ return pha, fgr, err, hid
99
+
100
+
101
+ class MattingRefine(MattingBase):
102
+ """
103
+ MattingRefine includes the refiner module to upsample coarse result to full resolution.
104
+ MattingRefine extends MattingBase.
105
+
106
+ Args:
107
+ backbone: ["resnet50", "resnet101", "mobilenetv2"]
108
+ backbone_scale: The image downsample scale for passing through backbone, default 1/4 or 0.25.
109
+ Must not be greater than 1/2.
110
+ refine_mode: refine area selection mode. Options:
111
+ "full" - No area selection, refine everywhere using regular Conv2d.
112
+ "sampling" - Refine fixed amount of pixels ranked by the top most errors.
113
+ "thresholding" - Refine varying amount of pixels that has more error than the threshold.
114
+ refine_sample_pixels: number of pixels to refine. Only used when mode == "sampling".
115
+ refine_threshold: error threshold ranged from 0 ~ 1. Refine where err > threshold. Only used when mode == "thresholding".
116
+ refine_kernel_size: the refiner's convolutional kernel size. Options: [1, 3]
117
+ refine_prevent_oversampling: prevent sampling more pixels than needed for sampling mode. Set False only for speedtest.
118
+
119
+ Input:
120
+ src: (B, 3, H, W) the source image. Channels are RGB values normalized to 0 ~ 1.
121
+ bgr: (B, 3, H, W) the background image. Channels are RGB values normalized to 0 ~ 1.
122
+
123
+ Output:
124
+ pha: (B, 1, H, W) the alpha prediction. Normalized to 0 ~ 1.
125
+ fgr: (B, 3, H, W) the foreground prediction. Channels are RGB values normalized to 0 ~ 1.
126
+ pha_sm: (B, 1, Hc, Wc) the coarse alpha prediction from matting base. Normalized to 0 ~ 1.
127
+ fgr_sm: (B, 3, Hc, Hc) the coarse foreground prediction from matting base. Normalized to 0 ~ 1.
128
+ err_sm: (B, 1, Hc, Wc) the coarse error prediction from matting base. Normalized to 0 ~ 1.
129
+ ref_sm: (B, 1, H/4, H/4) the quarter resolution refinement map. 1 indicates refined 4x4 patch locations.
130
+
131
+ Example:
132
+ model = MattingRefine(backbone='resnet50', backbone_scale=1/4, refine_mode='sampling', refine_sample_pixels=80_000)
133
+ model = MattingRefine(backbone='resnet50', backbone_scale=1/4, refine_mode='thresholding', refine_threshold=0.1)
134
+ model = MattingRefine(backbone='resnet50', backbone_scale=1/4, refine_mode='full')
135
+
136
+ pha, fgr, pha_sm, fgr_sm, err_sm, ref_sm = model(src, bgr) # for training
137
+ pha, fgr = model(src, bgr)[:2] # for inference
138
+ """
139
+
140
+ def __init__(self,
141
+ backbone: str,
142
+ backbone_scale: float = 1/4,
143
+ refine_mode: str = 'sampling',
144
+ refine_sample_pixels: int = 80_000,
145
+ refine_threshold: float = 0.1,
146
+ refine_kernel_size: int = 3,
147
+ refine_prevent_oversampling: bool = True,
148
+ refine_patch_crop_method: str = 'unfold',
149
+ refine_patch_replace_method: str = 'scatter_nd'):
150
+ assert backbone_scale <= 1/2, 'backbone_scale should not be greater than 1/2'
151
+ super().__init__(backbone)
152
+ self.backbone_scale = backbone_scale
153
+ self.refiner = Refiner(refine_mode,
154
+ refine_sample_pixels,
155
+ refine_threshold,
156
+ refine_kernel_size,
157
+ refine_prevent_oversampling,
158
+ refine_patch_crop_method,
159
+ refine_patch_replace_method)
160
+
161
+ def forward(self, src, bgr):
162
+ assert src.size() == bgr.size(), 'src and bgr must have the same shape'
163
+ assert src.size(2) // 4 * 4 == src.size(2) and src.size(3) // 4 * 4 == src.size(3), \
164
+ 'src and bgr must have width and height that are divisible by 4'
165
+
166
+ # Downsample src and bgr for backbone
167
+ src_sm = F.interpolate(src,
168
+ scale_factor=self.backbone_scale,
169
+ mode='bilinear',
170
+ align_corners=False,
171
+ recompute_scale_factor=True)
172
+ bgr_sm = F.interpolate(bgr,
173
+ scale_factor=self.backbone_scale,
174
+ mode='bilinear',
175
+ align_corners=False,
176
+ recompute_scale_factor=True)
177
+
178
+ # Base
179
+ x = torch.cat([src_sm, bgr_sm], dim=1)
180
+ x, *shortcuts = self.backbone(x)
181
+ x = self.aspp(x)
182
+ x = self.decoder(x, *shortcuts)
183
+ pha_sm = x[:, 0:1].clamp_(0., 1.)
184
+ fgr_sm = x[:, 1:4]
185
+ err_sm = x[:, 4:5].clamp_(0., 1.)
186
+ hid_sm = x[:, 5: ].relu_()
187
+
188
+ # Refiner
189
+ pha, fgr, ref_sm = self.refiner(src, bgr, pha_sm, fgr_sm, err_sm, hid_sm)
190
+
191
+ # Clamp outputs
192
+ pha = pha.clamp_(0., 1.)
193
+ fgr = fgr.add_(src).clamp_(0., 1.)
194
+ fgr_sm = src_sm.add_(fgr_sm).clamp_(0., 1.)
195
+
196
+ return pha, fgr, pha_sm, fgr_sm, err_sm, ref_sm
model/refiner.py ADDED
@@ -0,0 +1,282 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import torch
2
+ import torchvision
3
+ from torch import nn
4
+ from torch.nn import functional as F
5
+ from typing import Tuple
6
+
7
+
8
+ class Refiner(nn.Module):
9
+ """
10
+ Refiner refines the coarse output to full resolution.
11
+
12
+ Args:
13
+ mode: area selection mode. Options:
14
+ "full" - No area selection, refine everywhere using regular Conv2d.
15
+ "sampling" - Refine fixed amount of pixels ranked by the top most errors.
16
+ "thresholding" - Refine varying amount of pixels that have greater error than the threshold.
17
+ sample_pixels: number of pixels to refine. Only used when mode == "sampling".
18
+ threshold: error threshold ranged from 0 ~ 1. Refine where err > threshold. Only used when mode == "thresholding".
19
+ kernel_size: The convolution kernel_size. Options: [1, 3]
20
+ prevent_oversampling: True for regular cases, False for speedtest.
21
+
22
+ Compatibility Args:
23
+ patch_crop_method: the method for cropping patches. Options:
24
+ "unfold" - Best performance for PyTorch and TorchScript.
25
+ "roi_align" - Another way for croping patches.
26
+ "gather" - Another way for croping patches.
27
+ patch_replace_method: the method for replacing patches. Options:
28
+ "scatter_nd" - Best performance for PyTorch and TorchScript.
29
+ "scatter_element" - Another way for replacing patches.
30
+
31
+ Input:
32
+ src: (B, 3, H, W) full resolution source image.
33
+ bgr: (B, 3, H, W) full resolution background image.
34
+ pha: (B, 1, Hc, Wc) coarse alpha prediction.
35
+ fgr: (B, 3, Hc, Wc) coarse foreground residual prediction.
36
+ err: (B, 1, Hc, Hc) coarse error prediction.
37
+ hid: (B, 32, Hc, Hc) coarse hidden encoding.
38
+
39
+ Output:
40
+ pha: (B, 1, H, W) full resolution alpha prediction.
41
+ fgr: (B, 3, H, W) full resolution foreground residual prediction.
42
+ ref: (B, 1, H/4, W/4) quarter resolution refinement selection map. 1 indicates refined 4x4 patch locations.
43
+ """
44
+
45
+ # For TorchScript export optimization.
46
+ __constants__ = ['kernel_size', 'patch_crop_method', 'patch_replace_method']
47
+
48
+ def __init__(self,
49
+ mode: str,
50
+ sample_pixels: int,
51
+ threshold: float,
52
+ kernel_size: int = 3,
53
+ prevent_oversampling: bool = True,
54
+ patch_crop_method: str = 'unfold',
55
+ patch_replace_method: str = 'scatter_nd'):
56
+ super().__init__()
57
+ assert mode in ['full', 'sampling', 'thresholding']
58
+ assert kernel_size in [1, 3]
59
+ assert patch_crop_method in ['unfold', 'roi_align', 'gather']
60
+ assert patch_replace_method in ['scatter_nd', 'scatter_element']
61
+
62
+ self.mode = mode
63
+ self.sample_pixels = sample_pixels
64
+ self.threshold = threshold
65
+ self.kernel_size = kernel_size
66
+ self.prevent_oversampling = prevent_oversampling
67
+ self.patch_crop_method = patch_crop_method
68
+ self.patch_replace_method = patch_replace_method
69
+
70
+ channels = [32, 24, 16, 12, 4]
71
+ self.conv1 = nn.Conv2d(channels[0] + 6 + 4, channels[1], kernel_size, bias=False)
72
+ self.bn1 = nn.BatchNorm2d(channels[1])
73
+ self.conv2 = nn.Conv2d(channels[1], channels[2], kernel_size, bias=False)
74
+ self.bn2 = nn.BatchNorm2d(channels[2])
75
+ self.conv3 = nn.Conv2d(channels[2] + 6, channels[3], kernel_size, bias=False)
76
+ self.bn3 = nn.BatchNorm2d(channels[3])
77
+ self.conv4 = nn.Conv2d(channels[3], channels[4], kernel_size, bias=True)
78
+ self.relu = nn.ReLU(True)
79
+
80
+ def forward(self,
81
+ src: torch.Tensor,
82
+ bgr: torch.Tensor,
83
+ pha: torch.Tensor,
84
+ fgr: torch.Tensor,
85
+ err: torch.Tensor,
86
+ hid: torch.Tensor):
87
+ H_full, W_full = src.shape[2:]
88
+ H_half, W_half = H_full // 2, W_full // 2
89
+ H_quat, W_quat = H_full // 4, W_full // 4
90
+
91
+ src_bgr = torch.cat([src, bgr], dim=1)
92
+
93
+ if self.mode != 'full':
94
+ err = F.interpolate(err, (H_quat, W_quat), mode='bilinear', align_corners=False)
95
+ ref = self.select_refinement_regions(err)
96
+ idx = torch.nonzero(ref.squeeze(1))
97
+ idx = idx[:, 0], idx[:, 1], idx[:, 2]
98
+
99
+ if idx[0].size(0) > 0:
100
+ x = torch.cat([hid, pha, fgr], dim=1)
101
+ x = F.interpolate(x, (H_half, W_half), mode='bilinear', align_corners=False)
102
+ x = self.crop_patch(x, idx, 2, 3 if self.kernel_size == 3 else 0)
103
+
104
+ y = F.interpolate(src_bgr, (H_half, W_half), mode='bilinear', align_corners=False)
105
+ y = self.crop_patch(y, idx, 2, 3 if self.kernel_size == 3 else 0)
106
+
107
+ x = self.conv1(torch.cat([x, y], dim=1))
108
+ x = self.bn1(x)
109
+ x = self.relu(x)
110
+ x = self.conv2(x)
111
+ x = self.bn2(x)
112
+ x = self.relu(x)
113
+
114
+ x = F.interpolate(x, 8 if self.kernel_size == 3 else 4, mode='nearest')
115
+ y = self.crop_patch(src_bgr, idx, 4, 2 if self.kernel_size == 3 else 0)
116
+
117
+ x = self.conv3(torch.cat([x, y], dim=1))
118
+ x = self.bn3(x)
119
+ x = self.relu(x)
120
+ x = self.conv4(x)
121
+
122
+ out = torch.cat([pha, fgr], dim=1)
123
+ out = F.interpolate(out, (H_full, W_full), mode='bilinear', align_corners=False)
124
+ out = self.replace_patch(out, x, idx)
125
+ pha = out[:, :1]
126
+ fgr = out[:, 1:]
127
+ else:
128
+ pha = F.interpolate(pha, (H_full, W_full), mode='bilinear', align_corners=False)
129
+ fgr = F.interpolate(fgr, (H_full, W_full), mode='bilinear', align_corners=False)
130
+ else:
131
+ x = torch.cat([hid, pha, fgr], dim=1)
132
+ x = F.interpolate(x, (H_half, W_half), mode='bilinear', align_corners=False)
133
+ y = F.interpolate(src_bgr, (H_half, W_half), mode='bilinear', align_corners=False)
134
+ if self.kernel_size == 3:
135
+ x = F.pad(x, (3, 3, 3, 3))
136
+ y = F.pad(y, (3, 3, 3, 3))
137
+
138
+ x = self.conv1(torch.cat([x, y], dim=1))
139
+ x = self.bn1(x)
140
+ x = self.relu(x)
141
+ x = self.conv2(x)
142
+ x = self.bn2(x)
143
+ x = self.relu(x)
144
+
145
+ if self.kernel_size == 3:
146
+ x = F.interpolate(x, (H_full + 4, W_full + 4))
147
+ y = F.pad(src_bgr, (2, 2, 2, 2))
148
+ else:
149
+ x = F.interpolate(x, (H_full, W_full), mode='nearest')
150
+ y = src_bgr
151
+
152
+ x = self.conv3(torch.cat([x, y], dim=1))
153
+ x = self.bn3(x)
154
+ x = self.relu(x)
155
+ x = self.conv4(x)
156
+
157
+ pha = x[:, :1]
158
+ fgr = x[:, 1:]
159
+ ref = torch.ones((src.size(0), 1, H_quat, W_quat), device=src.device, dtype=src.dtype)
160
+
161
+ return pha, fgr, ref
162
+
163
+ def select_refinement_regions(self, err: torch.Tensor):
164
+ """
165
+ Select refinement regions.
166
+ Input:
167
+ err: error map (B, 1, H, W)
168
+ Output:
169
+ ref: refinement regions (B, 1, H, W). FloatTensor. 1 is selected, 0 is not.
170
+ """
171
+ if self.mode == 'sampling':
172
+ # Sampling mode.
173
+ b, _, h, w = err.shape
174
+ err = err.view(b, -1)
175
+ idx = err.topk(self.sample_pixels // 16, dim=1, sorted=False).indices
176
+ ref = torch.zeros_like(err)
177
+ ref.scatter_(1, idx, 1.)
178
+ if self.prevent_oversampling:
179
+ ref.mul_(err.gt(0).float())
180
+ ref = ref.view(b, 1, h, w)
181
+ else:
182
+ # Thresholding mode.
183
+ ref = err.gt(self.threshold).float()
184
+ return ref
185
+
186
+ def crop_patch(self,
187
+ x: torch.Tensor,
188
+ idx: Tuple[torch.Tensor, torch.Tensor, torch.Tensor],
189
+ size: int,
190
+ padding: int):
191
+ """
192
+ Crops selected patches from image given indices.
193
+
194
+ Inputs:
195
+ x: image (B, C, H, W).
196
+ idx: selection indices Tuple[(P,), (P,), (P,),], where the 3 values are (B, H, W) index.
197
+ size: center size of the patch, also stride of the crop.
198
+ padding: expansion size of the patch.
199
+ Output:
200
+ patch: (P, C, h, w), where h = w = size + 2 * padding.
201
+ """
202
+ if padding != 0:
203
+ x = F.pad(x, (padding,) * 4)
204
+
205
+ if self.patch_crop_method == 'unfold':
206
+ # Use unfold. Best performance for PyTorch and TorchScript.
207
+ return x.permute(0, 2, 3, 1) \
208
+ .unfold(1, size + 2 * padding, size) \
209
+ .unfold(2, size + 2 * padding, size)[idx[0], idx[1], idx[2]]
210
+ elif self.patch_crop_method == 'roi_align':
211
+ # Use roi_align. Best compatibility for ONNX.
212
+ idx = idx[0].type_as(x), idx[1].type_as(x), idx[2].type_as(x)
213
+ b = idx[0]
214
+ x1 = idx[2] * size - 0.5
215
+ y1 = idx[1] * size - 0.5
216
+ x2 = idx[2] * size + size + 2 * padding - 0.5
217
+ y2 = idx[1] * size + size + 2 * padding - 0.5
218
+ boxes = torch.stack([b, x1, y1, x2, y2], dim=1)
219
+ return torchvision.ops.roi_align(x, boxes, size + 2 * padding, sampling_ratio=1)
220
+ else:
221
+ # Use gather. Crops out patches pixel by pixel.
222
+ idx_pix = self.compute_pixel_indices(x, idx, size, padding)
223
+ pat = torch.gather(x.view(-1), 0, idx_pix.view(-1))
224
+ pat = pat.view(-1, x.size(1), size + 2 * padding, size + 2 * padding)
225
+ return pat
226
+
227
+ def replace_patch(self,
228
+ x: torch.Tensor,
229
+ y: torch.Tensor,
230
+ idx: Tuple[torch.Tensor, torch.Tensor, torch.Tensor]):
231
+ """
232
+ Replaces patches back into image given index.
233
+
234
+ Inputs:
235
+ x: image (B, C, H, W)
236
+ y: patches (P, C, h, w)
237
+ idx: selection indices Tuple[(P,), (P,), (P,)] where the 3 values are (B, H, W) index.
238
+
239
+ Output:
240
+ image: (B, C, H, W), where patches at idx locations are replaced with y.
241
+ """
242
+ xB, xC, xH, xW = x.shape
243
+ yB, yC, yH, yW = y.shape
244
+ if self.patch_replace_method == 'scatter_nd':
245
+ # Use scatter_nd. Best performance for PyTorch and TorchScript. Replacing patch by patch.
246
+ x = x.view(xB, xC, xH // yH, yH, xW // yW, yW).permute(0, 2, 4, 1, 3, 5)
247
+ x[idx[0], idx[1], idx[2]] = y
248
+ x = x.permute(0, 3, 1, 4, 2, 5).view(xB, xC, xH, xW)
249
+ return x
250
+ else:
251
+ # Use scatter_element. Best compatibility for ONNX. Replacing pixel by pixel.
252
+ idx_pix = self.compute_pixel_indices(x, idx, size=4, padding=0)
253
+ return x.view(-1).scatter_(0, idx_pix.view(-1), y.view(-1)).view(x.shape)
254
+
255
+ def compute_pixel_indices(self,
256
+ x: torch.Tensor,
257
+ idx: Tuple[torch.Tensor, torch.Tensor, torch.Tensor],
258
+ size: int,
259
+ padding: int):
260
+ """
261
+ Compute selected pixel indices in the tensor.
262
+ Used for crop_method == 'gather' and replace_method == 'scatter_element', which crop and replace pixel by pixel.
263
+ Input:
264
+ x: image: (B, C, H, W)
265
+ idx: selection indices Tuple[(P,), (P,), (P,),], where the 3 values are (B, H, W) index.
266
+ size: center size of the patch, also stride of the crop.
267
+ padding: expansion size of the patch.
268
+ Output:
269
+ idx: (P, C, O, O) long tensor where O is the output size: size + 2 * padding, P is number of patches.
270
+ the element are indices pointing to the input x.view(-1).
271
+ """
272
+ B, C, H, W = x.shape
273
+ S, P = size, padding
274
+ O = S + 2 * P
275
+ b, y, x = idx
276
+ n = b.size(0)
277
+ c = torch.arange(C)
278
+ o = torch.arange(O)
279
+ idx_pat = (c * H * W).view(C, 1, 1).expand([C, O, O]) + (o * W).view(1, O, 1).expand([C, O, O]) + o.view(1, 1, O).expand([C, O, O])
280
+ idx_loc = b * W * H + y * W * S + x * S
281
+ idx_pix = idx_loc.view(-1, 1, 1, 1).expand([n, C, O, O]) + idx_pat.view(1, C, O, O).expand([n, C, O, O])
282
+ return idx_pix
model/resnet.py ADDED
@@ -0,0 +1,48 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ from torch import nn
2
+ from torchvision.models.resnet import ResNet, Bottleneck
3
+
4
+
5
+ class ResNetEncoder(ResNet):
6
+ """
7
+ ResNetEncoder inherits from torchvision's official ResNet. It is modified to
8
+ use dilation on the last block to maintain output stride 16, and deleted the
9
+ global average pooling layer and the fully connected layer that was originally
10
+ used for classification. The forward method additionally returns the feature
11
+ maps at all resolutions for decoder's use.
12
+ """
13
+
14
+ layers = {
15
+ 'resnet50': [3, 4, 6, 3],
16
+ 'resnet101': [3, 4, 23, 3],
17
+ }
18
+
19
+ def __init__(self, in_channels, variant='resnet101', norm_layer=None):
20
+ super().__init__(
21
+ block=Bottleneck,
22
+ layers=self.layers[variant],
23
+ replace_stride_with_dilation=[False, False, True],
24
+ norm_layer=norm_layer)
25
+
26
+ # Replace first conv layer if in_channels doesn't match.
27
+ if in_channels != 3:
28
+ self.conv1 = nn.Conv2d(in_channels, 64, 7, 2, 3, bias=False)
29
+
30
+ # Delete fully-connected layer
31
+ del self.avgpool
32
+ del self.fc
33
+
34
+ def forward(self, x):
35
+ x0 = x # 1/1
36
+ x = self.conv1(x)
37
+ x = self.bn1(x)
38
+ x = self.relu(x)
39
+ x1 = x # 1/2
40
+ x = self.maxpool(x)
41
+ x = self.layer1(x)
42
+ x2 = x # 1/4
43
+ x = self.layer2(x)
44
+ x3 = x # 1/8
45
+ x = self.layer3(x)
46
+ x = self.layer4(x)
47
+ x4 = x # 1/16
48
+ return x4, x3, x2, x1, x0
model/utils.py ADDED
@@ -0,0 +1,14 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ def load_matched_state_dict(model, state_dict, print_stats=True):
2
+ """
3
+ Only loads weights that matched in key and shape. Ignore other weights.
4
+ """
5
+ num_matched, num_total = 0, 0
6
+ curr_state_dict = model.state_dict()
7
+ for key in curr_state_dict.keys():
8
+ num_total += 1
9
+ if key in state_dict and curr_state_dict[key].shape == state_dict[key].shape:
10
+ curr_state_dict[key] = state_dict[key]
11
+ num_matched += 1
12
+ model.load_state_dict(curr_state_dict)
13
+ if print_stats:
14
+ print(f'Loaded state_dict: {num_matched}/{num_total} matched')