import torch import torch.nn as nn from collections import OrderedDict class LayerNormFunction(torch.autograd.Function): @staticmethod def forward(ctx, x, weight, bias, eps): ctx.eps = eps N, C, H, W = x.size() mu = x.mean(1, keepdim=True) var = (x - mu).pow(2).mean(1, keepdim=True) y = (x - mu) / (var + eps).sqrt() ctx.save_for_backward(y, var, weight) y = weight.view(1, C, 1, 1) * y + bias.view(1, C, 1, 1) return y @staticmethod def backward(ctx, grad_output): eps = ctx.eps N, C, H, W = grad_output.size() y, var, weight = ctx.saved_tensors g = grad_output * weight.view(1, C, 1, 1) mean_g = g.mean(dim=1, keepdim=True) mean_gy = (g * y).mean(dim=1, keepdim=True) gx = 1. / torch.sqrt(var + eps) * (g - y * mean_gy - mean_g) return gx, (grad_output * y).sum(dim=3).sum(dim=2).sum(dim=0), grad_output.sum(dim=3).sum(dim=2).sum( dim=0), None class LayerNorm2d(nn.Module): def __init__(self, channels, eps=1e-6): super(LayerNorm2d, self).__init__() self.register_parameter('weight', nn.Parameter(torch.ones(channels))) self.register_parameter('bias', nn.Parameter(torch.zeros(channels))) self.eps = eps def forward(self, x): return LayerNormFunction.apply(x, self.weight, self.bias, self.eps) class SimpleGate(nn.Module): def forward(self, x): x1, x2 = x.chunk(2, dim=1) return x1 * x2 class CABlock(nn.Module): def __init__(self, channels): super(CABlock, self).__init__() self.ca = nn.Sequential( nn.AdaptiveAvgPool2d(1), nn.Conv2d(channels, channels, 1) ) def forward(self, x): return x * self.ca(x) class DualStreamGate(nn.Module): def forward(self, x, y): x1, x2 = x.chunk(2, dim=1) y1, y2 = y.chunk(2, dim=1) return x1 * y2, y1 * x2 class DualStreamSeq(nn.Sequential): def forward(self, x, y=None): y = y if y is not None else x for module in self: x, y = module(x, y) return x, y class DualStreamBlock(nn.Module): def __init__(self, *args): super(DualStreamBlock, self).__init__() self.seq_l = nn.Sequential() self.seq_r = nn.Sequential() if len(args) == 1 and isinstance(args[0], OrderedDict): for key, module in args[0].items(): self.seq_l.add_module(key, module) self.seq_r.add_module(key, module) else: for idx, module in enumerate(args): self.seq_l.add_module(str(idx), module) self.seq_r.add_module(str(idx), module) def forward(self, x, y): return self.seq_l(x), self.seq_r(y) class R2Block(nn.Module): def __init__(self, c): super().__init__() self.block1 = DualStreamSeq( DualStreamBlock( LayerNorm2d(c), nn.Conv2d(c, c * 2, 1), nn.Conv2d(c * 2, c * 2, 3, padding=1, groups=c * 2) ), DualStreamGate(), DualStreamBlock(CABlock(c)), DualStreamBlock(nn.Conv2d(c, c, 1)) ) self.a_l = nn.Parameter(torch.zeros((1, c, 1, 1)), requires_grad=True) self.a_r = nn.Parameter(torch.zeros((1, c, 1, 1)), requires_grad=True) self.block2 = DualStreamSeq( DualStreamBlock( LayerNorm2d(c), nn.Conv2d(c, c * 2, 1) ), DualStreamGate(), DualStreamBlock( nn.Conv2d(c, c, 1) ) ) self.b_l = nn.Parameter(torch.zeros((1, c, 1, 1)), requires_grad=True) self.b_r = nn.Parameter(torch.zeros((1, c, 1, 1)), requires_grad=True) def forward(self, inp_l, inp_r): x, y = self.block1(inp_l, inp_r) x_skip, y_skip = inp_l + x * self.a_l, inp_r + y * self.a_r x, y = self.block2(x_skip, y_skip) out_l, out_r = x_skip + x * self.b_l, y_skip + y * self.b_r return out_l, out_r class SinBlock(nn.Module): def __init__(self, c): super().__init__() self.block1 = nn.Sequential( LayerNorm2d(c), nn.Conv2d(c, c * 2, 1), nn.Conv2d(c * 2, c * 2, 3, padding=1, groups=c * 2), SimpleGate(), CABlock(c), nn.Conv2d(c, c, 1) ) self.block2 = nn.Sequential( LayerNorm2d(c), nn.Conv2d(c, c * 2, 1), SimpleGate(), nn.Conv2d(c, c, 1) ) self.a = nn.Parameter(torch.zeros((1, c, 1, 1)), requires_grad=True) self.b = nn.Parameter(torch.zeros((1, c, 1, 1)), requires_grad=True) def forward(self, inp): x = self.block1(inp) x_skip = inp + x * self.a x = self.block2(x_skip) out = x_skip + x * self.b return out class LRM(nn.Module): def __init__(self, in_channels=48, num_blocks=[2, 4]): super().__init__() self.device = 'cuda' channel = in_channels * 2 self.intro = DualStreamBlock(nn.Conv2d(in_channels, channel, 1)) self.blocks_inter = DualStreamSeq(*[R2Block(channel) for _ in range(num_blocks[0])]) self.blocks_merge = nn.Sequential(*[SinBlock(channel * 2) for _ in range(num_blocks[1])]) self.tail = nn.Sequential( nn.Conv2d(channel * 2, 3, 3, padding=1), nn.Tanh() ) def forward(self, ft, fr): ft, fr = self.intro(ft, fr) ft, fr = self.blocks_inter(ft, fr) fs = self.blocks_merge(torch.cat([ft, fr], dim=1)) out = self.tail(fs) return out